diff --git a/.gitignore b/.gitignore index f71c63d..8be0d69 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ dist/* # Ignore debug-dev dumped Binary files (Files saved to cwd for fast examination) src/scripts/*.bin ignore/texconv.exe +tests/relic/sga/sources.json diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..976ba02 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,2 @@ +[mypy] +ignore_missing_imports = True diff --git a/setup.cfg b/setup.cfg index aa09d57..0c4fc20 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,7 +21,7 @@ classifiers = include_package_data = True package_dir = = src -packages = find: +packages = find_namespace: python_requires = >=3.9 diff --git a/src/relic/chunky/chunk/header.py b/src/relic/chunky/chunk/header.py index 68c84ff..aa7f80d 100644 --- a/src/relic/chunky/chunk/header.py +++ b/src/relic/chunky/chunk/header.py @@ -8,7 +8,6 @@ from serialization_tools.vstruct import VStruct from ..chunky.header import ChunkyVersion -from ...common import VersionLike, VersionError class ChunkType(Enum): diff --git a/src/relic/chunky/chunky/header.py b/src/relic/chunky/chunky/header.py index 05b75b4..92fc258 100644 --- a/src/relic/chunky/chunky/header.py +++ b/src/relic/chunky/chunky/header.py @@ -1,23 +1,23 @@ from __future__ import annotations from dataclasses import dataclass +from enum import Enum from typing import BinaryIO, Dict, Type from serialization_tools.magic import MagicWordIO, MagicWord from serialization_tools.structx import Struct -from relic.common import VersionEnum, Version, VersionLike, VersionError - +VersionEnum = Version = VersionLike = None ChunkyVersionLayout = Struct("< 2L") -class ChunkyVersion(VersionEnum): +class ChunkyVersion(Enum): Unsupported = None - v0101 = Version(1, 1) + v0101 = None # Version(1, 1) Dow = v0101 # ALIAS for Prettiness - v0301 = Version(3, 1) + v0301 = None # Version(3, 1) Dow2 = v0301 # ALIAS for Prettiness - v4010 = Version(4, 1) + v4010 = None # Version(4, 1) @classmethod def unpack_version(cls, stream: BinaryIO) -> Version: diff --git a/src/relic/common.py b/src/relic/common.py deleted file mode 100644 index 3c2a745..0000000 --- a/src/relic/common.py +++ /dev/null @@ -1,88 +0,0 @@ -from dataclasses import dataclass -from enum import Enum -from typing import Union, Optional, Type, List - -from serialization_tools.structx import Struct - - -class ListableEnum(Enum): - @classmethod - def list(cls): - return list(map(lambda c: c.value, cls)) - - @staticmethod - def get_list(cls: Type[Enum]): - return list(map(lambda c: c.value, cls)) - - -class VersionEnum(ListableEnum): - def __eq__(self, other): - if isinstance(other, VersionEnum): - return self.value == other.value - elif isinstance(other, Version): - return self.value == other - else: - super().__eq__(other) - - def __ne__(self, other): - return not (self == other) - - def __hash__(self): - return self.value.__hash__() - - -@dataclass -class Version: - major: int - minor: Optional[int] = 0 - - _32 = Struct("< H H") - _64 = Struct("< L L") - - def __str__(self) -> str: - return f"Version {self.major}.{self.minor}" - - def __eq__(self, other): - if other is None: - return False - elif isinstance(other, VersionEnum): - return self.major == other.value.major and self.minor == other.value.minor - elif isinstance(other, Version): - return self.major == other.major and self.minor == other.minor - else: - return super().__eq__(other) - - def __ne__(self, other): - return not (self == other) - - def __hash__(self): - # Realistically; Version will always be <256 - # But we could manually set it to something much bigger by accident; and that may cause collisions - return self.major << 32 + self.minor - - -VersionLike = Union[Version, VersionEnum] - - -class VersionError(Exception): - def __init__(self, version: VersionLike = None, supported: Union[List[Version], Version, Type[VersionEnum], VersionEnum] = None, *args): - super().__init__(*args) - self.version = version - if supported: - if issubclass(supported, VersionEnum): - supported = ListableEnum.get_list(supported) - elif not isinstance(supported, list): - supported = [supported] - self.supported = supported - - def __str__(self): - msg = "Unexpected version" - if self.version or self.supported: - msg += ";" - if self.version: - msg += f" got {repr(self.version)}" - if self.version and self.supported: - msg += "," - if self.supported: - msg += f" expected {repr(self.supported)}" - return msg + "!" diff --git a/src/relic/config.py b/src/relic/config.py deleted file mode 100644 index c956374..0000000 --- a/src/relic/config.py +++ /dev/null @@ -1,115 +0,0 @@ -from enum import Enum -from os import PathLike -from os.path import join, exists, abspath -from pathlib import Path, PurePath -from typing import Optional, Iterable, Tuple, Set - -import serialization_tools.common_directories - -dll_folder = abspath(join(__file__, "..\\..\\..\\Required EXEs")) -aifc_decoder_path = join(dll_folder, "dec.exe") -aifc_encoder_path = join(dll_folder, "enc.exe") -texconv_path = join(dll_folder, "texconv.exe") - - -def get_path_to_steam_library(steam_directory: PathLike = None) -> Path: - steam_directory = (PurePath(steam_directory) if steam_directory else steam_directory) or archive_tools.common_directories.get_steam_install_dir() - return steam_directory / "steamapps" / "common" - - -class DowIIIGame(Enum): - BaseGame = 0 - - -class DowIIGame(Enum): - Retribution = 2 - ChaosRising = 1 - BaseGame = 0 - - -class DowGame(Enum): - SoulStorm = 4 - DarkCrusade = 3 - WinterAssault = 2 - Gold = 1 - BaseGame = 0 - - -dow_game_paths = { - DowIIIGame.BaseGame: "Dawn of War III", - - DowIIGame.Retribution: "Dawn of War II - Retribution", - - DowGame.SoulStorm: "Dawn of War Soulstorm", - DowGame.DarkCrusade: "Dawn of War Dark Crusade", - DowGame.WinterAssault: "Dawn of War Winter Assault", - DowGame.Gold: "Dawn of War Gold", - # DowGame.BaseGame:"Dawn of War", # The original dawn of war probably doesn't include 'Gold', IDK what it is specifically but this would be my first guess -} - - -def get_dow_root_directories() -> Iterable[Tuple[DowGame, Path]]: - steam_path = get_path_to_steam_library() - for game, partial_path in dow_game_paths.items(): - path = steam_path / partial_path - if exists(path): - yield game, path - - -def filter_unique_dow_game(dow_root_directories: Iterable[Tuple[DowGame, Path]]) -> Iterable[Tuple[DowGame, Path]]: - unique: Set[DowGame] = set() - for game, path in dow_root_directories: - if game in unique: - continue - yield game, path - unique.add(game) - - -# Allows us to get the most -# up-to-date dump of all assets: -# Gold (I believe) only contains Space Marines, Orks, Chaos, & Eldar -# Winter Assault Adds Imperial Guard -# Dark Crusade Adds Tau & Necrons -# SoulStorm Adds Dark Eldar & Sisters Of Battle -# If we only want to dump ONE game; we'd want to dump the latest to get all the assets from the previous one -# Except for campaign assets; which are unique to each install -# For Campaign assets, use get_unique and dump each to a separate directory (or order the dumps such that later games come after earlier games) -def filter_latest_dow_game(dow_root_directories: Iterable[Tuple[DowGame, Path]], series: Enum = DowGame) -> Optional[Tuple[DowGame, Path]]: - latest = latest_path = None - for game, path in dow_root_directories: - if not isinstance(game, series): - continue - if latest and latest.value > game.value: - continue - latest = game - latest_path = path - if latest: - return latest, latest_path - return None - - -def get_latest_dow_game() -> Optional[Tuple[DowGame, Path]]: - return filter_latest_dow_game(get_dow_root_directories(), series=DowGame) - - -def get_latest_dow2_game() -> Optional[Tuple[DowGame, Path]]: - return filter_latest_dow_game(get_dow_root_directories(), series=DowIIGame) - - -def get_latest_dow3_game() -> Optional[Tuple[DowGame, Path]]: - return filter_latest_dow_game(get_dow_root_directories(), series=DowIIIGame) - - -def get_unique_dow_game() -> Iterable[Tuple[DowGame, Path]]: - return filter_unique_dow_game(get_dow_root_directories()) - - -if __name__ == "__main__": - print("\nAll Dirs") - for game, path in get_dow_root_directories(): - print(game.name, ":\t", path) - - print("\nLatest") - dirs = get_dow_root_directories() - latest = filter_latest_dow_game(dirs) - print(latest) diff --git a/src/relic/sga/__init__.py b/src/relic/sga/__init__.py index 70428ba..af0b51b 100644 --- a/src/relic/sga/__init__.py +++ b/src/relic/sga/__init__.py @@ -1,19 +1,10 @@ -from .archive import * -from .file import * -from .folder import * -from .toc import * -from .vdrive import * -from . import common, hierarchy, writer -from . import archive, file, folder, toc, vdrive +from relic.sga._apis import apis as APIs +from relic.sga._core import Version, MagicWord, StorageType, VerificationType __all__ = [ - "common", - "hierarchy", - "writer", + "APIs", + "Version", + "MagicWord", + "StorageType", + "VerificationType" ] - -__all__.extend(archive.__all__) -__all__.extend(file.__all__) -__all__.extend(folder.__all__) -__all__.extend(toc.__all__) -__all__.extend(vdrive.__all__) diff --git a/src/relic/sga/_abc.py b/src/relic/sga/_abc.py new file mode 100644 index 0000000..9aa59a3 --- /dev/null +++ b/src/relic/sga/_abc.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +import zlib +from abc import ABC +from contextlib import contextmanager +from dataclasses import dataclass +from io import BytesIO +from pathlib import PurePath +from typing import List, Optional, Tuple, BinaryIO, Type, Generic, TypeVar + +from relic.sga import protocols as p +from relic.sga._core import StorageType, Version +from relic.sga.protocols import IONode, IOWalk, IOContainer + + +def _build_io_path(name: str, parent: Optional[p.IONode]) -> PurePath: + if parent is not None and isinstance(parent, p.IOPathable): + return parent.path / name + else: + return PurePath(name) + + +TFile = TypeVar("TFile", bound=p.File) +TFolder = TypeVar("TFolder", bound=p.Folder) +TDrive = TypeVar("TDrive", bound=p.Drive) +TArchive = TypeVar("TArchive", bound=p.Archive) +TMetadata = TypeVar("TMetadata") +TFileMetadata = TypeVar("TFileMetadata") + + +@dataclass +class _FileLazyInfo: + jump_to: int + packed_size: int + unpacked_size: int + stream: BinaryIO + decompress: bool + + def read(self, decompress: Optional[bool] = None) -> bytes: + decompress = self.decompress if decompress is None else decompress + jump_back = self.stream.tell() + self.stream.seek(self.jump_to) + buffer = self.stream.read(self.packed_size) + if decompress and self.packed_size != self.unpacked_size: + buffer = zlib.decompress(buffer) + assert len(buffer) == self.unpacked_size # TODO Raise Exception instead + self.stream.seek(jump_back) + return buffer + + +@dataclass +class DriveDef: + alias: str + name: str + root_folder: int + folder_range: Tuple[int, int] + file_range: Tuple[int, int] + + +@dataclass +class FolderDef: + name_pos: int + folder_range: Tuple[int, int] + file_range: Tuple[int, int] + + +@dataclass +class FileDefABC: + name_pos: int + data_pos: int + length_on_disk: int + length_in_archive: int + storage_type: StorageType + + +@dataclass +class File(Generic[TFileMetadata], p.File[TFileMetadata]): + name: str + _data: Optional[bytes] + storage_type: StorageType + _is_compressed: bool + metadata: TFileMetadata + parent: Optional[IOContainer] = None + _lazy_info: Optional[_FileLazyInfo] = None + + @property + def data(self) -> bytes: + if self._data is None: + if self._lazy_info is None: + raise TypeError("Data was not loaded!") + else: + self._data = self._lazy_info.read() + self._lazy_info = None + return self._data + + @data.setter + def data(self, value: bytes) -> None: + self._data = value + + @contextmanager + def open(self, read_only: bool = True): + data = self.data + with BytesIO(data) as stream: + yield stream + if not read_only: + stream.seek(0) + self.data = stream.read() + + @property + def is_compressed(self) -> bool: + return self._is_compressed + + def compress(self) -> None: + if self.data is None: + raise TypeError("Data was not loaded!") + if not self._is_compressed: + self.data = zlib.compress(self.data) + self._is_compressed = True + + def decompress(self) -> None: + if self._is_compressed: + self.data = zlib.decompress(self.data) + self._is_compressed = False + + @property + def path(self) -> PurePath: + return _build_io_path(self.name, self.parent) + + +@dataclass +class Folder(p.Folder): + name: str + sub_folders: List[Folder] + files: List[File] + parent: Optional[IONode] = None + + @property + def path(self) -> PurePath: + return _build_io_path(self.name, self.parent) + + def walk(self) -> IOWalk: + yield self, self.sub_folders, self.files + for folder in self.sub_folders: + for inner_walk in folder.walk(): + yield inner_walk + + +@dataclass +class Drive(p.Drive): + alias: str + name: str + sub_folders: List[Folder] + files: List[File] + parent: None = None + __ignore__ = ["parent"] + + @property + def path(self) -> PurePath: + return _build_io_path(f"{self.alias}:", None) + + def walk(self) -> IOWalk: + yield self, self.sub_folders, self.files + for folder in self.sub_folders: + for inner_walk in folder.walk(): + yield inner_walk + + +@dataclass +class Archive(Generic[TMetadata], p.Archive[TMetadata]): + name: str + metadata: TMetadata + drives: List[Drive] + + def walk(self) -> IOWalk: + for drive in self.drives: + for inner_walk in drive.walk(): + yield inner_walk + + +# for good typing; manually define dataclass attributes in construct +# it sucks, but good typing is better than no typing +class API(Generic[TArchive, TDrive, TFolder, TFile], p.API[TArchive, TDrive, TFolder, TFile], ABC): + def __init__(self, version: Version, archive: Type[TArchive], drive: Type[TDrive], folder: Type[TFolder], file: Type[TFile], serializer: APISerializer): + self.version = version + self.Archive = archive + self.Drive = drive + self.Folder = folder + self.File = file + self._serializer = serializer + + def read(self, stream: BinaryIO, lazy: bool = False, decompress: bool = True) -> TArchive: + return self._serializer.read(stream, lazy, decompress) + + def write(self, stream: BinaryIO, archive: TArchive) -> int: + return self._serializer.write(stream, archive) + + +class APISerializer(Generic[TArchive]): + def read(self, stream: BinaryIO, lazy: bool = False, decompress: bool = True) -> TArchive: + raise NotImplementedError + + def write(self, stream: BinaryIO, archive: TArchive) -> int: + raise NotImplementedError diff --git a/src/relic/sga/_apis.py b/src/relic/sga/_apis.py new file mode 100644 index 0000000..8b648a9 --- /dev/null +++ b/src/relic/sga/_apis.py @@ -0,0 +1,7 @@ +from typing import List, Dict + +from relic.sga import v2, v5, v7, v9, protocols +from relic.sga._core import Version + +_APIS: List[protocols.API] = [v2.API, v5.API, v7.API, v9.API] +apis: Dict[Version, protocols.API] = {api.version: api for api in _APIS} diff --git a/src/relic/sga/_core.py b/src/relic/sga/_core.py new file mode 100644 index 0000000..437720d --- /dev/null +++ b/src/relic/sga/_core.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + +from serialization_tools.structx import Struct +from typing import Optional, ClassVar, BinaryIO + +from serialization_tools.magic import MagicWordIO + +MagicWord = MagicWordIO(Struct("< 8s"), "_ARCHIVE".encode("ascii")) + + +@dataclass +class Version: + """ The Major Version; Relic refers to this as the 'Version' """ + major: int + """ The Minor Version; Relic refers to this as the 'Product' """ + minor: Optional[int] = 0 + + LAYOUT: ClassVar[Struct] = Struct("<2H") + + def __str__(self) -> str: + return f"Version {self.major}.{self.minor}" + + def __eq__(self, other): + if isinstance(other, Version): + return self.major == other.major and self.minor == other.minor + else: + return super().__eq__(other) + + def __hash__(self): + # Realistically; Version will always be <256 + # But we could manually set it to something much bigger by accident; and that may cause collisions + return self.major << (self.LAYOUT.size // 2) + self.minor + + @classmethod + def unpack(cls, stream: BinaryIO): + layout: Struct = cls.LAYOUT + args = layout.unpack_stream(stream) + return cls(*args) + + def pack(self, stream: BinaryIO): + layout: Struct = self.LAYOUT + args = (self.major, self.minor) + return layout.pack_stream(stream, *args) + + +class StorageType(int, Enum): + Store = 0 + BufferCompress = 1 + StreamCompress = 2 + + +class VerificationType(int, Enum): + None_ = 0 # unknown real values, assuming incremental + CRC = 1 # unknown real values, assuming incremental + CRCBlocks = 2 # unknown real values, assuming incremental + MD5Blocks = 3 # unknown real values, assuming incremental + SHA1Blocks = 4 # unknown real values, assuming incremental diff --git a/src/relic/sga/_serializers.py b/src/relic/sga/_serializers.py new file mode 100644 index 0000000..2d7c5e3 --- /dev/null +++ b/src/relic/sga/_serializers.py @@ -0,0 +1,233 @@ +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from typing import BinaryIO, List, Dict, Optional, Callable, Tuple, Iterable + +from serialization_tools.size import KiB +from serialization_tools.structx import Struct + +from relic.sga import _abc +from relic.sga._abc import DriveDef, FolderDef, FileDefABC as FileDef, _FileLazyInfo, FileDefABC +from relic.sga._core import StorageType +from relic.sga.errors import MD5MismatchError +from relic.sga.protocols import TFileMetadata, IOContainer, StreamSerializer, T, TFile, TDrive + + +@dataclass +class TocHeader: + drive_info: Tuple[int, int] + folder_info: Tuple[int, int] + file_info: Tuple[int, int] + name_info: Tuple[int, int] + + +class TocHeaderSerializer(StreamSerializer[TocHeader]): + def __init__(self, layout: Struct): + self.layout = layout + + def unpack(self, stream: BinaryIO) -> TocHeader: + drive_pos, \ + drive_count, \ + folder_pos, \ + folder_count, \ + file_pos, \ + file_count, \ + name_pos, \ + name_count = self.layout.unpack_stream(stream) + return TocHeader((drive_pos, drive_count), (folder_pos, folder_count), (file_pos, file_count), (name_pos, name_count)) + + def pack(self, stream: BinaryIO, value: TocHeader) -> int: + args = value.drive_info[0], \ + value.drive_info[1], \ + value.folder_info[0], \ + value.folder_info[1], \ + value.file_info[0], \ + value.file_info[1], \ + value.name_info[0], \ + value.name_info[1] + return self.layout.pack_stream(stream, *args) + + +class DriveDefSerializer(StreamSerializer[DriveDef]): + def __init__(self, layout: Struct): + self.layout = layout + + def unpack(self, stream: BinaryIO) -> DriveDef: + encoded_alias: bytes + encoded_name: bytes + encoded_alias, encoded_name, folder_start, folder_end, file_start, file_end, root_folder = self.layout.unpack_stream(stream) + alias: str = encoded_alias.rstrip(b"\0").decode("ascii") + name: str = encoded_name.rstrip(b"\0").decode("ascii") + folder_range = (folder_start, folder_end) + file_range = (file_start, file_end) + return DriveDef(alias=alias, name=name, root_folder=root_folder, folder_range=folder_range, file_range=file_range) + + def pack(self, stream: BinaryIO, value: DriveDef) -> int: + alias: bytes = value.alias.encode("ascii") + name: bytes = value.name.encode("ascii") + args = alias, name, value.folder_range[0], value.folder_range[1], value.file_range[0], value.file_range[1], value.root_folder + return self.layout.pack_stream(stream, *args) + + +class FolderDefSerializer(StreamSerializer[FolderDef]): + def __init__(self, layout: Struct): + self.layout = layout + + def unpack(self, stream: BinaryIO) -> FolderDef: + name_pos, folder_start, folder_end, file_start, file_end = self.layout.unpack_stream(stream) + folder_range = (folder_start, folder_end) + file_range = (file_start, file_end) + return FolderDef(name_pos=name_pos, folder_range=folder_range, file_range=file_range) + + def pack(self, stream: BinaryIO, value: FolderDef) -> int: + args = value.name_pos, value.folder_range[0], value.folder_range[1], value.file_range[0], value.file_range[1] + return self.layout.pack_stream(stream, *args) + + +def _assemble_io_from_defs(drive_defs: List[DriveDef], folder_defs: List[FolderDef], file_defs: List[FileDef], names: Dict[int, str], data_pos: int, stream: BinaryIO, build_file_meta: Optional[Callable[[FileDef], TFileMetadata]] = None, + decompress: bool = False) -> Tuple[List[_abc.Drive], List[_abc.File]]: + all_files: List[TFile] = [] + drives: List[TDrive] = [] + for drive_def in drive_defs: + local_folder_defs = folder_defs[drive_def.folder_range[0]:drive_def.folder_range[1]] + local_file_defs = file_defs[drive_def.file_range[0]:drive_def.file_range[1]] + + files: List[TFile] = [] + for file_def in local_file_defs: + name = names[file_def.name_pos] + metadata = build_file_meta(file_def) if build_file_meta is not None else None + lazy_info = _FileLazyInfo(data_pos + file_def.data_pos, file_def.length_in_archive, file_def.length_on_disk, stream, decompress) + file_compressed = file_def.storage_type != StorageType.Store + file = _abc.File(name=name, _data=None, storage_type=file_def.storage_type, _is_compressed=file_compressed, metadata=metadata, _lazy_info=lazy_info) + files.append(file) + + folders: List[_abc.Folder] = [] + for folder_def in local_folder_defs: + folder_name = names[folder_def.name_pos] + sub_files = files[folder_def.file_range[0]:folder_def.folder_range[1]] + folder = _abc.Folder(folder_name, [], sub_files, None) + folders.append(folder) + + for folder_def, folder in zip(local_folder_defs, folders): + folder.sub_folders = folders[folder_def.folder_range[0]:folder_def.folder_range[1]] + + for folder in folders: + _apply_self_as_parent(folder) + root_folder = drive_def.root_folder - drive_def.folder_range[0] # make root folder relative to our folder slice + drive_folder = folders[root_folder] + drive = _abc.Drive(drive_def.alias, drive_def.name, drive_folder.sub_folders, drive_folder.files) + _apply_self_as_parent(drive) + all_files.extend(files) + drives.append(drive) + return drives, all_files + + +def _apply_self_as_parent(collection: IOContainer): + for folder in collection.sub_folders: + folder.parent = collection + for file in collection.files: + file.parent = collection + + +def _unpack_helper(stream: BinaryIO, toc_info: Tuple[int, int], header_pos: int, serializer: StreamSerializer[T]) -> List[T]: + stream.seek(header_pos + toc_info[0]) + return [serializer.unpack(stream) for _ in range(toc_info[1])] + + +def _read_toc_definitions(stream: BinaryIO, toc: TocHeader, header_pos: int, drive_serializer: StreamSerializer[DriveDef], folder_serializer: StreamSerializer[FolderDef], file_serializer: StreamSerializer[FileDefABC]): + drives = _unpack_helper(stream, toc.drive_info, header_pos, drive_serializer) + folders = _unpack_helper(stream, toc.folder_info, header_pos, folder_serializer) + files = _unpack_helper(stream, toc.file_info, header_pos, file_serializer) + return drives, folders, files + + +def _read_toc_names_as_count(stream: BinaryIO, toc_info: Tuple[int, int], header_pos: int, buffer_size: int = 256) -> Dict[int, str]: + stream.seek(header_pos + toc_info[0]) + + names: Dict[int, str] = {} + running_buffer = bytearray() + offset = 0 + while len(names) < toc_info[1]: + buffer = stream.read(buffer_size) + if len(buffer) == 0: + raise Exception("Ran out of data!") # TODO, proper exception + terminal_null = buffer[-1] == b"\0" + parts = buffer.split(b"\0") + if len(parts) > 1: + parts[0] = running_buffer + parts[0] + running_buffer.clear() + if not terminal_null: + running_buffer.extend(parts[-1]) + parts = parts[:-1] + else: + if not terminal_null: + running_buffer.extend(parts[0]) + offset += len(buffer) + continue + + remaining = toc_info[1] - len(names) + available = min(len(parts), remaining) + for _ in range(available): + name = parts[_] + names[offset] = name.decode("ascii") + offset += len(name) + 1 + return names + + +def _read_toc_names_as_size(stream: BinaryIO, toc_info: Tuple[int, int], header_pos: int) -> Dict[int, str]: + stream.seek(header_pos + toc_info[0]) + name_buffer = stream.read(toc_info[1]) + parts = name_buffer.split(b"\0") + names: Dict[int, str] = {} + offset = 0 + for part in parts: + names[offset] = part.decode("ascii") + offset += len(part) + 1 + return names + + +def _chunked_read(stream: BinaryIO, size: Optional[int] = None, chunk_size: Optional[int] = None) -> Iterable[bytes]: + if size is None and chunk_size is None: + yield stream.read() + elif size is None and chunk_size is not None: + while True: + buffer = stream.read(chunk_size) + yield buffer + if len(buffer) != chunk_size: + break + elif size is not None and chunk_size is None: + yield stream.read(size) + elif size is not None and chunk_size is not None: # MyPy + chunks = size // chunk_size + for _ in range(chunks): + yield stream.read(chunk_size) + total_read = chunk_size * chunks + if total_read < size: + yield stream.read(size - total_read) + else: + raise Exception("Something impossible happened!") + + +@dataclass +class _Md5ChecksumHelper: + expected: bytes + stream: BinaryIO + start: int + size: Optional[int] = None + eigen: Optional[bytes] = None + + def read(self, stream: Optional[BinaryIO] = None) -> bytes: + stream = self.stream if stream is None else stream + stream.seek(self.start) + md5 = hashlib.md5(self.eigen) if self.eigen is not None else hashlib.md5() + # Safer for large files to read chunked + for chunk in _chunked_read(stream, self.size, 256 * KiB): + md5.update(chunk) + md5_str = md5.hexdigest() + return bytes.fromhex(md5_str) + + def validate(self, stream: Optional[BinaryIO] = None) -> None: + result = self.read(stream) + if self.expected != result: + raise MD5MismatchError(result, self.expected) diff --git a/src/relic/sga/archive/__init__.py b/src/relic/sga/archive/__init__.py deleted file mode 100644 index 995f3a1..0000000 --- a/src/relic/sga/archive/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -from .archive import Archive, DowIArchive, DowIIArchive, DowIIIArchive -from .header import ArchiveHeader, ArchiveVersion, DowIArchiveHeader, DowIIArchiveHeader, DowIIIArchiveHeader, ArchiveMagicWord - -__all__ = [ - "Archive", - "DowIArchive", - "DowIIArchive", - "DowIIIArchive", - "ArchiveHeader", - "ArchiveVersion", - "DowIArchiveHeader", - "DowIIArchiveHeader", - "DowIIIArchiveHeader", - - "ArchiveMagicWord", -] diff --git a/src/relic/sga/archive/archive.py b/src/relic/sga/archive/archive.py deleted file mode 100644 index 42fba30..0000000 --- a/src/relic/sga/archive/archive.py +++ /dev/null @@ -1,83 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import BinaryIO, List, Type, Dict, TYPE_CHECKING - -from .header import ArchiveHeader -from ..common import ArchiveVersion -from ..hierarchy import DriveCollection, ArchiveWalk, walk -from ...common import VersionLike - -if TYPE_CHECKING: - from ..toc.toc import ArchiveTableOfContents - from ..toc.toc_headers import ArchiveTableOfContentsHeaders - from ..toc.toc_ptr import ArchiveTableOfContentsPtr - from ..vdrive.virtual_drive import VirtualDrive - - -@dataclass -class Archive(DriveCollection): - header: ArchiveHeader - """Sparse represents whether data was loaded on creation.""" - _sparse: bool - - def __init__(self, header: ArchiveHeader, drives: List[VirtualDrive], _sparse: bool): - self.header = header - self._sparse = _sparse - self.drives = drives - - def walk(self) -> ArchiveWalk: - return walk(self) - - @classmethod - def _unpack(cls, stream: BinaryIO, header: ArchiveHeader, sparse: bool = True): - from ..toc import ArchiveTableOfContents, ArchiveTableOfContentsPtr, ArchiveTableOfContentsHeaders - version = header.version - with header.toc_ptr.stream_jump_to(stream) as handle: - toc_ptr = ArchiveTableOfContentsPtr.unpack_version(handle, version) - toc_headers = ArchiveTableOfContentsHeaders.unpack(handle, toc_ptr, version) - toc = ArchiveTableOfContents.create(toc_headers) - - toc.load_toc() - toc.build_tree() # ensures walk is unique; avoiding dupes and speeding things up - if not sparse: - with header.data_ptr.stream_jump_to(stream) as handle: - toc.load_data(handle) - - return cls(header, toc.drives, sparse) - - @classmethod - def unpack(cls, stream: BinaryIO, read_magic: bool = True, sparse: bool = True, *, validate: bool = True) -> Archive: - header = ArchiveHeader.unpack(stream, read_magic) - if validate: - header.validate_checksums(stream) - class_type = _VERSION_MAP[header.version] - return class_type._unpack(stream, header, sparse) # Defer to subclass (ensures packing works as expected) - - def pack(self, stream: BinaryIO, write_magic: bool = True) -> int: - raise NotImplementedError - - -@dataclass(init=False) -class DowIArchive(Archive): - def pack(self, stream: BinaryIO, write_magic: bool = True) -> int: - raise NotImplementedError - - -@dataclass(init=False) -class DowIIArchive(Archive): - def pack(self, stream: BinaryIO, write_magic: bool = True) -> int: - raise NotImplementedError - - -@dataclass(init=False) -class DowIIIArchive(Archive): - def pack(self, stream: BinaryIO, write_magic: bool = True) -> int: - raise NotImplementedError - - -_VERSION_MAP: Dict[VersionLike, Type[Archive]] = { - ArchiveVersion.Dow: DowIArchive, - ArchiveVersion.Dow2: DowIIArchive, - ArchiveVersion.Dow3: DowIIIArchive -} diff --git a/src/relic/sga/archive/header.py b/src/relic/sga/archive/header.py deleted file mode 100644 index fff58ec..0000000 --- a/src/relic/sga/archive/header.py +++ /dev/null @@ -1,246 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from hashlib import md5 -from typing import BinaryIO, Dict, Type, Tuple - -from serialization_tools.ioutil import WindowPtr, Ptr, iter_read, StreamPtr -from serialization_tools.magic import MagicWordIO -from serialization_tools.size import KiB -from serialization_tools.structx import Struct - -from ..common import ArchiveVersion -from ...common import VersionLike - -ArchiveMagicWord = MagicWordIO(Struct("< 8s"), "_ARCHIVE".encode("ascii")) - -_NAME_CHAR_COUNT = 64 # 64 characters max -_NAME_CHAR_SIZE = 2 # UTF-16-le ~ 2 bytes per character -_NAME_BYTE_SIZE = _NAME_CHAR_COUNT * _NAME_CHAR_SIZE - - -@dataclass -class ArchiveHeader: - name: str - - toc_ptr: Ptr - data_ptr: WindowPtr - - def validate_checksums(self, stream: BinaryIO, *, fast: bool = True, _assert: bool = True) -> bool: - """ - Validates header checksums against the content's of the stream. - - The stream should return to its original position when it was passed in. - - :param stream: The binary stream to read from - :param fast: When true, slow checksums may be skipped - :param _assert: When true, an assertion is raised instead of returning False - :returns: True if all checksums match (or the type does not have checksums to validate) - :raises AssertionError: if a checksum does not match and _assert is True - """ - raise NotImplementedError - - @property - def version(self) -> VersionLike: - raise NotImplementedError - - @classmethod - def _unpack(cls, stream: BinaryIO) -> 'ArchiveHeader': - raise NotImplementedError - - def _pack(self, stream: BinaryIO) -> int: - raise NotImplementedError - - @classmethod - def unpack(cls, stream: BinaryIO, read_magic: bool = True) -> 'ArchiveHeader': - # TODO move read_magic and unpack out of unpack - if read_magic: - ArchiveMagicWord.assert_magic_word(stream, True) - - version = ArchiveVersion.unpack_version(stream) - header_class = _HEADER_VERSION_MAP.get(version) - - if not header_class: - raise NotImplementedError(version) - - return header_class._unpack(stream) - - def pack(self, stream: BinaryIO, write_magic: bool = True) -> int: - written = 0 - - if write_magic: - written += ArchiveMagicWord.write_magic_word(stream) - - written += ArchiveVersion.pack_version(stream, self.version) - written += self._pack(stream) - return written - - -def _gen_md5_checksum(stream: BinaryIO, eigen: bytes, buffer_size: int = 64 * KiB, ptr: Ptr = None) -> bytes: - hasher = md5(eigen) if eigen else md5() - ptr = ptr or StreamPtr(stream) # Quick way to preserve stream integrity - with ptr.stream_jump_to(stream) as handle: - for buffer in iter_read(handle, buffer_size): - hasher.update(buffer) - return bytes.fromhex(hasher.hexdigest()) - - -def _validate_md5_checksum(stream: BinaryIO, ptr: WindowPtr, eigen: bytes, expected: bytes, buffer_size: int = 1024 * 64, _assert: bool = True) -> bool: - result = _gen_md5_checksum(stream, eigen, buffer_size, ptr=ptr) - if _assert: - assert expected == result, (expected, result) - return True - else: - return expected == result - - -@dataclass -class DowIArchiveHeader(ArchiveHeader): - # hash, name, hash (repeated), TOC_SIZE, DATA_OFFSET - LAYOUT = Struct(f"< 16s {_NAME_BYTE_SIZE}s 16s 2L") - # The eigen value is a guid? also knew that layout looked familiar - MD5_EIGENVALUES = ("E01519D6-2DB7-4640-AF54-0A23319C56C3".encode("ascii"), "DFC9AF62-FC1B-4180-BC27-11CCE87D3EFF".encode("ascii")) - toc_ptr: WindowPtr - checksums: Tuple[bytes, bytes] - - def validate_checksums(self, stream: BinaryIO, *, fast: bool = True, _assert: bool = True): - ptrs = [Ptr(self.toc_ptr.offset), self.toc_ptr] - valid = True - indexes = (1,) if fast else (0, 1) - for i in indexes: - valid &= _validate_md5_checksum(stream, ptrs[i], self.MD5_EIGENVALUES[i], self.checksums[i], _assert=_assert) - return valid - - @property - def version(self) -> VersionLike: - return ArchiveVersion.Dow - - @classmethod - def _unpack(cls, stream: BinaryIO) -> 'DowIArchiveHeader': - csum_a, name, csum_b, toc_size, data_offset = cls.LAYOUT.unpack_stream(stream) - - name = name.decode("utf-16-le").rstrip("\0") - toc_ptr = WindowPtr(offset=stream.tell(), size=toc_size) - - data_ptr = WindowPtr(offset=data_offset, size=None) - - return cls(name, toc_ptr, data_ptr, (csum_a, csum_b)) - - def _pack(self, stream: BinaryIO) -> int: - args = self.checksums[0], self.name.encode("utf-16-le"), self.checksums[1], self.toc_ptr.size, self.data_ptr.offset - return self.LAYOUT.pack_stream(stream, *args) - - def __eq__(self, other): - # TODO make issue to add equality to WindowPtr/Ptr - return self.name == other.name \ - and self.toc_ptr.size == other.toc_ptr.size and self.toc_ptr.offset == other.toc_ptr.offset \ - and self.data_ptr.size == other.data_ptr.size and self.data_ptr.offset == other.data_ptr.offset \ - and self.version == other.version and self.checksums[0] == other.checksums[0] and self.checksums[1] == other.checksums[1] - - -@dataclass -class DowIIArchiveHeader(ArchiveHeader): - # hash, name, hash (repeated), TOC_SIZE, DATA_OFFSET, TOC_POS, RESERVED:1, RESERVED:0?, UNK??? - LAYOUT = Struct(f"< 16s {_NAME_BYTE_SIZE}s 16s 3L 3L") - # Copied from DowI, may be different; praying it isn't - # UGH THIER DIFFERENT! Or the way to calculate them is different - # First, let's try no eigen # (None, None) # HAH TROLLED MYSELF, forgot to conert checksum to hex - MD5_EIGENVALUES = ("E01519D6-2DB7-4640-AF54-0A23319C56C3".encode("ascii"), "DFC9AF62-FC1B-4180-BC27-11CCE87D3EFF".encode("ascii")) - toc_ptr: WindowPtr - checksums: Tuple[bytes, bytes] - unk: int - - # This may not mirror DowI one-to-one, until it's verified, it stays here - # noinspection DuplicatedCode - def validate_checksums(self, stream: BinaryIO, *, fast: bool = True, _assert: bool = True): - # return True - ptrs = [Ptr(self.toc_ptr.offset), self.toc_ptr] - valid = True - indexes = (1,) if fast else (0, 1) - for i in indexes: - valid &= _validate_md5_checksum(stream, ptrs[i], self.MD5_EIGENVALUES[i], self.checksums[i], _assert=_assert) - return valid - - def __eq__(self, other): - # TODO make issue to add equality to WindowPtr/Ptr - return self.name == other.name and self.unk == other.unk \ - and self.toc_ptr.size == other.toc_ptr.size and self.toc_ptr.offset == other.toc_ptr.offset \ - and self.data_ptr.size == other.data_ptr.size and self.data_ptr.offset == other.data_ptr.offset \ - and self.version == other.version and self.checksums[0] == other.checksums[0] and self.checksums[1] == other.checksums[1] - - @property - def version(self) -> VersionLike: - return ArchiveVersion.Dow2 - - @classmethod - def _unpack(cls, stream: BinaryIO) -> 'DowIIArchiveHeader': - csum_a, name, csum_b, toc_size, data_offset, toc_pos, rsv_1, rsv_0, unk = cls.LAYOUT.unpack_stream(stream) - - assert rsv_1 == 1 - assert rsv_0 == 0 - - name = name.decode("utf-16-le").rstrip("\0") - toc_ptr = WindowPtr(offset=toc_pos, size=toc_size) - data_ptr = WindowPtr(offset=data_offset) - - return cls(name, toc_ptr, data_ptr, (csum_a, csum_b), unk) - - def _pack(self, stream: BinaryIO) -> int: - args = self.checksums[0], self.name.encode("utf-16-le"), self.checksums[1], self.toc_ptr.size, self.data_ptr.offset, self.toc_ptr.offset, 1, 0, self.unk - return self.LAYOUT.pack_stream(stream, *args) - - -@dataclass -class DowIIIArchiveHeader(ArchiveHeader): - # name, TOC_POS, TOC_SIZE, DATA_POS, DATA_SIZE, RESERVED:0?, RESERVED:1, RESERVED:0?, UNK??? - LAYOUT = Struct(f"<{_NAME_BYTE_SIZE}s Q L Q L 3L 256s") - toc_ptr: WindowPtr - data_ptr: WindowPtr - - unk: bytes - - def validate_checksums(self, stream: BinaryIO, *, fast: bool = True, _assert: bool = True) -> bool: - """ - Dawn of War III does not contain any checksums, and so will always return true. - - :param stream: Ignored - :param fast: Ignored - :param _assert: Ignored - :returns: True - """ - return True - - @property - def version(self) -> VersionLike: - return ArchiveVersion.Dow3 - - @classmethod - def _unpack(cls, stream: BinaryIO) -> ArchiveHeader: - name, toc_pos, toc_size, data_pos, data_size, rsv_0_a, rsv_1, rsv_0_b, unk = cls.LAYOUT.unpack_stream(stream) - - assert rsv_1 == 1 - assert rsv_0_a == 0 - assert rsv_0_b == 0 - - toc_ptr = WindowPtr(offset=toc_pos, size=toc_size) - data_ptr = WindowPtr(offset=data_pos, size=data_size) - name = name.decode("utf-16-le").rstrip("\0") - - return cls(name, toc_ptr, data_ptr, unk) - - def _pack(self, stream: BinaryIO) -> int: - args = self.name.encode("utf-16-le"), self.toc_ptr.offset, self.toc_ptr.size, self.data_ptr.offset, self.data_ptr.size, 0, 1, 0, self.unk - return self.LAYOUT.pack_stream(stream, *args) - - def __eq__(self, other): - # TODO make issue to add equality to WindowPtr/Ptr - return self.name == other.name and self.unk == other.unk \ - and self.toc_ptr.size == other.toc_ptr.size and self.toc_ptr.offset == other.toc_ptr.offset \ - and self.data_ptr.size == other.data_ptr.size and self.data_ptr.offset == other.data_ptr.offset \ - and self.version == other.version - -_HEADER_VERSION_MAP: Dict[VersionLike, Type[ArchiveHeader]] = { - ArchiveVersion.Dow: DowIArchiveHeader, - ArchiveVersion.Dow2: DowIIArchiveHeader, - ArchiveVersion.Dow3: DowIIIArchiveHeader -} diff --git a/src/relic/sga/common.py b/src/relic/sga/common.py deleted file mode 100644 index c864708..0000000 --- a/src/relic/sga/common.py +++ /dev/null @@ -1,53 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import Optional, Iterator, BinaryIO - -from serialization_tools.structx import Struct - -from ..common import VersionEnum, Version, VersionLike - -ArchiveVersionLayout = Struct("< 2H") - - -class ArchiveVersion(VersionEnum): - Unsupported = None - Dow = Version(2) - Dow2 = Version(5) - Dow3 = Version(9) - - @classmethod - def unpack_version(cls, stream: BinaryIO) -> Version: - return Version(*ArchiveVersionLayout.unpack_stream(stream)) - - @classmethod - def pack_version(cls, stream: BinaryIO, version: VersionLike) -> int: - if isinstance(version, VersionEnum): - version = version.value - return ArchiveVersionLayout.pack_stream(stream, version.major, version.minor) - - @classmethod - def unpack(cls, stream: BinaryIO) -> ArchiveVersion: - return ArchiveVersion(cls.unpack_version(stream)) - - def pack(self, stream: BinaryIO) -> int: - return self.pack_version(stream, self) - - -@dataclass -class ArchiveRange: - start: int - end: int - __iterable: Optional[Iterator] = None - - @property - def size(self) -> int: - return self.end - self.start - - # We don't use iterable to avoid x - def __iter__(self) -> ArchiveRange: - self.__iterable = iter(range(self.start, self.end)) - return self - - def __next__(self) -> int: - return next(self.__iterable) diff --git a/src/relic/sga/errors.py b/src/relic/sga/errors.py new file mode 100644 index 0000000..aa1d970 --- /dev/null +++ b/src/relic/sga/errors.py @@ -0,0 +1,59 @@ +from typing import List, Any + +from relic.sga._core import Version + + +def _print_mismatch(name: str, received, expected): + msg = f"Unexpected {name}" + if received or expected: + msg += ";" + if received: + msg += f" got `{str(received)}`" + if received and expected: + msg += "," + if expected: + msg += f" expected `{str(expected)}`" + return msg + "!" + + +class MismatchError(Exception): + def __init__(self, name: str, received: Any = None, expected: Any = None): + self.name = name + self.received = received + self.expected = expected + + def __str__(self): + return _print_mismatch(self.name, self.received, self.expected) + + +class VersionMismatchError(MismatchError): + def __init__(self, received: Version = None, expected: Version = None): + super().__init__("Version", received, expected) + + +class MD5MismatchError(MismatchError): + def __init__(self, received: bytes = None, expected: bytes = None): + super().__init__("MD5", received, expected) + + +class VersionNotSupportedError(Exception): + def __init__(self, received: Version, allowed: List[Version]): + self.received = received + self.allowed = allowed + + def __str__(self): + def str_ver(v: Version) -> str: # dont use str(version); too verbose + return f"{v.major}.{v.minor}" + + allowed_str = [str_ver(_) for _ in self.allowed] + return f"Version `{str_ver(self.received)}` is not supported. Versions supported: `{allowed_str}`" + + +# +__all__ = [ + "_print_mismatch", + "MismatchError", + "VersionMismatchError", + "MD5MismatchError", + "VersionNotSupportedError" +] diff --git a/src/relic/sga/file/__init__.py b/src/relic/sga/file/__init__.py deleted file mode 100644 index 899da3b..0000000 --- a/src/relic/sga/file/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from .file import File -from .header import FileHeader, DowIFileHeader, DowIIFileHeader, DowIIIFileHeader, FileCompressionFlag - -__all__ = [ - "File", - "FileHeader", - "FileCompressionFlag", - "DowIFileHeader", - "DowIIFileHeader", - "DowIIIFileHeader", -] diff --git a/src/relic/sga/file/file.py b/src/relic/sga/file/file.py deleted file mode 100644 index 83effdf..0000000 --- a/src/relic/sga/file/file.py +++ /dev/null @@ -1,85 +0,0 @@ -from __future__ import annotations - -import zlib -from dataclasses import dataclass -from pathlib import PurePosixPath -from typing import BinaryIO, Dict, Optional, TYPE_CHECKING - -from .header import FileHeader -if TYPE_CHECKING: - from ..folder.folder import Folder - from ..toc.toc import ArchiveTableOfContents - from ..vdrive.virtual_drive import VirtualDrive - - -@dataclass -class File: - header: FileHeader - name: str - data: Optional[bytes] = None - _decompressed: bool = False - _parent: Optional[Folder] = None - _drive: Optional[VirtualDrive] = None - - @property - def data_loaded(self) -> bool: - return self.data is not None - - @property - def expects_decompress(self) -> bool: - return self.header.compressed - - @property - def decompressed(self) -> bool: - if self.data_loaded: - return self._decompressed or not self.expects_decompress - else: - return False - - @property - def full_path(self) -> PurePosixPath: - if self._parent: - return self._parent.full_path / self.name - elif self._drive: - return self._drive.full_path / self.name - else: - return PurePosixPath(self.name) - - @classmethod - def create(cls, header: FileHeader) -> File: - _decompressed = False - # noinspection PyTypeChecker - return File(header, None, None, _decompressed) - - def load_name_from_lookup(self, name_lookup: Dict[int, str]): - self.name = name_lookup[self.header.name_sub_ptr.offset] - - def load_toc(self, toc: ArchiveTableOfContents): - self.load_name_from_lookup(toc.names) - - def read_data(self, stream: BinaryIO, decompress: bool = False) -> bytes: - with self.header.data_sub_ptr.stream_jump_to(stream) as handle: - buffer = handle.read(self.header.compressed_size) - if decompress and self.expects_decompress: - return zlib.decompress(buffer) - else: - return buffer - - def load_data(self, stream: BinaryIO, decompress: bool = False): - self.data = self.read_data(stream, decompress) - self._decompressed = decompress - - def get_decompressed_data(self) -> bytes: - if self.decompressed: - return self.data - else: - # zlib_header = Struct("2B").unpack(self.data[:2]) - # full_zlib_header = (zlib_header[0] & 0xF0) >> 4, zlib_header[0] & 0xF, \ - # (zlib_header[1] & 0b11000000) >> 6, (zlib_header[1] >> 5) & 0b1, zlib_header[1] & 0b11111 - # convert = {7: 32, 6: 16} - # assert convert[full_zlib_header[0]] == self.header.compression_flag.value - return zlib.decompress(self.data) - - def decompress(self): - self.data = self.get_decompressed_data() - self._decompressed = True diff --git a/src/relic/sga/file/header.py b/src/relic/sga/file/header.py deleted file mode 100644 index 61ddf0a..0000000 --- a/src/relic/sga/file/header.py +++ /dev/null @@ -1,151 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from enum import Enum -from typing import BinaryIO, ClassVar, Type, Dict - -from serialization_tools.ioutil import Ptr, WindowPtr -from serialization_tools.structx import Struct - -from ..common import ArchiveVersion -from ...common import VersionLike - - -class FileCompressionFlag(Enum): - # Compression flag is either 0 (Decompressed) or 16/32 which are both compressed - # Aside from 0; these appear to be the Window-Sizes for the Zlib Compression (In KibiBytes) - Decompressed = 0 - - Compressed16 = 16 - Compressed32 = 32 - - def compressed(self) -> bool: - return self != FileCompressionFlag.Decompressed - - -@dataclass -class FileHeader: - LAYOUT: ClassVar[Struct] - name_sub_ptr: Ptr # Sub ptr is expected to be used via window (E.G. 'WindowPtr() as handle', then, 'data_sub_ptr.stream_jump_to(handle)') - data_sub_ptr: Ptr - decompressed_size: int - compressed_size: int - - def __eq__(self, other): - # TODO ptr equality - return self.decompressed_size == other.decompressed_size and self.compressed_size == other.compressed_size - - @property - def compressed(self): - raise NotImplementedError - - @classmethod - def _unpack(cls, stream: BinaryIO) -> FileHeader: - raise NotImplementedError - - def _pack(self, stream: BinaryIO) -> int: - raise NotImplementedError - - def pack(self, stream: BinaryIO) -> int: - return self._pack(stream) - - @classmethod - def unpack(cls, stream: BinaryIO, version: VersionLike) -> FileHeader: - header_class = _HEADER_VERSION_MAP.get(version) - - if not header_class: - raise NotImplementedError(version) - - return header_class._unpack(stream) - - -@dataclass -class DowIFileHeader(FileHeader): - # name - LAYOUT = Struct(f"<5L") - compression_flag: FileCompressionFlag - - def __eq__(self, other): - return self.compression_flag == other.compression_flag and super().__eq__(other) - - - @classmethod - def _unpack(cls, stream: BinaryIO) -> DowIFileHeader: - name_offset, compression_flag_value, data_offset, decompressed_size, compressed_size = cls.LAYOUT.unpack_stream(stream) - compression_flag = FileCompressionFlag(compression_flag_value) - name_ptr = Ptr(name_offset) - data_ptr = WindowPtr(data_offset, compressed_size) - return cls(name_ptr, data_ptr, decompressed_size, compressed_size, compression_flag) - - def _pack(self, stream: BinaryIO) -> int: - return self.LAYOUT.pack_stream(stream, self.name_sub_ptr.offset, self.compression_flag.value, self.data_sub_ptr.offset, self.decompressed_size, self.compressed_size) - - @property - def compressed(self): - return self.compression_flag.compressed() - - -@dataclass -class DowIIFileHeader(FileHeader): - LAYOUT = Struct(f"<5L H") - unk_a: int - unk_b: int - - @property - def compressed(self): - return self.compressed_size < self.decompressed_size - - @classmethod - def _unpack(cls, stream: BinaryIO) -> DowIIFileHeader: - name_off, data_off, comp_size, decomp_size, unk_a, unk_b = cls.LAYOUT.unpack_stream(stream) - # Name, File, Compressed, Decompressed, ???, ??? - name_ptr = Ptr(name_off) - data_ptr = Ptr(data_off) - return cls(name_ptr, data_ptr, decomp_size, comp_size, unk_a, unk_b) - - def _pack(self, stream: BinaryIO) -> int: - return self.LAYOUT.pack_stream(stream, self.name_sub_ptr.offset, self.data_sub_ptr.offset, self.compressed_size, self.decompressed_size, self.unk_a, self.unk_b) - - def __eq__(self, other): - return self.unk_a == other.unk_a and self.unk_b == other.unk_b and super().__eq__(other) - -@dataclass -class DowIIIFileHeader(FileHeader): - LAYOUT = Struct("< 7L H L") - unk_a: int - unk_b: int - unk_c: int - unk_d: int # 256? - unk_e: int - - def __eq__(self, other): - return self.unk_a == other.unk_a and self.unk_b == other.unk_b and self.unk_c == other.unk_c and self.unk_d == other.unk_d and self.unk_e == other.unk_e and super().__eq__(other) - - @classmethod - def _unpack(cls, stream: BinaryIO) -> DowIIIFileHeader: - name_off, unk_a, data_off, unk_b, comp_size, decomp_size, unk_c, unk_d, unk_e = cls.LAYOUT.unpack_stream(stream) - # assert unk_a == 0, (unk_a, 0) - # assert unk_b == 0, (unk_b, 0) - # UNK_D is a new compression flag?! - # if comp_size != decomp_size: - # assert unk_d in [256,512], ((comp_size, decomp_size), (unk_d, [256,512]), (name_off, unk_a, data_off, unk_b, comp_size, decomp_size, unk_c, unk_d, unk_e)) - # Pulling stuff out of my ass; but dividing them by the max block size gets you 7, 6 respectively - # Name, File, Compressed, Decompressed, ???, ??? - name_ptr = Ptr(name_off) - data_ptr = Ptr(data_off) - return cls(name_ptr, data_ptr, decomp_size, comp_size, unk_a, unk_b, unk_c, unk_d, unk_e) - - def _pack(self, stream: BinaryIO) -> int: - args = self.name_sub_ptr.offset, self.unk_a, self.data_sub_ptr.offset, self.unk_b, self.compressed_size, self.decompressed_size, self.unk_c, self.unk_d, self.unk_e - return self.LAYOUT.pack_stream(stream, *args) - - @property - def compressed(self): - return self.compressed_size < self.decompressed_size - - -_HEADER_VERSION_MAP: Dict[VersionLike, Type[FileHeader]] = { - ArchiveVersion.Dow: DowIFileHeader, - ArchiveVersion.Dow2: DowIIFileHeader, - ArchiveVersion.Dow3: DowIIIFileHeader -} diff --git a/src/relic/sga/folder/__init__.py b/src/relic/sga/folder/__init__.py deleted file mode 100644 index a7f1432..0000000 --- a/src/relic/sga/folder/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from .folder import Folder -from .header import FolderHeader, DowIFolderHeader, DowIIFolderHeader, DowIIIFolderHeader - -__all__ = [ - "Folder", - "FolderHeader", - "DowIFolderHeader", - "DowIIFolderHeader", - "DowIIIFolderHeader" -] diff --git a/src/relic/sga/folder/folder.py b/src/relic/sga/folder/folder.py deleted file mode 100644 index 6322e3a..0000000 --- a/src/relic/sga/folder/folder.py +++ /dev/null @@ -1,68 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import PurePosixPath -from typing import Dict, List, Optional, TYPE_CHECKING - -from ..hierarchy import DriveChild, FolderCollection, FileCollection, FolderChild, walk - -if TYPE_CHECKING: - from ..file.file import File - from ..toc.toc import ArchiveTableOfContents - from ..vdrive.virtual_drive import VirtualDrive - from .header import FolderHeader - from ..hierarchy import ArchiveWalk - - -@dataclass -class Folder(FolderCollection, FileCollection, FolderChild, DriveChild): - header: FolderHeader - name: str - - def __init__(self, header: FolderHeader, name: str, sub_folders: List[Folder], files: List[File], parent_folder: Optional[Folder] = None, drive: Optional[VirtualDrive] = None): - self.header = header - self.name = name - self.sub_folders = sub_folders - self.files = files - self._drive = drive - self._parent = parent_folder - - @property - def full_path(self) -> PurePosixPath: - if self._drive: - return self._drive.full_path / self.name - else: - return PurePosixPath(self.name) - - def walk(self) -> ArchiveWalk: - return walk(self) - - @classmethod - def create(cls, header: FolderHeader) -> Folder: - name = None - folders = [None] * header.sub_folder_range.size - files = [None] * header.file_range.size - # noinspection PyTypeChecker - return Folder(header, name, folders, files) - - def load_toc(self, toc: ArchiveTableOfContents): - self.load_folders(toc.folders) - self.load_files(toc.files) - self.load_name_from_lookup(toc.names) - - def load_name_from_lookup(self, name_lookup: Dict[int, str]): - self.name = name_lookup[self.header.name_offset] - - def load_folders(self, folders: List[Folder]): - if self.header.sub_folder_range.start < len(folders): - for folder_index in self.header.sub_folder_range: - sub_folder_index = folder_index - self.header.sub_folder_range.start - f = self.sub_folders[sub_folder_index] = folders[folder_index] - f._parent = self - - def load_files(self, files: List[File]): - if self.header.file_range.start < len(files): - for file_index in self.header.file_range: - sub_file_index = file_index - self.header.file_range.start - f = self.files[sub_file_index] = files[file_index] - f._parent = self diff --git a/src/relic/sga/folder/header.py b/src/relic/sga/folder/header.py deleted file mode 100644 index 217e010..0000000 --- a/src/relic/sga/folder/header.py +++ /dev/null @@ -1,61 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import ClassVar, BinaryIO, Dict, Type - -from serialization_tools.structx import Struct - -from ...common import VersionLike -from ..common import ArchiveRange, ArchiveVersion - - -@dataclass -class FolderHeader: - LAYOUT: ClassVar[Struct] - - name_offset: int - sub_folder_range: ArchiveRange - file_range: ArchiveRange - - @classmethod - def unpack(cls, stream: BinaryIO, version: VersionLike) -> 'FolderHeader': - header_class = _HEADER_VERSION_MAP.get(version) - - if not header_class: - raise NotImplementedError(version) - - return header_class._unpack(stream) - - def _pack(self, stream: BinaryIO) -> int: - args = self.name_offset, self.sub_folder_range.start, self.sub_folder_range.end, \ - self.file_range.start, self.file_range.end - return self.LAYOUT.pack_stream(stream, *args) - - @classmethod - def _unpack(cls, stream: BinaryIO) -> 'FolderHeader': - name_offset, sub_folder_start, sub_folder_end, file_start, file_end = cls.LAYOUT.unpack_stream(stream) - sub_folder_range = ArchiveRange(sub_folder_start, sub_folder_end) - file_range = ArchiveRange(file_start, file_end) - return cls(name_offset, sub_folder_range, file_range) - - -@dataclass -class DowIFolderHeader(FolderHeader): - LAYOUT = Struct("< L 4H") - - -@dataclass -class DowIIFolderHeader(FolderHeader): - LAYOUT = Struct("< L 4H") - - -@dataclass -class DowIIIFolderHeader(FolderHeader): - LAYOUT = Struct("< 5L") - - -_HEADER_VERSION_MAP: Dict[VersionLike, Type[FolderHeader]] = { - ArchiveVersion.Dow: DowIFolderHeader, - ArchiveVersion.Dow2: DowIIFolderHeader, - ArchiveVersion.Dow3: DowIIIFolderHeader -} diff --git a/src/relic/sga/hierarchy.py b/src/relic/sga/hierarchy.py deleted file mode 100644 index a175c98..0000000 --- a/src/relic/sga/hierarchy.py +++ /dev/null @@ -1,77 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import PurePath -from typing import List, Optional, Union, Tuple, Iterable, TYPE_CHECKING - -if TYPE_CHECKING: - from .file import File - from .folder import Folder - from .vdrive import VirtualDrive - - -@dataclass -class DriveCollection: - drives: List[VirtualDrive] - - -@dataclass -class FolderCollection: - sub_folders: List[Folder] - - -@dataclass -class FileCollection: - files: List[File] - - -@dataclass -class FolderChild: - _parent: Optional[Folder] - - -@dataclass -class DriveChild: - _drive: Optional[VirtualDrive] - - -ArchivePath = PurePath - -if TYPE_CHECKING: - ArchiveWalk = Iterable[Tuple[Optional[VirtualDrive], Optional[Folder], Iterable[Folder], Iterable[File]]] -else: - ArchiveWalk = Iterable[Tuple['VirtualDrive', Optional['Folder'], Iterable['Folder'], Iterable['File']]] - - -def walk(collection: Union[DriveCollection, FolderCollection, FileCollection]) -> ArchiveWalk: - from .folder import Folder - from .vdrive import VirtualDrive - - drives = collection.drives if isinstance(collection, DriveCollection) else [] - sub_folders = collection.sub_folders if isinstance(collection, FolderCollection) else [] - files = collection.files if isinstance(collection, FileCollection) and not isinstance(collection,VirtualDrive) else [] - - root_drive = collection if isinstance(collection, VirtualDrive) else None - root_folder = collection if isinstance(collection, Folder) else None - - # TODO optimize - # logically, we can only walk folder OR drive - if root_drive is None and root_folder is None and len(sub_folders) == 0 and len(files) == 0: - # I don't think we need to return ANYTHING if we won't be iterating over it - pass - # if len(drives) == 0: # We will only yield this item, so we return this to always iterate over something - # yield root_drive, root_folder, sub_folders, files - else: - yield root_drive, root_folder, sub_folders, files # at least one of these isn't None/Empty so we yield iti - - for drive in drives: - for d, f, folds, files, in walk(drive): - d = d or drive or root_drive - f = f or root_folder - yield d, f, folds, files - - for folder in sub_folders: - for d, f, folds, files in walk(folder): - d = d or root_drive - f = f or folder or root_folder - yield d, f, folds, files diff --git a/src/relic/sga/protocols.py b/src/relic/sga/protocols.py new file mode 100644 index 0000000..fd6a9fd --- /dev/null +++ b/src/relic/sga/protocols.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +from contextlib import contextmanager +from pathlib import PurePath +from types import ModuleType +from typing import TypeVar, Protocol, List, Optional, ForwardRef, Tuple, Iterable, BinaryIO, Type, runtime_checkable, Sequence + +from relic.sga._core import StorageType, Version + +TFile = TypeVar("TFile") +TFolder = TypeVar("TFolder") +TDrive = TypeVar("TDrive") +TArchive = TypeVar("TArchive") +TMetadata = TypeVar("TMetadata") +TFileMetadata = TypeVar("TFileMetadata") +TFile_co = TypeVar("TFile_co", covariant=True) +TFolder_co = TypeVar("TFolder_co", covariant=True) +T = TypeVar("T") + + +@runtime_checkable +class StreamSerializer(Protocol[T]): + def unpack(self, stream: BinaryIO) -> T: + raise NotImplementedError + + def pack(self, stream: BinaryIO, value: T) -> int: + raise NotImplementedError + + +@runtime_checkable +class IOPathable(Protocol): + @property + def path(self) -> PurePath: + raise NotImplementedError + + +class IONode(Protocol): + parent: Optional[IOContainer] + + +class IOContainer(IONode, Protocol[TFolder, TFile]): + sub_folders: List[TFolder] + files: List[TFile] + + +IOWalkStep = Tuple[IOContainer, Sequence[TFolder_co], Sequence[TFile_co]] +IOWalk = Iterable[IOWalkStep] + + +class IOWalkable(Protocol[TFolder_co, TFile_co]): + def walk(self) -> IOWalk: + raise NotImplementedError + + +class File(IOPathable, IONode, Protocol[TFileMetadata]): + name: str + + @property + def data(self) -> bytes: + raise NotImplementedError + + @data.setter + def data(self, value: bytes) -> None: + raise NotImplementedError + + storage_type: StorageType + metadata: TFileMetadata + + @property + def is_compressed(self) -> bool: + raise NotImplementedError + + def compress(self) -> None: + raise NotImplementedError + + def decompress(self) -> None: + raise NotImplementedError + + @contextmanager + def open(self, read_only: bool = True) -> BinaryIO: + raise NotImplementedError + + +class Folder(IOWalkable, IOPathable, IOContainer, Protocol): + name: str + + +class Drive(IOWalkable, IOPathable, IOContainer, Protocol): + alias: str + name: str + + +class Archive(IOWalkable, Protocol[TMetadata]): + name: str + metadata: TMetadata + drives: List[Drive] + + +class API(Protocol[TArchive, TDrive, TFolder, TFile]): + version: Version + Archive: Type[TArchive] + Drive: Type[TDrive] + Folder: Type[TFolder] + File: Type[TFile] + + def read(self, stream: BinaryIO, lazy: bool = False, decompress: bool = True) -> TArchive: + raise NotImplementedError + + def write(self, stream: BinaryIO, archive: TArchive) -> int: + raise NotImplementedError + + +# Hard coded-ish but better then nothing +_required_api_attrs = API.__annotations__.keys() +_required_api_callables = ["read", "write"] + + +def is_module_api(module: ModuleType): + has_attr = all(hasattr(module, attr) for attr in _required_api_attrs) + funcs = dir(module) + has_callables = all(func in funcs for func in _required_api_callables) + return has_attr and has_callables diff --git a/src/relic/__init__.py b/src/relic/sga/py.typed similarity index 100% rename from src/relic/__init__.py rename to src/relic/sga/py.typed diff --git a/src/relic/sga/toc/__init__.py b/src/relic/sga/toc/__init__.py deleted file mode 100644 index 26ffa98..0000000 --- a/src/relic/sga/toc/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from .toc import ArchiveTableOfContents -from .toc_headers import ArchiveTableOfContentsHeaders -from .toc_ptr import ArchiveTableOfContentsPtr, TocItemPtr, DowIArchiveToCPtr, DowIIArchiveToCPtr, DowIIIArchiveToCPtr - -__all__ = [ - "ArchiveTableOfContentsHeaders", - "ArchiveTableOfContentsPtr", - "ArchiveTableOfContents", - "TocItemPtr", - "DowIArchiveToCPtr", - "DowIIArchiveToCPtr", - "DowIIIArchiveToCPtr", -] diff --git a/src/relic/sga/toc/toc.py b/src/relic/sga/toc/toc.py deleted file mode 100644 index c51cbde..0000000 --- a/src/relic/sga/toc/toc.py +++ /dev/null @@ -1,50 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import List, Dict, BinaryIO, TYPE_CHECKING - -from .toc_headers import ArchiveTableOfContentsHeaders - -if TYPE_CHECKING: - from ..file.file import File - from ..folder.folder import Folder - from ..vdrive.virtual_drive import VirtualDrive - - -@dataclass -class ArchiveTableOfContents: - drives: List[VirtualDrive] - folders: List[Folder] - files: List[File] - names: Dict[int, str] - - @classmethod - def create(cls, toc_headers: ArchiveTableOfContentsHeaders) -> ArchiveTableOfContents: - from ..vdrive.virtual_drive import VirtualDrive - from ..file.file import File - from ..folder.folder import Folder - - drives = [VirtualDrive.create(header) for header in toc_headers.drives] - folders = [Folder.create(header) for header in toc_headers.folders] - files = [File.create(header) for header in toc_headers.files] - - return ArchiveTableOfContents(drives, folders, files, toc_headers.names) - - def load_data(self, stream: BinaryIO): - for _ in self.files: - _.load_data(stream) - - def load_toc(self): - for _ in self.drives: - _.load_toc(self) - for _ in self.folders: - _.load_toc(self) - for _ in self.files: - _.load_toc(self) - - def build_tree(self): - for _ in self.drives: - _.build_tree() - - -ArchiveTOC = ArchiveTableOfContents diff --git a/src/relic/sga/toc/toc_headers.py b/src/relic/sga/toc/toc_headers.py deleted file mode 100644 index e0e1148..0000000 --- a/src/relic/sga/toc/toc_headers.py +++ /dev/null @@ -1,62 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import List, Dict, BinaryIO - -from ..file.header import FileHeader -from ..folder.header import FolderHeader -from .toc_ptr import ArchiveTableOfContentsPtr -from ..vdrive.header import VirtualDriveHeader -from ...common import VersionLike - -_NULL = "\0".encode("ascii") -_KIBI = 1024 -_BUFFER_SIZE = 64 * _KIBI - - -@dataclass -class ArchiveTableOfContentsHeaders: - drives: List[VirtualDriveHeader] - folders: List[FolderHeader] - files: List[FileHeader] - names: Dict[int, str] - - @classmethod - def unpack(cls, stream: BinaryIO, ptr: ArchiveTableOfContentsPtr, version: VersionLike = None) -> ArchiveTableOfContentsHeaders: - version = version or ptr.version # abusing the fact that the classes know their own version to avoid explicitly passing it in - - local_ptr = ptr.virtual_drive_ptr - with local_ptr.stream_jump_to(stream) as handle: - virtual_drives = [VirtualDriveHeader.unpack(handle, version) for _ in range(local_ptr.count)] - - local_ptr = ptr.folder_ptr - with local_ptr.stream_jump_to(stream) as handle: - folders = [FolderHeader.unpack(handle, version) for _ in range(local_ptr.count)] - - local_ptr = ptr.file_ptr - with local_ptr.stream_jump_to(stream) as handle: - files = [FileHeader.unpack(handle, version) for _ in range(local_ptr.count)] - - # This gets a bit wierd - local_ptr = ptr.name_ptr - names: Dict[int, str] = {} - with local_ptr.stream_jump_to(stream) as handle: - start = stream.tell() # use stream to avoid invalidating window - while len(names) < local_ptr.count: - remaining = local_ptr.count - len(names) - current = stream.tell() # Get relative pos to start - buffer = handle.read(_BUFFER_SIZE) - terminal_null = buffer.endswith(_NULL) - parts = buffer.split(_NULL, remaining) - - offset = 0 - for i, p in enumerate(parts): - if i == len(parts) - 1: - break - names[current - start + offset] = p.decode("ascii") - offset += len(p) + 1 # +1 to include null terminal - - if not terminal_null: - stream.seek(current + offset) - - return ArchiveTableOfContentsHeaders(virtual_drives, folders, files, names) diff --git a/src/relic/sga/toc/toc_ptr.py b/src/relic/sga/toc/toc_ptr.py deleted file mode 100644 index 0f3f839..0000000 --- a/src/relic/sga/toc/toc_ptr.py +++ /dev/null @@ -1,108 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import BinaryIO, Dict, Type, ClassVar, Tuple - -from serialization_tools.ioutil import Ptr -from serialization_tools.structx import Struct - -from ...common import VersionLike -from ..archive import ArchiveVersion - - -@dataclass -class TocItemPtr(Ptr): - def __init__(self, offset: int, count: int, whence: int = 0): - super().__init__(offset, whence) - self.count = count - - -@dataclass -class ArchiveTableOfContentsPtr: - # Virtual Drives (offset, count), Folder (offset, count), File (offset, count), Names (offset, count) - LAYOUT: ClassVar[Struct] - virtual_drive_ptr: TocItemPtr - folder_ptr: TocItemPtr - file_ptr: TocItemPtr - name_ptr: TocItemPtr - - @property - def version(self) -> ArchiveVersion: - raise NotImplementedError - - @classmethod - def _unpack_tuple(cls, stream: BinaryIO) -> Tuple[TocItemPtr, TocItemPtr, TocItemPtr, TocItemPtr]: - vd_offset, vd_count, fold_offset, fold_count, file_offset, file_count, name_offset, name_count = cls.LAYOUT.unpack_stream(stream) - vd_ptr = TocItemPtr(vd_offset, vd_count) - fold_ptr = TocItemPtr(fold_offset, fold_count) - file_ptr = TocItemPtr(file_offset, file_count) - name_ptr = TocItemPtr(name_offset, name_count) - return vd_ptr, fold_ptr, file_ptr, name_ptr - - def _pack_tuple(self) -> Tuple[int, int, int, int, int, int, int, int]: - return self.virtual_drive_ptr.offset, self.virtual_drive_ptr.count, \ - self.folder_ptr.offset, self.folder_ptr.count, \ - self.file_ptr.offset, self.file_ptr.count, \ - self.name_ptr.offset, self.name_ptr.count - - @classmethod - def unpack_version(cls, stream: BinaryIO, version: VersionLike) -> 'ArchiveTableOfContentsPtr': - toc_ptr_class = _ToCPtr_VERSION_MAP.get(version) - - if not toc_ptr_class: - raise NotImplementedError(version) - - return toc_ptr_class.unpack(stream) - - @classmethod - def unpack(cls, stream: BinaryIO) -> 'ArchiveTableOfContentsPtr': - args = cls._unpack_tuple(stream) - return cls(*args) - - def pack(self, stream: BinaryIO) -> int: - args = self._pack_tuple() - return self.LAYOUT.pack_stream(stream, *args) - - def __str__(self): - parts = [f"{k}={v}" for k,v in self.__dict__.items()] - return f"{self.__class__.__name__}({', '.join(parts)})" - - def __repr__(self): - return str(self) - -# Alias -ArchiveToCPtr = ArchiveTableOfContentsPtr - - -@dataclass -class DowIArchiveToCPtr(ArchiveToCPtr): - @property - def version(self) -> ArchiveVersion: - return ArchiveVersion.Dow - - LAYOUT = Struct("< LH LH LH LH") - - -@dataclass -class DowIIArchiveToCPtr(ArchiveToCPtr): - LAYOUT = DowIArchiveToCPtr.LAYOUT - - @property - def version(self) -> ArchiveVersion: - return ArchiveVersion.Dow2 - - -@dataclass -class DowIIIArchiveToCPtr(ArchiveToCPtr): - LAYOUT = Struct("< 8L") - - @property - def version(self) -> ArchiveVersion: - return ArchiveVersion.Dow3 - - -_ToCPtr_VERSION_MAP: Dict[VersionLike, Type[ArchiveToCPtr]] = { - ArchiveVersion.Dow: DowIArchiveToCPtr, - ArchiveVersion.Dow2: DowIIArchiveToCPtr, - ArchiveVersion.Dow3: DowIIIArchiveToCPtr -} diff --git a/src/relic/sga/v2/__init__.py b/src/relic/sga/v2/__init__.py new file mode 100644 index 0000000..04c1ba2 --- /dev/null +++ b/src/relic/sga/v2/__init__.py @@ -0,0 +1,22 @@ +from relic.sga import _abc, protocols +from relic.sga.v2._serializers import APISerializers +from relic.sga.v2.core import Archive, Drive, Folder, File, ArchiveMetadata, version + + +def _create_api(): + serializer = APISerializers() + api = _abc.API(version, Archive, Drive, Folder, File, serializer) + return api + + +API: protocols.API[Archive, Drive, Folder, File] = _create_api() + +__all__ = [ + "Archive", + "Drive", + "Folder", + "File", + "API", + "version", + "ArchiveMetadata" +] diff --git a/src/relic/sga/v2/_serializers.py b/src/relic/sga/v2/_serializers.py new file mode 100644 index 0000000..9423f65 --- /dev/null +++ b/src/relic/sga/v2/_serializers.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +from typing import BinaryIO, Dict, ClassVar, Optional + +from serialization_tools.structx import Struct + +import relic.sga._serializers +from relic.sga import _abc, _serializers as _s +from relic.sga._abc import FileDefABC as FileDef, Archive +from relic.sga.errors import VersionMismatchError +from relic.sga.v2 import core +from relic.sga._core import MagicWord, Version, StorageType +from relic.sga.protocols import StreamSerializer + +folder_layout = Struct(" FileDef: + storage_type_val: int + name_pos, storage_type_val, data_pos, length_on_disk, length_in_archive = self.layout.unpack_stream(stream) + storage_type: StorageType = self.INT2STORAGE[storage_type_val] + return FileDef(name_pos, data_pos, length_on_disk, length_in_archive, storage_type) + + def pack(self, stream: BinaryIO, value: FileDef) -> int: + storage_type = self.STORAGE2INT[value.storage_type] + args = value.name_pos, storage_type, value.data_pos, value.length_on_disk, value.length_in_archive + return self.layout.pack_stream(stream, *args) + + +file_serializer = FileDefSerializer(file_layout) +toc_layout = Struct(" Archive: + MagicWord.read_magic_word(stream) + version = Version.unpack(stream) + if version != self.version: + raise VersionMismatchError(version,self.version) + + encoded_name: bytes + file_md5, encoded_name, header_md5, header_size, data_pos = self.layout.unpack_stream(stream) + header_pos = stream.tell() + # Seek to header; but we skip that because we are already there + toc_header = self.TocHeader.unpack(stream) + drive_defs, folder_defs, file_defs = _s._read_toc_definitions(stream, toc_header, header_pos, self.DriveDef, self.FolderDef, self.FileDef) + names = _s._read_toc_names_as_count(stream, toc_header.name_info, header_pos) + drives, files = _s._assemble_io_from_defs(drive_defs, folder_defs, file_defs, names, data_pos, stream,decompress=decompress) + + if not lazy: + for file in files: + lazy_info: Optional[_abc._FileLazyInfo] = file._lazy_info + if lazy_info is None: + raise Exception("API read files, but failed to create lazy info!") + else: + file.data = lazy_info.read(decompress) + file._lazy_info = None + + name: str = encoded_name.rstrip(b"").decode("utf-16-le") + file_md5_helper = relic.sga._serializers._Md5ChecksumHelper(file_md5, stream, header_pos, eigen=self.FILE_MD5_EIGEN) + header_md5_helper = relic.sga._serializers._Md5ChecksumHelper(file_md5, stream, header_pos, header_size, eigen=self.FILE_MD5_EIGEN) + metadata = core.ArchiveMetadata(file_md5_helper, header_md5_helper) + + return Archive(name, metadata, drives) + + def write(self, stream: BinaryIO, archive: Archive) -> int: + raise NotImplementedError + + def __init__(self): + self.DriveDef = drive_serializer + self.FolderDef = folder_serializer + self.FileDef = file_serializer + self.TocHeader = toc_header_serializer + self.version = core.version + self.layout = Struct("<16s 128s 16s 2I") diff --git a/src/relic/sga/v2/core.py b/src/relic/sga/v2/core.py new file mode 100644 index 0000000..50f4f35 --- /dev/null +++ b/src/relic/sga/v2/core.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Type + +from relic.sga import _abc +from relic.sga._serializers import _Md5ChecksumHelper +from relic.sga.errors import Version + +version = Version(2) + + +@dataclass +class ArchiveMetadata: + @property + def file_md5(self) -> bytes: + return self._file_md5.expected + + @property + def header_md5(self) -> bytes: + return self._header_md5.expected + + _file_md5: _Md5ChecksumHelper + _header_md5: _Md5ChecksumHelper + + +Archive: Type[_abc.Archive[ArchiveMetadata]] = _abc.Archive +Folder = _abc.Folder +File = _abc.File +Drive = _abc.Drive diff --git a/src/relic/sga/v5/__init__.py b/src/relic/sga/v5/__init__.py new file mode 100644 index 0000000..a1d8d2b --- /dev/null +++ b/src/relic/sga/v5/__init__.py @@ -0,0 +1,22 @@ +from relic.sga import _abc +from relic.sga.v5._serializers import APISerializers +from relic.sga.v5.core import Archive, Drive, Folder, File, ArchiveMetadata, version + + +def _create_api(): + serializer = APISerializers() + api = _abc.API(version, Archive, Drive, Folder, File, serializer) + return api + + +API = _create_api() + +__all__ = [ + "Archive", + "Drive", + "Folder", + "File", + "API", + "version", + "ArchiveMetadata" +] diff --git a/src/relic/sga/v5/_serializers.py b/src/relic/sga/v5/_serializers.py new file mode 100644 index 0000000..199e469 --- /dev/null +++ b/src/relic/sga/v5/_serializers.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import BinaryIO, ClassVar, Optional + +from serialization_tools.structx import Struct + +import relic.sga._serializers +from relic.sga import _abc, _serializers as _s +from relic.sga._abc import Archive +from relic.sga.errors import MismatchError, VersionMismatchError +from relic.sga.protocols import StreamSerializer +from relic.sga._core import StorageType, VerificationType, MagicWord, Version +from relic.sga.v5 import core + +folder_layout = Struct(" int: + modified: int = int(value.modified.timestamp()) + storage_type = value.storage_type.value # convert enum to value + verification_type = value.verification.value # convert enum to value + args = value.name_pos, value.data_pos, value.length_on_disk, value.length_in_archive, storage_type, modified, verification_type + return self.layout.pack_stream(stream, *args) + + +file_serializer = FileDefSerializer(file_layout) +toc_layout = Struct(" Archive: + MagicWord.read_magic_word(stream) + version = Version.unpack(stream) + if version != self.version: + raise VersionMismatchError(version,self.version) + + encoded_name: bytes + file_md5, encoded_name, header_md5, header_size, data_pos, header_pos, RSV_1, RSV_0, unk_a = self.layout.unpack_stream(stream) + if (RSV_1, RSV_0) != (1, 0): + raise MismatchError("Reserved Field", (RSV_1, RSV_0), (1, 0)) + # header_pos = stream.tell() + stream.seek(header_pos) + toc_header = self.TocHeader.unpack(stream) + drive_defs, folder_defs, file_defs = _s._read_toc_definitions(stream, toc_header, header_pos, self.DriveDef, self.FolderDef, self.FileDef) + names = _s._read_toc_names_as_count(stream, toc_header.name_info, header_pos) + drives, files = _s._assemble_io_from_defs(drive_defs, folder_defs, file_defs, names, data_pos, stream,decompress=decompress) + + if not lazy: + for file in files: + lazy_info: Optional[_abc._FileLazyInfo] = file._lazy_info + if lazy_info is None: + raise Exception("API read files, but failed to create lazy info!") + else: + file.data = lazy_info.read(decompress) + file._lazy_info = None + + name: str = encoded_name.rstrip(b"").decode("utf-16-le") + file_md5_helper = relic.sga._serializers._Md5ChecksumHelper(file_md5, stream, header_pos, eigen=self.FILE_MD5_EIGEN) + header_md5_helper = relic.sga._serializers._Md5ChecksumHelper(file_md5, stream, header_pos, header_size, eigen=self.FILE_MD5_EIGEN) + metadata = core.ArchiveMetadata(file_md5_helper, header_md5_helper, unk_a) + + return Archive(name, metadata, drives) + + def write(self, stream: BinaryIO, archive: Archive) -> int: + raise NotImplementedError + + def __init__(self): + self.DriveDef = drive_serializer + self.FolderDef = folder_serializer + self.FileDef = file_serializer + self.TocHeader = toc_header_serializer + self.version = core.version + self.layout = Struct("<16s 128s 16s 6I") diff --git a/src/relic/sga/v5/core.py b/src/relic/sga/v5/core.py new file mode 100644 index 0000000..c7e2698 --- /dev/null +++ b/src/relic/sga/v5/core.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + +from relic.sga import _abc +from relic.sga._abc import FileDefABC +from relic.sga._core import VerificationType +from relic.sga._serializers import _Md5ChecksumHelper +from relic.sga.errors import Version + +version = Version(5) + + +@dataclass +class ArchiveMetadata: + @property + def file_md5(self) -> bytes: + return self._file_md5.expected + + @property + def header_md5(self) -> bytes: + return self._header_md5.expected + + _file_md5: _Md5ChecksumHelper + _header_md5: _Md5ChecksumHelper + unk_a:int + + +@dataclass +class FileDef(FileDefABC): + modified: datetime + verification: VerificationType + + +@dataclass +class FileMetadata: + modified: datetime + verification: VerificationType + + +Archive = _abc.Archive[ArchiveMetadata] +Folder = _abc.Folder +File = _abc.File[FileMetadata] +Drive = _abc.Drive diff --git a/src/relic/sga/v7/__init__.py b/src/relic/sga/v7/__init__.py new file mode 100644 index 0000000..4d5d7ca --- /dev/null +++ b/src/relic/sga/v7/__init__.py @@ -0,0 +1,22 @@ +from relic.sga import _abc +from relic.sga.v7._serializers import APISerializers +from relic.sga.v7.core import Archive, Drive, Folder, File, ArchiveMetadata, version + + +def _create_api(): + serializer = APISerializers() + api = _abc.API(version, Archive, Drive, Folder, File, serializer) + return api + + +API = _create_api() + +__all__ = [ + "Archive", + "Drive", + "Folder", + "File", + "API", + "version", + "ArchiveMetadata" +] diff --git a/src/relic/sga/v7/_serializers.py b/src/relic/sga/v7/_serializers.py new file mode 100644 index 0000000..e9ba1a3 --- /dev/null +++ b/src/relic/sga/v7/_serializers.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import BinaryIO, Optional + +from serialization_tools.structx import Struct + +from relic.sga import _abc, _serializers as _s +from relic.sga._abc import Archive +from relic.sga.errors import MismatchError, VersionMismatchError +from relic.sga.protocols import StreamSerializer +from relic.sga._core import StorageType, VerificationType, MagicWord, Version +from relic.sga.v7 import core + +folder_layout = Struct(" int: + modified: int = int(value.modified.timestamp()) + storage_type = value.storage_type.value # convert enum to value + verification_type = value.verification.value # convert enum to value + args = value.name_pos, value.data_pos, value.length_on_disk, value.length_in_archive, modified, verification_type, storage_type, value.crc, value.hash_pos + return self.layout.pack_stream(stream, *args) + + +file_serializer = FileDefSerializer(file_layout) +toc_layout = Struct("<8I") +toc_header_serializer = _s.TocHeaderSerializer(toc_layout) + + +class APISerializers(_abc.APISerializer): + def read(self, stream: BinaryIO, lazy: bool = False, decompress: bool = True) -> Archive: + MagicWord.read_magic_word(stream) + version = Version.unpack(stream) + if version != self.version: + raise VersionMismatchError(version,self.version) + + + encoded_name: bytes + encoded_name, header_size, data_pos, RSV_1 = self.layout.unpack_stream(stream) + if RSV_1 != 1: + raise MismatchError("Reserved Field", RSV_1, 1) + header_pos = stream.tell() + # stream.seek(header_pos) + toc_header = self.TocHeader.unpack(stream) + unk_a, block_size = self.metadata_layout.unpack_stream(stream) + drive_defs, folder_defs, file_defs = _s._read_toc_definitions(stream, toc_header, header_pos, self.DriveDef, self.FolderDef, self.FileDef) + names = _s._read_toc_names_as_count(stream, toc_header.name_info, header_pos) + drives, files = _s._assemble_io_from_defs(drive_defs, folder_defs, file_defs, names, data_pos, stream,decompress=decompress) + + if not lazy: + for file in files: + lazy_info: Optional[_abc._FileLazyInfo] = file._lazy_info + if lazy_info is None: + raise Exception("API read files, but failed to create lazy info!") + else: + file.data = lazy_info.read(decompress) + file._lazy_info = None + + name: str = encoded_name.rstrip(b"").decode("utf-16-le") + metadata = core.ArchiveMetadata(unk_a, block_size) + + return Archive(name, metadata, drives) + + def write(self, stream: BinaryIO, archive: Archive) -> int: + raise NotImplementedError + + def __init__(self): + self.DriveDef = drive_serializer + self.FolderDef = folder_serializer + self.FileDef = file_serializer + self.TocHeader = toc_header_serializer + self.version = core.version + self.layout = Struct("<128s 3I") + self.metadata_layout = Struct("<2I") diff --git a/src/relic/sga/v7/core.py b/src/relic/sga/v7/core.py new file mode 100644 index 0000000..1f11f15 --- /dev/null +++ b/src/relic/sga/v7/core.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + +from relic.sga import _abc +from relic.sga._abc import FileDefABC +from relic.sga.errors import Version +from relic.sga._core import VerificationType + +version = Version(7) + + +@dataclass +class ArchiveMetadata: + unk_a: int + block_size:int + + +@dataclass +class FileDef(FileDefABC): + modified: datetime + verification: VerificationType + crc: int + hash_pos: int + + +@dataclass +class FileMetadata: + modified: datetime + verification: VerificationType + crc: int + hash_pos: int + + +Archive = _abc.Archive[ArchiveMetadata] +Folder = _abc.Folder +File = _abc.File[FileMetadata] +Drive = _abc.Drive diff --git a/src/relic/sga/v9/__init__.py b/src/relic/sga/v9/__init__.py new file mode 100644 index 0000000..519fe96 --- /dev/null +++ b/src/relic/sga/v9/__init__.py @@ -0,0 +1,22 @@ +from relic.sga import _abc +from relic.sga.v9._serializers import APISerializers +from relic.sga.v9.core import Archive, Drive, Folder, File, ArchiveMetadata, version + + +def _create_api(): + serializer = APISerializers() + api = _abc.API(version, Archive, Drive, Folder, File, serializer) + return api + + +API = _create_api() + +__all__ = [ + "Archive", + "Drive", + "Folder", + "File", + "API", + "version", + "ArchiveMetadata" +] diff --git a/src/relic/sga/v9/_serializers.py b/src/relic/sga/v9/_serializers.py new file mode 100644 index 0000000..13e1a46 --- /dev/null +++ b/src/relic/sga/v9/_serializers.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import BinaryIO, Optional + +from serialization_tools.structx import Struct + +from relic.sga import _abc, _serializers as _s +from relic.sga._abc import Archive +from relic.sga.errors import MismatchError, VersionMismatchError +from relic.sga.protocols import StreamSerializer +from relic.sga._core import StorageType, VerificationType, Version, MagicWord +from relic.sga.v9 import core + +folder_layout = Struct("<5I") +folder_serializer = _s.FolderDefSerializer(folder_layout) + +drive_layout = Struct("<64s 64s 5I") +drive_serializer = _s.DriveDefSerializer(drive_layout) + +file_layout = Struct("<2I Q 3I 2B I") + + +class FileDefSerializer(StreamSerializer[core.FileDef]): + def __init__(self, layout: Struct): + self.layout = layout + + def unpack(self, stream: BinaryIO): + name_rel_pos, hash_pos, data_rel_pos, length, store_length, modified_seconds,verification_type_val, storage_type_val, crc = self.layout.unpack_stream(stream) + + modified = datetime.fromtimestamp(modified_seconds, timezone.utc) + storage_type: StorageType = StorageType(storage_type_val) + verification_type: VerificationType = VerificationType(verification_type_val) + + return core.FileDef(name_rel_pos, data_rel_pos, length, store_length, storage_type, modified, verification_type,crc, hash_pos) + + def pack(self, stream: BinaryIO, value: core.FileDef) -> int: + modified: int = int(value.modified.timestamp()) + storage_type = value.storage_type.value # convert enum to value + verification_type = value.verification.value # convert enum to value + args = value.name_pos, value.hash_pos, value.data_pos, value.length_on_disk, value.length_in_archive, storage_type, modified, verification_type, value.crc + return self.layout.pack_stream(stream, *args) + + +file_serializer = FileDefSerializer(file_layout) +toc_layout = Struct("<8I") +toc_header_serializer = _s.TocHeaderSerializer(toc_layout) + + +class APISerializers(_abc.APISerializer): + def read(self, stream: BinaryIO, lazy: bool = False, decompress: bool = True) -> Archive: + MagicWord.read_magic_word(stream) + version = Version.unpack(stream) + if version != self.version: + raise VersionMismatchError(version,self.version) + + + encoded_name: bytes + encoded_name, header_pos, header_size, data_pos, data_pos, RSV_1, sha_256 = self.layout.unpack_stream(stream) + if RSV_1 != 1: + raise MismatchError("Reserved Field", RSV_1, 1) + # header_pos = stream.tell() + stream.seek(header_pos) + toc_header = self.TocHeader.unpack(stream) + unk_a, unk_b, block_size = self.metadata_layout.unpack_stream(stream) + drive_defs, folder_defs, file_defs = _s._read_toc_definitions(stream, toc_header, header_pos, self.DriveDef, self.FolderDef, self.FileDef) + names = _s._read_toc_names_as_size(stream, toc_header.name_info, header_pos) + drives, files = _s._assemble_io_from_defs(drive_defs, folder_defs, file_defs, names, data_pos, stream, decompress=decompress) + + if not lazy: + for file in files: + lazy_info: Optional[_abc._FileLazyInfo] = file._lazy_info + if lazy_info is None: + raise Exception("API read files, but failed to create lazy info!") + else: + file.data = lazy_info.read(decompress) + file._lazy_info = None + + name: str = encoded_name.rstrip(b"").decode("utf-16-le") + metadata = core.ArchiveMetadata(sha_256, unk_a, unk_b, block_size) + + return Archive(name, metadata, drives) + + def write(self, stream: BinaryIO, archive: Archive) -> int: + raise NotImplementedError + + def __init__(self): + self.DriveDef = drive_serializer + self.FolderDef = folder_serializer + self.FileDef = file_serializer + self.TocHeader = toc_header_serializer + self.version = core.version + self.layout = Struct("<128s QIQQ I 256s") + self.metadata_layout = Struct("<3I") diff --git a/src/relic/sga/v9/core.py b/src/relic/sga/v9/core.py new file mode 100644 index 0000000..fdce714 --- /dev/null +++ b/src/relic/sga/v9/core.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + +from relic.sga import _abc +from relic.sga._abc import FileDefABC +from relic.sga._core import VerificationType +from relic.sga.errors import Version + +version = Version(9) + + +@dataclass +class ArchiveMetadata: + sha_256:bytes + unk_a: int + unk_b: int + block_size:int + + +@dataclass +class FileDef(FileDefABC): + modified: datetime + verification: VerificationType + crc: int + hash_pos: int + + +@dataclass +class FileMetadata: + modified: datetime + verification: VerificationType + crc: int + hash_pos: int + + +Archive = _abc.Archive[ArchiveMetadata] +Folder = _abc.Folder +File = _abc.File[FileMetadata] +Drive = _abc.Drive diff --git a/src/relic/sga/vdrive/__init__.py b/src/relic/sga/vdrive/__init__.py deleted file mode 100644 index d105b6e..0000000 --- a/src/relic/sga/vdrive/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from .virtual_drive import VirtualDrive -from .header import VirtualDriveHeader, DowIVirtualDriveHeader, DowIIVirtualDriveHeader, DowIIIVirtualDriveHeader - -__all__ = [ - "VirtualDrive", - "VirtualDriveHeader", - "DowIVirtualDriveHeader", - "DowIIVirtualDriveHeader", - "DowIIIVirtualDriveHeader", -] diff --git a/src/relic/sga/vdrive/header.py b/src/relic/sga/vdrive/header.py deleted file mode 100644 index 547b816..0000000 --- a/src/relic/sga/vdrive/header.py +++ /dev/null @@ -1,65 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import ClassVar, BinaryIO, Dict, Type - -from serialization_tools.structx import Struct - -from ...common import VersionLike -from ..common import ArchiveVersion, ArchiveRange - - -@dataclass -class VirtualDriveHeader: - LAYOUT: ClassVar[Struct] - - path: str - name: str - - sub_folder_range: ArchiveRange - file_range: ArchiveRange - unk: bytes - - @classmethod - def unpack(cls, stream: BinaryIO, version: VersionLike) -> 'VirtualDriveHeader': - header_class = _HEADER_VERSION_MAP.get(version) - - if not header_class: - raise NotImplementedError(version) - - return header_class._unpack(stream) - - def _pack(self, stream: BinaryIO) -> int: - args = self.path.encode("ascii"), self.name.encode("ascii"), self.sub_folder_range.start, self.sub_folder_range.end, \ - self.file_range.start, self.file_range.end, 0 - return self.LAYOUT.pack_stream(stream, *args) - - @classmethod - def _unpack(cls, stream: BinaryIO) -> 'VirtualDriveHeader': - path, name, sub_folder_start, sub_folder_end, file_start, file_end, unk = cls.LAYOUT.unpack_stream(stream) - path, name = path.decode("ascii").rstrip("\00"), name.decode("ascii").rstrip("\00") - sub_folder_range = ArchiveRange(sub_folder_start, sub_folder_end) - file_range = ArchiveRange(file_start, file_end) - return cls(path, name, sub_folder_range, file_range, unk) - - -@dataclass -class DowIVirtualDriveHeader(VirtualDriveHeader): - LAYOUT = Struct("< 64s 64s 4H 2s") - - -@dataclass -class DowIIVirtualDriveHeader(VirtualDriveHeader): - LAYOUT = Struct("< 64s 64s 4H 2s") - - -@dataclass -class DowIIIVirtualDriveHeader(VirtualDriveHeader): - LAYOUT = Struct("< 64s 64s 4L 4s") - - -_HEADER_VERSION_MAP: Dict[VersionLike, Type[VirtualDriveHeader]] = { - ArchiveVersion.Dow: DowIVirtualDriveHeader, - ArchiveVersion.Dow2: DowIIVirtualDriveHeader, - ArchiveVersion.Dow3: DowIIIVirtualDriveHeader -} diff --git a/src/relic/sga/vdrive/virtual_drive.py b/src/relic/sga/vdrive/virtual_drive.py deleted file mode 100644 index 3a8da0d..0000000 --- a/src/relic/sga/vdrive/virtual_drive.py +++ /dev/null @@ -1,67 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import PurePosixPath -from typing import List, TYPE_CHECKING - -from ..hierarchy import FileCollection, FolderCollection, ArchiveWalk, walk - -if TYPE_CHECKING: - from ..file.file import File - from ..folder.folder import Folder - from ..vdrive.header import VirtualDriveHeader - from ..toc.toc import ArchiveTableOfContents - - -@dataclass -class VirtualDrive(FolderCollection, FileCollection): - header: VirtualDriveHeader - - def __init__(self, header: VirtualDriveHeader, sub_folders: List[Folder], files: List[File]): - self.header = header - self.sub_folders = sub_folders - self.files = files - - @property - def path(self) -> str: - return self.header.path - - @property - def name(self) -> str: - return self.header.name - - def walk(self) -> ArchiveWalk: - return walk(self) - - @property - def full_path(self) -> PurePosixPath: - return PurePosixPath(self.path + ":") - - @classmethod - def create(cls, header: VirtualDriveHeader) -> VirtualDrive: - folders = [None] * header.sub_folder_range.size - files = [None] * header.file_range.size - # noinspection PyTypeChecker - return VirtualDrive(header, folders, files) - - def load_toc(self, toc: ArchiveTableOfContents): - self.load_folders(toc.folders) - self.load_files(toc.files) - - def load_folders(self, folders: List[Folder]): - if self.header.sub_folder_range.start < len(folders): - for folder_index in self.header.sub_folder_range: - sub_folder_index = folder_index - self.header.sub_folder_range.start - f = self.sub_folders[sub_folder_index] = folders[folder_index] - f._drive = self - - def load_files(self, files: List[File]): - if self.header.file_range.start < len(files): - for file_index in self.header.file_range: - sub_file_index = file_index - self.header.file_range.start - f = self.files[sub_file_index] = files[file_index] - f._drive = self - - def build_tree(self): - self.sub_folders = [f for f in self.sub_folders if not f._parent] - self.files = [f for f in self.files if not f._parent] diff --git a/src/relic/sga/writer.py b/src/relic/sga/writer.py deleted file mode 100644 index 44120fb..0000000 --- a/src/relic/sga/writer.py +++ /dev/null @@ -1,297 +0,0 @@ -# TODO Dig through this and see how much can be moved to TOC and if any of it must be a separate file - -# # Cycles aren't supported (and will crash) -# # Multiple parents will be copied -# -# -# def flatten_folders(collection: AbstractDirectory, flattened: List[Folder]) -> Tuple[int, int]: -# start = len(flattened) -# flattened.extend(collection.folders) -# stop = len(flattened) -# return start, stop -# -# -# def flatten_files(collection: AbstractDirectory, flattened: List[File]) -> Tuple[int, int]: -# start = len(flattened) -# flattened.extend(collection.files) -# stop = len(flattened) -# return start, stop -# -# -# # Offset, Count (Items), Size (Bytes) -# def write_virtual_drives(stream: BinaryIO, archive: Archive, version: Version, name_table: Dict[any, int], -# recalculate: bool = False) -> Tuple[int, int, int]: -# running_folder = 0 -# running_file = 0 -# written = 0 -# -# offset = stream.tell() -# for drive in archive.drives: -# folder_count = drive.folder_count(recalculate) -# file_count = drive.file_count(recalculate) -# -# folder = ArchiveRange(running_folder, running_folder + folder_count) -# files = ArchiveRange(running_file, running_file + file_count) -# -# running_folder += folder_count -# running_file += file_count -# -# header = VirtualDriveHeader(drive.path, drive.name, folder, files, folder.start) -# written += header.pack(stream, version) -# -# return offset, len(archive.drives), written -# -# -# def write_names(stream: BinaryIO, archive: Archive) -> Tuple[int, int, int, Dict[str, int]]: -# offset = stream.tell() -# running_total = 0 -# lookup = {} -# written = 0 -# -# def try_write_null_terminated(name: str) -> int: -# if name in lookup: -# return 0 -# # We must use relative offset to data_origin -# lookup[name] = stream.tell() - offset -# terminated_name = name -# if name[-1] != "\0": -# terminated_name += "\0" -# encoded = terminated_name.encode("ascii") -# return stream.write(encoded) -# -# # This will not re-use repeated names; we could change it, but I won't since my brain is over-optimizing this -# # By allowing names to repeat, we avoid perform hash checks in a dictionary (or equality comparisons in a list) -# for drive in archive.drives: -# for _, folders, files in drive.walk(): -# for f in folders: -# written += try_write_null_terminated(f.name) -# running_total += 1 -# for f in files: -# written += try_write_null_terminated(f.name) -# running_total += 1 -# -# return offset, running_total, written, lookup -# -# -# # Offset, Count (Items), Size (Bytes) -# def write_folders(stream: BinaryIO, archive: Archive, version: Version, name_lookup: Dict[str, int], -# recalculate: bool = False) -> Tuple[ -# int, int, int]: -# running_folder = 0 -# running_file = 0 -# written = 0 -# total_folders = 0 -# offset = stream.tell() -# for drive in archive.drives: -# for _, folders, _ in drive.walk(): -# for folder in folders: -# total_folders += 1 -# folder_count = folder.folder_count(recalculate) -# file_count = folder.file_count(recalculate) -# -# folder_range = ArchiveRange(running_folder, running_folder + folder_count) -# file_range = ArchiveRange(running_file, running_file + file_count) -# -# running_folder += folder_count -# running_file += file_count -# -# name_offset = name_lookup[folder.name] -# -# header = FolderHeader(name_offset, folder_range, file_range) -# written += header.pack(stream, version) -# -# return offset, total_folders, written -# -# -# def get_v2_compflag(comp_data: bytes, decomp_data: bytes): -# if len(comp_data) == len(decomp_data): -# return FileCompressionFlag.Decompressed -# flag = (comp_data[0] & 0xF0) >> 4 -# lookup = {7: FileCompressionFlag.Compressed32, 6: FileCompressionFlag.Compressed16} -# return lookup[flag] -# -# -# def get_v9_compflag(comp_data: bytes, decomp_data: bytes): -# if len(comp_data) == len(decomp_data): -# return 0 -# flag = (comp_data[0] & 0xF0) >> 4 -# lookup = {7: FileCompressionFlag.Compressed32, 6: FileCompressionFlag.Compressed16} -# return lookup[flag] -# -# -# # Lookup ~ Offset, Copmressed, Decompressed, Version Args -# # Offset, Count, Byte Size -# def write_file_data(stream: BinaryIO, archive: Archive, version: Version, auto_compress: bool = True) -> Tuple[ -# int, int, int, Dict[File, FileHeader]]: -# offset = stream.tell() -# -# KIBI = 1024 -# Kb16 = 16 * KIBI -# Kb32 = 32 * KIBI -# -# lookup = {} -# -# def write_info(compressed_data: bytes, decompressed_data: bytes) -> FileHeader: -# # We must use relative offset to data_origin -# data_offset = stream.tell() - offset -# -# if version == ArchiveVersion.Dow: -# compression_flag = get_v2_compflag(decompressed_data, decompressed_data) -# header = DowIFileHeader(None, data_offset, len(decompressed_data), len(compressed_data), compression_flag) -# elif version == ArchiveVersion.Dow2: -# header = DowIIFileHeader(None, data_offset, len(decompressed_data), len(compressed_data), 0, 0) -# elif version == ArchiveVersion.Dow3: -# # TODO rename unk_d to compression_flag -# compression_flag = get_v9_compflag(decompressed_data, decompressed_data) -# header = DowIIIFileHeader(None, data_offset, len(decompressed_data), len(compressed_data), 0, 0, 0, compression_flag, 0) -# else: -# raise NotImplementedError(version) -# stream.write(compressed_data) -# return header -# -# for drive in archive.drives: -# for _, _, files in drive.walk(): -# for file in files: -# comp_data = file.data -# decomp_data = file.get_decompressed() -# -# if not auto_compress: # Just dump it and GO! -# header = write_info(comp_data, decomp_data) -# else: -# # This is rather arbitrary, but these are my rules for auto-copmression: -# # Don't compress files that... -# # Are compressed (duh) -# # Are smaller than the largest (16-KibiBytes) compression window -# # When Compressing Files... -# # If the data size is less than 256 KibiBytes -# # Use 16-KbB Window -# # Otherwise -# # Use 32-KbB Window -# if len(comp_data) != len(decomp_data): # Compressed; just write as is -# header = write_info(comp_data, decomp_data) -# elif len(decomp_data) < Kb16: # Too small -# header = write_info(comp_data, decomp_data) -# else: -# if len(decomp_data) < KIBI: # Use Window 16KbB -# compressor = zlib.compressobj(wbits=14) -# else: # Use Window 32KbB -# compressor = zlib.compressobj(wbits=15) -# # Compress; because we are using compression obj, we need to use a temp -# with BytesIO() as temp: -# temp.write(compressor.compress(comp_data)) -# temp.write(compressor.flush()) -# temp.seek(0) -# comp_data = temp.read() -# header = write_info(comp_data, decomp_data) -# lookup[file] = header -# -# stop = stream.tell() -# size = stop - offset -# return offset, len(lookup), size, lookup -# -# -# def write_files(stream: BinaryIO, archive: Archive, version: Version, name_lookup: Dict[str, int], -# data_lookup: Dict[File, FileHeader]) -> Tuple[int, int, int]: -# offset = stream.tell() -# written = 0 -# file_count = 0 -# -# for drive in archive.drives: -# for _, _, files in drive.walk(): -# for file in files: -# header = data_lookup[file] -# header.name_subptr = name_lookup[file.name] -# written += header.pack_version(stream, version) -# file_count += 1 -# -# return offset, file_count, written -# -# -# def write_table_of_contents(stream: BinaryIO, archive: Archive, version: Version, -# data_lookup: Dict[File, FileHeader], recalculate_totals: bool = True) -> Tuple[int, int]: -# if recalculate_totals: -# for d in archive.drives: -# d.folder_count(True) -# d.file_count(True) -# -# toc_offset = stream.tell() -# toc_size = ArchiveToC.get_size(version) -# stream.write(bytes([0x00] * toc_size)) -# -# # Names needs to be computer first, but DOW's layout is Drives, Folders, Files, Names (not that it HAS to be) -# # I follow their pattern for consistency if nothing else -# # THIS ONLY WORKS BECAUSE OFFSETS ARE RELATIVE TO THE NAME OFFSET -# with BytesIO() as name_buffer: -# _, name_count, name_size, name_lookup = write_names(name_buffer, archive) -# -# vd_offset, vd_count, vd_size = write_virtual_drives(stream, archive, version, name_lookup) -# vd_part = OffsetInfo(toc_offset, vd_offset - toc_offset, vd_count) -# -# fold_offset, fold_count, fold_size = write_folders(stream, archive, version, name_lookup) -# fold_part = OffsetInfo(toc_offset, fold_offset - toc_offset, fold_count) -# -# file_offset, file_count, file_size = write_files(stream, archive, version, name_lookup, data_lookup) -# file_part = OffsetInfo(toc_offset, file_offset - toc_offset, file_count) -# -# name_offset = stream.tell() -# name_buffer.seek(0) -# stream.write(name_buffer.read()) -# name_part = FilenameOffsetInfo(toc_offset, name_offset - toc_offset, name_count, name_size) -# -# end = stream.tell() -# # Writeback proper TOC -# toc = ArchiveTableOfContents(vd_part, fold_part, file_part, name_part) -# stream.seek(toc_offset) -# toc.pack(stream, version) -# -# stream.seek(end) -# return toc_offset, end - toc_offset -# -# -# def write_archive(stream: BinaryIO, archive: Archive, auto_compress: bool = True, recalculate_totals: bool = True) -> int: -# version = archive.info.header.version -# -# if version not in [ArchiveVersion.Dow, ArchiveVersion.Dow2, ArchiveVersion.Dow3]: -# raise NotImplementedError(version) -# -# start = stream.tell() -# # PRIMARY HEADER -# archive.info.header.pack(stream) -# -# # SUB HEADER SETUP -# # We need to do a write-back once we know the offsets, sizes, what have you -# subheader_offset = stream.tell() -# -# subheader = ArchiveSubHeader.default(version) -# subheader.pack(stream, version) # Write filler data -# -# # TOC & DATA -# if version == ArchiveVersion.Dow: -# # Unfortunately, we depend on Data Buffer to write TOC, and TOC 'MUST' come immediately after the Sub Header in Sga-V2.0 -# # So we write data to a memory buffer before rewriting to a -# with BytesIO() as data_buffer: -# _, _, _, data_lookup = write_file_data(data_buffer, archive, version, auto_compress) -# toc_offset, toc_size = write_table_of_contents(stream, archive, version, data_lookup, recalculate_totals) -# data_offset = stream.tell() -# data_buffer.seek(0) -# stream.write(data_buffer.read()) -# subheader = ArchiveSubHeader(toc_size, data_offset, toc_offset) -# -# elif version in [ArchiveVersion.Dow2, ArchiveVersion.Dow3]: -# # Since these formats can point to TOC specifically, I write to the stream directly -# data_offset, _, data_size, data_lookup = write_file_data(stream, archive, version, auto_compress) -# toc_offset, toc_size = write_table_of_contents(stream, archive, version, data_lookup, recalculate_totals) -# if version == ArchiveVersion.Dow2: -# subheader = ArchiveSubHeader(toc_size, data_offset, toc_offset, 1, 0, 0) -# elif version == ArchiveVersion.Dow3: -# subheader = ArchiveSubHeader(toc_size, data_offset, toc_offset, None, None, None, 0, 0, 1, -# bytes([0x00] * 256), data_size) -# else: -# raise NotImplementedError(version) # In case I add to the list in the above if and forget to add it here -# -# end = stream.tell() -# stream.seek(subheader_offset) -# subheader.pack(stream, version) -# -# stream.seek(end) -# return end - start diff --git a/src/relic/ucs.py b/src/relic/ucs.py index db40f92..c71acf1 100644 --- a/src/relic/ucs.py +++ b/src/relic/ucs.py @@ -1,19 +1,15 @@ from __future__ import annotations -import json import re -from os import PathLike, walk from collections import UserDict +from os import PathLike, walk from os.path import join, splitext, split -from pathlib import Path from typing import TextIO, Optional, Iterable, Union, Mapping # UCS probably stands for UnicodeString # I personally think that's a horribly misleading name for this file from serialization_tools.walkutil import filter_by_file_extension, collapse_walk_on_files, filter_by_path -from relic.config import DowIIIGame, DowGame, DowIIGame, filter_latest_dow_game, get_dow_root_directories - class UcsDict(UserDict): def write_stream(self, stream: TextIO, ordered: bool = False) -> int: @@ -175,33 +171,3 @@ def get_lang_string_for_file(environment: Union[LangEnvironment, LangFile], file replacement = _file_safe_string(replacement) return join(dir_path, replacement + f" ~ Clip {num}" + ext) - - -if __name__ == "__main__": - # A compromise between an automatic location and NOT the local directory - # PyCharm will hang trying to reload the files (just to update the hierarchy, not update references) - # To avoid that, we DO NOT use a local directory, but an external directory - # TODO add a persistent_data path to archive tools - Root = Path(r"~\Appdata\Local\ModernMAK\ArchiveTools\Relic-SGA").expanduser() - dump_type = "UCS_DUMP" - path_lookup = { - DowIIIGame: Root / r"DOW_III", - DowIIGame: Root / r"DOW_II", - DowGame: Root / r"DOW_I" - } - series = DowGame - out_path = path_lookup[series] / dump_type - r = filter_latest_dow_game(get_dow_root_directories(), series=series) - if r: - game, in_path = r - else: - raise FileNotFoundError("Couldn't find any suitable DOW games!") - - print("Loading Locale Environment...") - lang_env = LangEnvironment.load_environment(in_path) - print(f"\tReading from '{in_path}'") - out_path = out_path.with_suffix(".json") - with open(out_path, "w") as handle: - lang_env_sorted = dict(sorted(lang_env.items())) - json.dump(lang_env_sorted, handle, indent=4) - print(f"\tSaved to '{out_path}'") diff --git a/src/scripts/dump_sga.py b/src/scripts/dump_sga.py deleted file mode 100644 index 108849c..0000000 --- a/src/scripts/dump_sga.py +++ /dev/null @@ -1,107 +0,0 @@ -import os -from os.path import splitext, dirname, basename -from pathlib import Path -from typing import Iterable - -from serialization_tools.walkutil import BlackList, WhiteList, filter_by_path, filter_by_file_extension, collapse_walk_on_files - -from relic.config import DowIIIGame, DowIIGame, DowGame, filter_latest_dow_game, get_dow_root_directories - -from relic.sga.archive import ArchiveMagicWord, Archive - - -def __safe_makedirs(path: str, use_dirname: bool = True): - if use_dirname: - path = dirname(path) - try: - os.makedirs(path) - except FileExistsError: - pass - - -# walk all archives in the given directory, custom whitelist, blacklist, and extensions will overwrite defaults -# Defaults: .sga, No *-Med, *-Low archives -def walk_archive_paths(folder: os.PathLike, extensions: WhiteList = None, whitelist: WhiteList = None, blacklist: BlackList = None) -> Iterable[str]: - # Default EXT and Blacklist - extensions = extensions or "sga" # Default to sga, it shouldn't ever be different, so I could probably - blacklist = blacklist or ["-Low", "-Med"] # Typically, beside -High files, we only want the biggest - # Flattened long call to make it easy to read - walk = os.walk(folder) - walk = filter_by_path(walk, whitelist=whitelist, blacklist=blacklist, prune=True) - walk = filter_by_file_extension(walk, whitelist=extensions) - walk = ArchiveMagicWord.walk(walk) - return collapse_walk_on_files(walk) - - -def dump_archive(input_folder: os.PathLike, output_folder: os.PathLike, overwrite: bool = False, update: bool = False): - if overwrite and update: - raise NotImplementedError("Both write options selected, would you like to overwrite files? Or only update non-matching files?") - - output_folder_path = Path(output_folder) - for input_file_path in walk_archive_paths(input_folder): - with open(input_file_path, "rb") as in_handle: - archive = Archive.unpack(in_handle) - archive_name = splitext(basename(input_file_path))[0] - with archive.header.data_ptr.stream_jump_to(in_handle) as data_stream: - print(f"\tDumping '{archive_name}'") - for _, _, _, files in archive.walk(): - for file in files: - relative_file_path = file.full_path - - if ':' in relative_file_path.parts[0]: - relative_file_path = str(relative_file_path).replace(":", "") - - output_file_path = output_folder_path / archive_name / relative_file_path - - msg = f"Writing '{relative_file_path}'" - skip = False - if output_file_path.exists(): - if update: - if output_file_path.stat().st_size == file.header.decompressed_size: - msg = f"Skipping (Up to date - Decompressed Size Match)" - skip = True - elif output_file_path.stat().st_size == file.header.compressed_size: - msg = f"Skipping (Up to date - Compressed Size Match)" - skip = True - else: - msg = f"Updating" - elif not overwrite: - msg = f"Skipping (Exists)" - skip = True - - print(f"\t\t{msg} '{relative_file_path}'") - if skip: - continue - __safe_makedirs(str(output_file_path)) - with open(output_file_path, "wb") as out_handle: - data = file.read_data(data_stream, True) - out_handle.write(data) - print(f"\t\t\tWrote to '{output_file_path}'") - - # write_binary(walk, output_folder, decompress, write_ext) - - -if __name__ == "__main__": - # A compromise between an automatic location and NOT the local directory - # PyCharm will hang trying to reload the files (just to update the hierarchy, not update references) - # To avoid that, we DO NOT use a local directory, but an external directory - # TODO add a persistent_data path to archive tools - Root = Path(r"~\Appdata\Local\ModernMAK\ArchiveTools\Relic-SGA").expanduser() - dump_type = "SGA_DUMP" - path_lookup = { - DowIIIGame: Root / r"DOW_III", - DowIIGame: Root / r"DOW_II", - DowGame: Root / r"DOW_I" - } - series = DowGame - out_path = path_lookup[series] / dump_type - r = filter_latest_dow_game(get_dow_root_directories(), series=series) - if r: - game, in_path = r - else: - raise FileNotFoundError("Couldn't find any suitable DOW games!") - print(f"Dumping game '{game}' from '{in_path}' to '{out_path}'\n") - dump_archive(in_path, out_path, update=True) - print(f"\nDumped game '{game}' from '{in_path}' to '{out_path}'") - # dump_all_sga(root, blacklist=[r"-Low", "-Med"], - # out_dir=r"D:/Dumps/DOW I/sga", verbose=True) diff --git a/src/scripts/universal/sga/common.py b/src/scripts/universal/sga/common.py index 27d4ca6..bac6b57 100644 --- a/src/scripts/universal/sga/common.py +++ b/src/scripts/universal/sga/common.py @@ -5,7 +5,7 @@ from serialization_tools.walkutil import blacklisted -from relic.sga import ArchiveMagicWord +from relic.sga.errors import MagicWord from scripts.universal.common import print_error, print_wrote, print_reading, PrintOptions, SharedExtractorParser SharedSgaParser = argparse.ArgumentParser(parents=[SharedExtractorParser], add_help=False) @@ -29,7 +29,7 @@ def is_sga(input_file: str, ext: Union[str, List[str]] = None, magic: bool = Fal # Make sure magic word is present if magic: with open(input_file, "rb") as check_handle: - return ArchiveMagicWord.check_magic_word(check_handle) + return MagicWord.check_magic_word(check_handle) return True diff --git a/src/scripts/universal/sga/unpack.py b/src/scripts/universal/sga/unpack.py index 0d5f5c2..1cb31e0 100644 --- a/src/scripts/universal/sga/unpack.py +++ b/src/scripts/universal/sga/unpack.py @@ -3,7 +3,8 @@ from pathlib import Path from typing import Dict -from relic.sga import Archive +from relic.sga.errors import FileABC +from relic.sga.apis import read_archive from scripts.universal.common import PrintOptions, print_error, print_any, SharedExtractorParser from scripts.universal.sga.common import get_runner @@ -25,37 +26,39 @@ def extract_args(args: argparse.Namespace) -> Dict: def unpack_archive(in_path: str, out_path: str, print_opts: PrintOptions = None, prepend_archive_path: bool = True, indent_level: int = 0, **kwargs): out_path = Path(out_path) with open(in_path, "rb") as in_handle: - archive = Archive.unpack(in_handle) + archive = read_archive(in_handle, True) archive_name = splitext(basename(in_path))[0] - with archive.header.data_ptr.stream_jump_to(in_handle) as data_stream: - print_any(f"Unpacking \"{archive_name}\"...", indent_level, print_opts) - for _, _, _, files in archive.walk(): - for file in files: - try: - relative_file_path = file.full_path - - if ':' in relative_file_path.parts[0]: - relative_file_path = str(relative_file_path).replace(":", "") - - rel_out_path = Path(out_path) - if prepend_archive_path: - rel_out_path /= archive_name - - rel_out_path /= relative_file_path - - rel_out_path.parent.mkdir(parents=True, exist_ok=True) - print_any(f"Reading \"{relative_file_path}\"...", indent_level + 1, print_opts) - with open(rel_out_path, "wb") as out_handle: - data = file.read_data(data_stream, True) - out_handle.write(data) - print_any(f"Writing \"{rel_out_path}\"...", indent_level + 2, print_opts) - except KeyboardInterrupt: + # with archive.header.data_ptr.stream_jump_to(in_handle) as data_stream: + print_any(f"Unpacking \"{archive_name}\"...", indent_level, print_opts) + for _, _, _, files in archive.walk(): + for file in files: + file: FileABC + try: + relative_file_path = file.path + + # Cant use drive since our 'drive' isn't one letter + if ':' in relative_file_path.parts[0]: + relative_file_path = str(relative_file_path).replace(":", "") # Valid on windows systems, on posix; idk + + rel_out_path = Path(out_path) + if prepend_archive_path: + rel_out_path /= archive_name + + rel_out_path /= relative_file_path + + rel_out_path.parent.mkdir(parents=True, exist_ok=True) + print_any(f"Reading \"{relative_file_path}\"...", indent_level + 1, print_opts) + with open(rel_out_path, "wb") as out_handle: + file.read_data(in_handle) + out_handle.write(file.data) + print_any(f"Writing \"{rel_out_path}\"...", indent_level + 2, print_opts) + except KeyboardInterrupt: + raise + except BaseException as e: + if not print_opts or print_opts.error_fail: raise - except BaseException as e: - if not print_opts or print_opts.error_fail: - raise - else: - print_error(e, indent_level, print_opts) + else: + print_error(e, indent_level, print_opts) Runner = get_runner(unpack_archive, extract_args) diff --git a/tests/relic/sga/archive/test_archive.py b/tests/relic/sga/archive/test_archive.py deleted file mode 100644 index 585848b..0000000 --- a/tests/relic/sga/archive/test_archive.py +++ /dev/null @@ -1,168 +0,0 @@ -from abc import abstractmethod -from io import BytesIO - -import pytest - -from relic.sga.archive import Archive, ArchiveMagicWord -from relic.sga.hierarchy import ArchiveWalk -from tests.helpers import TF -from tests.relic.sga.datagen import DowII, DowI, DowIII - - -class ArchiveTests: - def assert_equal(self, expected: Archive, result: Archive, sparse: bool): - assert expected.header == result.header - if sparse: - assert result._sparse - # TODO - - @abstractmethod - def test_walk(self, archive: Archive, expected: ArchiveWalk): - archive_walk = archive.walk() - for (a_vdrive, a_folder, a_folders, a_files), (e_vdrive, e_folder, e_folders, e_files) in zip(archive_walk, expected): - assert a_vdrive == e_vdrive - assert a_folder == e_folder - assert a_folders == e_folders - assert a_files == e_files - - @abstractmethod - def test_inner_unpack(self, stream_data: bytes, expected: Archive): - for sparse in TF: - with BytesIO(stream_data) as stream: - archive = expected.__class__._unpack(stream, expected.header, sparse) - assert expected.__class__ == archive.__class__ - self.assert_equal(expected, archive, sparse) - - @abstractmethod - def test_unpack(self, stream_data: bytes, expected: Archive, valid_checksums: bool): - for read_magic in TF: - for sparse in TF: - for validate in ([False] if not valid_checksums else TF): - with BytesIO(stream_data) as stream: - if not read_magic: - stream.seek(ArchiveMagicWord.layout.size) - archive = Archive.unpack(stream, read_magic, sparse, validate=validate) - assert expected.__class__ == archive.__class__ - self.assert_equal(expected, archive, sparse) - - @abstractmethod - def test_pack(self, archive: Archive, expected: bytes): - for write_magic in TF: - try: - with BytesIO() as stream: - packed = archive.pack(stream, write_magic) - except NotImplementedError: - pass # Currently not implemented; we'll expect this for now - else: - assert expected == packed - - -def fast_gen_dow1_archive(*args): - return DowI.gen_sample_archive(*args), DowI.gen_sample_archive_buffer(*args) - - -DOW1_ARCHIVE, DOW1_ARCHIVE_PACKED = fast_gen_dow1_archive("Dow1 Test Archive", "Tests", "And Now For Something Completely Different.txt", b"Just kidding, it's Monty Python.") - - -def DOW1_ARCHIVE_WALK() -> ArchiveWalk: - a = DOW1_ARCHIVE - d = a.drives[0] - sfs = d.sub_folders - yield d, None, sfs, [] - yield d, sfs[0], [], sfs[0].files - - -class TestDowIArchive(ArchiveTests): - @pytest.mark.parametrize(["stream_data", "expected"], - [(DOW1_ARCHIVE_PACKED, DOW1_ARCHIVE)]) - def test_inner_unpack(self, stream_data: bytes, expected: Archive): - super().test_inner_unpack(stream_data, expected) - - @pytest.mark.parametrize(["stream_data", "expected", "valid_checksums"], - [(DOW1_ARCHIVE_PACKED, DOW1_ARCHIVE, True)]) - def test_unpack(self, stream_data: bytes, expected: Archive, valid_checksums: bool): - super().test_unpack(stream_data, expected, valid_checksums) - - @pytest.mark.parametrize(["archive", "expected"], - [(DOW1_ARCHIVE, DOW1_ARCHIVE_PACKED)]) - def test_pack(self, archive: Archive, expected: bytes): - super().test_pack(archive, expected) - - @pytest.mark.parametrize(["archive", "expected"], - [(DOW1_ARCHIVE, DOW1_ARCHIVE_WALK())]) - def test_walk(self, archive: Archive, expected: ArchiveWalk): - super().test_walk(archive, expected) - - -def fast_gen_dow2_archive(*args): - return DowII.gen_sample_archive(*args), DowII.gen_sample_archive_buffer(*args) - - -DOW2_ARCHIVE, DOW2_ARCHIVE_PACKED = fast_gen_dow2_archive("Dow2 Test Archive", "Tests", "A Favorite Guardsmen VL.txt", b"Where's that artillery!?") - - -def DOW2_ARCHIVE_WALK() -> ArchiveWalk: - a = DOW2_ARCHIVE - d = a.drives[0] - sfs = d.sub_folders - yield d, None, sfs, [] - yield d, sfs[0], [], sfs[0].files - - -class TestDowIIArchive(ArchiveTests): - @pytest.mark.parametrize(["stream_data", "expected"], - [(DOW2_ARCHIVE_PACKED, DOW2_ARCHIVE)]) - def test_inner_unpack(self, stream_data: bytes, expected: Archive): - super().test_inner_unpack(stream_data, expected) - - @pytest.mark.parametrize(["stream_data", "expected", "valid_checksums"], - [(DOW2_ARCHIVE_PACKED, DOW2_ARCHIVE, True)]) - def test_unpack(self, stream_data: bytes, expected: Archive, valid_checksums: bool): - super().test_unpack(stream_data, expected, valid_checksums) - - @pytest.mark.parametrize(["archive", "expected"], - [(DOW2_ARCHIVE, DOW2_ARCHIVE_PACKED)]) - def test_pack(self, archive: Archive, expected: bytes): - super().test_pack(archive, expected) - - @pytest.mark.parametrize(["archive", "expected"], - [(DOW2_ARCHIVE, DOW2_ARCHIVE_WALK())]) - def test_walk(self, archive: Archive, expected: ArchiveWalk): - super().test_walk(archive, expected) - - -def fast_gen_dow3_archive(*args): - return DowIII.gen_sample_archive(*args), DowIII.gen_sample_archive_buffer(*args) - - -DOW3_ARCHIVE, DOW3_ARCHIVE_PACKED = fast_gen_dow3_archive("Dow3 Test Archive", "Tests", "Some Witty FileName.txt", b"NGL; I'm running out of dumb/clever test data.") - - -def DOW3_ARCHIVE_WALK() -> ArchiveWalk: - a = DOW3_ARCHIVE - d = a.drives[0] - sfs = d.sub_folders - yield d, None, sfs, [] - yield d, sfs[0], [], sfs[0].files - - -class TestDowIIIArchive(ArchiveTests): - @pytest.mark.parametrize(["stream_data", "expected"], - [(DOW3_ARCHIVE_PACKED, DOW3_ARCHIVE)]) - def test_inner_unpack(self, stream_data: bytes, expected: Archive): - super().test_inner_unpack(stream_data, expected) - - @pytest.mark.parametrize(["stream_data", "expected", "valid_checksums"], - [(DOW3_ARCHIVE_PACKED, DOW3_ARCHIVE, True)]) - def test_unpack(self, stream_data: bytes, expected: Archive, valid_checksums: bool): - super().test_unpack(stream_data, expected, valid_checksums) - - @pytest.mark.parametrize(["archive", "expected"], - [(DOW3_ARCHIVE, DOW3_ARCHIVE_PACKED)]) - def test_pack(self, archive: Archive, expected: bytes): - super().test_pack(archive, expected) - - @pytest.mark.parametrize(["archive", "expected"], - [(DOW3_ARCHIVE, DOW3_ARCHIVE_WALK())]) - def test_walk(self, archive: Archive, expected: ArchiveWalk): - super().test_walk(archive, expected) diff --git a/tests/relic/sga/archive/test_archive_header.py b/tests/relic/sga/archive/test_archive_header.py deleted file mode 100644 index a240d71..0000000 --- a/tests/relic/sga/archive/test_archive_header.py +++ /dev/null @@ -1,275 +0,0 @@ -from abc import abstractmethod -from io import BytesIO -from typing import List - -import pytest -from serialization_tools.ioutil import WindowPtr, Ptr -from serialization_tools.size import KiB, MiB, GiB - -from tests.relic.sga.datagen import DowI, DowII, DowII, DowIII -from tests.helpers import TF -from relic.common import Version -from relic.sga import ArchiveHeader, ArchiveVersion, DowIIIArchiveHeader, ArchiveMagicWord -from relic.sga.archive import header - - -class ArchiveHeaderTests: - @abstractmethod # Trick PyCharm into requiring us to redefine this - def test_validate_checksums(self, archive: bytes): - for fast in TF: - for _assert in TF: - with BytesIO(archive) as stream: - archive_header = ArchiveHeader.unpack(stream) - archive_header.validate_checksums(stream, fast=fast, _assert=_assert) - - @abstractmethod # Trick PyCharm into requiring us to redefine this - def test_version(self, archive: ArchiveHeader, expected: Version): - assert archive.version == expected - - @abstractmethod # Trick PyCharm into requiring us to redefine this - def test_private_unpack(self, buffer: bytes, expected: ArchiveHeader): - with BytesIO(buffer) as stream: - result = expected.__class__._unpack(stream) - assert result == expected - - @abstractmethod # Trick PyCharm into requiring us to redefine this - def test_private_pack(self, inst: ArchiveHeader, expected: bytes): - with BytesIO() as stream: - inst._pack(stream) - stream.seek(0) - result = stream.read() - assert result == expected - - @abstractmethod # Trick PyCharm into requiring us to redefine this - def test_unpack(self, buffer: bytes, expected: ArchiveHeader, bad_magic_word: bool): - for read_magic in TF: - with BytesIO(buffer) as stream: - if not read_magic: - ArchiveMagicWord.read_magic_word(stream) # read past magic - - try: - unpacked = ArchiveHeader.unpack(stream, read_magic) - except AssertionError as e: - if read_magic and bad_magic_word: - return # Test passed - else: - raise e - else: - assert expected.__class__ == unpacked.__class__ - assert expected == unpacked - - @abstractmethod # Trick PyCharm into requiring us to redefine this - def test_pack(self, inst: ArchiveHeader, expected: bytes): - magic_size = ArchiveMagicWord.layout.size - for write_magic in TF: - with BytesIO() as stream: - written = inst.pack(stream, write_magic) - # assert len(expected) == written - (0 if write_magic else magic_size) - if not write_magic: - true_expected = expected[magic_size:] - else: - true_expected = expected - stream.seek(0) - result = stream.read() - assert true_expected == result - - -_KNOWN_EIGEN = b'06BEF126-4E3C-48D3-8D2E-430BF125B54F' -_KNOWN_DATA = b'\xf3\x0cGjx:"\xb7O\x89\xc1\x82H\xb2\xa1\xaa\x82-\xe4\\{\xe2\x905\x0c\xdbT\x0c\x82\xa3y\xdat\xd5\xdf\xb7\x04\x1e\xd0\xaa\xf6\xc9|U%\xf7\x0c\xb9\x92\xc9\xbf\xa9\xa3\xaaQ]\xb6\x8c\x10\x87\xc3r\xe3\x89\x16T\x936\xc5l/(\xbd\xbc\x08\xa2\x9b`|\xec\xd5\xf3\xfd\x83\x85\xadHY\xf4U\xb8\x85\x92\xcd\x1d\xc1\xa2\x0f\xbam!\xd5\xacnft>\'\xf0\x12\x9c\x0c\x1c{\xa2\x15VI\xb0\x13\x89\xde\x889\xdc\x15_\xc8\\\x97\x06\xa7\xde\xc0p\xf9o\t\xd3_\x9d\xa7@.\x81\xed\xdd\x13\x9b m9\xf5\x1bV\xc3\xe0\xd4@\x99\xa2\x8aGr\x04\xff\x05\xedIs\x15\t0\x98G\x87O\x9c\xa1\xd2\tcS\xb3\x1eI\xf5\xe3Qp\xe0\xd0m\xbf;\xfb\x856\xa7\\\xb8\xad\x19\xc1\xa3\xaf+\xd4\x08\xd5Y4\x87p|p`dQ\x1c|>is\x17;\xa6\x8d\xa2\xa4\xdc\xe0\xd6\xaf\xc3\x93\xf59\x9a[\x19J\xc88\xb8\xfd/\xe4\xc6J\x8c\xddCY&\x8f' -_KNOWN_BAD_DATA = b'\xe9F{\x17\xc2\x118\xe4\x0c\xbd\x07\xf2\x07\x03:\xee%\xabx<\xc3\xb5\x98\x7f\xa6[\xc53+Y]t' -_KNOWN_DATA_MD5 = b'\x0f\xd3\xc3|\xb2d\x16U\xfd\xc2<\x98\x0b\xf1\x91\xde' - - -@pytest.mark.parametrize( - ["stream_data", "eigen", "md5_checksum"], - [(_KNOWN_DATA, _KNOWN_EIGEN, _KNOWN_DATA_MD5)] -) -def test_gen_md5_checksum(stream_data: bytes, eigen: bytes, md5_checksum: bytes, buffer_sizes: List[int] = None, ptr: Ptr = None): - buffer_sizes = [KiB, MiB, GiB] if buffer_sizes is None else buffer_sizes - ptr = WindowPtr(0, len(stream_data)) if ptr is None else ptr - for buffer_size in buffer_sizes: - with BytesIO(stream_data) as stream: - result = header._gen_md5_checksum(stream, eigen, buffer_size, ptr) - assert md5_checksum == result - - -@pytest.mark.parametrize( - ["stream_data", "eigen", "md5_checksum", "fail_expected"], - [(_KNOWN_DATA, _KNOWN_EIGEN, _KNOWN_DATA_MD5, False), - (_KNOWN_BAD_DATA, _KNOWN_EIGEN, _KNOWN_DATA_MD5, True)] -) -def test_validate_md5_checksum(stream_data: bytes, eigen: bytes, md5_checksum: bytes, fail_expected: bool, ptr: WindowPtr = None, buffer_sizes: List[int] = None, ): - buffer_sizes = [KiB, MiB, GiB] if buffer_sizes is None else buffer_sizes - ptr = WindowPtr(0, len(stream_data)) if ptr is None else ptr - for _assert in TF: - for buffer_size in buffer_sizes: - try: - with BytesIO(stream_data) as stream: - result = header._validate_md5_checksum(stream, ptr, eigen, md5_checksum, buffer_size, _assert) - # Own lines to make assertions clearer - except AssertionError as e: - if not fail_expected: # MD5 mismatch; if fail_expected we - raise e - else: - if fail_expected: - # Invalid and should have asserted - assert not result and not _assert - else: - assert result - - -# Not garunteed to be a valid header - -def fast_dow1_archive_header(name, toc_pos, bad_magic: bytes): - _AB = 0, 120 # Random values - return DowI.gen_archive_header(name, *_AB, toc_pos=toc_pos), DowI.gen_archive_header_buffer(name, *_AB), DowI.gen_archive_header_buffer(name, *_AB, magic=bad_magic) - - -DOW1_HEADER, DOW1_HEADER_DATA, DOW1_HEADER_DATA_BAD_MAGIC = fast_dow1_archive_header("Dawn Of War 1 Test Header", 180, b"deadbeef") -# By not writing Magic/Archive TOC-Pos must be changed in the generated DowIIArchiveHeader; the buffers (should be) identical given the same input -DOW1_HEADER_INNER, DOW1_HEADER_INNER_DATA, _ = fast_dow1_archive_header("Dawn Of War 1 Test Header (Inner Pack)", 168, b"deaddead") -DOW1_ARCHIVE_BUFFER = DowI.gen_sample_archive_buffer("Dawn Of War 1 Test Archive", "Tests", "Dow1 Header Tests.txt", b"You thought this was a test, but it was me, DIO!") - - -class TestDowIArchiveHeader(ArchiveHeaderTests): - @pytest.mark.parametrize( - ["archive"], - [(DOW1_ARCHIVE_BUFFER,)]) - def test_validate_checksums(self, archive: bytes): - super().test_validate_checksums(archive) - - @pytest.mark.parametrize( - ["expected", "inst"], - [(DOW1_HEADER_INNER_DATA[12:], DOW1_HEADER_INNER)] - ) - def test_private_pack(self, inst: ArchiveHeader, expected: bytes): - super().test_private_pack(inst, expected) - - @pytest.mark.parametrize( - ["buffer", "expected"], - [(DOW1_HEADER_INNER_DATA[12:], DOW1_HEADER_INNER)] - ) - def test_private_unpack(self, buffer: bytes, expected: ArchiveHeader): - super().test_private_unpack(buffer, expected) - - @pytest.mark.parametrize( - ["buffer", "expected", "bad_magic_word"], - [(DOW1_HEADER_DATA, DOW1_HEADER, False), - (DOW1_HEADER_DATA_BAD_MAGIC, DOW1_HEADER, True)] - ) - def test_unpack(self, buffer: bytes, expected: ArchiveHeader, bad_magic_word: bool): - super().test_unpack(buffer, expected, bad_magic_word) - - @pytest.mark.parametrize( - ["inst", "expected"], - [(DOW1_HEADER, DOW1_HEADER_DATA)]) - def test_pack(self, inst: ArchiveHeader, expected: bytes): - super().test_pack(inst, expected) - - @pytest.mark.parametrize(["archive", "expected"], [(DOW1_HEADER, ArchiveVersion.Dow)]) - def test_version(self, archive: ArchiveHeader, expected: Version): - super().test_version(archive, expected) - - -# Not garunteed to be a valid header - - -def fast_dow2_archive_header(name, bad_magic: bytes): - _ABC = 0, 0, 0 - return DowII.gen_archive_header(name, *_ABC), DowII.gen_archive_header_buffer(name, *_ABC), DowII.gen_archive_header_buffer(name, *_ABC, magic=bad_magic) - - -DOW2_HEADER, DOW2_HEADER_DATA, DOW2_HEADER_DATA_BAD_MAGIC = fast_dow2_archive_header("Dawn Of War 2 Test Header", b"Garbage!") -DOW2_ARCHIVE_BUFFER = DowII.gen_sample_archive_buffer("Dawn Of War 2 Test Archive", "Dow2 Tests", "Imperial Propoganda.txt", b"By the Emperor, we're ready to unleash eleven barrels, m' lord, sir!") - - -class TestDowIIArchiveHeader(ArchiveHeaderTests): - @pytest.mark.parametrize( - ["expected", "inst"], - [(DOW2_HEADER_DATA[12:], DOW2_HEADER)], - ) - def test_private_pack(self, inst: ArchiveHeader, expected: bytes): - super().test_private_pack(inst, expected) - - @pytest.mark.parametrize( - ["buffer", "expected"], - [(DOW2_HEADER_DATA[12:], DOW2_HEADER)], - ) - def test_private_unpack(self, buffer: bytes, expected: ArchiveHeader): - super().test_private_unpack(buffer, expected) - - @pytest.mark.parametrize( - ["buffer", "expected", "bad_magic_word"], - [(DOW2_HEADER_DATA, DOW2_HEADER, False), - (DOW2_HEADER_DATA_BAD_MAGIC, DOW2_HEADER, True)], - ) - def test_unpack(self, buffer: bytes, expected: ArchiveHeader, bad_magic_word: bool): - super().test_unpack(buffer, expected, bad_magic_word) - - @pytest.mark.parametrize( - ["inst", "expected"], - [(DOW2_HEADER, DOW2_HEADER_DATA)]) - def test_pack(self, inst: ArchiveHeader, expected: bytes): - super().test_pack(inst, expected) - - @pytest.mark.parametrize( - ["archive"], - [(DOW2_ARCHIVE_BUFFER,)], - ) - def test_validate_checksums(self, archive: bytes): - super().test_validate_checksums(archive) - - @pytest.mark.parametrize(["archive", "expected"], [(DOW2_HEADER, ArchiveVersion.Dow2)]) - def test_version(self, archive: ArchiveHeader, expected: Version): - super().test_version(archive, expected) - - -def fast_dow3_archive_header(name, bad_magic: bytes): - _ABCD = 0, 1, 2, 3 - return DowIII.gen_archive_header(name, *_ABCD), DowIII.gen_archive_header_buffer(name, *_ABCD), DowIII.gen_archive_header_buffer(name, *_ABCD, magic=bad_magic) - - -DOW3_HEADER, DOW3_HEADER_DATA, DOW3_HEADER_DATA_BAD_MAGIC = fast_dow3_archive_header("Dawn Of War 3 Test Header", b" Marine!") # Big Brain Pun in ` Marine!` - - -class TestDowIIIArchiveHeader(ArchiveHeaderTests): - @pytest.mark.parametrize( - ["archive"], - [(None,)]) - def test_validate_checksums(self, archive: bytes): - for fast in TF: - for _assert in TF: - # HACK but if it fails it means logic has changed - assert DowIIIArchiveHeader.validate_checksums(None, None, fast=fast, _assert=_assert) - - @pytest.mark.parametrize( - ["expected", "inst"], - [(DOW3_HEADER_DATA[12:], DOW3_HEADER)], - ) - def test_private_pack(self, inst: ArchiveHeader, expected: bytes): - super().test_private_pack(inst, expected) - - @pytest.mark.parametrize( - ["buffer", "expected"], - [(DOW3_HEADER_DATA[12:], DOW3_HEADER)], - ) - def test_private_unpack(self, buffer: bytes, expected: ArchiveHeader): - super().test_private_unpack(buffer, expected) - - @pytest.mark.parametrize( - ["buffer", "expected", "bad_magic_word"], - [(DOW3_HEADER_DATA, DOW3_HEADER, False), - (DOW3_HEADER_DATA_BAD_MAGIC, DOW3_HEADER, True)], - ) - def test_unpack(self, buffer: bytes, expected: ArchiveHeader, bad_magic_word: bool): - super().test_unpack(buffer, expected, bad_magic_word) - - @pytest.mark.parametrize( - ["inst", "expected"], - [(DOW3_HEADER, DOW3_HEADER_DATA)]) - def test_pack(self, inst: ArchiveHeader, expected: bytes): - super().test_pack(inst, expected) - - @pytest.mark.parametrize(["archive", "expected"], [(DOW3_HEADER, ArchiveVersion.Dow3)]) - def test_version(self, archive: ArchiveHeader, expected: Version): - super().test_version(archive, expected) diff --git a/tests/relic/sga/datagen.py b/tests/relic/sga/datagen.py index 83c7eb3..34206bd 100644 --- a/tests/relic/sga/datagen.py +++ b/tests/relic/sga/datagen.py @@ -1,12 +1,49 @@ import hashlib -from typing import Tuple, Dict +from typing import Tuple, Dict, ClassVar -from serialization_tools.ioutil import WindowPtr, Ptr +from relic.sga._core import StorageType -from relic.sga import ArchiveHeader, DowIArchiveHeader, DowIIArchiveHeader, DowIIIArchiveHeader, VirtualDrive, Folder, File, DowIIArchive, DowIArchive, DowIIIArchive, \ - DowIIIFolderHeader, DowIIIFileHeader, DowIIIVirtualDriveHeader, DowIVirtualDriveHeader, DowIFolderHeader, DowIFileHeader, FileCompressionFlag, DowIIFolderHeader, DowIIVirtualDriveHeader, DowIIFileHeader -from relic.sga.common import ArchiveRange -from relic.sga.toc.toc import ArchiveTOC + +class VirtualDriveABC: + pass + + +class ArchiveHeader: + pass + + +class v9: + VirtualDriveHeader = None + FolderHeader: ClassVar = None + Archive: ClassVar = None + FileHeader: ClassVar = None + + +class v5: + VirtualDriveHeader = None + FolderHeader: ClassVar = None + Archive: ClassVar = None + FileHeader: ClassVar = None + + +class v2: + VirtualDriveHeader = None + FolderHeader: ClassVar = None + Archive: ClassVar = None + FileCompressionFlag: ClassVar = None + FileHeader: ClassVar = None + + +class FolderABC: + pass + + +class FileABC: + pass + + +class ArchiveTOC: + pass def encode_and_pad(v: str, byte_size: int, encoding: str) -> bytes: @@ -35,11 +72,12 @@ def splice_toc_offsets(vdrive: int, folders: int, files: int, names: int, offset class DowI: DEFAULT_CSUMS = (b"\x01\x02\0\x04\0\0\0\x08\0\0\0\0\0\0\0\0", b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f") - VDRIVE_UNK = b"\xde\xad" # Arbitrary value + DEF_ROOT_FOLDER = ushort(0) # b"\xde\xad" # Arbitrary value @staticmethod def gen_archive_header(name: str, toc_size: int, data_offset: int, csums: Tuple[bytes, bytes] = DEFAULT_CSUMS, toc_pos: int = 180) -> ArchiveHeader: - return DowIArchiveHeader(name, WindowPtr(toc_pos, toc_size), WindowPtr(data_offset), csums) + raise TypeError("Not currently supported") + # return v2.ArchiveHeader(name, WindowPtr(toc_pos, toc_size), WindowPtr(data_offset), csums) @staticmethod def gen_archive_header_buffer(name: str, toc_size: int, data_offset: int, csums: Tuple[bytes, bytes] = DEFAULT_CSUMS, magic: bytes = b"_ARCHIVE") -> bytes: @@ -50,42 +88,45 @@ def gen_archive_header_buffer(name: str, toc_size: int, data_offset: int, csums: return magic + version + csums[0] + encoded_name + csums[1] + encoded_toc_size + encoded_data_offset @staticmethod - def gen_vdrive_header(archive_name: str, subfolder_offset: int = 0, subfolder_count: int = 0, file_offset: int = 0, file_count: int = 0, path: str = "data", unk: bytes = VDRIVE_UNK) -> DowIVirtualDriveHeader: - return DowIVirtualDriveHeader(path, archive_name, ArchiveRange(subfolder_offset, subfolder_offset + subfolder_count), ArchiveRange(file_offset, file_offset + file_count), unk) + def gen_vdrive_header(archive_name: str, subfolder_offset: int = 0, subfolder_count: int = 0, file_offset: int = 0, file_count: int = 0, path: str = "data", unk: bytes = DEF_ROOT_FOLDER) -> v2.VirtualDriveHeader: + raise TypeError("Not currently supported") + # return v2.VirtualDriveHeader(path, archive_name, ArchiveRange(subfolder_offset, subfolder_offset + subfolder_count), ArchiveRange(file_offset, file_offset + file_count), unk) @staticmethod - def gen_vdrive_header_buffer(name: str, subfolder_offset: int = 0, subfolder_count: int = 0, file_offset: int = 0, file_count: int = 0, path: str = "data", unk: bytes = VDRIVE_UNK): - return encode_and_pad(path, 64, "ascii") + encode_and_pad(name, 64, "ascii") + ushort(subfolder_offset) + ushort(subfolder_offset + subfolder_count) + ushort(file_offset) + ushort(file_count + file_offset) + unk + def gen_vdrive_header_buffer(name: str, subfolder_offset: int = 0, subfolder_count: int = 0, file_offset: int = 0, file_count: int = 0, path: str = "data", root_folder: bytes = DEF_ROOT_FOLDER): + return encode_and_pad(path, 64, "ascii") + encode_and_pad(name, 64, "ascii") + ushort(subfolder_offset) + ushort(subfolder_offset + subfolder_count) + ushort(file_offset) + ushort(file_count + file_offset) + root_folder @staticmethod - def gen_folder_header(name_offset: int, subfolder_offset: int = 0, subfolder_count: int = 0, file_offset: int = 0, file_count: int = 0) -> DowIFolderHeader: - return DowIFolderHeader(name_offset, ArchiveRange(subfolder_offset, subfolder_offset + subfolder_count), ArchiveRange(file_offset, file_offset + file_count)) + def gen_folder_header(name_offset: int, subfolder_offset: int = 0, subfolder_count: int = 0, file_offset: int = 0, file_count: int = 0) -> v2.FolderHeader: + raise TypeError("Not currently supported") + # return v2.FolderHeader(name_offset, ArchiveRange(subfolder_offset, subfolder_offset + subfolder_count), ArchiveRange(file_offset, file_offset + file_count)) @staticmethod def gen_folder_header_buffer(name_offset: int, subfolder_offset: int = 0, subfolder_count: int = 0, file_offset: int = 0, file_count: int = 0) -> bytes: return uint(name_offset) + ushort(subfolder_offset) + ushort(subfolder_offset + subfolder_count) + ushort(file_offset) + ushort(file_count + file_offset) @staticmethod - def gen_file_header(name_offset: int, data_offset: int, decomp_size: int, comp_size: int = None, comp_flag: FileCompressionFlag = None) -> DowIFileHeader: - if comp_size is None: - comp_size = decomp_size - if comp_flag is None: - if comp_size != decomp_size: - comp_flag = FileCompressionFlag.Compressed16 # IDK, just choose one - else: - comp_flag = FileCompressionFlag.Decompressed - return DowIFileHeader(Ptr(name_offset), WindowPtr(data_offset, comp_size), decomp_size, comp_size, comp_flag) + def gen_file_header(name_offset: int, data_offset: int, decomp_size: int, comp_size: int = None, comp_flag: v2.FileCompressionFlag = None) -> v2.FileHeader: + raise TypeError("Not currently supported") + # if comp_size is None: + # comp_size = decomp_size + # if comp_flag is None: + # if comp_size != decomp_size: + # comp_flag = v2.FileCompressionFlag.Compressed16 # IDK, just choose one + # else: + # comp_flag = v2.FileCompressionFlag.Decompressed + # return v2.FileHeader(Ptr(name_offset), WindowPtr(data_offset, comp_size), decomp_size, comp_size, comp_flag) @staticmethod - def gen_file_header_buffer(name_offset: int, data_offset: int, decomp_size: int, comp_size: int = None, comp_flag: FileCompressionFlag = None) -> bytes: + def gen_file_header_buffer(name_offset: int, data_offset: int, decomp_size: int, comp_size: int = None, comp_flag: StorageType = None) -> bytes: if comp_size is None: comp_size = decomp_size if comp_flag is None: if comp_size != decomp_size: - comp_flag = FileCompressionFlag.Compressed16 # IDK, just choose one + comp_flag = 32 # StorageType.StreamCompress # IDK, just choose one else: - comp_flag = FileCompressionFlag.Decompressed - return uint(name_offset) + uint(comp_flag.value) + uint(data_offset) + uint(decomp_size) + uint(comp_size) + comp_flag = 0 # StorageType.Store + return uint(name_offset) + uint(comp_flag) + uint(data_offset) + uint(decomp_size) + uint(comp_size) @staticmethod def gen_name_buffer(*names: str, encoding: str = "ascii") -> Tuple[bytes, Dict[str, int]]: @@ -113,8 +154,9 @@ def gen_toc_ptr_buffer(vdrive: Tuple[int, int], folders: Tuple[int, int], files: return b"".join(parts) @staticmethod - def gen_toc(vdrive: VirtualDrive, folder: Folder, file: File, names: Dict[int, str]) -> ArchiveTOC: - return ArchiveTOC([vdrive], [folder], [file], names) + def gen_toc(vdrive: VirtualDriveABC, folder: FolderABC, file: FileABC, names: Dict[int, str]) -> ArchiveTOC: + raise TypeError("Not currently supported") + # return ArchiveTOC([vdrive], [folder], [file], names) @classmethod def gen_archive_buffer(cls, archive_name: str, toc_ptrs: bytes, toc: bytes, data: bytes, magic: bytes = "_ARCHIVE") -> bytes: @@ -146,56 +188,58 @@ def gen_sample_archive_buffer(cls, archive_name: str, folder: str, file: str, fi return cls.gen_archive_buffer(archive_name, toc_ptr_buf, toc_buf, file_uncomp_data, magic) @classmethod - def gen_sample_archive(cls, archive_name: str, folder: str, file: str, file_uncomp_data: bytes, toc_pos: int = 180) -> DowIArchive: - def dirty_toc_hack(): - name_buf, name_offsets = cls.gen_name_buffer(folder, file) - vdrive_buf = cls.gen_vdrive_header_buffer(archive_name, 0, 1, 0, 1) - folder_buf = cls.gen_folder_header_buffer(name_offsets[folder], 0, 0, 0, 1) - file_buf = cls.gen_file_header_buffer(name_offsets[file], 0, len(file_uncomp_data)) - toc_buf, toc_offsets = cls.gen_toc_buffer_and_offsets(vdrive_buf, folder_buf, file_buf, name_buf) - toc_ptrs = splice_toc_offsets(1, 1, 1, 2, toc_offsets) - return cls.gen_toc_ptr_buffer(*toc_ptrs) + toc_buf - - toc_buf = dirty_toc_hack() - - def dirty_csum_hack(): - EIGENS = ("E01519D6-2DB7-4640-AF54-0A23319C56C3".encode("ascii"), "DFC9AF62-FC1B-4180-BC27-11CCE87D3EFF".encode("ascii")) - - def gen_csum(buffer: bytes, eigen: bytes) -> bytes: - hasher = hashlib.md5(eigen) - hasher.update(buffer) - return bytes.fromhex(hasher.hexdigest()) - - csum2 = gen_csum(toc_buf, EIGENS[1]) - toc_and_data = toc_buf + file_uncomp_data - csum1 = gen_csum(toc_and_data, EIGENS[0]) - return csum1, csum2 - - csums = dirty_csum_hack() - - _, name_offsets = cls.gen_name_buffer(folder, file) - vdrive_h = cls.gen_vdrive_header(archive_name, 0, 1, 0, 1) - folder_h = cls.gen_folder_header(name_offsets[folder], 0, 0, 0, 1) - file_h = cls.gen_file_header(name_offsets[file], 0, len(file_uncomp_data)) - file_ = File(file_h, file, file_uncomp_data, True) - folder_ = Folder(folder_h, folder, [], [file_]) - vdrive_ = VirtualDrive(vdrive_h, [folder_], [file_]) - folder_._drive = file_._drive = vdrive_ - file_._parent = folder_ - header = cls.gen_archive_header(archive_name, len(toc_buf), len(toc_buf) + toc_pos, csums, toc_pos) - return DowIArchive(header, [vdrive_], False) + def gen_sample_archive(cls, archive_name: str, folder: str, file: str, file_uncomp_data: bytes, toc_pos: int = 180) -> v2.Archive: + raise TypeError("Currently not supported") + # def dirty_toc_hack(): + # name_buf, name_offsets = cls.gen_name_buffer(folder, file) + # vdrive_buf = cls.gen_vdrive_header_buffer(archive_name, 0, 1, 0, 1) + # folder_buf = cls.gen_folder_header_buffer(name_offsets[folder], 0, 0, 0, 1) + # file_buf = cls.gen_file_header_buffer(name_offsets[file], 0, len(file_uncomp_data)) + # toc_buf, toc_offsets = cls.gen_toc_buffer_and_offsets(vdrive_buf, folder_buf, file_buf, name_buf) + # toc_ptrs = splice_toc_offsets(1, 1, 1, 2, toc_offsets) + # return cls.gen_toc_ptr_buffer(*toc_ptrs) + toc_buf + # + # toc_buf = dirty_toc_hack() + # + # def dirty_csum_hack(): + # EIGENS = ("E01519D6-2DB7-4640-AF54-0A23319C56C3".encode("ascii"), "DFC9AF62-FC1B-4180-BC27-11CCE87D3EFF".encode("ascii")) + # + # def gen_csum(buffer: bytes, eigen: bytes) -> bytes: + # hasher = hashlib.md5(eigen) + # hasher.update(buffer) + # return bytes.fromhex(hasher.hexdigest()) + # + # csum2 = gen_csum(toc_buf, EIGENS[1]) + # toc_and_data = toc_buf + file_uncomp_data + # csum1 = gen_csum(toc_and_data, EIGENS[0]) + # return csum1, csum2 + # + # csums = dirty_csum_hack() + # + # _, name_offsets = cls.gen_name_buffer(folder, file) + # vdrive_h = cls.gen_vdrive_header(archive_name, 0, 1, 0, 1) + # folder_h = cls.gen_folder_header(name_offsets[folder], 0, 0, 0, 1) + # file_h = cls.gen_file_header(name_offsets[file], 0, len(file_uncomp_data)) + # file_ = v2.File(file_h, file, file_uncomp_data, True) + # folder_ = v2.Folder(folder_h, folder, [], [file_]) + # vdrive_ = v2.VirtualDrive(vdrive_h, [folder_], [file_]) + # folder_.parent_drive = file_.parent_drive = vdrive_ + # file_.parent_folder = folder_ + # header = cls.gen_archive_header(archive_name, len(toc_buf), len(toc_buf) + toc_pos, csums, toc_pos) + # return v2.Archive(header, [vdrive_], False) class DowII: DEFAULT_CSUMS = (b"\x01\x02\0\x04\0\0\0\x08\0\0\0\0\0\0\0\0", b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f") - VDRIVE_UNK = b"\xde\xad" # Arbitrary value + DEF_ROOT_FOLDER = ushort(0) # b"\xde\xad" # Arbitrary value ARCHIVE_HEADER_UNK = bytes.fromhex("4d41dFFF") # F in place of unknowns ARCHIVE_HEADER_UNK_INT = int.from_bytes(ARCHIVE_HEADER_UNK, byteorder="little", signed=False) # F in place of unknowns ARCHIVE_HEADER_SIZE = 196 @classmethod def gen_archive_header(cls, name: str, toc_size: int, data_offset: int, toc_offset: int, csums: Tuple[bytes, bytes] = DEFAULT_CSUMS) -> ArchiveHeader: - return DowIIArchiveHeader(name, WindowPtr(toc_offset, toc_size), WindowPtr(data_offset), csums, cls.ARCHIVE_HEADER_UNK_INT) + raise TypeError("Not currently supported") + # return v5.ArchiveHeader(name, WindowPtr(toc_offset, toc_size), WindowPtr(data_offset), csums, cls.ARCHIVE_HEADER_UNK_INT) @classmethod def gen_archive_header_buffer(cls, name: str, toc_size: int, data_offset: int, toc_offset: int, csums: Tuple[bytes, bytes] = DEFAULT_CSUMS, magic: bytes = b"_ARCHIVE") -> bytes: @@ -207,21 +251,24 @@ def gen_archive_header_buffer(cls, name: str, toc_size: int, data_offset: int, t return magic + version + csums[0] + encoded_name + csums[1] + encoded_toc_size + encoded_data_offset + encoded_toc_offset + uint(1) + uint(0) + cls.ARCHIVE_HEADER_UNK @staticmethod - def gen_vdrive_header(archive_name: str, subfolder_offset: int = 0, subfolder_count: int = 0, file_offset: int = 0, file_count: int = 0, path: str = "data", unk: bytes = VDRIVE_UNK) -> DowIIVirtualDriveHeader: - return DowIIVirtualDriveHeader(path, archive_name, ArchiveRange(subfolder_offset, subfolder_offset + subfolder_count), ArchiveRange(file_offset, file_offset + file_count), unk) + def gen_vdrive_header(archive_name: str, subfolder_offset: int = 0, subfolder_count: int = 0, file_offset: int = 0, file_count: int = 0, path: str = "data", root_folder: bytes = DEF_ROOT_FOLDER) -> v5.VirtualDriveHeader: + raise TypeError("Not currently supported") + # return v5.VirtualDriveHeader(path, archive_name, ArchiveRange(subfolder_offset, subfolder_offset + subfolder_count), ArchiveRange(file_offset, file_offset + file_count), unk) gen_vdrive_header_buffer = DowI.gen_vdrive_header_buffer # Same exact layout; @staticmethod - def gen_folder_header(name_offset: int, subfolder_offset: int = 0, subfolder_count: int = 0, file_offset: int = 0, file_count: int = 0) -> DowIIFolderHeader: - return DowIIFolderHeader(name_offset, ArchiveRange(subfolder_offset, subfolder_offset + subfolder_count), ArchiveRange(file_offset, file_offset + file_count)) + def gen_folder_header(name_offset: int, subfolder_offset: int = 0, subfolder_count: int = 0, file_offset: int = 0, file_count: int = 0) -> v5.FolderHeader: + raise TypeError("Not currently supported") + # return v5.FolderHeader(name_offset, ArchiveRange(subfolder_offset, subfolder_offset + subfolder_count), ArchiveRange(file_offset, file_offset + file_count)) gen_folder_header_buffer = DowI.gen_folder_header_buffer # Same exact layout; @staticmethod - def gen_file_header(name_offset: int, data_offset: int, decomp_size: int, comp_size: int = None) -> DowIIFileHeader: - comp_size = decomp_size if comp_size is None else comp_size - return DowIIFileHeader(Ptr(name_offset), Ptr(data_offset, comp_size), decomp_size, comp_size, 0, 0) + def gen_file_header(name_offset: int, data_offset: int, decomp_size: int, comp_size: int = None) -> v5.FileHeader: + raise TypeError("Not currently supported") + # comp_size = decomp_size if comp_size is None else comp_size + # return v5.FileHeader(Ptr(name_offset), Ptr(data_offset, comp_size), decomp_size, comp_size, 0, 0) @staticmethod def gen_file_header_buffer(name_offset: int, data_offset: int, decomp_size: int, comp_size: int = None) -> bytes: @@ -262,54 +309,56 @@ def gen_sample_archive_buffer(cls, archive_name: str, folder: str, file: str, fi return cls.gen_archive_buffer(archive_name, toc_ptr_buf, toc_buf, file_uncomp_data, magic) @classmethod - def gen_sample_archive(cls, archive_name: str, folder: str, file: str, file_uncomp_data: bytes, toc_pos: int = 180) -> DowIIArchive: - def dirty_toc_hack(): - name_buf, name_offsets = cls.gen_name_buffer(folder, file) - vdrive_buf = cls.gen_vdrive_header_buffer(archive_name, 0, 1, 0, 1) - folder_buf = cls.gen_folder_header_buffer(name_offsets[folder], 0, 0, 0, 1) - file_buf = cls.gen_file_header_buffer(name_offsets[file], 0, len(file_uncomp_data)) - toc_buf, toc_offsets = cls.gen_toc_buffer_and_offsets(vdrive_buf, folder_buf, file_buf, name_buf) - toc_ptrs = splice_toc_offsets(1, 1, 1, 2, toc_offsets) - return cls.gen_toc_ptr_buffer(*toc_ptrs) + toc_buf - - full_toc = dirty_toc_hack() - - def dirty_csum_hack(): - EIGENS = ("E01519D6-2DB7-4640-AF54-0A23319C56C3".encode("ascii"), "DFC9AF62-FC1B-4180-BC27-11CCE87D3EFF".encode("ascii")) - - def gen_csum(buffer: bytes, eigen: bytes) -> bytes: - hasher = hashlib.md5(eigen) - hasher.update(buffer) - return bytes.fromhex(hasher.hexdigest()) - - csum2 = gen_csum(full_toc, EIGENS[1]) - toc_and_data = full_toc + file_uncomp_data - csum1 = gen_csum(toc_and_data, EIGENS[0]) - return csum1, csum2 - - csums = dirty_csum_hack() - - _, name_offsets = cls.gen_name_buffer(folder, file) - vdrive_h = cls.gen_vdrive_header(archive_name, 0, 1, 0, 1) - folder_h = cls.gen_folder_header(name_offsets[folder], 0, 0, 0, 1) - file_h = cls.gen_file_header(name_offsets[file], 0, len(file_uncomp_data)) - file_ = File(file_h, file, file_uncomp_data, True) - folder_ = Folder(folder_h, folder, [], [file_]) - vdrive_ = VirtualDrive(vdrive_h, [folder_], [file_]) - folder_._drive = file_._drive = vdrive_ - file_._parent = folder_ - header = cls.gen_archive_header(archive_name, len(full_toc), cls.ARCHIVE_HEADER_SIZE + len(full_toc), cls.ARCHIVE_HEADER_SIZE, csums) - return DowIIArchive(header, [vdrive_], False) + def gen_sample_archive(cls, archive_name: str, folder: str, file: str, file_uncomp_data: bytes, toc_pos: int = 180) -> v5.Archive: + raise TypeError("Not currently supported") + # def dirty_toc_hack(): + # name_buf, name_offsets = cls.gen_name_buffer(folder, file) + # vdrive_buf = cls.gen_vdrive_header_buffer(archive_name, 0, 1, 0, 1) + # folder_buf = cls.gen_folder_header_buffer(name_offsets[folder], 0, 0, 0, 1) + # file_buf = cls.gen_file_header_buffer(name_offsets[file], 0, len(file_uncomp_data)) + # toc_buf, toc_offsets = cls.gen_toc_buffer_and_offsets(vdrive_buf, folder_buf, file_buf, name_buf) + # toc_ptrs = splice_toc_offsets(1, 1, 1, 2, toc_offsets) + # return cls.gen_toc_ptr_buffer(*toc_ptrs) + toc_buf + # + # full_toc = dirty_toc_hack() + # + # def dirty_csum_hack(): + # EIGENS = ("E01519D6-2DB7-4640-AF54-0A23319C56C3".encode("ascii"), "DFC9AF62-FC1B-4180-BC27-11CCE87D3EFF".encode("ascii")) + # + # def gen_csum(buffer: bytes, eigen: bytes) -> bytes: + # hasher = hashlib.md5(eigen) + # hasher.update(buffer) + # return bytes.fromhex(hasher.hexdigest()) + # + # csum2 = gen_csum(full_toc, EIGENS[1]) + # toc_and_data = full_toc + file_uncomp_data + # csum1 = gen_csum(toc_and_data, EIGENS[0]) + # return csum1, csum2 + # + # csums = dirty_csum_hack() + # + # _, name_offsets = cls.gen_name_buffer(folder, file) + # vdrive_h = cls.gen_vdrive_header(archive_name, 0, 1, 0, 1) + # folder_h = cls.gen_folder_header(name_offsets[folder], 0, 0, 0, 1) + # file_h = cls.gen_file_header(name_offsets[file], 0, len(file_uncomp_data)) + # file_ = FileABC(file_h, file, file_uncomp_data, True) + # folder_ = FolderABC(folder_h, folder, [], [file_]) + # vdrive_ = VirtualDriveABC(vdrive_h, [folder_], [file_]) + # folder_.parent_drive = file_.parent_drive = vdrive_ + # file_.parent_folder = folder_ + # header = cls.gen_archive_header(archive_name, len(full_toc), cls.ARCHIVE_HEADER_SIZE + len(full_toc), cls.ARCHIVE_HEADER_SIZE, csums) + # return v5.Archive(header, [vdrive_], False) class DowIII: - VDRIVE_UNK = bytes.fromhex("dead") # Arbitrary value - ARCHIVE_HEADER_SIZE = 432 + DEF_ROOT_FOLDER = ushort(0) # bytes.fromhex("dead") # Arbitrary value + ARCHIVE_HEADER_SIZE = 428 ARCHIVE_HEADER_UNK = b"dead " * 51 + b"\0" # 256 bytes spamming `dead ` in ascii; with one byte '\0' to pad to 256 @classmethod def gen_archive_header(cls, name: str, toc_offset: int, toc_size: int, data_offset: int, data_size: int) -> ArchiveHeader: - return DowIIIArchiveHeader(name, WindowPtr(toc_offset, toc_size), WindowPtr(data_offset, data_size), cls.ARCHIVE_HEADER_UNK) + raise TypeError("Not currently supported") + # return v9.ArchiveHeader(name, WindowPtr(toc_offset, toc_size), WindowPtr(data_offset, data_size), cls.ARCHIVE_HEADER_UNK) @classmethod def gen_archive_header_buffer(cls, name: str, toc_offset: int, toc_size: int, data_offset: int, data_size: int, magic: bytes = b"_ARCHIVE") -> bytes: @@ -319,29 +368,32 @@ def gen_archive_header_buffer(cls, name: str, toc_offset: int, toc_size: int, da encoded_toc_size = uint(toc_size) encoded_data_offset = ulong(data_offset) encoded_data_size = uint(data_size) - return magic + version + encoded_name + encoded_toc_offset + encoded_toc_size + encoded_data_offset + encoded_data_size + uint(0) + uint(1) + uint(0) + cls.ARCHIVE_HEADER_UNK + return magic + version + encoded_name + encoded_toc_offset + encoded_toc_size + encoded_data_offset + encoded_data_size + uint(0) + uint(1) + cls.ARCHIVE_HEADER_UNK @staticmethod - def gen_vdrive_header(archive_name: str, subfolder_offset: int = 0, subfolder_count: int = 0, file_offset: int = 0, file_count: int = 0, path: str = "data", unk: bytes = VDRIVE_UNK) -> DowIIIVirtualDriveHeader: - return DowIIIVirtualDriveHeader(path, archive_name, ArchiveRange(subfolder_offset, subfolder_offset + subfolder_count), ArchiveRange(file_offset, file_offset + file_count), unk) + def gen_vdrive_header(archive_name: str, subfolder_offset: int = 0, subfolder_count: int = 0, file_offset: int = 0, file_count: int = 0, path: str = "data", unk: bytes = DEF_ROOT_FOLDER) -> v9.VirtualDriveHeader: + raise TypeError("Not currently supported") + # return v9.VirtualDriveHeader(path, archive_name, ArchiveRange(subfolder_offset, subfolder_offset + subfolder_count), ArchiveRange(file_offset, file_offset + file_count), unk) @staticmethod - def gen_vdrive_header_buffer(name: str, subfolder_offset: int = 0, subfolder_count: int = 0, file_offset: int = 0, file_count: int = 0, path: str = "data", unk: bytes = VDRIVE_UNK): + def gen_vdrive_header_buffer(name: str, subfolder_offset: int = 0, subfolder_count: int = 0, file_offset: int = 0, file_count: int = 0, path: str = "data", unk: bytes = DEF_ROOT_FOLDER): return encode_and_pad(path, 64, "ascii") + encode_and_pad(name, 64, "ascii") + uint(subfolder_offset) + uint(subfolder_offset + subfolder_count) + uint(file_offset) + uint(file_count + file_offset) + unk @staticmethod - def gen_folder_header(name_offset: int, subfolder_offset: int = 0, subfolder_count: int = 0, file_offset: int = 0, file_count: int = 0) -> DowIIIFolderHeader: - return DowIIIFolderHeader(name_offset, ArchiveRange(subfolder_offset, subfolder_offset + subfolder_count), ArchiveRange(file_offset, file_offset + file_count)) + def gen_folder_header(name_offset: int, subfolder_offset: int = 0, subfolder_count: int = 0, file_offset: int = 0, file_count: int = 0) -> v9.FolderHeader: + raise TypeError("Not currently supported") + # return v9.FolderHeader(name_offset, ArchiveRange(subfolder_offset, subfolder_offset + subfolder_count), ArchiveRange(file_offset, file_offset + file_count)) @staticmethod def gen_folder_header_buffer(name_offset: int, subfolder_offset: int = 0, subfolder_count: int = 0, file_offset: int = 0, file_count: int = 0) -> bytes: return uint(name_offset) + uint(subfolder_offset) + uint(subfolder_offset + subfolder_count) + uint(file_offset) + uint(file_count + file_offset) @staticmethod - def gen_file_header(name_offset: int, data_offset: int, decomp_size: int, comp_size: int = None) -> DowIIIFileHeader: + def gen_file_header(name_offset: int, data_offset: int, decomp_size: int, comp_size: int = None) -> v9.FileHeader: if comp_size is None: comp_size = decomp_size - return DowIIIFileHeader(Ptr(name_offset), Ptr(data_offset), decomp_size, comp_size, 0, 0, 0, 0, 0) + raise TypeError("Not currently supported") + # return v9.FileHeader(Ptr(name_offset), Ptr(data_offset), decomp_size, comp_size, 0, 0, 0, 0, 0) @staticmethod def gen_file_header_buffer(name_offset: int, data_offset: int, decomp_size: int, comp_size: int = None) -> bytes: @@ -368,8 +420,9 @@ def gen_toc_ptr_buffer(vdrive: Tuple[int, int], folders: Tuple[int, int], files: return b"".join(parts) @staticmethod - def gen_toc(vdrive: VirtualDrive, folder: Folder, file: File, names: Dict[int, str]) -> ArchiveTOC: - return ArchiveTOC([vdrive], [folder], [file], names) + def gen_toc(vdrive: VirtualDriveABC, folder: FolderABC, file: FileABC, names: Dict[int, str]) -> ArchiveTOC: + raise TypeError("Not currently supported") + # return ArchiveTOC([vdrive], [folder], [file], names) @classmethod def gen_archive_buffer(cls, archive_name: str, toc_ptrs: bytes, toc: bytes, data: bytes, magic: bytes = "_ARCHIVE") -> bytes: @@ -385,31 +438,30 @@ def gen_sample_archive_buffer(cls, archive_name: str, folder: str, file: str, fi folder_buf = cls.gen_folder_header_buffer(name_offsets[folder], 0, 0, 0, 1) file_buf = cls.gen_file_header_buffer(name_offsets[file], 0, len(file_uncomp_data)) toc_buf, toc_offsets = cls.gen_toc_buffer_and_offsets(vdrive_buf, folder_buf, file_buf, name_buf) - # toc_ptrs = splice_toc_offsets(1, 1, 1, len(name_buf), toc_offsets) # WE NEED TO USE BYTE-SIZE of NAME BUFFER!!!! - toc_ptrs = splice_toc_offsets(1, 1, 1, 2, toc_offsets) # According to my notes; V9 uses this to store the size of the name buffer; but Archive Unpacking assumes it's a count; must test on real V9 files. TODO + toc_ptrs = splice_toc_offsets(1, 1, 1, len(name_buf), toc_offsets) # WE NEED TO USE BYTE-SIZE of NAME BUFFER!!!! toc_ptr_buf = cls.gen_toc_ptr_buffer(*toc_ptrs) return cls.gen_archive_buffer(archive_name, toc_ptr_buf, toc_buf, file_uncomp_data, magic) @classmethod - def gen_sample_archive(cls, archive_name: str, folder: str, file: str, file_uncomp_data: bytes) -> DowIIIArchive: - name_buf, name_offsets = cls.gen_name_buffer(folder, file) - vdrive_h = cls.gen_vdrive_header(archive_name, 0, 1, 0, 1) - folder_h = cls.gen_folder_header(name_offsets[folder], 0, 0, 0, 1) - file_h = cls.gen_file_header(name_offsets[file], 0, len(file_uncomp_data)) - file_ = File(file_h, file, file_uncomp_data, True) - folder_ = Folder(folder_h, folder, [], [file_]) - vdrive_ = VirtualDrive(vdrive_h, [folder_], [file_]) - folder_._drive = file_._drive = vdrive_ - file_._parent = folder_ - - vdrive_buf = cls.gen_vdrive_header_buffer(archive_name, 0, 1, 0, 1) - folder_buf = cls.gen_folder_header_buffer(name_offsets[folder], 0, 0, 0, 1) - file_buf = cls.gen_file_header_buffer(name_offsets[file], 0, len(file_uncomp_data)) - toc_buf, toc_offsets = cls.gen_toc_buffer_and_offsets(vdrive_buf, folder_buf, file_buf, name_buf) + def gen_sample_archive(cls, archive_name: str, folder: str, file: str, file_uncomp_data: bytes) -> v9.Archive: + raise TypeError("Not currently supported") + # name_buf, name_offsets = cls.gen_name_buffer(folder, file) + # vdrive_h = cls.gen_vdrive_header(archive_name, 0, 1, 0, 1) + # folder_h = cls.gen_folder_header(name_offsets[folder], 0, 0, 0, 1) + # file_h = cls.gen_file_header(name_offsets[file], 0, len(file_uncomp_data)) + # file_ = FileABC(file_h, file, file_uncomp_data, True) + # folder_ = FolderABC(folder_h, folder, [], [file_]) + # vdrive_ = VirtualDriveABC(vdrive_h, [folder_], [file_]) + # folder_.parent_drive = file_.parent_drive = vdrive_ + # file_.parent_folder = folder_ + # + # vdrive_buf = cls.gen_vdrive_header_buffer(archive_name, 0, 1, 0, 1) + # folder_buf = cls.gen_folder_header_buffer(name_offsets[folder], 0, 0, 0, 1) + # file_buf = cls.gen_file_header_buffer(name_offsets[file], 0, len(file_uncomp_data)) + # toc_buf, toc_offsets = cls.gen_toc_buffer_and_offsets(vdrive_buf, folder_buf, file_buf, name_buf) # toc_ptrs = splice_toc_offsets(1, 1, 1, len(name_buf), toc_offsets) # WE NEED TO USE BYTE-SIZE of NAME BUFFER!!!! - toc_ptrs = splice_toc_offsets(1, 1, 1, 2, toc_offsets) # According to my notes; V9 uses this to store the size of the name buffer; but Archive Unpacking assumes it's a count; must test on real V9 files. TODO - toc_ptr_buf = cls.gen_toc_ptr_buffer(*toc_ptrs) - full_toc = toc_ptr_buf + toc_buf - - header = cls.gen_archive_header(archive_name, cls.ARCHIVE_HEADER_SIZE, len(full_toc), cls.ARCHIVE_HEADER_SIZE + len(full_toc), len(file_uncomp_data)) - return DowIIIArchive(header, [vdrive_], False) + # toc_ptr_buf = cls.gen_toc_ptr_buffer(*toc_ptrs) + # full_toc = toc_ptr_buf + toc_buf + # + # header = cls.gen_archive_header(archive_name, cls.ARCHIVE_HEADER_SIZE, len(full_toc), cls.ARCHIVE_HEADER_SIZE + len(full_toc), len(file_uncomp_data)) + # return v9.Archive(header, [vdrive_], False) diff --git a/tests/relic/sga/datagen/v2.py b/tests/relic/sga/datagen/v2.py new file mode 100644 index 0000000..5f44f28 --- /dev/null +++ b/tests/relic/sga/datagen/v2.py @@ -0,0 +1,36 @@ +from typing import Optional, List, BinaryIO +from relic.sga._serializers import _Md5ChecksumHelper +from relic.sga import StorageType +from relic.sga.protocols import IOContainer +from relic.sga.v2 import API, core, _serializers as _s + + +def generate_file(name: str, data: bytes, storage: StorageType, compressed: bool = False, parent: Optional[IOContainer] = None) -> API.File: + return core.File(name, data, storage, compressed, None, parent, None) + + +def generate_folder(name: str, folders: Optional[List[API.Folder]] = None, files: Optional[List[API.File]] = None, parent: Optional[IOContainer] = None) -> API.Folder: + folders = [] if folders is None else folders + files = [] if files is None else files + return core.Folder(name, folders, files, parent=parent) + + +def generate_drive(name: str, folders: Optional[List[API.Folder]] = None, files: Optional[List[API.File]] = None, alias: str = "data") -> API.Drive: + folders = [] if folders is None else folders + files = [] if files is None else files + return core.Drive(alias, name, folders, files) + + +def generate_archive_meta(stream: BinaryIO, header_pos: int, header_size: int) -> core.ArchiveMetadata: + header_helper = _Md5ChecksumHelper(None, None, header_pos, header_size, _s.APISerializers.HEADER_MD5_EIGEN) + file_helper = _Md5ChecksumHelper(None, None, header_pos, None, _s.APISerializers.FILE_MD5_EIGEN) + # Setup expected MD5 results + header_helper.expected = header_helper.read(stream) + file_helper.expected = file_helper.read(stream) + return core.ArchiveMetadata(file_helper, header_helper) + + +def generate_archive(name: str, meta: core.ArchiveMetadata =None, drives: Optional[List[API.Drive]] = None) -> API.Archive: + drives = [] if drives is None else drives + return core.Archive(name,meta,drives) + diff --git a/tests/relic/sga/file/test_file_header.py b/tests/relic/sga/file/test_file_header.py deleted file mode 100644 index 3d4032e..0000000 --- a/tests/relic/sga/file/test_file_header.py +++ /dev/null @@ -1,101 +0,0 @@ -from abc import abstractmethod -from io import BytesIO - -import pytest - -from relic.common import VersionLike -from relic.sga import FileHeader, ArchiveVersion -from tests.relic.sga.datagen import DowI, DowII, DowIII - - -class FileHeaderTests: - @abstractmethod - def test_pack(self, header: FileHeader, expected: bytes): - with BytesIO() as stream: - written = header.pack(stream) - assert written == len(expected) - stream.seek(0) - assert stream.read() == expected - - @abstractmethod - def test_inner_pack(self, header: FileHeader, expected: bytes): - with BytesIO() as stream: - written = header._pack(stream) - assert written == len(expected) - stream.seek(0) - assert stream.read() == expected - - @abstractmethod - def test_inner_unpack(self, data_stream: bytes, expected: FileHeader): - with BytesIO(data_stream) as stream: - header = expected.__class__._unpack(stream) - assert header == expected - - @abstractmethod - def test_unpack(self, data_stream: bytes, expected: FileHeader, version: VersionLike): - with BytesIO(data_stream) as stream: - header = FileHeader.unpack(stream, version) - assert header == expected - - -DOW1_HEADER, DOW1_HEADER_BUFFER = DowI.gen_file_header(0, 0, 0), DowI.gen_file_header_buffer(0, 0, 0) - - -class TestDowIFileHeader(FileHeaderTests): - @pytest.mark.parametrize(["header", "expected"], [(DOW1_HEADER, DOW1_HEADER_BUFFER)]) - def test_pack(self, header: FileHeader, expected: bytes): - super().test_pack(header, expected) - - @pytest.mark.parametrize(["header", "expected"], [(DOW1_HEADER, DOW1_HEADER_BUFFER)]) - def test_inner_pack(self, header: FileHeader, expected: bytes): - super().test_pack(header, expected) - - @pytest.mark.parametrize(["expected", "data_stream", "version"], [(DOW1_HEADER, DOW1_HEADER_BUFFER, ArchiveVersion.Dow)]) - def test_unpack(self, data_stream: bytes, expected: FileHeader, version: VersionLike): - super().test_unpack(data_stream, expected, version) - - @pytest.mark.parametrize(["expected", "data_stream"], [(DOW1_HEADER, DOW1_HEADER_BUFFER)]) - def test_inner_unpack(self, data_stream: bytes, expected: FileHeader): - super().test_inner_unpack(data_stream, expected) - - -DOW2_HEADER, DOW2_HEADER_BUFFER = DowII.gen_file_header(0, 0, 0), DowII.gen_file_header_buffer(0, 0, 0) - - -class TestDowIIFileHeader(FileHeaderTests): - @pytest.mark.parametrize(["header", "expected"], [(DOW2_HEADER, DOW2_HEADER_BUFFER)]) - def test_pack(self, header: FileHeader, expected: bytes): - super().test_pack(header, expected) - - @pytest.mark.parametrize(["header", "expected"], [(DOW2_HEADER, DOW2_HEADER_BUFFER)]) - def test_inner_pack(self, header: FileHeader, expected: bytes): - super().test_pack(header, expected) - - @pytest.mark.parametrize(["expected", "data_stream", "version"], [(DOW2_HEADER, DOW2_HEADER_BUFFER, ArchiveVersion.Dow2)]) - def test_unpack(self, data_stream: bytes, expected: FileHeader, version: VersionLike): - super().test_unpack(data_stream, expected, version) - - @pytest.mark.parametrize(["expected", "data_stream"], [(DOW2_HEADER, DOW2_HEADER_BUFFER)]) - def test_inner_unpack(self, data_stream: bytes, expected: FileHeader): - super().test_inner_unpack(data_stream, expected) - - -DOW3_HEADER, DOW3_HEADER_BUFFER = DowIII.gen_file_header(0x0f, 0xf0, 0x09, 0x90), DowIII.gen_file_header_buffer(0x0f, 0xf0, 0x09, 0x90) - - -class TestDowIIIFileHeader(FileHeaderTests): - @pytest.mark.parametrize(["header", "expected"], [(DOW3_HEADER, DOW3_HEADER_BUFFER)]) - def test_pack(self, header: FileHeader, expected: bytes): - super().test_pack(header, expected) - - @pytest.mark.parametrize(["header", "expected"], [(DOW3_HEADER, DOW3_HEADER_BUFFER)]) - def test_inner_pack(self, header: FileHeader, expected: bytes): - super().test_pack(header, expected) - - @pytest.mark.parametrize(["expected", "data_stream", "version"], [(DOW3_HEADER, DOW3_HEADER_BUFFER, ArchiveVersion.Dow3)]) - def test_unpack(self, data_stream: bytes, expected: FileHeader, version: VersionLike): - super().test_unpack(data_stream, expected, version) - - @pytest.mark.parametrize(["expected", "data_stream"], [(DOW3_HEADER, DOW3_HEADER_BUFFER)]) - def test_inner_unpack(self, data_stream: bytes, expected: FileHeader): - super().test_inner_unpack(data_stream, expected) diff --git a/tests/relic/sga/test_apis.py b/tests/relic/sga/test_apis.py new file mode 100644 index 0000000..eccd298 --- /dev/null +++ b/tests/relic/sga/test_apis.py @@ -0,0 +1,139 @@ +import json +from abc import abstractmethod +from io import BytesIO +from pathlib import Path +from typing import Union, Iterable, Tuple, List + +import pytest + +from relic.sga import v2, v5, v9, MagicWord, Version, v7 +from relic.sga.protocols import API +from tests.relic.sga.datagen import DowII, DowI, DowIII + + +class APITests: + @abstractmethod + def test_read(self, buffer: Union[bytes, str], api: API): + if isinstance(buffer, str): + with open(buffer, "rb") as stream: + api.read(stream, True) + else: + with BytesIO(buffer) as stream: + api.read(stream, True) + + +def scan_directory(root_dir: str, desired_version: Version) -> Iterable[str]: + root_directory = Path(root_dir) + for path_object in root_directory.glob('**/*.sga'): + with path_object.open("rb") as stream: + if not MagicWord.check_magic_word(stream, advance=True): + continue + version = Version.unpack(stream) + if version != desired_version: + continue + yield str(path_object) + + +def fast_gen_dow1_archive(*args): + return None, DowI.gen_sample_archive_buffer(*args) + + +def prepare_for_parametrize(files: Iterable[str]) -> Iterable[Tuple[str]]: + return [(_,) for _ in files] + + +_path = Path(__file__).parent +# Explicit path locations +try: + path = _path / "sources.json" + with path.open() as stream: + file_sources = json.load(stream) +except IOError as e: + file_sources = {} + + +# Implicit path locations +def _update_implicit_file_sources(src_key: str): + if src_key not in file_sources: + file_sources[src_key] = {} + if "dirs" not in file_sources[src_key]: + file_sources[src_key]["dirs"] = [] + dirs:List[str] = file_sources[src_key]["dirs"] + dirs.append(str(_path / "test_data" / src_key)) + + +def _helper(src_key: str, version: Version): + _update_implicit_file_sources(src_key) + try: + local_sources = file_sources.get(src_key, {}) + files = set() + for src_dir in local_sources.get("dirs", []): + for f in scan_directory(src_dir, version): + files.add(f) + for src_file in local_sources.get("files", []): + files.add(src_file) + return prepare_for_parametrize(files) + except IOError as e: + return tuple() + + +v2Files = _helper("v2", v2.version) +v5Files = _helper("v5", v5.version) +v7Files = _helper("v7", v7.version) +v9Files = _helper("v9", v9.version) + +DOW1_ARCHIVE, DOW1_ARCHIVE_PACKED = fast_gen_dow1_archive("Dow1 Test Archive", "Tests", "And Now For Something Completely Different.txt", b"Just kidding, it's Monty Python.") + + +class TestV2(APITests): + @pytest.fixture() + def api(self) -> API: + return v2.API + + @pytest.mark.parametrize(["buffer"], [(DOW1_ARCHIVE_PACKED,), *v2Files]) + def test_read(self, buffer: Union[bytes, str], api: API): + super(TestV2, self).test_read(buffer, api) + + +def fast_gen_dow2_archive(*args): + return None, DowII.gen_sample_archive_buffer(*args) + + +DOW2_ARCHIVE, DOW2_ARCHIVE_PACKED = fast_gen_dow2_archive("Dow2 Test Archive", "Tests", "A Favorite Guardsmen VL.txt", b"Where's that artillery!?") + + +class TestV5(APITests): + @pytest.fixture() + def api(self) -> API: + return v5.API + + @pytest.mark.parametrize(["buffer"], [*v5Files, (DOW2_ARCHIVE_PACKED,)]) + def test_read(self, buffer: Union[bytes, str], api: API): + super(TestV5, self).test_read(buffer, api) + + +def fast_gen_dow3_archive(*args): + return None, DowIII.gen_sample_archive_buffer(*args) + + +DOW3_ARCHIVE, DOW3_ARCHIVE_PACKED = fast_gen_dow3_archive("Dow3 Test Archive", "Tests", "Some Witty FileName.txt", b"NGL; I'm running out of dumb/clever test data.") + + +class TestV9(APITests): + @pytest.fixture() + def api(self) -> API: + return v9.API + + @pytest.mark.parametrize(["buffer"], [*v9Files, (DOW3_ARCHIVE_PACKED,)]) + def test_read(self, buffer: Union[bytes, str], api: API): + super(TestV9, self).test_read(buffer, api) + + +class TestV7(APITests): + @pytest.fixture() + def api(self) -> API: + return v7.API + + @pytest.mark.parametrize(["buffer"], [*v7Files]) + def test_read(self, buffer: Union[bytes, str], api: API): + super(TestV7, self).test_read(buffer, api) diff --git a/test_data/sga/archive-v2_0.sga b/tests/relic/sga/test_data/v2/DowI Test Data.sga similarity index 100% rename from test_data/sga/archive-v2_0.sga rename to tests/relic/sga/test_data/v2/DowI Test Data.sga diff --git a/tests/relic/sga/test_sga.py b/tests/relic/sga/test_sga.py deleted file mode 100644 index cda3fa5..0000000 --- a/tests/relic/sga/test_sga.py +++ /dev/null @@ -1,52 +0,0 @@ -from __future__ import annotations - -# from io import BytesIO -# from relic.sga import Archive -# from write_sga_samples import build_sample_dow1_archive, build_sample_dow3_archive, build_sample_dow2_archive -# -# -# def assert_archives(left: Archive, right: Archive): -# # ASSERT HEADER -# assert left.header == right.header, (left.header, right.header) -# v = left.header.version -# -# # # ASSERT TOC -# # l_toc, r_toc = left.info.table_of_contents, right.info.table_of_contents -# # assert l_toc.drive_info.count == r_toc.drive_info.count -# # assert l_toc.files_info.count == r_toc.files_info.count -# # if l_toc.filenames_info.count: -# # assert l_toc.filenames_info.count == r_toc.filenames_info.count -# # else: -# # assert l_toc.filenames_info.byte_size == r_toc.filenames_info.byte_size -# # assert l_toc.folders_info.count == r_toc.folders_info.count -# -# r_lookup = {d.path: d for d in right.drives} -# l_lookup = {d.path: d for d in left.drives} -# assert len(r_lookup) == len(l_lookup), "Drive Count" -# for key in l_lookup: -# assert key in r_lookup, "Drive Not Found" -# -# # TODO assert folders and files -# -# -# def run_test(archive: Archive): -# with BytesIO() as buffer: -# archive.pack(buffer, True) -# buffer.seek(0) -# gen_archive = Archive.unpack(buffer) -# assert_archives(archive, gen_archive) -# -# -# def test_archive_DowI(): -# archive = build_sample_dow1_archive() -# run_test(archive) -# -# -# def test_archive_Dow2(): -# archive = build_sample_dow2_archive() -# run_test(archive) -# -# -# def test_archive_Dow3(): -# archive = build_sample_dow3_archive() -# run_test(archive) diff --git a/tests/relic/sga/write_sga_samples.py b/tests/relic/sga/write_sga_samples.py deleted file mode 100644 index 73d6ac8..0000000 --- a/tests/relic/sga/write_sga_samples.py +++ /dev/null @@ -1,103 +0,0 @@ -# import zlib -# from io import BytesIO -# from os.path import join -# -# from archive_tools.structio import WindowPtr, Ptr -# -# from relic.sga.archive.header import DowIArchiveHeader -# from relic.sga.file.file import File -# from relic.sga.file.header import DowIFileHeader -# from relic.sga.folder.folder import Folder -# from relic.sga.vdrive.virtual_drive import VirtualDrive -# from tests.helpers import get_testdata_root_folder, lorem_ipsum -# -# -# def compress16(b: bytes) -> bytes: -# compressor = zlib.compressobj(wbits=14) -# with BytesIO() as stream: -# stream.write(compressor.compress(b)) -# stream.write(compressor.flush()) -# stream.seek(0) -# return stream.read() -# -# -# def compress32(b: bytes) -> bytes: -# compressor = zlib.compressobj(wbits=15) -# with BytesIO() as stream: -# stream.write(compressor.compress(b)) -# stream.write(compressor.flush()) -# stream.seek(0) -# return stream.read() -# -# -# def build_sample_dow1_archive(): -# header = DowIArchiveHeader("DowI Test Data", WindowPtr(None, None), Ptr(None), 0) -# -# raw_content = lorem_ipsum.encode("ascii") -# comp_16_content = compress16(raw_content) -# comp_32_content = compress32(raw_content) -# -# rf = DowIFileHeader(None,None,#, len(raw_content), len(raw_content), FileCompressionFlag.Decompressed) -# raw_file = File(, "Lorem Ipsum Raw", raw_content)) -# comp16_file = File(DowIFileHeader(None, None, len(raw_content), len(comp_16_content), FileCompressionFlag.Compressed16), "Lorem Ipsum Zlib-16", comp_16_content) -# comp32_file = File(DowIFileHeader(None, None, len(raw_content), len(comp_32_content), FileCompressionFlag.Compressed32), "Lorem Ipsum Zlib-32", comp_32_content) -# lorem_folder = Folder([], [raw_file, comp16_file, comp32_file], 0, 3, FolderHeader(None, None, None), "Lorem Ipsum") -# test_drive = VirtualDrive([lorem_folder], [], 1, 0, "test", "Test Drive", None) -# -# archive = Archive(info, [test_drive]) -# return archive -# -# # -# # def build_sample_dow2_archive(): -# # header = ArchiveHeader(SgaVersion.Dow2.value, "DowII Test Data", bytes([0x00] * 16), bytes([0x00] * 16)) -# # -# # info = ArchiveInfo(header, None, None) -# # raw_content = lorem_ipsum.encode("ascii") -# # comp_16_content = compress16(raw_content) -# # comp_32_content = compress32(raw_content) -# # -# # raw_file = File(DowIIFileHeader(None, None, len(raw_content), len(raw_content), 0, 0), "Lorem Ipsum Raw", raw_content) -# # comp16_file = File(DowIIFileHeader(None, None, len(raw_content), len(comp_16_content), 0, 0), "Lorem Ipsum Zlib-16", comp_16_content) -# # comp32_file = File(DowIIFileHeader(None, None, len(raw_content), len(comp_32_content), 0, 0), "Lorem Ipsum Zlib-32", comp_32_content) -# # lorem_folder = Folder([], [raw_file, comp16_file, comp32_file], 0, 3, FolderHeader(None, None, None), "Lorem Ipsum") -# # test_drive = VirtualDrive([lorem_folder], [], 1, 0, "test", "Test Drive", None) -# # -# # archive = Archive(info, [test_drive]) -# # return archive -# # -# # -# # def build_sample_dow3_archive(): -# # header = ArchiveHeader(SgaVersion.Dow3.value, "DowIII Test Data") -# # -# # info = ArchiveInfo(header, None, None) -# # raw_content = lorem_ipsum.encode("ascii") -# # comp_16_content = compress16(raw_content) -# # comp_32_content = compress32(raw_content) -# # -# # raw_file = File(DowIIIFileHeader(None, None, len(raw_content), len(raw_content), 0, 0, 0, 0, 0), -# # "Lorem Ipsum Raw", raw_content) -# # comp16_file = File(DowIIIFileHeader(None, None, len(raw_content), len(comp_16_content), 0, 0, 0, 0, 0), -# # "Lorem Ipsum Zlib-16", comp_16_content) -# # comp32_file = File(DowIIIFileHeader(None, None, len(raw_content), len(comp_32_content), 0, 0, 0, 0, 0), -# # "Lorem Ipsum Zlib-32", comp_32_content) -# # lorem_folder = Folder([], [raw_file, comp16_file, comp32_file], 0, 3, FolderHeader(None, None, None), "Lorem Ipsum") -# # test_drive = VirtualDrive([lorem_folder], [], 1, 0, "test", "Test Drive", None) -# # -# # archive = Archive(info, [test_drive]) -# # return archive -# -# -# if __name__ == "__main__": -# root = get_testdata_root_folder() -# archive = build_sample_dow1_archive() -# with open(join(root, "archive-v2_0.sga"), "wb") as file: -# archive.pack(file) -# # write_archive(file, archive) -# -# # archive2 = build_sample_dow2_archive() -# # with open(join(root, "archive-v5_0.sga"), "wb") as file: -# # write_archive(file, archive2) -# # -# # archive3 = build_sample_dow3_archive() -# # with open(join(root, "archive-v9_0.sga"), "wb") as file: -# # write_archive(file, archive3) diff --git a/tests/relic_chunky/test_relic_chunky.py b/tests/relic_chunky/test_relic_chunky.py deleted file mode 100644 index 948e153..0000000 --- a/tests/relic_chunky/test_relic_chunky.py +++ /dev/null @@ -1,74 +0,0 @@ -# from io import BytesIO -# -# from relic.chunky import RelicChunky, DataChunk, FolderChunk, ChunkyVersion -# # from relic.sga import Archive, writer, File, Folder -# from write_chunky_samples import build_sample_chunky_v1_1 -# -# -# def assert_chunies(left: RelicChunky, right: RelicChunky): -# # ASSERT HEADER -# assert left.header == right.header, "Chunky Version Mismatch" -# v = left.header.version -# assert len(left.chunks) == len(right.chunks), "Chunk Count Mismatch" -# for left, right in zip(left.walk_chunks(True, False), right.walk_chunks(True, False)): -# l_path, l_folders, l_datas = left -# r_path, r_folders, r_datas = right -# -# assert l_path == r_path, "Chunk Path Mismatch" -# assert len(l_folders) == len(r_folders), "Chunk Folder Count Mismatch" -# for l_folder, r_folder in zip(l_folders, r_folders): -# l_folder: FolderChunk -# r_folder: FolderChunk -# # WE do it manualy since we don't expect size to be correct for manually built chunkies -# assert l_folder.header.version == r_folder.header.version, "Chunk Folder Header ('Version') Mismatch" -# assert l_folder.header.name == r_folder.header.name, "Chunk Folder Header ('Name') Mismatch" -# assert l_folder.header.type == r_folder.header.type, "Chunk Folder Header ('Type') Mismatch" -# assert l_folder.header.id == r_folder.header.id, "Chunk Folder Header ('Id') Mismatch" -# if v == ChunkyVersion.v3_1: -# assert l_folder.header.unk_v3_1 == r_folder.header.unk_v3_1, "Chunk Folder Header ('Unks v3.1') Mismatch" -# -# assert len(l_datas) == len(r_datas), "Chunk Data Count Mismatch" -# for l_data, r_data in zip(l_datas, r_datas): -# l_data: DataChunk -# r_data: DataChunk -# assert l_data.header.equal(r_data.header, v), "Chunk Data Header Mismatch" -# assert len(l_data.data) == len(r_data.data), "Chunk Data Size Mismatch" -# for i in range(len(l_data.data)): -# assert l_data.data[i] == r_data.data[i], f"Chunk Data Mismatch @{i}" -# -# # for path, folders, datas in left.walk_chunks() -# -# # assert len(r_lookup) == len(l_lookup), "Drive Count" -# # for key in l_lookup: -# # assert key in r_lookup, "Drive Not Found" -# # -# # for path, folders, files in left.walk(True): -# # for l_folder in folders: -# # r_folder: Folder = right.get_from_path(path, l_folder.name) -# # assert r_folder is not None, "Folder Not Found" -# # assert r_folder.name == l_folder.name, "Foldern Name" -# # assert r_folder.folder_count() == l_folder.folder_count(), "Folder Subfolder Count" -# # assert r_folder.file_count() == l_folder.file_count(), "Folder File Count" -# # for l_file in files: -# # r_data: File = right.get_from_path(path, l_file.name) -# # assert r_data is not None, "File Not Found" -# # assert r_data.name == l_file.name, "File Name" -# # assert len(r_data.data) == len(l_file.data), f"File Data Length" -# # for i in range(len(r_data.data)): -# # assert r_data.data[i] == l_file.data[i], f"File Data Mismatch @{i}" -# # assert r_data.header.decompressed_size == l_file.header.decompressed_size, "File Decompressed Size" -# # assert r_data.header.compressed_size == l_file.header.compressed_size, "File Compressed Size" -# # # TODO assert file flags -# -# -# def run_test(chunky: RelicChunky): -# with BytesIO() as buffer: -# chunky._pack(buffer) -# buffer.seek(0) -# generated = RelicChunky._unpack(buffer) -# assert_chunies(chunky, generated) -# -# -# def test_chunky_v1_1(): -# archive = build_sample_chunky_v1_1() -# run_test(archive) diff --git a/tests/relic_chunky/write_chunky_samples.py b/tests/relic_chunky/write_chunky_samples.py deleted file mode 100644 index 9dec7b6..0000000 --- a/tests/relic_chunky/write_chunky_samples.py +++ /dev/null @@ -1,24 +0,0 @@ -# import zlib -# -# from relic.chunky import RelicChunky, DataChunk, ChunkHeader, ChunkType, FolderChunk, RelicChunkyHeader, ChunkyVersion -# from tests.helpers import lorem_ipsum -# -# -# def build_sample_chunky_v1_1() -> RelicChunky: -# EXD = "EXD " -# EXDC = "EXDC" -# EXDF = "EXDF" -# -# lorem_ipsum_data = lorem_ipsum.encode("ascii") -# lorem_ipsum_compressed = zlib.compress(lorem_ipsum_data) -# -# uncomp_header = ChunkHeader(ChunkType.Data, EXD, 1, len(lorem_ipsum_data), "Lorem Ipsum") -# lorem_ipsum_uncomp = DataChunk(uncomp_header, lorem_ipsum_data) -# comp_header = ChunkHeader(ChunkType.Data, EXDC, 1, len(lorem_ipsum_compressed), "Lorem Ipsum Compressed") -# lorem_ipsum_comp = DataChunk(comp_header, lorem_ipsum_compressed) -# folder_header = ChunkHeader(ChunkType.Folder, EXDF, 1, 0, "Lorem Ipsum Test Data") # size wil be fixed when writing, and is ignored in assetions -# folder = FolderChunk([lorem_ipsum_uncomp,lorem_ipsum_comp],folder_header) -# -# chunky_header = RelicChunkyHeader.default(version=ChunkyVersion.v1_1.value) -# chunky = RelicChunky([folder],chunky_header) -# return chunky