Merge pull request #69 from CapsCollective/feature/asset-compression

Added compression and switched to recursive binary serialisation for asset packing

Author: jonjondev

.clang-format

@@ -29,7 +29,7 @@ AttributeMacros:
 BinPackArguments: false
 BinPackParameters: false
 BraceWrapping:
-  AfterCaseLabel:  false
+  AfterCaseLabel:  true
   AfterClass:      true
   AfterControlStatement: Always
   AfterEnum:       true

engine/core/Makefile

@@ -15,13 +15,6 @@ coreSources := $(call rwildcard,$(coreSrcDir)/,*.cpp)
 coreObjects := $(call findobjs,$(coreSrcDir),$(coreBinDir),$(coreSources))
 coreDepends := $(patsubst %.o, %.d, $(call rwildcard,$(coreBinDir)/,*.o))
 
-# Set build vars
-ifeq ($(platform), windows)
-    libGenDir := src
-else ifeq ($(platform), macos)
-    libGenDir := src
-endif
-
 # Build the static library
 $(coreLib): $(coreObjects)
 	$(call MKDIR,$(call platformpth,$(libDir)))

engine/core/scene/SceneFile.cpp

@@ -10,6 +10,7 @@
 #include "SceneFile.h"
 
 #include <resources/PackFile.h>
+#include <resources/PackFileData.h>
 #include <resources/ResourceSystem.h>
 #include <resources/SceneData.h>
 #include <utils/Logging.h>
@@ -105,35 +106,36 @@ bool SceneFile::Deserialise(std::vector<Entity*>& entities)
         };
 
     // TODO: Implement proper write-mode handling for scene system
-    String ScenePath = MakeScenePath(sceneName);
+    String scenePath = MakeScenePath(sceneName);
     if (!SceneSystem::GetBaseDirectory().IsEmpty())
     {
         bool result = FileSystem::ForEachFileInDir(
-            ScenePath,
+            scenePath,
             [&deserialiseEntityString](const std::filesystem::path& path) {
                 if (path.extension() != ENTITY_FILE_EXT) return;
                 String entityData = FileSystem::Read(path.c_str());
                 deserialiseEntityString(entityData, path);
             });
         if (!result)
         {
-            CC_LOG_ERROR("Failed to read scene file at path \"{}\"", ScenePath)
+            CC_LOG_ERROR("Failed to read scene file at path \"{}\"", scenePath)
             return false;
         }
     }
     else
     {
         PackFile* packFile = ResourceSystem::GetInstance().GetPackFile();
 
-        SceneData* sceneData = packFile->FindData<SceneData>(ScenePath);
+        std::shared_ptr<SceneData> sceneData = packFile->FindDataDeserialised<SceneData>(scenePath);
         if (!sceneData)
         {
-            CC_LOG_WARNING("Failed to find scene \"{}\" in pack file", ScenePath)
+            CC_LOG_WARNING(
+                "Failed to retrieve deserialised scene data for scene \"{}\" from pack file",
+                scenePath)
             return false;
         }
 
-        String sceneString(sceneData->data);
-        for (const String& entityData : sceneString.Split('|'))
+        for (const String& entityData : sceneData->entities)
         {
             deserialiseEntityString(entityData.Str(), "");
         }

engine/render/Makefile

@@ -15,18 +15,18 @@ renderSources := $(call rwildcard,$(renderSrcDir)/,*.cpp)
 renderObjects := $(call findobjs,$(renderSrcDir),$(renderBinDir),$(renderSources))
 renderDepends := $(patsubst %.o, %.d, $(call rwildcard,$(renderBinDir)/,*.o))
 renderBuildDir := $(renderBinDir)/build
-renderLibs := $(vendorDir)/zlib/build/lib/libz.a $(vendorDir)/libpng/build/libpng.a $(vendorDir)/freetype/build/libfreetype.a
+renderLibs := $(vendorDir)/libpng/build/libpng.a $(vendorDir)/freetype/build/libfreetype.a
 
 # Set shader build vars
 vertSources := $(call rwildcard,assets/shaders,*.vert)
 fragSources := $(call rwildcard,assets/shaders,*.frag)
 vertObjects := $(patsubst %.vert,$(renderBuildDir)/%.vert.spv,$(vertSources))
 fragObjects := $(patsubst %.frag,$(renderBuildDir)/%.frag.spv,$(fragSources))
 
+# Set build vars
 linkFlags += -l utils -l window
-
-compileFlags += -I $(vendorDir)/vulkan/include -I $(vendorDir)/glfw/include -I $(vendorDir)/glm \
-				-I $(vendorDir)/zlib/build/include -I $(vendorDir)/libpng -I $(vendorDir)/include/freetype
+compileFlags += -I $(vendorDir)/vulkan/include -I $(vendorDir)/glfw/include \
+				-I $(vendorDir)/libpng -I $(vendorDir)/include/freetype
 
 .PHONY: all
 

engine/render/renderer/platform/vulkan/Font.cpp

@@ -10,8 +10,8 @@
 #include "Font.h"
 
 #include <freetype/freetype.h>
-#include <resources/GenericFileData.h>
 #include <resources/PackFile.h>
+#include <resources/PackFileData.h>
 #include <resources/ResourceSystem.h>
 #include <utils/Defer.h>
 #include <utils/Logging.h>
@@ -27,7 +27,7 @@ namespace Siege::Vulkan
 Font::Font(const char* filePath)
 {
     PackFile* packFile = ResourceSystem::GetInstance().GetPackFile();
-    GenericFileData* fileData = packFile->FindData<GenericFileData>(filePath);
+    std::shared_ptr<PackFileData> fileData = packFile->FindData(filePath);
 
     FT_Open_Args args;
     args.flags = FT_OPEN_MEMORY;

engine/render/renderer/platform/vulkan/Shader.cpp

@@ -9,8 +9,8 @@
 
 #include "Shader.h"
 
-#include <resources/GenericFileData.h>
 #include <resources/PackFile.h>
+#include <resources/PackFileData.h>
 #include <resources/ResourceSystem.h>
 #include <utils/Logging.h>
 
@@ -174,7 +174,7 @@ void Shader::Destroy()
 MHArray<char> Shader::ReadFileAsBinary(const String& filePath)
 {
     PackFile* packFile = ResourceSystem::GetInstance().GetPackFile();
-    GenericFileData* fileData = packFile->FindData<GenericFileData>(filePath);
+    std::shared_ptr<PackFileData> fileData = packFile->FindData(filePath);
     MHArray<char> buffer(fileData->data, fileData->dataSize);
     return buffer;
 }

engine/render/renderer/platform/vulkan/StaticMesh.cpp

@@ -11,10 +11,10 @@
 
 #include <resources/PackFile.h>
 #include <resources/ResourceSystem.h>
+#include <resources/StaticMeshData.h>
 #include <utils/Logging.h>
 
 #include "Swapchain.h"
-#include "resources/StaticMeshData.h"
 
 namespace Siege::Vulkan
 {
@@ -43,22 +43,25 @@ StaticMesh::StaticMesh(const char* filePath, Material* material)
 {
     // TODO(Aryeh): How to extract material data from object files?
     PackFile* packFile = ResourceSystem::GetInstance().GetPackFile();
-    StaticMeshData* vertexData = packFile->FindData<StaticMeshData>(filePath);
+    std::shared_ptr<StaticMeshData> staticMeshData =
+        packFile->FindDataDeserialised<StaticMeshData>(filePath);
 
-    CC_ASSERT(vertexData->verticesCount > 0, "Cannot load in a file with no vertices!")
-    CC_ASSERT(vertexData->indicesCount > 0, "Cannot load in a file with no indices!")
+    CC_ASSERT(!staticMeshData->vertices.empty(), "Cannot load in a file with no vertices!")
+    CC_ASSERT(!staticMeshData->indices.empty(), "Cannot load in a file with no indices!")
 
-    CC_ASSERT(vertexData->verticesCount < MAX_VERTICES, "The provided model has too many vertices!")
-    CC_ASSERT(vertexData->indicesCount < MAX_INDICES, "The provided model has too many indices!")
+    CC_ASSERT(staticMeshData->vertices.size() < MAX_VERTICES,
+              "The provided model has too many vertices!")
+    CC_ASSERT(staticMeshData->indices.size() < MAX_INDICES,
+              "The provided model has too many indices!")
 
-    vertexBuffer = VertexBuffer(sizeof(BaseVertex) * vertexData->verticesCount);
-    vertexBuffer.Copy(vertexData->GetVertices(), sizeof(BaseVertex) * vertexData->verticesCount);
+    vertexCount = staticMeshData->vertices.size();
+    indexCount = staticMeshData->indices.size();
 
-    indexBuffer = IndexBuffer(sizeof(unsigned int) * vertexData->indicesCount);
-    indexBuffer.Copy(vertexData->GetIndices(), sizeof(unsigned int) * vertexData->indicesCount);
+    vertexBuffer = VertexBuffer(sizeof(BaseVertex) * vertexCount);
+    vertexBuffer.Copy(staticMeshData->vertices.data(), sizeof(BaseVertex) * vertexCount);
 
-    vertexCount = vertexData->verticesCount;
-    indexCount = vertexData->indicesCount;
+    indexBuffer = IndexBuffer(sizeof(unsigned int) * indexCount);
+    indexBuffer.Copy(staticMeshData->indices.data(), sizeof(unsigned int) * indexCount);
 
     subMeshes = MHArray<SubMesh>(1);
     materials = MHArray<Material*>(1);

engine/render/renderer/platform/vulkan/Texture2D.cpp

@@ -9,13 +9,14 @@
 
 #include "Texture2D.h"
 
+#include <resources/PackFile.h>
+#include <resources/ResourceSystem.h>
+#include <resources/Texture2DData.h>
 #include <utils/Defer.h>
 
 #include "Constants.h"
 #include "Context.h"
 #include "render/renderer/buffer/Buffer.h"
-#include "resources/ResourceSystem.h"
-#include "resources/Texture2DData.h"
 #include "utils/Descriptor.h"
 #include "utils/TypeAdaptor.h"
 
@@ -93,14 +94,13 @@ Texture2D& Texture2D::operator=(Texture2D&& other)
 void Texture2D::LoadFromFile(const char* filePath)
 {
     PackFile* packFile = ResourceSystem::GetInstance().GetPackFile();
-    Texture2DData* texture2dData = packFile->FindData<Texture2DData>(filePath);
-    uint64_t imageSize = texture2dData->GetImageSize();
-    const uint8_t* pixelPtr = texture2dData->GetPixels();
+    std::shared_ptr<Texture2DData> texture2dData =
+        packFile->FindDataDeserialised<Texture2DData>(filePath);
 
     Buffer::Buffer stagingBuffer;
     defer([&stagingBuffer] { Buffer::DestroyBuffer(stagingBuffer); });
 
-    Buffer::CreateBuffer(imageSize,
+    Buffer::CreateBuffer(texture2dData->GetImageSize(),
                          VK_BUFFER_USAGE_TRANSFER_SRC_BIT,
                          // specifies that data is accessible on the CPU.
                          VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
@@ -109,7 +109,7 @@ void Texture2D::LoadFromFile(const char* filePath)
                          OUT stagingBuffer.buffer,
                          OUT stagingBuffer.bufferMemory);
 
-    Buffer::CopyData(stagingBuffer, imageSize, pixelPtr);
+    Buffer::CopyData(stagingBuffer, texture2dData->GetImageSize(), texture2dData->pixels.data());
 
     extent = {static_cast<uint32_t>(texture2dData->texWidth),
               static_cast<uint32_t>(texture2dData->texHeight)};

engine/resources/Makefile

@@ -14,13 +14,16 @@ resourcesBinDir := $(binDir)/engine/resources
 resourcesSources := $(call rwildcard,$(resourcesSrcDir)/,*.cpp)
 resourcesObjects := $(call findobjs,$(resourcesSrcDir),$(resourcesBinDir),$(resourcesSources))
 resourcesDepends := $(patsubst %.o, %.d, $(call rwildcard,$(resourcesBinDir)/,*.o))
+resourcesLibs := $(vendorDir)/zlib/build/lib/libz.a
 
-linkFlags += -l utils
+# Set build vars
+linkFlags += -l utils -l z
+compileFlags += -I $(vendorDir)/zlib/build/include
 
 # Build the static library
 $(resourcesLib): $(resourcesObjects)
 	$(call MKDIR,$(call platformpth,$(libDir)))
-	ar -crs $(resourcesLib) $(resourcesObjects)
+	$(call COMBINE_LIBS, $(resourcesLibs), $(resourcesObjects), $(libDir), resources)
 
 # Add all rules from dependency files
 -include $(resourcesDepends)

engine/resources/PackFile.cpp

@@ -28,7 +28,7 @@ bool PackFile::LoadFromPath(const String& filepath)
 {
     std::ifstream inputFileStream;
     inputFileStream.open(filepath, std::ios::in | std::ios::binary);
-    inputFileStream.read(reinterpret_cast<char*>(&header), sizeof(PackFile::Header));
+    inputFileStream.read(reinterpret_cast<char*>(&header), sizeof(Header));
 
     body = reinterpret_cast<char*>(malloc(header.bodySize));
     inputFileStream.read(body, (uint32_t) header.bodySize);
@@ -41,7 +41,7 @@ bool PackFile::LoadFromPath(const String& filepath)
     char* tocEnd = body + header.bodySize;
     while (tocCurr < tocEnd)
     {
-        PackFile::TocEntry* toc = reinterpret_cast<PackFile::TocEntry*>(tocCurr);
+        TocEntry* toc = reinterpret_cast<TocEntry*>(tocCurr);
         if (!toc)
         {
             break;
@@ -52,6 +52,27 @@ bool PackFile::LoadFromPath(const String& filepath)
     return true;
 }
 
+std::shared_ptr<PackFileData> PackFile::FindData(const String& filepath)
+{
+    const TocEntry* toc = entries[filepath];
+    if (!toc)
+    {
+        return nullptr;
+    }
+
+    uLongf bodyDataSizeUncompressed = toc->dataSize;
+    uLongf bodyDataSizeCompressed = toc->dataSizeCompressed;
+
+    PackFileData* packFileData = new (malloc(bodyDataSizeUncompressed)) PackFileData();
+    int result = uncompress(reinterpret_cast<Bytef*>(packFileData),
+                            &bodyDataSizeUncompressed,
+                            reinterpret_cast<Bytef*>(body + toc->dataOffset),
+                            bodyDataSizeCompressed);
+    CC_ASSERT(result == Z_OK, "Decompression failed for filepath: " + filepath);
+
+    return {packFileData, free};
+}
+
 const std::map<String, PackFile::TocEntry*>& PackFile::GetEntries()
 {
     return entries;
@@ -69,7 +90,7 @@ PackFile::TocEntry* PackFile::TocEntry::Create(const String& name,
     uint32_t nameDataSize = name.Size() + 1;
     void* mem = malloc(sizeof(TocEntry) + nameDataSize);
 
-    PackFile::TocEntry* tocEntry = new (mem) TocEntry();
+    TocEntry* tocEntry = new (mem) TocEntry();
     tocEntry->dataOffset = dataOffset;
     tocEntry->dataSize = dataSize;
     strcpy(&tocEntry->name[0], name.Str());