Directly parse SerializeFile to detect format

Author: SkowronskiAndrew

Analyzer.Tests/FileDetectionTests.cs

@@ -0,0 +1,244 @@
+using System;
+using System.IO;
+using NUnit.Framework;
+using UnityDataTools.Analyzer.Util;
+using UnityDataTools.FileSystem;
+
+namespace UnityDataTools.Analyzer.Tests;
+
+/// <summary>
+/// Tests for file format detection utilities (ArchiveDetector and SerializedFileDetector).
+/// </summary>
+public class FileDetectionTests
+{
+    private string m_TestDataPath;
+
+    [OneTimeSetUp]
+    public void OneTimeSetUp()
+    {
+        m_TestDataPath = Path.Combine(TestContext.CurrentContext.TestDirectory, "Data");
+        UnityFileSystem.Init();
+    }
+
+    [OneTimeTearDown]
+    public void OneTimeTearDown()
+    {
+        UnityFileSystem.Cleanup();
+    }
+
+    #region SerializedFile Detection Tests
+
+    [Test]
+    public void TryDetectSerializedFile_ValidPlayerDataFile_ReturnsTrue()
+    {
+        var testFile = Path.Combine(m_TestDataPath, "PlayerData", "2022.1.20f1", "level0");
+
+        bool result = SerializedFileDetector.TryDetectSerializedFile(testFile, out var info);
+
+        Assert.IsTrue(result, "level0 should be detected as a valid SerializedFile");
+        Assert.IsNotNull(info);
+
+        // Verify exact values from the level0 file header
+        Assert.That(info.Version, Is.EqualTo(22u), "Version should be 22");
+        Assert.That(info.FileSize, Is.EqualTo(31988UL), "FileSize should be 31988");
+        Assert.That(info.MetadataSize, Is.EqualTo(24580UL), "MetadataSize should be 24580");
+        Assert.That(info.DataOffset, Is.EqualTo(24640UL), "DataOffset should be 24640");
+        Assert.That(info.Endianness, Is.EqualTo((byte)1), "Endianness should be 1 (BigEndian)");
+        Assert.IsFalse(info.IsLegacyFormat, "Version 22 uses modern format (64-bit header)");
+    }
+
+    [Test]
+    public void TryDetectSerializedFile_SerializedFileInsideArchive_ReturnsTrue()
+    {
+        // This tests a serialized file extracted from the alienprefab archive
+        // The file was originally at CAB-c5053efeda8860d7e7b7ce4b4c66705b inside the archive
+        var testFile = Path.Combine(m_TestDataPath, "LegacyFormats", "CAB-c5053efeda8860d7e7b7ce4b4c66705b");
+
+        bool result = SerializedFileDetector.TryDetectSerializedFile(testFile, out var info);
+
+        Assert.IsTrue(result, "CAB-c5053efeda8860d7e7b7ce4b4c66705b should be detected as a valid SerializedFile");
+        Assert.IsNotNull(info);
+
+        // Verify exact values from the CAB file header
+        Assert.That(info.Version, Is.EqualTo(17u), "Version should be 17");
+        Assert.That(info.FileSize, Is.EqualTo(595380UL), "FileSize should be 595380");
+        Assert.That(info.MetadataSize, Is.EqualTo(61328UL), "MetadataSize should be 61328");
+        Assert.That(info.DataOffset, Is.EqualTo(61360UL), "DataOffset should be 61360");
+        Assert.That(info.Endianness, Is.EqualTo((byte)1), "Endianness should be 1 (BigEndian)");
+        Assert.IsTrue(info.IsLegacyFormat, "Version 17 uses legacy format (32-bit header)");
+    }
+
+    [Test]
+    public void TryDetectSerializedFile_JsonFile_ReturnsFalse()
+    {
+        var testFiles = Directory.GetFiles(Path.Combine(m_TestDataPath, "AddressableBuildLayouts"), "*.json");
+        Assert.Greater(testFiles.Length, 0, "Should have at least one JSON test file");
+
+        foreach (var testFile in testFiles)
+        {
+            bool result = SerializedFileDetector.TryDetectSerializedFile(testFile, out var info);
+
+            Assert.IsFalse(result, $"{Path.GetFileName(testFile)} should not be detected as a SerializedFile");
+            Assert.IsNull(info, "Info should be null for non-SerializedFile");
+        }
+    }
+
+    [Test]
+    public void TryDetectSerializedFile_TextFile_ReturnsFalse()
+    {
+        var testFile = Path.Combine(m_TestDataPath, "PlayerNoTypeTree", "README.md");
+
+        bool result = SerializedFileDetector.TryDetectSerializedFile(testFile, out var info);
+
+        Assert.IsFalse(result, "README.md should not be detected as a SerializedFile");
+        Assert.IsNull(info);
+    }
+
+    [Test]
+    public void TryDetectSerializedFile_EmptyFile_ReturnsFalse()
+    {
+        // Create a temporary empty file
+        var tempFile = Path.GetTempFileName();
+        try
+        {
+            bool result = SerializedFileDetector.TryDetectSerializedFile(tempFile, out var info);
+
+            Assert.IsFalse(result, "Empty file should not be detected as a SerializedFile");
+            Assert.IsNull(info);
+        }
+        finally
+        {
+            File.Delete(tempFile);
+        }
+    }
+
+    [Test]
+    public void TryDetectSerializedFile_TruncatedHeader_ReturnsFalse()
+    {
+        // Create a temporary file with only partial header (10 bytes)
+        var tempFile = Path.GetTempFileName();
+        try
+        {
+            File.WriteAllBytes(tempFile, new byte[10]); // Less than minimum header size (20 bytes)
+
+            bool result = SerializedFileDetector.TryDetectSerializedFile(tempFile, out var info);
+
+            Assert.IsFalse(result, "Truncated file should not be detected as a SerializedFile");
+            Assert.IsNull(info);
+        }
+        finally
+        {
+            File.Delete(tempFile);
+        }
+    }
+
+    [Test]
+    public void TryDetectSerializedFile_RandomBytes_ReturnsFalse()
+    {
+        // Create a temporary file with random bytes
+        var tempFile = Path.GetTempFileName();
+        try
+        {
+            var random = new Random(12345); // Fixed seed for reproducibility
+            byte[] randomData = new byte[100];
+            random.NextBytes(randomData);
+            File.WriteAllBytes(tempFile, randomData);
+
+            bool result = SerializedFileDetector.TryDetectSerializedFile(tempFile, out var info);
+
+            Assert.IsFalse(result, "Random bytes should not be detected as a SerializedFile");
+            Assert.IsNull(info);
+        }
+        finally
+        {
+            File.Delete(tempFile);
+        }
+    }
+
+    [Test]
+    public void TryDetectSerializedFile_NonExistentFile_ReturnsFalse()
+    {
+        var nonExistentFile = Path.Combine(m_TestDataPath, "ThisFileDoesNotExist.xyz");
+
+        bool result = SerializedFileDetector.TryDetectSerializedFile(nonExistentFile, out var info);
+
+        Assert.IsFalse(result, "Non-existent file should not be detected as a SerializedFile");
+        Assert.IsNull(info);
+    }
+
+    #endregion
+
+    #region Archive Detection Tests
+
+    [Test]
+    public void IsUnityArchive_ValidAssetBundle_ReturnsTrue()
+    {
+        var testFile = Path.Combine(m_TestDataPath, "AssetBundles", "2022.1.20f1", "assetbundle");
+
+        bool result = ArchiveDetector.IsUnityArchive(testFile);
+
+        Assert.IsTrue(result, "assetbundle should be detected as a Unity Archive");
+    }
+
+    [Test]
+    public void IsUnityArchive_OldFormatArchive_ReturnsTrue()
+    {
+        var testFile = Path.Combine(m_TestDataPath, "LegacyFormats", "alienprefab");
+
+        bool result = ArchiveDetector.IsUnityArchive(testFile);
+
+        Assert.IsTrue(result, "alienprefab should be detected as a Unity Archive");
+    }
+
+    [Test]
+    public void IsUnityArchive_SerializedFile_ReturnsFalse()
+    {
+        var testFile = Path.Combine(m_TestDataPath, "PlayerData", "2022.1.20f1", "level0");
+
+        bool result = ArchiveDetector.IsUnityArchive(testFile);
+
+        Assert.IsFalse(result, "level0 (SerializedFile) should not be detected as an archive");
+    }
+
+    [Test]
+    public void IsUnityArchive_JsonFile_ReturnsFalse()
+    {
+        var testFiles = Directory.GetFiles(Path.Combine(m_TestDataPath, "AddressableBuildLayouts"), "*.json");
+        Assert.Greater(testFiles.Length, 0, "Should have at least one JSON test file");
+
+        foreach (var testFile in testFiles)
+        {
+            bool result = ArchiveDetector.IsUnityArchive(testFile);
+
+            Assert.IsFalse(result, $"{Path.GetFileName(testFile)} should not be detected as an archive");
+        }
+    }
+
+    [Test]
+    public void IsUnityArchive_EmptyFile_ReturnsFalse()
+    {
+        var tempFile = Path.GetTempFileName();
+        try
+        {
+            bool result = ArchiveDetector.IsUnityArchive(tempFile);
+
+            Assert.IsFalse(result, "Empty file should not be detected as an archive");
+        }
+        finally
+        {
+            File.Delete(tempFile);
+        }
+    }
+
+    [Test]
+    public void IsUnityArchive_NonExistentFile_ReturnsFalse()
+    {
+        var nonExistentFile = Path.Combine(m_TestDataPath, "ThisFileDoesNotExist.xyz");
+
+        bool result = ArchiveDetector.IsUnityArchive(nonExistentFile);
+
+        Assert.IsFalse(result, "Non-existent file should not be detected as an archive");
+    }
+
+    #endregion
+}

Analyzer/SQLite/Parsers/SerializedFileParser.cs

@@ -4,6 +4,7 @@
 using Microsoft.Data.Sqlite;
 using UnityDataTools.Analyzer.SQLite.Handlers;
 using UnityDataTools.Analyzer.SQLite.Writers;
+using UnityDataTools.Analyzer.Util;
 using UnityDataTools.FileSystem;
 
 namespace UnityDataTools.Analyzer.SQLite.Parsers
@@ -17,7 +18,14 @@ public class SerializedFileParser : ISQLiteFileParser
 
         public bool CanParse(string filename)
         {
-            return ShouldIgnoreFile(filename) == false;
+            // First check if the file is in the ignore list (by extension or filename)
+            if (ShouldIgnoreFile(filename))
+                return false;
+
+            // Then validate that it's actually a Unity file by checking its format
+            // This prevents ugly exceptions when processing non-Unity files
+            return ArchiveDetector.IsUnityArchive(filename)
+                || SerializedFileDetector.TryDetectSerializedFile(filename, out _);
         }
 
 
@@ -40,12 +48,12 @@ public void Parse(string filename)
 
         bool ShouldIgnoreFile(string file)
         {
-            // Unfortunately there is no standard extension for AssetBundles, and SerializedFiles often have no extension at all.
-            // Also there is also no distinctive signature at the start of a SerializedFile to immediately recognize it based on its first bytes.
-            // This makes it difficult to use the "--search-pattern" argument to only pick those files.
-
-            // Hence to reduce noise in UnityDataTool output we filter out files that we have a high confidence are
-            // NOT SerializedFiles or Unity Archives.
+            // Filter out common non-Unity files by extension or filename.
+            // This is a fast initial filter before we perform format detection.
+            //
+            // Note: AssetBundles have no standard extension, and SerializedFiles often have no extension at all.
+            // Format detection (via ArchiveDetector and SerializedFileDetector) is performed after this filter
+            // to definitively identify Unity files.
 
             string fileName = Path.GetFileName(file);
             string extension = Path.GetExtension(file);
@@ -69,7 +77,7 @@ bool ShouldIgnoreFile(string file)
 
         void ProcessFile(string file, string rootDirectory)
         {
-            if (IsUnityArchive(file))
+            if (ArchiveDetector.IsUnityArchive(file))
             {
                 bool archiveHadErrors = false;
                 using (UnityArchive archive = UnityFileSystem.MountArchive(file, "archive:" + Path.DirectorySeparatorChar))
@@ -122,45 +130,12 @@ void ProcessFile(string file, string rootDirectory)
             }
             else
             {
-                // This isn't a Unity Archive file.  Try to open it as a SerializedFile.
-                // Unfortunately there is no standard file extension, or clear signature at the start of the file,
-                // to test if it truly is a SerializedFile.  So this will process files that are clearly not unity build files,
-                // and there is a chance for crashes and freezes if the parser misinterprets the file content.
+                // This isn't a Unity Archive file, so process it as a SerializedFile.
+                // Note: The file has already been validated in CanParse() via SerializedFileDetector,
+                // so we're confident it's a valid SerializedFile at this point.
                 var relativePath = Path.GetRelativePath(rootDirectory, file);
                 m_Writer.WriteSerializedFile(relativePath, file, Path.GetDirectoryName(file));
             }
         }
-
-        private static bool IsUnityArchive(string filePath)
-        {
-            // Check whether a file is a Unity Archive (AssetBundle) by looking for known signatures at the start of the file.
-            // "UnifyFS" is the current signature, but some older formats of the file are still supported
-            string[] signatures = { "UnityFS", "UnityWeb", "UnityRaw", "UnityArchive" };
-            int maxLen = 12; // "UnityArchive".Length
-            byte[] buffer = new byte[maxLen];
-
-            using (var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read))
-            {
-                int read = fs.Read(buffer, 0, buffer.Length);
-                foreach (var sig in signatures)
-                {
-                    if (read >= sig.Length)
-                    {
-                        bool match = true;
-                        for (int i = 0; i < sig.Length; ++i)
-                        {
-                            if (buffer[i] != sig[i])
-                            {
-                                match = false;
-                                break;
-                            }
-                        }
-                        if (match)
-                            return true;
-                    }
-                }
-                return false;
-            }
-        }
     }
 }