Detect and specific warning for text format SerializedFiles

SkowronskiAndrew · SkowronskiAndrew · commit 877fd3064b82 · 2026-02-20T08:42:23.000-05:00
diff --git a/Analyzer.Tests/FileDetectionTests.cs b/Analyzer.Tests/FileDetectionTests.cs
@@ -168,6 +168,64 @@ public void TryDetectSerializedFile_NonExistentFile_ReturnsFalse()
 
     #endregion
 
+    #region YAML SerializedFile Detection Tests
+
+    [Test]
+    public void IsYamlSerializedFile_ValidYamlAsset_ReturnsTrue()
+    {
+        var testFile = Path.Combine(m_TestDataPath, "YamlFormat.asset");
+
+        bool result = YamlSerializedFileDetector.IsYamlSerializedFile(testFile);
+
+        Assert.IsTrue(result, "YamlFormat.asset should be detected as a YAML SerializedFile");
+    }
+
+    [Test]
+    public void IsYamlSerializedFile_BinarySerializedFile_ReturnsFalse()
+    {
+        var testFile = Path.Combine(m_TestDataPath, "PlayerData", "2022.1.20f1", "level0");
+
+        bool result = YamlSerializedFileDetector.IsYamlSerializedFile(testFile);
+
+        Assert.IsFalse(result, "Binary SerializedFile should not be detected as YAML");
+    }
+
+    [Test]
+    public void IsYamlSerializedFile_Archive_ReturnsFalse()
+    {
+        var testFile = Path.Combine(m_TestDataPath, "AssetBundles", "2022.1.20f1", "assetbundle");
+
+        bool result = YamlSerializedFileDetector.IsYamlSerializedFile(testFile);
+
+        Assert.IsFalse(result, "AssetBundle should not be detected as YAML");
+    }
+
+    [Test]
+    public void IsYamlSerializedFile_JsonFile_ReturnsFalse()
+    {
+        var testFiles = Directory.GetFiles(Path.Combine(m_TestDataPath, "AddressableBuildLayouts"), "*.json");
+        Assert.Greater(testFiles.Length, 0, "Should have at least one JSON test file");
+
+        foreach (var testFile in testFiles)
+        {
+            bool result = YamlSerializedFileDetector.IsYamlSerializedFile(testFile);
+
+            Assert.IsFalse(result, $"JSON file should not be detected as YAML SerializedFile: {Path.GetFileName(testFile)}");
+        }
+    }
+
+    [Test]
+    public void IsYamlSerializedFile_NonExistentFile_ReturnsFalse()
+    {
+        var nonExistentFile = Path.Combine(m_TestDataPath, "ThisFileDoesNotExist.asset");
+
+        bool result = YamlSerializedFileDetector.IsYamlSerializedFile(nonExistentFile);
+
+        Assert.IsFalse(result, "Non-existent file should not be detected as YAML");
+    }
+
+    #endregion
+
     #region Archive Detection Tests
 
     [Test]
diff --git a/Analyzer/Util/YamlSerializedFileDetector.cs b/Analyzer/Util/YamlSerializedFileDetector.cs
@@ -0,0 +1,97 @@
+using System;
+using System.IO;
+using System.Text;
+
+namespace UnityDataTools.Analyzer.Util;
+
+/// <summary>
+/// Utility for detecting YAML-format Unity SerializedFiles.
+///
+/// Unity SerializedFiles can be stored in two formats:
+/// 1. Binary format (used in builds and by Unity Runtime) - detected by SerializedFileDetector
+/// 2. YAML format (text format used in Editor for .asset, .prefab, .unity files) - detected by this class
+///
+/// The YAML format is not supported by Unity Runtime and cannot be read by UnityDataTool,
+/// which only supports the binary format.
+///
+/// YAML SerializedFiles begin with the magic string "%YAML 1.1", optionally preceded by
+/// a UTF-8 BOM (byte order mark: 0xEF 0xBB 0xBF).
+///
+/// This implementation is based on Unity's SerializedFile.cpp::FinalizeInitRead() and
+/// IsSerializedFileTextFile() from Runtime/Serialize/SerializedFile.cpp (lines 101-117, 668-687).
+/// </summary>
+public static class YamlSerializedFileDetector
+{
+    // Magic string that identifies YAML-format SerializedFiles
+    private const string UnityTextMagicString = "%YAML 1.1";
+
+    // UTF-8 BOM (Byte Order Mark): EF BB BF
+    private static readonly byte[] Utf8Bom = new byte[] { 0xEF, 0xBB, 0xBF };
+
+    /// <summary>
+    /// Checks if a file is a YAML-format Unity SerializedFile by reading the magic string
+    /// at the beginning of the file.
+    /// </summary>
+    /// <param name="filePath">Path to the file to check</param>
+    /// <returns>True if the file starts with "%YAML 1.1" (optionally with UTF-8 BOM), false otherwise</returns>
+    public static bool IsYamlSerializedFile(string filePath)
+    {
+        if (!File.Exists(filePath))
+            return false;
+
+        try
+        {
+            using var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
+
+            // Unity checks for UTF-8 BOM (3 bytes) + magic string (9 bytes) = 12 bytes total
+            const int bomLength = 3;
+            int magicLength = UnityTextMagicString.Length;
+            int bufferSize = bomLength + magicLength;
+
+            if (stream.Length < magicLength)
+                return false;
+
+            byte[] buffer = new byte[bufferSize];
+            int bytesRead = stream.Read(buffer, 0, Math.Min(bufferSize, (int)stream.Length));
+
+            if (bytesRead < magicLength)
+                return false;
+
+            // Check if file starts with UTF-8 BOM
+            int offset = 0;
+            if (bytesRead >= bomLength && HasUtf8Bom(buffer))
+            {
+                offset = bomLength;
+            }
+
+            // Check for magic string after BOM (if present)
+            if (bytesRead - offset < magicLength)
+                return false;
+
+            string fileStart = Encoding.ASCII.GetString(buffer, offset, magicLength);
+            return fileStart == UnityTextMagicString;
+        }
+        catch
+        {
+            // Any exception during file reading means this isn't a valid YAML file
+            return false;
+        }
+    }
+
+    /// <summary>
+    /// Checks if a byte array starts with UTF-8 BOM (0xEF 0xBB 0xBF).
+    /// </summary>
+    private static bool HasUtf8Bom(byte[] buffer)
+    {
+        if (buffer.Length < Utf8Bom.Length)
+            return false;
+
+        for (int i = 0; i < Utf8Bom.Length; i++)
+        {
+            if (buffer[i] != Utf8Bom[i])
+                return false;
+        }
+
+        return true;
+    }
+}
diff --git a/UnityDataTool.Tests/SerializedFileCommandTests.cs b/UnityDataTool.Tests/SerializedFileCommandTests.cs
@@ -679,6 +679,80 @@ public async Task ErrorHandling_InvalidFile_ShowsHelpfulMessage()
         }
     }
 
+    [Test]
+    public async Task ErrorHandling_YamlFile_ReturnsHelpfulError()
+    {
+        var path = Path.Combine(TestContext.CurrentContext.TestDirectory, "Data", "YamlFormat.asset");
+
+        using var sw = new StringWriter();
+        var currentErr = Console.Error;
+        try
+        {
+            Console.SetError(sw);
+
+            var result = await Program.Main(new string[] { "serialized-file", "header", path });
+
+            Assert.AreNotEqual(0, result, "Should return error code for YAML file");
+
+            var errorOutput = sw.ToString();
+            StringAssert.Contains("YAML-format SerializedFile", errorOutput, "Error message should mention YAML format");
+            StringAssert.Contains("not supported", errorOutput, "Error message should explain YAML is not supported");
+            StringAssert.Contains("binary-format", errorOutput, "Error message should mention binary format is supported");
+        }
+        finally
+        {
+            Console.SetError(currentErr);
+        }
+    }
+
+    [Test]
+    public async Task ErrorHandling_YamlFile_ExternalRefs_ReturnsHelpfulError()
+    {
+        var path = Path.Combine(TestContext.CurrentContext.TestDirectory, "Data", "YamlFormat.asset");
+
+        using var sw = new StringWriter();
+        var currentErr = Console.Error;
+        try
+        {
+            Console.SetError(sw);
+
+            var result = await Program.Main(new string[] { "serialized-file", "externalrefs", path });
+
+            Assert.AreNotEqual(0, result, "Should return error code for YAML file");
+
+            var errorOutput = sw.ToString();
+            StringAssert.Contains("YAML-format SerializedFile", errorOutput, "Error message should mention YAML format");
+        }
+        finally
+        {
+            Console.SetError(currentErr);
+        }
+    }
+
+    [Test]
+    public async Task ErrorHandling_YamlFile_ObjectList_ReturnsHelpfulError()
+    {
+        var path = Path.Combine(TestContext.CurrentContext.TestDirectory, "Data", "YamlFormat.asset");
+
+        using var sw = new StringWriter();
+        var currentErr = Console.Error;
+        try
+        {
+            Console.SetError(sw);
+
+            var result = await Program.Main(new string[] { "sf", "objectlist", path });
+
+            Assert.AreNotEqual(0, result, "Should return error code for YAML file");
+
+            var errorOutput = sw.ToString();
+            StringAssert.Contains("YAML-format SerializedFile", errorOutput, "Error message should mention YAML format");
+        }
+        finally
+        {
+            Console.SetError(currentErr);
+        }
+    }
+
     #endregion
 }
 
diff --git a/UnityDataTool/SerializedFileCommands.cs b/UnityDataTool/SerializedFileCommands.cs
@@ -94,6 +94,19 @@ private static bool ValidateSerializedFile(string filePath, out SerializedFileIn
             return false;
         }
 
+        if (YamlSerializedFileDetector.IsYamlSerializedFile(filePath))
+        {
+            Console.Error.WriteLine($"Error: The file is a YAML-format SerializedFile, which is not supported.");
+            Console.Error.WriteLine($"File: {filePath}");
+            Console.Error.WriteLine();
+            Console.Error.WriteLine("UnityDataTool only supports binary-format SerializedFiles (used in builds).");
+            Console.Error.WriteLine("YAML-format SerializedFiles are used by Unity Editor for .asset, .prefab, and .unity files.");
+            Console.Error.WriteLine();
+            Console.Error.WriteLine("To analyze YAML files, you must first build your project. Then run UnityDataTool");
+            Console.Error.WriteLine("on the build output, which contains binary-format SerializedFiles.");
+            return false;
+        }
+
         if (!SerializedFileDetector.TryDetectSerializedFile(filePath, out fileInfo))
         {
             Console.Error.WriteLine($"Error: The file does not appear to be a valid Unity SerializedFile.");
