Improve error reporting from serialized-file command

Use detection helpers to print more informative messages when command is run against the wrong file type.

Author: SkowronskiAndrew

UnityDataTool.Tests/SerializedFileCommandTests.cs

@@ -495,6 +495,73 @@ public async Task ErrorHandling_NonExistentFile_ReturnsError()
         Assert.AreNotEqual(0, result, "Should return error code for non-existent file");
     }
 
+    [Test]
+    public async Task ErrorHandling_ArchiveFile_ReturnsHelpfulError()
+    {
+        // Use an AssetBundle from test data
+        var assetBundlesDir = Path.Combine(TestContext.CurrentContext.TestDirectory, "Data", "AssetBundles", "2022.1.20f1");
+
+        // Skip if the test data doesn't exist (CI environments might not have all test data)
+        if (!Directory.Exists(assetBundlesDir))
+        {
+            Assert.Ignore("AssetBundle test data not found");
+            return;
+        }
+
+        var archiveFiles = Directory.GetFiles(assetBundlesDir, "*", SearchOption.TopDirectoryOnly);
+        if (archiveFiles.Length == 0)
+        {
+            Assert.Ignore("No AssetBundle test files found");
+            return;
+        }
+
+        var archivePath = archiveFiles[0]; // Use first archive file found
+
+        using var sw = new StringWriter();
+        var currentErr = Console.Error;
+        try
+        {
+            Console.SetError(sw);
+
+            var result = await Program.Main(new string[] { "serialized-file", "objectlist", archivePath });
+
+            Assert.AreNotEqual(0, result, "Should return error code for archive file");
+
+            var errorOutput = sw.ToString();
+            StringAssert.Contains("Unity Archive", errorOutput, "Error message should mention Unity Archive");
+            StringAssert.Contains("archive extract", errorOutput, "Error message should suggest using archive extract command");
+        }
+        finally
+        {
+            Console.SetError(currentErr);
+        }
+    }
+
+    [Test]
+    public async Task ErrorHandling_InvalidFile_ShowsHelpfulMessage()
+    {
+        var path = Path.Combine(m_TestDataFolder, "README.md");
+
+        using var sw = new StringWriter();
+        var currentErr = Console.Error;
+        try
+        {
+            Console.SetError(sw);
+
+            var result = await Program.Main(new string[] { "serialized-file", "objectlist", path });
+
+            Assert.AreNotEqual(0, result, "Should return error code for invalid file");
+
+            var errorOutput = sw.ToString();
+            StringAssert.Contains("not appear to be a valid Unity SerializedFile", errorOutput,
+                "Error message should explain that the file is not a valid SerializedFile");
+        }
+        finally
+        {
+            Console.SetError(currentErr);
+        }
+    }
+
     #endregion
 }
 

UnityDataTool/SerializedFileCommands.cs

@@ -1,6 +1,7 @@
 using System;
 using System.IO;
 using System.Text.Json;
+using UnityDataTools.Analyzer.Util;
 using UnityDataTools.FileSystem;
 
 namespace UnityDataTools.UnityDataTool;
@@ -9,44 +10,77 @@ public static class SerializedFileCommands
 {
     public static int HandleExternalRefs(FileInfo filename, OutputFormat format)
     {
-        try
+        return OpenSerializedFileAndProcess(filename, sf =>
         {
-            using var sf = UnityFileSystem.OpenSerializedFile(filename.FullName);
-
             if (format == OutputFormat.Json)
                 OutputExternalRefsJson(sf);
             else
                 OutputExternalRefsText(sf);
-        }
-        catch (Exception err) when (err is NotSupportedException || err is FileFormatException)
-        {
-            Console.Error.WriteLine($"Error opening serialized file: {filename.FullName}");
-            Console.Error.WriteLine(err.Message);
-            return 1;
-        }
-
-        return 0;
+        });
     }
 
     public static int HandleObjectList(FileInfo filename, OutputFormat format)
     {
-        try
+        return OpenSerializedFileAndProcess(filename, sf =>
         {
-            using var sf = UnityFileSystem.OpenSerializedFile(filename.FullName);
-
             if (format == OutputFormat.Json)
                 OutputObjectListJson(sf);
             else
                 OutputObjectListText(sf);
+        });
+    }
+
+    /// <summary>
+    /// Opens a SerializedFile and processes it with the given action.
+    /// Provides helpful error messages if the file is not a valid SerializedFile.
+    /// </summary>
+    private static int OpenSerializedFileAndProcess(FileInfo filename, Action<SerializedFile> processAction)
+    {
+        string filePath = filename.FullName;
+
+        // Check if file exists (should already be validated by System.CommandLine, but be defensive)
+        if (!File.Exists(filePath))
+        {
+            Console.Error.WriteLine($"Error: File not found: {filePath}");
+            return 1;
+        }
+
+        // Common misuse would be to try to use the command with an AssetBundle
+        // so give an instructive error message for that case.
+        if (ArchiveDetector.IsUnityArchive(filePath))
+        {
+            Console.Error.WriteLine($"Error: The file is an AssetBundle or other Unity Archive, not a SerializedFile.");
+            Console.Error.WriteLine($"File: {filePath}");
+            Console.Error.WriteLine();
+            Console.Error.WriteLine("Unity Archives can contain SerializedFiles inside them.");
+            Console.Error.WriteLine("To access the SerializedFiles, first extract the archive using:");
+            Console.Error.WriteLine($"  UnityDataTool archive extract \"{filePath}\" -o <output-directory>");
+            Console.Error.WriteLine();
+            Console.Error.WriteLine("Then you can run serialized-file commands on the extracted files.");
+            return 1;
+        }
+
+        bool isSerializedFile = SerializedFileDetector.TryDetectSerializedFile(filePath, out var _);
+        if (!isSerializedFile)
+        {
+            Console.Error.WriteLine($"Error: The file does not appear to be a valid Unity SerializedFile.");
+            Console.Error.WriteLine($"File: {filePath}");
+            return 1;
+        }
+
+        // Try to open and process the SerializedFile
+        try
+        {
+            using var sf = UnityFileSystem.OpenSerializedFile(filePath);
+            processAction(sf);
+            return 0;
         }
         catch (Exception err) when (err is NotSupportedException || err is FileFormatException)
         {
-            Console.Error.WriteLine($"Error opening serialized file: {filename.FullName}");
+            Console.Error.WriteLine($"Error opening SerializedFile: {filePath}");
             Console.Error.WriteLine(err.Message);
             return 1;
         }
-
-        return 0;
     }
 
     private static void OutputExternalRefsText(SerializedFile sf)