[#70] Fix analyze CRC for cah:/ resources; add independent --skip-crc

Recognize content-addressed stream paths (cah:/<hash>) produced by Unity
6.6 ContentDirectory builds: fold the path (which contains the content
hash) into the CRC instead of opening the differently named resource
file, which was failing with "Error opening resource file". Legacy
CAB-*.resS/.resource streams still read their bytes for the CRC.

Also addresses the performance issues from PR 66 without losing CRC
coverage of external streams:
- Fix UnityFileReader.ComputeCRC chunking (advance the file offset and
  handle the partial final chunk) so ranges larger than the buffer no
  longer produce a wrong CRC or over-read.
- Fix the ProcessManagedReferenceData CRC size argument (stringSize + 4,
  not m_Offset + stringSize + 4).
- Keep journal_mode = MEMORY (drop PR 66's ineffective WAL change).

Add a --skip-crc option, fully independent of --skip-references:
--skip-references now only skips reference extraction and no longer
skips the CRC. The reference walk still resolves referenced object ids
(so the CRC stays stable) but only inserts refs rows when extracting.

Add a ComputeCRC unit test for buffer-boundary ranges and update the
analyze documentation for the new flag semantics.

Fixes #70.

Author: SkowronskiAndrew

Analyzer/AnalyzerTool.cs

@@ -26,6 +26,7 @@ public int Analyze(
         string databaseName,
         string searchPattern,
         bool skipReferences,
+        bool skipCrc,
         bool verbose,
         bool noRecursion)
     {
@@ -40,6 +41,7 @@ public int Analyze(
             {
                 parser.Verbose = verbose;
                 parser.SkipReferences = skipReferences;
+                parser.SkipCrc = skipCrc;
                 parser.Init(writer.Connection);
 
             }

Analyzer/PPtrAndCrcProcessor.cs

@@ -13,12 +13,18 @@ public class PPtrAndCrcProcessor : IDisposable
 {
     public delegate int CallbackDelegate(long objectId, int fileId, long pathId, string propertyPath, string propertyType);
 
+    // Content-addressed stream paths (new ContentDirectory build output) look like
+    // "cah:/<hash>". The hash already identifies the content, so the path itself is
+    // folded into the CRC instead of opening the (differently named) resource file.
+    private const string ContentAddressedPrefix = "cah:/";
+
     private SerializedFile m_SerializedFile;
     private UnityFileReader m_Reader;
     private long m_Offset;
     private long m_ObjectId;
     private uint m_Crc32;
     private string m_Folder;
+    private bool m_SkipCrc;
     private StringBuilder m_StringBuilder = new();
     private byte[] m_pptrBytes = new byte[4];
 
@@ -27,11 +33,12 @@ public class PPtrAndCrcProcessor : IDisposable
     private Dictionary<string, UnityFileReader> m_resourceReaders = new();
 
     public PPtrAndCrcProcessor(SerializedFile serializedFile, UnityFileReader reader, string folder,
-        CallbackDelegate callback)
+        bool skipCrc, CallbackDelegate callback)
     {
         m_SerializedFile = serializedFile;
         m_Reader = reader;
         m_Folder = folder;
+        m_SkipCrc = skipCrc;
         m_Callback = callback;
     }
 
@@ -79,6 +86,32 @@ private UnityFileReader GetResourceReader(string filename)
         return reader;
     }
 
+    // Extends the CRC with a range of the main serialized file, unless CRC is disabled.
+    private void AppendCrc(long offset, int size)
+    {
+        if (!m_SkipCrc)
+            m_Crc32 = m_Reader.ComputeCRC(offset, size, m_Crc32);
+    }
+
+    // Extends the CRC with the content of an external stream segment (StreamingInfo /
+    // StreamedResource), unless CRC is disabled. Content-addressed paths fold in the path
+    // string; other paths read the actual bytes from the companion resource file.
+    private void AppendStreamCrc(long offset, int size, string path)
+    {
+        if (m_SkipCrc)
+            return;
+
+        if (path.StartsWith(ContentAddressedPrefix))
+        {
+            m_Crc32 = Crc32Algorithm.Append(m_Crc32, Encoding.UTF8.GetBytes(path));
+            return;
+        }
+
+        var resourceFile = GetResourceReader(path);
+        if (resourceFile != null)
+            m_Crc32 = resourceFile.ComputeCRC(offset, size, m_Crc32);
+    }
+
     public uint Process(long objectId, long offset, TypeTreeNode node)
     {
         m_Offset = offset;
@@ -99,7 +132,7 @@ private void ProcessNode(TypeTreeNode node, bool isInManagedReferenceRegistry)
     {
         if (node.IsBasicType)
         {
-            m_Crc32 = m_Reader.ComputeCRC(m_Offset, node.Size, m_Crc32);
+            AppendCrc(m_Offset, node.Size);
             m_Offset += node.Size;
         }
         else if (node.IsArray)
@@ -136,12 +169,7 @@ private void ProcessNode(TypeTreeNode node, bool isInManagedReferenceRegistry)
 
             if (size > 0)
             {
-                var resourceFile = GetResourceReader(filename);
-
-                if (resourceFile != null)
-                {
-                    m_Crc32 = resourceFile.ComputeCRC(offset, size, m_Crc32);
-                }
+                AppendStreamCrc(offset, size, filename);
             }
         }
         else if (node.Type == "StreamedResource")
@@ -162,19 +190,14 @@ private void ProcessNode(TypeTreeNode node, bool isInManagedReferenceRegistry)
 
             if (size > 0)
             {
-                var resourceFile = GetResourceReader(filename);
-
-                if (resourceFile != null)
-                {
-                    m_Crc32 = resourceFile.ComputeCRC(offset, size, m_Crc32);
-                }
+                AppendStreamCrc(offset, size, filename);
             }
         }
         else if (node.CSharpType == typeof(string))
         {
             var prevOffset = m_Offset;
             m_Offset += m_Reader.ReadInt32(m_Offset) + 4;
-            m_Crc32 = m_Reader.ComputeCRC(prevOffset, (int)(m_Offset - prevOffset), m_Crc32);
+            AppendCrc(prevOffset, (int)(m_Offset - prevOffset));
         }
         else if (node.IsManagedReferenceRegistry)
         {
@@ -210,12 +233,12 @@ private void ProcessArray(TypeTreeNode node, bool isManagedReferenceRegistry, bo
         if (dataNode.IsBasicType)
         {
             var arraySize = m_Reader.ReadInt32(m_Offset);
-            m_Crc32 = m_Reader.ComputeCRC(m_Offset, dataNode.Size * arraySize + 4, m_Crc32);
+            AppendCrc(m_Offset, dataNode.Size * arraySize + 4);
             m_Offset += dataNode.Size * arraySize + 4;
         }
         else
         {
-            m_Crc32 = m_Reader.ComputeCRC(m_Offset, 4, m_Crc32);
+            AppendCrc(m_Offset, 4);
             var arraySize = m_Reader.ReadInt32(m_Offset);
             m_Offset += 4;
 
@@ -239,7 +262,7 @@ private void ProcessArray(TypeTreeNode node, bool isManagedReferenceRegistry, bo
 
                     // First child is rid.
                     long rid = m_Reader.ReadInt64(m_Offset);
-                    m_Crc32 = m_Reader.ComputeCRC(m_Offset, 8, m_Crc32);
+                    AppendCrc(m_Offset, 8);
                     m_Offset += 8;
 
                     ProcessManagedReferenceData(dataNode.Children[1], dataNode.Children[2], rid);
@@ -255,7 +278,7 @@ private void ProcessManagedReferenceRegistry(TypeTreeNode node)
 
         // First child is version number.
         var version = m_Reader.ReadInt32(m_Offset);
-        m_Crc32 = m_Reader.ComputeCRC(m_Offset, node.Children[0].Size, m_Crc32);
+        AppendCrc(m_Offset, node.Children[0].Size);
         m_Offset += node.Children[0].Size;
 
         if (version == 1)
@@ -301,19 +324,19 @@ bool ProcessManagedReferenceData(TypeTreeNode refTypeNode, TypeTreeNode referenc
             throw new Exception("Invalid ReferencedManagedType");
 
         var stringSize = m_Reader.ReadInt32(m_Offset);
-        m_Crc32 = m_Reader.ComputeCRC(m_Offset, (int)(m_Offset + stringSize + 4), m_Crc32);
+        AppendCrc(m_Offset, stringSize + 4);
         var className = m_Reader.ReadString(m_Offset + 4, stringSize);
         m_Offset += stringSize + 4;
         m_Offset = (m_Offset + 3) & ~(3);
 
         stringSize = m_Reader.ReadInt32(m_Offset);
-        m_Crc32 = m_Reader.ComputeCRC(m_Offset, (int)(m_Offset + stringSize + 4), m_Crc32);
+        AppendCrc(m_Offset, stringSize + 4);
         var namespaceName = m_Reader.ReadString(m_Offset + 4, stringSize);
         m_Offset += stringSize + 4;
         m_Offset = (m_Offset + 3) & ~(3);
 
         stringSize = m_Reader.ReadInt32(m_Offset);
-        m_Crc32 = m_Reader.ComputeCRC(m_Offset, (int)(m_Offset + stringSize + 4), m_Crc32);
+        AppendCrc(m_Offset, stringSize + 4);
         var assemblyName = m_Reader.ReadString(m_Offset + 4, stringSize);
         m_Offset += stringSize + 4;
         m_Offset = (m_Offset + 3) & ~(3);
@@ -347,11 +370,15 @@ private void ExtractPPtr(string referencedType)
         if (fileId != 0 || pathId != 0)
         {
             var refId = m_Callback(m_ObjectId, fileId, pathId, m_StringBuilder.ToString(), referencedType);
-            m_pptrBytes[0] = (byte)(refId >> 24);
-            m_pptrBytes[1] = (byte)(refId >> 16);
-            m_pptrBytes[2] = (byte)(refId >> 8);
-            m_pptrBytes[3] = (byte)(refId);
-            m_Crc32 = Crc32Algorithm.Append(m_Crc32, m_pptrBytes);
+
+            if (!m_SkipCrc)
+            {
+                m_pptrBytes[0] = (byte)(refId >> 24);
+                m_pptrBytes[1] = (byte)(refId >> 16);
+                m_pptrBytes[2] = (byte)(refId >> 8);
+                m_pptrBytes[3] = (byte)(refId);
+                m_Crc32 = Crc32Algorithm.Append(m_Crc32, m_pptrBytes);
+            }
         }
     }
 }

Analyzer/SQLite/Handlers/ISQLiteHandler.cs

@@ -29,4 +29,5 @@ public interface ISQLiteFileParser : IDisposable
     void Parse(string filename);
     public bool Verbose { get; set; }
     public bool SkipReferences { get; set; }
+    public bool SkipCrc { get; set; }
 }

Analyzer/SQLite/Parsers/AddressablesBuildLayoutParser.cs

@@ -15,6 +15,7 @@ public class AddressablesBuildLayoutParser : ISQLiteFileParser
 
         public bool Verbose { get; set; }
         public bool SkipReferences { get; set; }
+        public bool SkipCrc { get; set; }
 
         public void Dispose()
         {

Analyzer/SQLite/Parsers/SerializedFileParser.cs

@@ -15,6 +15,7 @@ public class SerializedFileParser : ISQLiteFileParser
 
         public bool Verbose { get; set; }
         public bool SkipReferences { get; set; }
+        public bool SkipCrc { get; set; }
 
         public bool CanParse(string filename)
         {
@@ -36,7 +37,7 @@ public void Dispose()
 
         public void Init(SqliteConnection db)
         {
-            m_Writer = new SerializedFileSQLiteWriter(db, SkipReferences);
+            m_Writer = new SerializedFileSQLiteWriter(db, SkipReferences, SkipCrc);
         }
 
         public void Parse(string filename)

Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs

@@ -19,6 +19,7 @@ public class SerializedFileSQLiteWriter : IDisposable
     private int m_NextAssetBundleId = 0;
 
     private bool m_SkipReferences;
+    private bool m_SkipCrc;
 
     private IdProvider<string> m_SerializedFileIdProvider = new();
     private ObjectIdProvider m_ObjectIdProvider = new();
@@ -54,11 +55,12 @@ public class SerializedFileSQLiteWriter : IDisposable
     private SqliteConnection m_Database;
     private SqliteCommand m_LastId = new SqliteCommand();
     private SqliteTransaction m_CurrentTransaction = null;
-    public SerializedFileSQLiteWriter(SqliteConnection database, bool skipReferences)
+    public SerializedFileSQLiteWriter(SqliteConnection database, bool skipReferences, bool skipCrc)
     {
         m_Initialized = false;
         m_Database = database;
         m_SkipReferences = skipReferences;
+        m_SkipCrc = skipCrc;
     }
 
     public void Init()
@@ -116,7 +118,7 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con
     {
         using var sf = UnityFileSystem.OpenSerializedFile(fullPath);
         using var reader = new UnityFileReader(fullPath, 64 * 1024 * 1024);
-        using var pptrReader = new PPtrAndCrcProcessor(sf, reader, containingFolder, AddReference);
+        using var pptrReader = new PPtrAndCrcProcessor(sf, reader, containingFolder, m_SkipCrc, AddReference);
         int serializedFileId = m_SerializedFileIdProvider.GetId(Path.GetFileName(fullPath).ToLower());
         int sceneId = -1;
 
@@ -228,7 +230,10 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con
                     m_AddObjectCommand.SetValue("game_object", "");
                 }
 
-                if (!m_SkipReferences)
+                // The walk both extracts references and accumulates the CRC, so it is needed
+                // unless both are disabled. When CRC is on but references are off, the walk
+                // still resolves referenced object ids (AddReference skips the insert).
+                if (!m_SkipReferences || !m_SkipCrc)
                 {
                     crc32 = pptrReader.Process(currentObjectId, offset, root);
                 }
@@ -266,13 +271,20 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con
 
     private int AddReference(long objectId, int fileId, long pathId, string propertyPath, string propertyType)
     {
+        // Always resolve the id so the CRC stays stable; only persist the row when references
+        // are being extracted.
         var referencedObjectId = m_ObjectIdProvider.GetId((m_LocalToDbFileId[fileId], pathId));
-        m_AddReferenceCommand.SetTransaction(m_CurrentTransaction);
-        m_AddReferenceCommand.SetValue("object", objectId);
-        m_AddReferenceCommand.SetValue("referenced_object", referencedObjectId);
-        m_AddReferenceCommand.SetValue("property_path", propertyPath);
-        m_AddReferenceCommand.SetValue("property_type", propertyType);
-        m_AddReferenceCommand.ExecuteNonQuery();
+
+        if (!m_SkipReferences)
+        {
+            m_AddReferenceCommand.SetTransaction(m_CurrentTransaction);
+            m_AddReferenceCommand.SetValue("object", objectId);
+            m_AddReferenceCommand.SetValue("referenced_object", referencedObjectId);
+            m_AddReferenceCommand.SetValue("property_path", propertyPath);
+            m_AddReferenceCommand.SetValue("property_type", propertyType);
+            m_AddReferenceCommand.ExecuteNonQuery();
+        }
+
         return referencedObjectId;
     }
 

Documentation/analyzer.md

@@ -47,7 +47,7 @@ case, Unity will include the asset in all the AssetBundles with a reference to i
 view_potential_duplicates provides the number of instances and the total size of the potentially
 duplicated assets. It also lists all the AssetBundles where the asset was found.
 
-If the skipReferences option is used, there will be a lot of false positives in that view. Otherwise,
+If the `--skip-crc` option is used, there will be a lot of false positives in that view. Otherwise,
 it should be very accurate because CRCs are used to determine if objects are identical.
 
 ## asset_view (AssetBundleProcessor)

Documentation/command-analyze.md

@@ -13,7 +13,8 @@ UnityDataTool analyze <path> [options]
 | `<path>` | Path to folder containing files to analyze | *(required)* |
 | `-o, --output-file <file>` | Output database filename | `database.db` |
 | `-p, --search-pattern <pattern>` | File search pattern (`*` and `?` supported) | `*` |
-| `-s, --skip-references` | Skip CRC and reference extraction (faster, smaller DB) | `false` |
+| `-s, --skip-references` | Do not extract references (smaller DB, no `refs` table). CRC is still computed. | `false` |
+| `--skip-crc` | Skip the CRC32 checksum calculation (faster; `objects.crc32` will be 0) | `false` |
 | `-v, --verbose` | Show more information during analysis | `false` |
 | `--no-recurse` | Do not recurse into sub-directories | `false` |
 | `-d, --typetree-data <file>` | Load an external TypeTree data file before processing (Unity 6.5+) | — |
@@ -30,9 +31,9 @@ Analyze only `.bundle` files and specify a custom database name:
 UnityDataTool analyze /path/to/asset/bundles -o my_database.db -p "*.bundle"
 ```
 
-Fast analysis (skip reference tracking):
+Fastest analysis (skip both reference extraction and CRC):
 ```bash
-UnityDataTool analyze /path/to/bundles -s
+UnityDataTool analyze /path/to/bundles --skip-references --skip-crc
 ```
 
 See also [Analyze Examples](../../Documentation/analyze-examples.md).
@@ -121,23 +122,27 @@ See [Comparing Builds](../../Documentation/comparing-builds.md) for strategies t
 
 ### Slow Analyze times, large output database
 
-Consider using the `--skip-references` argument.
+Two independent flags reduce analyze time and database size:
 
-A real life analyze of a big Addressables build shows how large a difference this can make:
+* `--skip-crc` skips the CRC32 calculation. This is usually the largest time saver, because computing a CRC requires reading the full content of every object, including large texture, mesh and audio data in companion `.resS`/`.resource` files.
+* `--skip-references` skips reference extraction, which is the largest contributor to database size (the `refs` table). The references are not needed for core asset inventory and size information.
 
-* 208 seconds and producted a 500MB database (not specifying --skip-reference)
-* 9 seconds and produced a 68 MB file (with --skip-reference)
+For the fastest, smallest result, combine them.
 
-The references are not needed for core asset inventory and size information. 
+A real life analyze of a big Addressables build, skipping both references and CRC, shows how large a difference this can make:
 
-Note: When specifying `--skip-reference` some functionality is lost:
+* 208 seconds and produced a 500MB database (default)
+* 9 seconds and produced a 68 MB file (with `--skip-references --skip-crc`)
+
+When `--skip-references` is used, some functionality is lost:
 
 * the `find-refs` command will not work
 * `view_material_shader_refs` and `view_material_texture_refs` will be empty
+* `script_object_view` will be empty
 * Queries that look at the relationship between objects will not work.  For example the refs table is required to link between a `MonoBehaviour` and its `MonoScript`.
-* The `objects.crc32` column will be NULL/0 for all objects. This means:
-  * No detection of identical objects by content hash (See [Comparing Builds](../../Documentation/comparing-builds.md))
-  * The `view_potential_duplicates` view relies partially on CRC32 to distinguish true duplicates
 
-Future work: The refs table contains a lot of repeated strings and could be made smaller and more efficient.  It might also be prudent to control the CRC32 calculation using an independent flag.
+When `--skip-crc` is used, the `objects.crc32` column will be 0 for all objects. This means:
+
+* No detection of identical objects by content hash (See [Comparing Builds](../../Documentation/comparing-builds.md))
+* The `view_potential_duplicates` view relies partially on CRC32 to distinguish true duplicates