Skip to content

Commit b448c63

Browse files
UnityZappyUnityZappy
committed
Improve SQLite write speed for large bundles.
- Set DB connection journal mode to use Write-Ahea log to speed up insert transactions. - Fix buffer offset bugs in CRC checksum calculation, use fast CRC fingerprint for external file references.
1 parent 32a31e4 commit b448c63

4 files changed

Lines changed: 35 additions & 72 deletions

File tree

‎Analyzer/PPtrAndCrcProcessor.cs‎

Lines changed: 20 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,15 @@
11
using System;
2-
using System.Collections.Generic;
3-
using System.IO;
42
using System.Text;
53
using Force.Crc32;
64
using UnityDataTools.FileSystem;
75

86
namespace UnityDataTools.Analyzer;
97

10-
// This class is used to extract all the PPtrs in a serialized object. It executes a callback whenever a PPtr is found.
11-
// It provides a string representing the property path of the property (e.g. "m_MyObject.m_MyArray[2].m_PPtrProperty").
8+
/// <summary>
9+
/// Walks serialized object TypeTrees to extract PPtr references and compute a rolling CRC32.
10+
/// External stream segments (StreamingInfo / StreamedResource) extend the CRC using offset, size, and path only,
11+
/// avoiding full reads of large companion .resS data.
12+
/// </summary>
1213
public class PPtrAndCrcProcessor : IDisposable
1314
{
1415
public delegate int CallbackDelegate(long objectId, int fileId, long pathId, string propertyPath, string propertyType);
@@ -18,65 +19,31 @@ public class PPtrAndCrcProcessor : IDisposable
1819
private long m_Offset;
1920
private long m_ObjectId;
2021
private uint m_Crc32;
21-
private string m_Folder;
2222
private StringBuilder m_StringBuilder = new();
2323
private byte[] m_pptrBytes = new byte[4];
2424

2525
private CallbackDelegate m_Callback;
2626

27-
private Dictionary<string, UnityFileReader> m_resourceReaders = new();
28-
29-
public PPtrAndCrcProcessor(SerializedFile serializedFile, UnityFileReader reader, string folder,
30-
CallbackDelegate callback)
27+
public PPtrAndCrcProcessor(SerializedFile serializedFile, UnityFileReader reader, CallbackDelegate callback)
3128
{
3229
m_SerializedFile = serializedFile;
3330
m_Reader = reader;
34-
m_Folder = folder;
3531
m_Callback = callback;
3632
}
3733

3834
public void Dispose()
3935
{
40-
foreach (var r in m_resourceReaders.Values)
41-
{
42-
r?.Dispose();
43-
}
44-
45-
m_resourceReaders.Clear();
4636
}
4737

48-
private UnityFileReader GetResourceReader(string filename)
38+
/// <summary>
39+
/// Extends CRC32 with a stable fingerprint for an external stream segment without reading blob bytes.
40+
/// </summary>
41+
private static uint AppendExternalStreamFingerprint(uint crc32, long offset, int size, string filename)
4942
{
50-
var slashPos = filename.LastIndexOf('/');
51-
if (slashPos > 0)
52-
{
53-
filename = filename.Remove(0, slashPos + 1);
54-
}
55-
56-
if (!m_resourceReaders.TryGetValue(filename, out var reader))
57-
{
58-
try
59-
{
60-
reader = new UnityFileReader("archive:/" + filename, 4 * 1024 * 1024);
61-
}
62-
catch (Exception)
63-
{
64-
try
65-
{
66-
reader = new UnityFileReader(Path.Join(m_Folder, filename), 4 * 1024 * 1024);
67-
}
68-
catch (Exception)
69-
{
70-
Console.Error.WriteLine();
71-
Console.Error.WriteLine($"Error opening resource file {filename}");
72-
reader = null;
73-
}
74-
}
75-
76-
m_resourceReaders[filename] = reader;
77-
}
78-
79-
return reader;
43+
crc32 = Crc32Algorithm.Append(crc32, BitConverter.GetBytes(offset));
44+
crc32 = Crc32Algorithm.Append(crc32, BitConverter.GetBytes(size));
45+
crc32 = Crc32Algorithm.Append(crc32, Encoding.UTF8.GetBytes(filename));
46+
return crc32;
8047
}
8148

8249
public uint Process(long objectId, long offset, TypeTreeNode node)
@@ -123,7 +90,7 @@ private void ProcessNode(TypeTreeNode node, bool isInManagedReferenceRegistry)
12390
if (node.Children.Count != 3)
12491
throw new Exception("Invalid StreamingInfo");
12592

126-
var offset = node.Children[0].Size == 4 ? m_Reader.ReadInt32(m_Offset) : m_Reader.ReadInt64(m_Offset);
93+
var streamOffset = node.Children[0].Size == 4 ? m_Reader.ReadInt32(m_Offset) : m_Reader.ReadInt64(m_Offset);
12794
m_Offset += node.Children[0].Size;
12895

12996
var size = m_Reader.ReadInt32(m_Offset);
@@ -136,12 +103,7 @@ private void ProcessNode(TypeTreeNode node, bool isInManagedReferenceRegistry)
136103

137104
if (size > 0)
138105
{
139-
var resourceFile = GetResourceReader(filename);
140-
141-
if (resourceFile != null)
142-
{
143-
m_Crc32 = resourceFile.ComputeCRC(offset, size, m_Crc32);
144-
}
106+
m_Crc32 = AppendExternalStreamFingerprint(m_Crc32, streamOffset, size, filename);
145107
}
146108
}
147109
else if (node.Type == "StreamedResource")
@@ -162,12 +124,7 @@ private void ProcessNode(TypeTreeNode node, bool isInManagedReferenceRegistry)
162124

163125
if (size > 0)
164126
{
165-
var resourceFile = GetResourceReader(filename);
166-
167-
if (resourceFile != null)
168-
{
169-
m_Crc32 = resourceFile.ComputeCRC(offset, size, m_Crc32);
170-
}
127+
m_Crc32 = AppendExternalStreamFingerprint(m_Crc32, offset, size, filename);
171128
}
172129
}
173130
else if (node.CSharpType == typeof(string))
@@ -301,19 +258,19 @@ bool ProcessManagedReferenceData(TypeTreeNode refTypeNode, TypeTreeNode referenc
301258
throw new Exception("Invalid ReferencedManagedType");
302259

303260
var stringSize = m_Reader.ReadInt32(m_Offset);
304-
m_Crc32 = m_Reader.ComputeCRC(m_Offset, (int)(m_Offset + stringSize + 4), m_Crc32);
261+
m_Crc32 = m_Reader.ComputeCRC(m_Offset, stringSize + 4, m_Crc32);
305262
var className = m_Reader.ReadString(m_Offset + 4, stringSize);
306263
m_Offset += stringSize + 4;
307264
m_Offset = (m_Offset + 3) & ~(3);
308265

309266
stringSize = m_Reader.ReadInt32(m_Offset);
310-
m_Crc32 = m_Reader.ComputeCRC(m_Offset, (int)(m_Offset + stringSize + 4), m_Crc32);
267+
m_Crc32 = m_Reader.ComputeCRC(m_Offset, stringSize + 4, m_Crc32);
311268
var namespaceName = m_Reader.ReadString(m_Offset + 4, stringSize);
312269
m_Offset += stringSize + 4;
313270
m_Offset = (m_Offset + 3) & ~(3);
314271

315272
stringSize = m_Reader.ReadInt32(m_Offset);
316-
m_Crc32 = m_Reader.ComputeCRC(m_Offset, (int)(m_Offset + stringSize + 4), m_Crc32);
273+
m_Crc32 = m_Reader.ComputeCRC(m_Offset, stringSize + 4, m_Crc32);
317274
var assemblyName = m_Reader.ReadString(m_Offset + 4, stringSize);
318275
m_Offset += stringSize + 4;
319276
m_Offset = (m_Offset + 3) & ~(3);

‎Analyzer/SQLite/Writers/SQLiteWriter.cs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,13 +34,16 @@ public void Begin()
3434
try
3535
{
3636
m_Database.Open();
37+
38+
using var walCommand = m_Database.CreateCommand();
39+
walCommand.CommandText = "PRAGMA journal_mode=WAL";
40+
walCommand.ExecuteNonQuery();
3741
}
3842
catch (Exception e)
3943
{
4044
Console.Error.WriteLine($"Error creating database: {e.Message}");
4145
}
4246

43-
// this does all the legacy import of Init.sql
4447
using var command = m_Database.CreateCommand();
4548
command.CommandText = Resources.Init;
4649
command.ExecuteNonQuery();

‎Analyzer/SQLite/Writers/SerializedFileSQLiteWriter.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ public void WriteSerializedFile(string relativePath, string fullPath, string con
116116
{
117117
using var sf = UnityFileSystem.OpenSerializedFile(fullPath);
118118
using var reader = new UnityFileReader(fullPath, 64 * 1024 * 1024);
119-
using var pptrReader = new PPtrAndCrcProcessor(sf, reader, containingFolder, AddReference);
119+
using var pptrReader = new PPtrAndCrcProcessor(sf, reader, AddReference);
120120
int serializedFileId = m_SerializedFileIdProvider.GetId(Path.GetFileName(fullPath).ToLower());
121121
int sceneId = -1;
122122

‎UnityFileSystem/UnityFileReader.cs‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -117,16 +117,19 @@ public byte ReadUInt8(long fileOffset)
117117
return m_Buffer[offset];
118118
}
119119

120+
/// <summary>
121+
/// Computes CRC32 over a contiguous byte range, reading the file in buffer-sized chunks.
122+
/// </summary>
120123
public uint ComputeCRC(long fileOffset, int size, uint crc32 = 0)
121124
{
122-
var readSize = size > m_Buffer.Length ? m_Buffer.Length : size;
123-
var readBytes = 0;
124-
125-
while (readBytes < size)
125+
var remaining = size;
126+
while (remaining > 0)
126127
{
127-
var offset = GetBufferOffset(fileOffset, readSize);
128-
crc32 = Crc32Algorithm.Append(crc32, m_Buffer, offset, readSize);
129-
readBytes += readSize;
128+
var chunk = (int)Math.Min((long)m_Buffer.Length, remaining);
129+
var offset = GetBufferOffset(fileOffset, chunk);
130+
crc32 = Crc32Algorithm.Append(crc32, m_Buffer, offset, chunk);
131+
fileOffset += chunk;
132+
remaining -= chunk;
130133
}
131134

132135
return crc32;

0 commit comments

Comments
 (0)