Skip to content

Commit e5b8180

Browse files
Linwenxuan04Linwenxuan04
committed
[Proto] Implement SIMD-optimized encoding for two 32-bit varint values and add performance tests
1 parent 273d648 commit e5b8180

5 files changed

Lines changed: 649 additions & 7 deletions

File tree

‎Lagrange.Proto.Runner/Program.cs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,15 @@ internal static partial class Program
66
{
77
private static void Main(string[] args)
88
{
9+
// Run VarInt performance test
10+
VarIntPerformanceTest.Run();
11+
12+
Console.WriteLine("\n=== Original ProtoObject Test ===");
913
var test = new ProtoObject()
1014
{
1115
{ 1, new ProtoObject{ { 1, 2 } } },
1216
{ 1, new ProtoObject{ { 1, 2 } } },
13-
{ 3, 4 },
17+
{ 3, 4 },
1418
{ 5, 6 }
1519
};
1620

@@ -19,5 +23,6 @@ private static void Main(string[] args)
1923
var parsed = ProtoObject.Parse(bytes);
2024

2125
int value = parsed[1][0][1].GetValue<int>();
26+
Console.WriteLine($"Parsed value: {value}");
2227
}
2328
}

‎Lagrange.Proto.Runner/VarIntPerformanceTest.cs‎

Lines changed: 249 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,249 @@
1+
using System.Buffers;
2+
using System.Diagnostics;
3+
using Lagrange.Proto.Primitives;
4+
5+
namespace Lagrange.Proto.Runner;
6+
7+
public static class VarIntPerformanceTest
8+
{
9+
public static void Run()
10+
{
11+
const int count = 1_000_000;
12+
var random = new Random(42);
13+
14+
// Generate test data with various sizes
15+
var values1 = new uint[count];
16+
var values2 = new uint[count];
17+
18+
for (int i = 0; i < count; i++)
19+
{
20+
// Mix of different sized values to test various varint lengths
21+
values1[i] = (uint)(i % 5) switch
22+
{
23+
0 => (uint)random.Next(0, 128), // 1 byte varint
24+
1 => (uint)random.Next(128, 16384), // 2 byte varint
25+
2 => (uint)random.Next(16384, 2097152), // 3 byte varint
26+
3 => (uint)random.Next(2097152, 268435456), // 4 byte varint
27+
_ => (uint)random.Next(268435456, int.MaxValue) // 5 byte varint
28+
};
29+
30+
values2[i] = (uint)((i + 2) % 5) switch
31+
{
32+
0 => (uint)random.Next(0, 128),
33+
1 => (uint)random.Next(128, 16384),
34+
2 => (uint)random.Next(16384, 2097152),
35+
3 => (uint)random.Next(2097152, 268435456),
36+
_ => (uint)random.Next(268435456, int.MaxValue)
37+
};
38+
}
39+
40+
// Validation test - ensure both methods produce identical output
41+
Console.WriteLine("=== Validation Test ===");
42+
Console.WriteLine("Verifying that Sequential and SIMD encoding produce identical output...");
43+
44+
bool validationPassed = ValidateEncodingEquality(values1, values2, Math.Min(10000, count));
45+
46+
if (validationPassed)
47+
{
48+
Console.WriteLine("✓ Validation PASSED: Both methods produce identical output");
49+
}
50+
else
51+
{
52+
Console.WriteLine("✗ Validation FAILED: Methods produce different output!");
53+
return; // Don't continue with performance tests if validation fails
54+
}
55+
56+
// Warm-up
57+
Console.WriteLine("\nWarming up...");
58+
RunSequentialTest(values1, values2, 1000);
59+
RunSimdTest(values1, values2, 1000);
60+
61+
Console.WriteLine("\n=== VarInt Performance Test ===");
62+
Console.WriteLine($"Encoding {count:N0} pairs of uint32 values ({count * 2:N0} total values)\n");
63+
64+
// Test 1: Sequential encoding
65+
Console.WriteLine("Test 1: Sequential EncodeVarInt (2x per iteration)");
66+
var sequentialTime = RunSequentialTest(values1, values2, count);
67+
Console.WriteLine($" Time: {sequentialTime.TotalMilliseconds:F2} ms");
68+
Console.WriteLine($" Throughput: {count * 2 / sequentialTime.TotalSeconds:F0} values/sec");
69+
Console.WriteLine($" Per value: {sequentialTime.TotalNanoseconds / (count * 2):F1} ns");
70+
71+
// Test 2: SIMD encoding
72+
Console.WriteLine("\nTest 2: EncodeTwo32VarIntUnsafe (SIMD)");
73+
var simdTime = RunSimdTest(values1, values2, count);
74+
Console.WriteLine($" Time: {simdTime.TotalMilliseconds:F2} ms");
75+
Console.WriteLine($" Throughput: {count * 2 / simdTime.TotalSeconds:F0} values/sec");
76+
Console.WriteLine($" Per value: {simdTime.TotalNanoseconds / (count * 2):F1} ns");
77+
78+
// Calculate speedup
79+
var speedup = sequentialTime.TotalMilliseconds / simdTime.TotalMilliseconds;
80+
Console.WriteLine($"\nSpeedup: {speedup:F2}x");
81+
Console.WriteLine($"Time saved: {(sequentialTime - simdTime).TotalMilliseconds:F2} ms ({(1 - 1/speedup) * 100:F1}%)");
82+
83+
// Test 3: Mixed strategy (small values sequential, large SIMD)
84+
Console.WriteLine("\nTest 3: Mixed Strategy (small sequential, large SIMD)");
85+
var mixedTime = RunMixedTest(values1, values2, count);
86+
Console.WriteLine($" Time: {mixedTime.TotalMilliseconds:F2} ms");
87+
Console.WriteLine($" Throughput: {count * 2 / mixedTime.TotalSeconds:F0} values/sec");
88+
Console.WriteLine($" Per value: {mixedTime.TotalNanoseconds / (count * 2):F1} ns");
89+
Console.WriteLine($" Speedup vs Sequential: {sequentialTime.TotalMilliseconds / mixedTime.TotalMilliseconds:F2}x");
90+
}
91+
92+
private static TimeSpan RunSequentialTest(uint[] values1, uint[] values2, int count)
93+
{
94+
var buffer = new ArrayBufferWriter<byte>(count * 10);
95+
var writer = new ProtoWriter(buffer);
96+
97+
var stopwatch = Stopwatch.StartNew();
98+
99+
for (int i = 0; i < count; i++)
100+
{
101+
writer.EncodeVarInt(values1[i]);
102+
writer.EncodeVarInt(values2[i]);
103+
}
104+
writer.Flush();
105+
106+
stopwatch.Stop();
107+
return stopwatch.Elapsed;
108+
}
109+
110+
private static TimeSpan RunSimdTest(uint[] values1, uint[] values2, int count)
111+
{
112+
var buffer = new ArrayBufferWriter<byte>(count * 10);
113+
var writer = new ProtoWriter(buffer);
114+
115+
var stopwatch = Stopwatch.StartNew();
116+
117+
for (int i = 0; i < count; i++)
118+
{
119+
writer.EncodeTwo32VarIntUnsafe(values1[i], values2[i]);
120+
}
121+
writer.Flush();
122+
123+
stopwatch.Stop();
124+
return stopwatch.Elapsed;
125+
}
126+
127+
private static TimeSpan RunMixedTest(uint[] values1, uint[] values2, int count)
128+
{
129+
var buffer = new ArrayBufferWriter<byte>(count * 10);
130+
var writer = new ProtoWriter(buffer);
131+
132+
var stopwatch = Stopwatch.StartNew();
133+
134+
for (int i = 0; i < count; i++)
135+
{
136+
uint v1 = values1[i];
137+
uint v2 = values2[i];
138+
139+
// Use sequential for small values, SIMD for larger
140+
if (v1 < 128 && v2 < 128)
141+
{
142+
writer.EncodeVarInt(v1);
143+
writer.EncodeVarInt(v2);
144+
}
145+
else
146+
{
147+
writer.EncodeTwo32VarIntUnsafe(v1, v2);
148+
}
149+
}
150+
writer.Flush();
151+
152+
stopwatch.Stop();
153+
return stopwatch.Elapsed;
154+
}
155+
156+
private static bool ValidateEncodingEquality(uint[] values1, uint[] values2, int count)
157+
{
158+
// Encode using sequential method
159+
var sequentialBuffer = new ArrayBufferWriter<byte>(count * 10);
160+
var sequentialWriter = new ProtoWriter(sequentialBuffer);
161+
162+
for (int i = 0; i < count; i++)
163+
{
164+
sequentialWriter.EncodeVarInt(values1[i]);
165+
sequentialWriter.EncodeVarInt(values2[i]);
166+
}
167+
sequentialWriter.Flush();
168+
169+
var sequentialBytes = sequentialBuffer.WrittenMemory.ToArray();
170+
171+
// Encode using SIMD method
172+
var simdBuffer = new ArrayBufferWriter<byte>(count * 10);
173+
var simdWriter = new ProtoWriter(simdBuffer);
174+
175+
for (int i = 0; i < count; i++)
176+
{
177+
simdWriter.EncodeTwo32VarIntUnsafe(values1[i], values2[i]);
178+
}
179+
simdWriter.Flush();
180+
181+
var simdBytes = simdBuffer.WrittenMemory.ToArray();
182+
183+
// Compare byte arrays
184+
if (sequentialBytes.Length != simdBytes.Length)
185+
{
186+
Console.WriteLine($" Length mismatch: Sequential={sequentialBytes.Length}, SIMD={simdBytes.Length}");
187+
return false;
188+
}
189+
190+
for (int i = 0; i < sequentialBytes.Length; i++)
191+
{
192+
if (sequentialBytes[i] != simdBytes[i])
193+
{
194+
Console.WriteLine($" First difference at byte {i} of {sequentialBytes.Length}:");
195+
Console.WriteLine($" Sequential[{i}]: 0x{sequentialBytes[i]:X2}");
196+
Console.WriteLine($" SIMD[{i}]: 0x{simdBytes[i]:X2}");
197+
198+
// Show surrounding bytes for context
199+
int start = Math.Max(0, i - 5);
200+
int end = Math.Min(sequentialBytes.Length, i + 6);
201+
Console.WriteLine($" Context (bytes {start}-{end - 1}):");
202+
Console.WriteLine($" Sequential: {BitConverter.ToString(sequentialBytes, start, end - start)}");
203+
Console.WriteLine($" SIMD: {BitConverter.ToString(simdBytes, start, end - start)}");
204+
205+
// Try to decode and find which pair it might be
206+
var reader = new ProtoReader(sequentialBytes.AsSpan());
207+
int pairIndex = 0;
208+
int currentPos = 0;
209+
210+
try
211+
{
212+
while (!reader.IsCompleted && currentPos < i)
213+
{
214+
var v1 = reader.DecodeVarInt<uint>();
215+
var v2 = reader.DecodeVarInt<uint>();
216+
217+
// Estimate position based on varint sizes
218+
currentPos += GetVarIntSize(v1) + GetVarIntSize(v2);
219+
220+
if (currentPos > i)
221+
{
222+
Console.WriteLine($" Likely pair index: {pairIndex} (values: {values1[pairIndex]}, {values2[pairIndex]})");
223+
break;
224+
}
225+
pairIndex++;
226+
}
227+
}
228+
catch
229+
{
230+
// If we can't decode, just show the byte difference
231+
}
232+
233+
return false;
234+
}
235+
}
236+
237+
Console.WriteLine($" Validated {count} pairs ({count * 2} values, {sequentialBytes.Length} bytes)");
238+
return true;
239+
}
240+
241+
private static int GetVarIntSize(uint value)
242+
{
243+
if (value < 128) return 1;
244+
if (value < 16384) return 2;
245+
if (value < 2097152) return 3;
246+
if (value < 268435456) return 4;
247+
return 5;
248+
}
249+
}

0 commit comments

Comments
 (0)