Skip to content

Commit c2845df

Browse files
Linwenxuan04Linwenxuan04
committed
[Proto] Fix the varint encoding & decoding for uint32
1 parent 33e4e97 commit c2845df

5 files changed

Lines changed: 223 additions & 11 deletions

File tree

‎Lagrange.Proto.Test/ProtoNodeTest.cs‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,17 @@ public void TestProtoValue_Serialization()
107107

108108
bufferWriter.Dispose();
109109
}
110+
111+
[Test]
112+
public void TestProtoReader_DecodeVarIntUInt32_FiveByteValue()
113+
{
114+
byte[] bytes = [0xBB, 0x8C, 0x85, 0xBF, 0x06, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
115+
var reader = new ProtoReader(bytes);
116+
117+
uint decodedValue = reader.DecodeVarInt<uint>();
118+
119+
Assert.That(decodedValue, Is.EqualTo(1742816827u));
120+
}
110121

111122
#endregion
112123

@@ -598,4 +609,4 @@ public partial class TestNodeHybridClass
598609
[ProtoMember(3)] public ProtoObject? Test3 { get; set; }
599610
}
600611

601-
#endregion
612+
#endregion

‎Lagrange.Proto.Test/ProtoSerializationTest.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ public void TestSerialize_SignedNumbers()
197197
{
198198
SignedInt = -123456,
199199
SignedLong = -9876543210L,
200-
UnsignedInt = 123456,
200+
UnsignedInt = 1742816827,
201201
UnsignedLong = 9876543210L,
202202
Fixed32Value = -42,
203203
Fixed64Value = -999999999999L
@@ -698,4 +698,4 @@ public partial class ComparisionTestClass
698698
[ProtoMember(9)] public long Test9 { get; set; }
699699
}
700700

701-
#endregion
701+
#endregion

‎Lagrange.Proto.Test/ProtoWriterReaderTest.cs‎

Lines changed: 202 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,41 @@ namespace Lagrange.Proto.Test;
1111
[TestFixture]
1212
public class ProtoWriterReaderTest
1313
{
14+
private static readonly ulong[] VarInt64Values =
15+
[
16+
0,
17+
1,
18+
127,
19+
128,
20+
1UL << 28,
21+
1UL << 32,
22+
1UL << 56,
23+
1UL << 63,
24+
0x0102030405060708,
25+
0xFEDCBA9876543210,
26+
ulong.MaxValue
27+
];
28+
29+
private static readonly uint[] VarInt32Values =
30+
[
31+
0,
32+
1,
33+
127,
34+
128,
35+
16_383,
36+
16_384,
37+
2_097_151,
38+
2_097_152,
39+
268_435_455,
40+
268_435_456,
41+
1_742_816_827,
42+
uint.MaxValue
43+
];
44+
45+
private static readonly byte[] VarInt8Values = [0, 1, 127, 128, byte.MaxValue];
46+
47+
private static readonly ushort[] VarInt16Values = [0, 1, 127, 128, 16_383, 16_384, ushort.MaxValue];
48+
1449
#region VarInt Tests
1550

1651
[Test]
@@ -87,6 +122,136 @@ public void TestVarInt_MaxValues()
87122
Assert.That(ulongVal, Is.EqualTo(ulong.MaxValue));
88123
});
89124
}
125+
126+
[Test]
127+
public void TestEncodeVarInt_UInt8AndUInt16MatchReference()
128+
{
129+
foreach (byte value in VarInt8Values)
130+
{
131+
AssertEncodedVarIntMatchesReference(value, value);
132+
}
133+
134+
foreach (ushort value in VarInt16Values)
135+
{
136+
AssertEncodedVarIntMatchesReference(value, value);
137+
}
138+
}
139+
140+
[Test]
141+
public void TestDecodeVarIntUInt8AndUInt16_UnsafePathMatchesReference()
142+
{
143+
foreach (byte value in VarInt8Values)
144+
{
145+
var reader = new ProtoReader(PadForUnsafePath(EncodeVarIntReference(value)));
146+
Assert.That(reader.DecodeVarInt<byte>(), Is.EqualTo(value));
147+
}
148+
149+
foreach (ushort value in VarInt16Values)
150+
{
151+
var reader = new ProtoReader(PadForUnsafePath(EncodeVarIntReference(value)));
152+
Assert.That(reader.DecodeVarInt<ushort>(), Is.EqualTo(value));
153+
}
154+
}
155+
156+
[TestCaseSource(nameof(VarInt32Values))]
157+
public void TestEncodeVarInt_UInt32MatchesReference(uint value)
158+
{
159+
var buffer = new ArrayBufferWriter<byte>();
160+
var writer = new ProtoWriter(buffer);
161+
162+
writer.WriteRawByte(0xAA);
163+
writer.EncodeVarInt(value);
164+
writer.Flush();
165+
166+
byte[] expected = [0xAA, .. EncodeVarIntReference(value)];
167+
Assert.That(buffer.WrittenMemory.ToArray(), Is.EqualTo(expected));
168+
}
169+
170+
[TestCaseSource(nameof(VarInt32Values))]
171+
public void TestDecodeVarIntUInt32_UnsafePathMatchesReference(uint value)
172+
{
173+
byte[] encoded = EncodeVarIntReference(value);
174+
byte[] padded = PadForUnsafePath(encoded);
175+
var reader = new ProtoReader(padded);
176+
177+
uint decoded = reader.DecodeVarInt<uint>();
178+
179+
Assert.That(decoded, Is.EqualTo(value));
180+
}
181+
182+
[TestCaseSource(nameof(VarInt32Values))]
183+
public void TestDecodeVarIntUnsafePair_UInt32MatchesReference(uint value)
184+
{
185+
if (!Ssse3.X64.IsSupported && !AdvSimd.Arm64.IsSupported)
186+
{
187+
Assert.Ignore("SSSE3/NEON is not supported on this platform.");
188+
}
189+
190+
byte[] first = EncodeVarIntReference(value);
191+
byte[] second = EncodeVarIntReference(42);
192+
byte[] padded = new byte[first.Length + second.Length + 16];
193+
first.CopyTo(padded, 0);
194+
second.CopyTo(padded, first.Length);
195+
var reader = new ProtoReader(padded);
196+
197+
var (decodedFirst, decodedSecond) = reader.DecodeVarIntUnsafe<uint, uint>(padded);
198+
199+
Assert.Multiple(() =>
200+
{
201+
Assert.That(decodedFirst, Is.EqualTo(value));
202+
Assert.That(decodedSecond, Is.EqualTo(42u));
203+
});
204+
}
205+
206+
[TestCaseSource(nameof(VarInt64Values))]
207+
public void TestEncodeVarInt_UInt64MatchesReference(ulong value)
208+
{
209+
var buffer = new ArrayBufferWriter<byte>();
210+
var writer = new ProtoWriter(buffer);
211+
212+
writer.WriteRawByte(0xAA);
213+
writer.EncodeVarInt(value);
214+
writer.Flush();
215+
216+
byte[] expected = [0xAA, .. EncodeVarIntReference(value)];
217+
Assert.That(buffer.WrittenMemory.ToArray(), Is.EqualTo(expected));
218+
}
219+
220+
[TestCaseSource(nameof(VarInt64Values))]
221+
public void TestDecodeVarIntUInt64_UnsafePathMatchesReference(ulong value)
222+
{
223+
byte[] encoded = EncodeVarIntReference(value);
224+
byte[] padded = PadForUnsafePath(encoded);
225+
var reader = new ProtoReader(padded);
226+
227+
ulong decoded = reader.DecodeVarInt<ulong>();
228+
229+
Assert.That(decoded, Is.EqualTo(value));
230+
}
231+
232+
[TestCaseSource(nameof(VarInt64Values))]
233+
public void TestDecodeVarIntUnsafePair_UInt64MatchesReference(ulong value)
234+
{
235+
if (!Ssse3.X64.IsSupported && !AdvSimd.Arm64.IsSupported)
236+
{
237+
Assert.Ignore("SSSE3/NEON is not supported on this platform.");
238+
}
239+
240+
byte[] first = EncodeVarIntReference(value);
241+
byte[] second = EncodeVarIntReference(42);
242+
byte[] padded = new byte[first.Length + second.Length + 16];
243+
first.CopyTo(padded, 0);
244+
second.CopyTo(padded, first.Length);
245+
var reader = new ProtoReader(padded);
246+
247+
var (decodedFirst, decodedSecond) = reader.DecodeVarIntUnsafe<ulong, uint>(padded);
248+
249+
Assert.Multiple(() =>
250+
{
251+
Assert.That(decodedFirst, Is.EqualTo(value));
252+
Assert.That(decodedSecond, Is.EqualTo(42u));
253+
});
254+
}
90255

91256
[Test]
92257
public void TestVarInt_NegativeNumbers()
@@ -1002,4 +1167,40 @@ public void TestPerformance_MixedOperations()
10021167
}
10031168

10041169
#endregion
1005-
}
1170+
1171+
private static byte[] EncodeVarIntReference(ulong value)
1172+
{
1173+
var bytes = new List<byte>();
1174+
1175+
do
1176+
{
1177+
byte next = (byte)(value & 0x7F);
1178+
value >>= 7;
1179+
if (value != 0) next |= 0x80;
1180+
bytes.Add(next);
1181+
}
1182+
while (value != 0);
1183+
1184+
return bytes.ToArray();
1185+
}
1186+
1187+
private static byte[] PadForUnsafePath(byte[] encoded)
1188+
{
1189+
byte[] padded = new byte[encoded.Length + 16];
1190+
encoded.CopyTo(padded, 0);
1191+
return padded;
1192+
}
1193+
1194+
private static void AssertEncodedVarIntMatchesReference<T>(T value, ulong expectedValue) where T : unmanaged, System.Numerics.INumber<T>
1195+
{
1196+
var buffer = new ArrayBufferWriter<byte>();
1197+
var writer = new ProtoWriter(buffer);
1198+
1199+
writer.WriteRawByte(0xAA);
1200+
writer.EncodeVarInt(value);
1201+
writer.Flush();
1202+
1203+
byte[] expected = [0xAA, .. EncodeVarIntReference(expectedValue)];
1204+
Assert.That(buffer.WrittenMemory.ToArray(), Is.EqualTo(expected));
1205+
}
1206+
}

‎Lagrange.Proto/Primitives/ProtoReader.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ public unsafe T DecodeVarIntUnsafe<T>() where T : unmanaged, INumber<T>
135135
{
136136
sizeof(byte) => T.CreateTruncating((varintPart & 0x000000000000007f) | ((varintPart & 0x0000000000000100) >> 1)),
137137
sizeof(ushort) => T.CreateTruncating((varintPart & 0x000000000000007f) | ((varintPart & 0x0000000000030000) >> 2) | ((varintPart & 0x0000000000007f00) >> 1)),
138-
sizeof(uint) => T.CreateTruncating((varintPart & 0x000000000000007f) | (varintPart & 0x0000000f00000000 >> 4) | ((varintPart & 0x000000007f000000) >> 3) | ((varintPart & 0x00000000007f0000) >> 2) | ((varintPart & 0x0000000000007f00) >> 1)),
138+
sizeof(uint) => T.CreateTruncating((varintPart & 0x000000000000007f) | ((varintPart & 0x0000000f00000000) >> 4) | ((varintPart & 0x000000007f000000) >> 3) | ((varintPart & 0x00000000007f0000) >> 2) | ((varintPart & 0x0000000000007f00) >> 1)),
139139
_ => throw new NotSupportedException()
140140
};
141141
}
@@ -192,7 +192,7 @@ private static T ExtractFromVector<T>(ulong varintPart0, ulong varintPart1) wher
192192
var e = Sse2.Or(d, Sse2.ShiftRightLogical128BitLane(d, 8));
193193
ulong pt1 = Sse41.X64.Extract(e, 0);
194194

195-
return T.CreateTruncating(pt1 | (varintPart1 & 0x0000000000000100) << 56 | (varintPart1 & 0x000000000000007f) << 56);
195+
return T.CreateTruncating(pt1 | ((varintPart1 & 0x0000000000000100) << 55) | ((varintPart1 & 0x000000000000007f) << 56));
196196
}
197197
else if (AdvSimd.Arm64.IsSupported)
198198
{
@@ -356,7 +356,7 @@ private void SkipVarInt()
356356
ulong b1 = Unsafe.As<byte, ulong>(ref Unsafe.Add(ref _first, _offset + 8));
357357
ulong msbs0 = ~b0 & ~0x7f7f7f7f7f7f7f7ful;
358358
ulong msbs1 = ~b1 & ~0x7f7f7f7f7f7f7f7ful;
359-
_offset += msbs0 == 0 ? (BitOperations.TrailingZeroCount(msbs1) + 1 + 64) >> 3 : BitOperations.TrailingZeroCount(msbs0) + 1 >> 3;
359+
_offset += msbs0 == 0 ? (BitOperations.TrailingZeroCount(msbs1) + 1 + 64) >> 3 : (BitOperations.TrailingZeroCount(msbs0) + 1) >> 3;
360360
}
361361

362362
private void SkipFixed32()
@@ -383,4 +383,4 @@ private void SkipLengthDelimited()
383383

384384
[ExcludeFromCodeCoverage]
385385
private string DebuggerDisplay => $"Offset: {_offset}, Length: {_length}, Hex : {Convert.ToHexString(MemoryMarshal.CreateSpan(ref _first, _length))}";
386-
}
386+
}

‎Lagrange.Proto/Primitives/ProtoWriter.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -277,15 +277,15 @@ private static unsafe Vector128<ulong> PackVector<T>(ulong v) where T : unmanage
277277
);
278278
var d = Sse2.Or(c, Sse2.ShiftRightLogical128BitLane(c, 8));
279279
x = Sse41.X64.Extract(d, 0);
280-
y = (v & 0x7f00000000000000) >> 56 | (v & 0x8000000000000000) >> 55;
280+
y = ((v & 0x7f00000000000000) >> 56) | ((v & 0x8000000000000000) >> 55);
281281
}
282282
else if (AdvSimd.Arm64.IsSupported)
283283
{
284284
var b = Vector128.Create(v, v);
285285
var c = (AdvSimd.ShiftLogical(b & Vector128.Create(0x00000007f0000000ul, 0x000003f800000000ul), Vector128.Create(4L, 5L)) | AdvSimd.ShiftLogical(b & Vector128.Create(0x0001fc0000000000ul, 0x00fe000000000000ul), Vector128.Create(6L, 7L))) | (AdvSimd.ShiftLogical(b & Vector128.Create(0x000000000000007ful, 0x0000000000003f80ul), Vector128.Create(0L, 1L)) | AdvSimd.ShiftLogical(b & Vector128.Create(0x00000000001fc000ul, 0x000000000fe00000ul), Vector128.Create(2L, 3L)));
286286
var d = c | Vector128.Create(c.GetElement(1), 0ul);
287287
x = d.ToScalar();
288-
y = (v & 0x7f00000000000000) >> 56 | (v & 0x8000000000000000) >> 55;
288+
y = ((v & 0x7f00000000000000) >> 56) | ((v & 0x8000000000000000) >> 55);
289289
}
290290
else
291291
{
@@ -468,4 +468,4 @@ public void Dispose()
468468
[ExcludeFromCodeCoverage]
469469
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
470470
private string DebuggerDisplay => $"BytesCommitted = {BytesCommitted} BytesPending = {BytesPending}";
471-
}
471+
}

0 commit comments

Comments
 (0)