Skip to content

Commit ed44c98

Browse files
tyrielvtyrielv
authored
Merge pull request #2003 from tyrielv/tyrielv/prefetch-idx-strategy
Add pack-index object existence checker strategy for prefetch
2 parents 93de0e3 + 13d4935 commit ed44c98

11 files changed

Lines changed: 1438 additions & 4 deletions

‎GVFS/GVFS.Common/GVFSConstants.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,9 @@ public static class GitConfig
4848
public const bool ShowHydrationStatusDefault = false;
4949

5050
public const string MaxHttpConnectionsConfig = GVFSPrefix + "max-http-connections";
51+
52+
public const string PrefetchUseIdx = GVFSPrefix + "prefetch-use-idx";
53+
public const bool PrefetchUseIdxDefault = false;
5154
}
5255

5356
public static class LocalGVFSConfig

‎GVFS/GVFS.Common/Git/IObjectExistenceChecker.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
using System;
2+
3+
namespace GVFS.Common.Git
4+
{
5+
/// <summary>
6+
/// Strategy interface for checking whether git objects exist locally.
7+
/// Implementations must be safe to call from a single worker thread.
8+
/// Thread-safety across multiple workers depends on the implementation.
9+
/// </summary>
10+
public interface IObjectExistenceChecker : IDisposable
11+
{
12+
bool ObjectExists(string sha);
13+
}
14+
}

‎GVFS/GVFS.Common/Git/LibGit2ObjectExistenceChecker.cs‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
using GVFS.Common.Tracing;
2+
3+
namespace GVFS.Common.Git
4+
{
5+
/// <summary>
6+
/// Object existence checker backed by libgit2 — one instance per worker thread.
7+
/// </summary>
8+
public class LibGit2ObjectExistenceChecker : IObjectExistenceChecker
9+
{
10+
private readonly LibGit2Repo repo;
11+
12+
public LibGit2ObjectExistenceChecker(ITracer tracer, string repoPath)
13+
{
14+
this.repo = new LibGit2Repo(tracer, repoPath);
15+
}
16+
17+
public bool ObjectExists(string sha)
18+
{
19+
return this.repo.ObjectExists(sha);
20+
}
21+
22+
public void Dispose()
23+
{
24+
this.repo.Dispose();
25+
}
26+
}
27+
}

‎GVFS/GVFS.Common/Git/MidxReader.cs‎

Lines changed: 283 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,283 @@
1+
using System;
2+
using System.Buffers.Binary;
3+
using System.Collections.Generic;
4+
using System.IO;
5+
using System.IO.MemoryMappedFiles;
6+
using System.Runtime.CompilerServices;
7+
8+
namespace GVFS.Common.Git
9+
{
10+
/// <summary>
11+
/// Reads a git multi-pack-index (MIDX) file and performs binary search
12+
/// lookups against the sorted OID table. Pure managed code, thread-safe.
13+
/// </summary>
14+
public sealed class MidxReader : IDisposable
15+
{
16+
private const uint MidxMagic = 0x4D494458; // "MIDX"
17+
private const uint ChunkIdPNAM = 0x504E414D; // Pack Names
18+
private const uint ChunkIdOIDF = 0x4F494446; // OID Fanout
19+
private const uint ChunkIdOIDL = 0x4F49444C; // OID Lookup
20+
21+
private readonly MemoryMappedFile mmf;
22+
private readonly MemoryMappedViewAccessor accessor;
23+
private int hashLen;
24+
private long fanoutOffset;
25+
private long oidLookupOffset;
26+
private int totalObjects;
27+
private HashSet<string> packStems;
28+
29+
public int TotalObjects => this.totalObjects;
30+
31+
public MidxReader(string path)
32+
{
33+
long fileLength = new FileInfo(path).Length;
34+
this.mmf = MemoryMappedFile.CreateFromFile(path, FileMode.Open, null, 0, MemoryMappedFileAccess.Read);
35+
try
36+
{
37+
this.accessor = this.mmf.CreateViewAccessor(0, fileLength, MemoryMappedFileAccess.Read);
38+
try
39+
{
40+
this.InitializeFromAccessor();
41+
}
42+
catch
43+
{
44+
this.accessor.Dispose();
45+
throw;
46+
}
47+
}
48+
catch
49+
{
50+
this.mmf.Dispose();
51+
throw;
52+
}
53+
}
54+
55+
private void InitializeFromAccessor()
56+
{
57+
// Header: MIDX(4) version(1) oidVersion(1) numChunks(1) reserved(1) numPacks(4)
58+
uint magic = this.ReadUInt32BE(0);
59+
if (magic != MidxMagic)
60+
{
61+
throw new InvalidDataException($"Not a MIDX file (magic=0x{magic:X8})");
62+
}
63+
64+
byte version = this.ReadByte(4);
65+
if (version != 1)
66+
{
67+
throw new InvalidDataException($"Unsupported MIDX version {version}");
68+
}
69+
70+
byte oidVersion = this.ReadByte(5);
71+
this.hashLen = oidVersion == 2 ? 32 : 20;
72+
int numChunks = this.ReadByte(6);
73+
74+
// Parse chunk TOC at offset 12
75+
long tocStart = 12;
76+
long pnamOffset = 0;
77+
long pnamEnd = 0;
78+
this.fanoutOffset = 0;
79+
this.oidLookupOffset = 0;
80+
81+
// Read all chunk entries + terminator to get chunk boundaries
82+
long[] chunkOffsets = new long[numChunks + 1];
83+
uint[] chunkIds = new uint[numChunks];
84+
for (int i = 0; i < numChunks; i++)
85+
{
86+
long entryOff = tocStart + ((long)i * 12);
87+
chunkIds[i] = this.ReadUInt32BE(entryOff);
88+
chunkOffsets[i] = this.ReadInt64BE(entryOff + 4);
89+
}
90+
91+
// Terminator entry
92+
long terminatorOff = tocStart + ((long)numChunks * 12);
93+
chunkOffsets[numChunks] = this.ReadInt64BE(terminatorOff + 4);
94+
95+
for (int i = 0; i < numChunks; i++)
96+
{
97+
switch (chunkIds[i])
98+
{
99+
case ChunkIdPNAM:
100+
pnamOffset = chunkOffsets[i];
101+
pnamEnd = chunkOffsets[i + 1];
102+
break;
103+
case ChunkIdOIDF:
104+
this.fanoutOffset = chunkOffsets[i];
105+
break;
106+
case ChunkIdOIDL:
107+
this.oidLookupOffset = chunkOffsets[i];
108+
break;
109+
}
110+
}
111+
112+
if (this.fanoutOffset == 0 || this.oidLookupOffset == 0)
113+
{
114+
throw new InvalidDataException("MIDX missing required OIDF/OIDL chunks");
115+
}
116+
117+
// Total objects from fanout[255]
118+
this.totalObjects = (int)this.ReadUInt32BE(this.fanoutOffset + (255 * 4));
119+
120+
// Parse pack names from PNAM chunk
121+
this.packStems = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
122+
if (pnamOffset > 0 && pnamEnd > pnamOffset)
123+
{
124+
int pnamLen = (int)(pnamEnd - pnamOffset);
125+
byte[] pnamBuf = new byte[pnamLen];
126+
this.accessor.ReadArray(pnamOffset, pnamBuf, 0, pnamLen);
127+
string pnamStr = System.Text.Encoding.ASCII.GetString(pnamBuf);
128+
foreach (string name in pnamStr.Split('\0', StringSplitOptions.RemoveEmptyEntries))
129+
{
130+
// PNAM stores .idx names; strip extension to get stem
131+
string stem = name;
132+
if (stem.EndsWith(".idx", StringComparison.OrdinalIgnoreCase))
133+
{
134+
stem = stem.Substring(0, stem.Length - 4);
135+
}
136+
137+
this.packStems.Add(stem);
138+
}
139+
}
140+
}
141+
142+
/// <summary>
143+
/// Returns the set of pack file stems (without extension) covered by this MIDX.
144+
/// </summary>
145+
public HashSet<string> GetPackStems()
146+
{
147+
return this.packStems;
148+
}
149+
150+
/// <summary>
151+
/// Check if an object with the given SHA-1 hex string exists in the MIDX.
152+
/// Thread-safe.
153+
/// </summary>
154+
public bool Exists(string shaHex)
155+
{
156+
if (shaHex == null || shaHex.Length < this.hashLen * 2)
157+
{
158+
return false;
159+
}
160+
161+
Span<byte> oid = stackalloc byte[this.hashLen];
162+
HexToBytes(shaHex, oid);
163+
return this.Exists(oid);
164+
}
165+
166+
/// <summary>
167+
/// Check if an object with the given binary OID exists in the MIDX.
168+
/// Thread-safe.
169+
/// </summary>
170+
public bool Exists(ReadOnlySpan<byte> oid)
171+
{
172+
int firstByte = oid[0];
173+
174+
uint lo = firstByte == 0 ? 0 : this.ReadUInt32BE(this.fanoutOffset + ((firstByte - 1) * 4));
175+
uint hi = this.ReadUInt32BE(this.fanoutOffset + (firstByte * 4));
176+
177+
if (lo >= hi)
178+
{
179+
return false;
180+
}
181+
182+
return this.BinarySearchOid(oid, (int)lo, (int)hi - 1);
183+
}
184+
185+
private bool BinarySearchOid(ReadOnlySpan<byte> target, int lo, int hi)
186+
{
187+
while (lo <= hi)
188+
{
189+
int mid = lo + ((hi - lo) / 2);
190+
long offset = this.oidLookupOffset + ((long)mid * this.hashLen);
191+
192+
int cmp = this.CompareOidAtOffset(target, offset);
193+
if (cmp == 0)
194+
{
195+
return true;
196+
}
197+
else if (cmp < 0)
198+
{
199+
hi = mid - 1;
200+
}
201+
else
202+
{
203+
lo = mid + 1;
204+
}
205+
}
206+
207+
return false;
208+
}
209+
210+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
211+
private int CompareOidAtOffset(ReadOnlySpan<byte> target, long fileOffset)
212+
{
213+
for (int i = 0; i < this.hashLen; i++)
214+
{
215+
int diff = target[i] - this.accessor.ReadByte(fileOffset + i);
216+
if (diff != 0)
217+
{
218+
return diff;
219+
}
220+
}
221+
222+
return 0;
223+
}
224+
225+
internal static void HexToBytes(string hex, Span<byte> output)
226+
{
227+
for (int i = 0; i < output.Length; i++)
228+
{
229+
output[i] = (byte)((HexVal(hex[i * 2]) << 4) | HexVal(hex[(i * 2) + 1]));
230+
}
231+
}
232+
233+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
234+
private static int HexVal(char c)
235+
{
236+
if (c >= 'a')
237+
{
238+
return c - 'a' + 10;
239+
}
240+
241+
if (c >= 'A')
242+
{
243+
return c - 'A' + 10;
244+
}
245+
246+
return c - '0';
247+
}
248+
249+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
250+
private byte ReadByte(long offset)
251+
{
252+
return this.accessor.ReadByte(offset);
253+
}
254+
255+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
256+
private uint ReadUInt32BE(long offset)
257+
{
258+
byte b0 = this.accessor.ReadByte(offset);
259+
byte b1 = this.accessor.ReadByte(offset + 1);
260+
byte b2 = this.accessor.ReadByte(offset + 2);
261+
byte b3 = this.accessor.ReadByte(offset + 3);
262+
return ((uint)b0 << 24) | ((uint)b1 << 16) | ((uint)b2 << 8) | b3;
263+
}
264+
265+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
266+
private long ReadInt64BE(long offset)
267+
{
268+
Span<byte> buf = stackalloc byte[8];
269+
for (int i = 0; i < 8; i++)
270+
{
271+
buf[i] = this.accessor.ReadByte(offset + i);
272+
}
273+
274+
return BinaryPrimitives.ReadInt64BigEndian(buf);
275+
}
276+
277+
public void Dispose()
278+
{
279+
this.accessor.Dispose();
280+
this.mmf.Dispose();
281+
}
282+
}
283+
}

0 commit comments

Comments
 (0)