Skip to content

Commit 4429f6d

Browse files
tyrielvtyrielv
committed
Offload prefetch to mount process for warm auth
When a GVFS mount is running, all prefetch operations now offload to the mount process via named pipe IPC, using its already-warm authentication to skip the slow cold-auth path (anonymous HTTP probe + git credential helper invocation). Commits prefetch (--commits): PrefetchCommits IPC message tells the mount to run PrefetchStep with its warm GitObjectsHttpRequestor. A post-fetch callback is injected to avoid re-entrant named pipe IPC when SchedulePostFetchJob would otherwise call back into the same mount. Blob prefetch (--files/--folders): PrefetchBlobs IPC message carries file/folder lists, HEAD commit ID, and hydrate flag. The mount creates a fresh GitObjectsHttpRequestor with warm auth, runs BlobPrefetcher with capped thread counts (ProcessorCount/2), validates inputs, and properly disposes HTTP resources. LastBlobPrefetch.dat is passed through for noop state. Hydration (--hydrate): Two-phase approach: mount downloads blobs (no hydrate), then the verb process hydrates files locally using Parallel.ForEach with ProcessorCount/2 parallelism. This avoids the mount writing to ProjFS-virtualized files (self-callback risk) while ensuring all blobs are cached before hydration starts, minimizing the ProjFS expansion race window. Fallback: If the mount is not running, not ready, or is an older version that does not recognize the new IPC messages, the verb falls back to the existing direct-auth path transparently. Mount-side failures are surfaced directly (no fallback on real errors). Benchmarks on os.2020 (144 files in tools/nmakejs+Razzle+signing): Mounted offload: 158s (warm auth) Unmounted direct: 171s (cold auth) Mounted offload+hydrate: 153s (two-phase) Auth savings: ~13s per prefetch call Assisted-by: Claude Opus 4.6 Signed-off-by: Tyrie Vella <tyrielv@gmail.com>
1 parent 73e8e29 commit 4429f6d

8 files changed

Lines changed: 762 additions & 16 deletions

File tree

‎GVFS/GVFS.Common/GVFSJsonContext.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@ namespace GVFS.Common
3939
[JsonSerializable(typeof(NamedPipeMessages.GetActiveRepoListRequest))]
4040
[JsonSerializable(typeof(NamedPipeMessages.GetActiveRepoListRequest.Response), TypeInfoPropertyName = "GetActiveRepoListResponse")]
4141
[JsonSerializable(typeof(NamedPipeMessages.BaseResponse<string>))]
42+
[JsonSerializable(typeof(NamedPipeMessages.PrefetchCommits.Response), TypeInfoPropertyName = "PrefetchCommitsResponse")]
43+
[JsonSerializable(typeof(NamedPipeMessages.PrefetchBlobs.Request), TypeInfoPropertyName = "PrefetchBlobsRequest")]
44+
[JsonSerializable(typeof(NamedPipeMessages.PrefetchBlobs.Response), TypeInfoPropertyName = "PrefetchBlobsResponse")]
4245
[JsonSerializable(typeof(TelemetryDaemonEventListener.PipeMessage))]
4346
[JsonSerializable(typeof(PrettyConsoleEventListener.ConsoleOutputPayload))]
4447
internal partial class GVFSJsonContext : JsonSerializerContext

‎GVFS/GVFS.Common/Maintenance/PrefetchStep.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,18 @@ public class PrefetchStep : GitMaintenanceStep
1919
private const int NoExistingPrefetchPacks = -1;
2020
private readonly TimeSpan timeBetweenPrefetches = TimeSpan.FromMinutes(70);
2121

22+
private readonly Action<List<string>> postFetchCallback;
23+
2224
public PrefetchStep(GVFSContext context, GitObjects gitObjects, bool requireCacheLock = true)
25+
: this(context, gitObjects, requireCacheLock, postFetchCallback: null)
26+
{
27+
}
28+
29+
public PrefetchStep(GVFSContext context, GitObjects gitObjects, bool requireCacheLock, Action<List<string>> postFetchCallback)
2330
: base(context, requireCacheLock)
2431
{
2532
this.GitObjects = gitObjects;
33+
this.postFetchCallback = postFetchCallback;
2634
}
2735

2836
public override string Area => "PrefetchStep";
@@ -283,6 +291,14 @@ private void SchedulePostFetchJob(List<string> packIndexes)
283291
return;
284292
}
285293

294+
// When running inside the mount process, use the injected callback to
295+
// enqueue the post-fetch step directly (avoids re-entrant named pipe IPC).
296+
if (this.postFetchCallback != null)
297+
{
298+
this.postFetchCallback(packIndexes);
299+
return;
300+
}
301+
286302
// We make a best-effort request to run MIDX and commit-graph writes
287303
using (NamedPipeClient pipeClient = new NamedPipeClient(this.Context.Enlistment.NamedPipeName))
288304
{

‎GVFS/GVFS.Common/NamedPipes/NamedPipeMessages.cs‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -313,6 +313,71 @@ public Message CreateMessage()
313313
}
314314
}
315315

316+
public static class PrefetchCommits
317+
{
318+
public const string Request = "PrefetchCommits";
319+
public const string CompleteResult = "PrefetchCommitsComplete";
320+
public const string MountNotReadyResult = "MountNotReady";
321+
322+
public class Response
323+
{
324+
public bool Success { get; set; }
325+
public string Error { get; set; }
326+
327+
public static Response FromMessage(Message message)
328+
{
329+
return GVFSJsonOptions.Deserialize<Response>(message.Body);
330+
}
331+
332+
public Message CreateMessage()
333+
{
334+
return new Message(CompleteResult, GVFSJsonOptions.Serialize(this));
335+
}
336+
}
337+
}
338+
339+
public static class PrefetchBlobs
340+
{
341+
public const string RequestHeader = "PrefetchBlobs";
342+
public const string CompleteResult = "PrefetchBlobsComplete";
343+
public const string MountNotReadyResult = "MountNotReady";
344+
345+
public class Request
346+
{
347+
public List<string> Files { get; set; }
348+
public List<string> Folders { get; set; }
349+
public string HeadCommitId { get; set; }
350+
351+
public static Request FromMessage(Message message)
352+
{
353+
return GVFSJsonOptions.Deserialize<Request>(message.Body);
354+
}
355+
356+
public Message CreateMessage()
357+
{
358+
return new Message(RequestHeader, GVFSJsonOptions.Serialize(this));
359+
}
360+
}
361+
362+
public class Response
363+
{
364+
public bool Success { get; set; }
365+
public string Error { get; set; }
366+
public int MatchedBlobCount { get; set; }
367+
public int DownloadedBlobCount { get; set; }
368+
369+
public static Response FromMessage(Message message)
370+
{
371+
return GVFSJsonOptions.Deserialize<Response>(message.Body);
372+
}
373+
374+
public Message CreateMessage()
375+
{
376+
return new Message(CompleteResult, GVFSJsonOptions.Serialize(this));
377+
}
378+
}
379+
}
380+
316381
public static class Notification
317382
{
318383
public class Request

‎GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchBlobsOffloadTests.cs‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
using GVFS.FunctionalTests.FileSystemRunners;
2+
using GVFS.FunctionalTests.Should;
3+
using GVFS.FunctionalTests.Tools;
4+
using GVFS.Tests.Should;
5+
using NUnit.Framework;
6+
using System.IO;
7+
8+
namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture
9+
{
10+
[TestFixture]
11+
public class PrefetchBlobsOffloadTests : TestsWithEnlistmentPerFixture
12+
{
13+
private FileSystemRunner fileSystem;
14+
15+
public PrefetchBlobsOffloadTests()
16+
{
17+
this.fileSystem = new SystemIORunner();
18+
}
19+
20+
[TestCase, Order(1)]
21+
public void PrefetchBlobsMountedUsesOffload()
22+
{
23+
// With the enlistment mounted, blob prefetch should succeed
24+
// by offloading to the mount process (using its warm auth).
25+
string output = this.Enlistment.Prefetch($"--files {Path.Combine("GVFS", "GVFS", "Program.cs")}");
26+
output.ShouldContain("Matched blobs:");
27+
output.ShouldContain("Downloaded:");
28+
}
29+
30+
[TestCase, Order(2)]
31+
public void PrefetchBlobsMountedReportsStats()
32+
{
33+
// Prefetch multiple files and verify stats are reported
34+
string output = this.Enlistment.Prefetch(
35+
$"--files {Path.Combine("GVFS", "GVFS", "Program.cs")};{Path.Combine("GVFS", "GVFS.FunctionalTests", "GVFS.FunctionalTests.csproj")}");
36+
output.ShouldContain("Matched blobs:");
37+
output.ShouldContain("Already cached:");
38+
output.ShouldContain("Downloaded:");
39+
}
40+
41+
[TestCase, Order(3)]
42+
public void PrefetchBlobsUnmountedFallsBackToDirectAuth()
43+
{
44+
// Unmount, then blob prefetch should fall back to direct auth
45+
// and still succeed.
46+
this.Enlistment.UnmountGVFS();
47+
48+
try
49+
{
50+
string output = this.Enlistment.Prefetch($"--files {Path.Combine("GVFS", "GVFS", "Program.cs")}");
51+
output.ShouldContain("Matched blobs:");
52+
output.ShouldContain("Downloaded:");
53+
}
54+
finally
55+
{
56+
this.Enlistment.MountGVFS();
57+
}
58+
}
59+
60+
[TestCase, Order(4)]
61+
public void PrefetchBlobsMountedWithFolders()
62+
{
63+
// Prefetch a folder while mounted
64+
string output = this.Enlistment.Prefetch("--folders GVFS/GVFS");
65+
output.ShouldContain("Matched blobs:");
66+
}
67+
68+
[TestCase, Order(5)]
69+
public void PrefetchBlobsMountedAfterRemount()
70+
{
71+
// After unmount + remount, blob prefetch should work via
72+
// the mount process again.
73+
this.Enlistment.UnmountGVFS();
74+
this.Enlistment.MountGVFS();
75+
76+
string output = this.Enlistment.Prefetch($"--files {Path.Combine("GVFS", "GVFS", "Program.cs")}");
77+
output.ShouldContain("Matched blobs:");
78+
}
79+
}
80+
}

‎GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchCommitsOffloadTests.cs‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
using GVFS.FunctionalTests.FileSystemRunners;
2+
using GVFS.FunctionalTests.Should;
3+
using GVFS.FunctionalTests.Tools;
4+
using GVFS.Tests.Should;
5+
using NUnit.Framework;
6+
using System.IO;
7+
8+
namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture
9+
{
10+
[TestFixture]
11+
public class PrefetchCommitsOffloadTests : TestsWithEnlistmentPerFixture
12+
{
13+
private const string PrefetchPackPrefix = "prefetch";
14+
15+
private FileSystemRunner fileSystem;
16+
17+
public PrefetchCommitsOffloadTests()
18+
: base(forcePerRepoObjectCache: true, skipPrefetchDuringClone: true)
19+
{
20+
this.fileSystem = new SystemIORunner();
21+
}
22+
23+
private string PackRoot
24+
{
25+
get
26+
{
27+
return this.Enlistment.GetPackRoot(this.fileSystem);
28+
}
29+
}
30+
31+
[TestCase, Order(1)]
32+
public void PrefetchCommitsMountedUsesOffload()
33+
{
34+
// With the enlistment mounted, prefetch --commits should succeed
35+
// by offloading to the mount process (using its warm auth).
36+
this.Enlistment.Prefetch("--commits");
37+
this.PostFetchJobShouldComplete();
38+
39+
string[] prefetchPacks = this.ReadPrefetchPackFileNames();
40+
prefetchPacks.Length.ShouldBeAtLeast(1, "There should be at least one prefetch pack after mounted prefetch");
41+
this.AllPrefetchPacksShouldHaveIdx(prefetchPacks);
42+
}
43+
44+
[TestCase, Order(2)]
45+
public void PrefetchCommitsMountedIsIdempotent()
46+
{
47+
// Running prefetch --commits again while mounted should succeed
48+
// (may be a no-op if packs are already up to date).
49+
string[] packsBefore = this.ReadPrefetchPackFileNames();
50+
51+
this.Enlistment.Prefetch("--commits");
52+
this.PostFetchJobShouldComplete();
53+
54+
string[] packsAfter = this.ReadPrefetchPackFileNames();
55+
packsAfter.Length.ShouldBeAtLeast(packsBefore.Length, "Pack count should not decrease after idempotent prefetch");
56+
this.AllPrefetchPacksShouldHaveIdx(packsAfter);
57+
}
58+
59+
[TestCase, Order(3)]
60+
public void PrefetchCommitsUnmountedFallsBackToDirectAuth()
61+
{
62+
// Unmount, then prefetch --commits should fall back to direct auth
63+
// and still succeed.
64+
this.Enlistment.UnmountGVFS();
65+
66+
try
67+
{
68+
this.Enlistment.Prefetch("--commits");
69+
70+
string[] prefetchPacks = this.ReadPrefetchPackFileNames();
71+
prefetchPacks.Length.ShouldBeAtLeast(1, "There should be at least one prefetch pack after unmounted prefetch");
72+
this.AllPrefetchPacksShouldHaveIdx(prefetchPacks);
73+
}
74+
finally
75+
{
76+
this.Enlistment.MountGVFS();
77+
}
78+
}
79+
80+
[TestCase, Order(4)]
81+
public void PrefetchCommitsMountedAfterRemount()
82+
{
83+
// After unmount + remount, prefetch --commits should work via
84+
// the mount process again.
85+
this.Enlistment.UnmountGVFS();
86+
this.Enlistment.MountGVFS();
87+
88+
this.Enlistment.Prefetch("--commits");
89+
this.PostFetchJobShouldComplete();
90+
91+
string[] prefetchPacks = this.ReadPrefetchPackFileNames();
92+
prefetchPacks.Length.ShouldBeAtLeast(1, "There should be at least one prefetch pack after remount prefetch");
93+
this.AllPrefetchPacksShouldHaveIdx(prefetchPacks);
94+
}
95+
96+
private string[] ReadPrefetchPackFileNames()
97+
{
98+
return Directory.GetFiles(this.PackRoot, $"{PrefetchPackPrefix}*.pack");
99+
}
100+
101+
private void AllPrefetchPacksShouldHaveIdx(string[] prefetchPacks)
102+
{
103+
foreach (string prefetchPack in prefetchPacks)
104+
{
105+
string idxPath = Path.ChangeExtension(prefetchPack, ".idx");
106+
idxPath.ShouldBeAFile(this.fileSystem);
107+
}
108+
}
109+
110+
private void PostFetchJobShouldComplete()
111+
{
112+
string objectDir = this.Enlistment.GetObjectRoot(this.fileSystem);
113+
string postFetchLock = Path.Combine(objectDir, "git-maintenance-step.lock");
114+
115+
System.Diagnostics.Stopwatch timeout = System.Diagnostics.Stopwatch.StartNew();
116+
while (this.fileSystem.FileExists(postFetchLock))
117+
{
118+
timeout.Elapsed.TotalSeconds.ShouldBeAtMost(60, "Post-fetch lock file was not released within 60 seconds");
119+
System.Threading.Thread.Sleep(500);
120+
}
121+
122+
ProcessResult graphResult = GitProcess.InvokeProcess(
123+
this.Enlistment.RepoRoot,
124+
"commit-graph verify --shallow --object-dir=\"" + objectDir + "\"");
125+
graphResult.ExitCode.ShouldEqual(0);
126+
}
127+
}
128+
}

0 commit comments

Comments
 (0)