Skip to content

Commit 3560504

Browse files
critesjoshclaude
critesjosh
and
claude
committed
feat: split sparsePathOverrides into separate clone and parallelize sync
Instead of overlaying docs from the `next` branch into the blobless tag clone of aztec-packages (which triggered slow one-by-one lazy blob fetching or SIGSEGV), create a separate shallow clone (`aztec-packages-docs`) for the docs paths. All independent repos now clone in parallel via Promise.all after the blocking aztec-packages clone completes. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 20e5827 commit 3560504

8 files changed

Lines changed: 109 additions & 142 deletions

File tree

‎src/tools/search.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,12 +73,12 @@ export function searchAztecDocs(options: {
7373
} {
7474
const { query, section, maxResults = 20 } = options;
7575

76-
if (!isRepoCloned("aztec-packages")) {
76+
if (!isRepoCloned("aztec-packages-docs")) {
7777
return {
7878
success: false,
7979
results: [],
8080
message:
81-
"aztec-packages is not cloned. Run aztec_sync_repos first to get documentation.",
81+
"aztec-packages-docs is not cloned. Run aztec_sync_repos first to get documentation.",
8282
};
8383
}
8484

‎src/tools/sync.ts‎

Lines changed: 52 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -46,14 +46,35 @@ export async function syncRepos(options: {
4646
};
4747
}
4848

49-
log?.(`Starting sync: ${reposToSync.length} repos, version=${effectiveVersion}, force=${force}`, "info");
49+
// Generate synthetic repo configs from sparsePathOverrides
50+
const syntheticRepos: RepoConfig[] = [];
51+
for (const repo of reposToSync) {
52+
if (repo.sparsePathOverrides) {
53+
for (const override of repo.sparsePathOverrides) {
54+
syntheticRepos.push({
55+
name: `${repo.name}-docs`,
56+
url: repo.url,
57+
branch: override.branch,
58+
sparse: override.paths,
59+
description: `${repo.description} (docs from ${override.branch})`,
60+
});
61+
}
62+
}
63+
}
64+
65+
// Include synthetic repos in total count
66+
const totalRepos = reposToSync.length + syntheticRepos.length;
67+
log?.(`Starting sync: ${totalRepos} repos, version=${effectiveVersion}, force=${force}`, "info");
5068

5169
const results: SyncResult["repos"] = [];
52-
let syncIndex = 0;
5370

54-
async function syncRepo(config: RepoConfig, statusTransform?: (s: string) => string): Promise<void> {
55-
syncIndex++;
56-
log?.(`Syncing ${syncIndex}/${reposToSync.length}: ${config.name}`, "info");
71+
async function syncRepo(
72+
config: RepoConfig,
73+
index: number,
74+
total: number,
75+
statusTransform?: (s: string) => string,
76+
): Promise<void> {
77+
log?.(`Syncing ${index}/${total}: ${config.name}`, "info");
5778
try {
5879
const status = log ? await cloneRepo(config, force, log) : await cloneRepo(config, force);
5980
results.push({ name: config.name, status: statusTransform ? statusTransform(status) : status });
@@ -66,16 +87,11 @@ export async function syncRepos(options: {
6687
}
6788
}
6889

69-
// Sort repos so aztec-packages is cloned first (needed to determine Noir version)
90+
// Clone aztec-packages first (blocking - needed to determine Noir version)
7091
const aztecPackages = reposToSync.find((r) => r.name === "aztec-packages");
71-
const noirRepos = reposToSync.filter((r) => r.url.includes("noir-lang"));
72-
const otherRepos = reposToSync.filter(
73-
(r) => r.name !== "aztec-packages" && !r.url.includes("noir-lang")
74-
);
75-
76-
// Clone aztec-packages first if present
92+
let nextIndex = 1;
7793
if (aztecPackages) {
78-
await syncRepo(aztecPackages);
94+
await syncRepo(aztecPackages, nextIndex++, totalRepos);
7995
}
8096

8197
// Get the Noir commit from aztec-packages (if available)
@@ -84,24 +100,39 @@ export async function syncRepos(options: {
84100
log?.(`Resolved Noir commit from aztec-packages: ${noirCommit.substring(0, 7)}`, "info");
85101
}
86102

87-
// Clone Noir repos with the commit from aztec-packages
103+
// Build list of all remaining repos to clone in parallel
104+
const parallelBatch: { config: RepoConfig; index: number; statusTransform?: (s: string) => string }[] = [];
105+
106+
const noirRepos = reposToSync.filter((r) => r.url.includes("noir-lang"));
107+
const otherRepos = reposToSync.filter(
108+
(r) => r.name !== "aztec-packages" && !r.url.includes("noir-lang")
109+
);
110+
88111
for (const config of noirRepos) {
89112
const useAztecCommit = config.name === "noir" && noirCommit;
90113
const noirConfig: RepoConfig = useAztecCommit
91114
? { ...config, commit: noirCommit, branch: undefined }
92115
: config;
93-
94-
await syncRepo(
95-
noirConfig,
96-
useAztecCommit ? (s) => s.replace("(commit", "(commit from aztec-packages") : undefined
97-
);
116+
parallelBatch.push({
117+
config: noirConfig,
118+
index: nextIndex++,
119+
statusTransform: useAztecCommit ? (s) => s.replace("(commit", "(commit from aztec-packages") : undefined,
120+
});
98121
}
99122

100-
// Clone other repos
101123
for (const config of otherRepos) {
102-
await syncRepo(config);
124+
parallelBatch.push({ config, index: nextIndex++ });
103125
}
104126

127+
for (const config of syntheticRepos) {
128+
parallelBatch.push({ config, index: nextIndex++ });
129+
}
130+
131+
// Clone all remaining repos in parallel
132+
await Promise.all(
133+
parallelBatch.map((item) => syncRepo(item.config, item.index, totalRepos, item.statusTransform))
134+
);
135+
105136
const allSuccess = results.every(
106137
(r) => !r.status.toLowerCase().includes("error")
107138
);

‎src/utils/git.ts‎

Lines changed: 5 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,12 @@ export async function cloneRepo(
5858
rmSync(repoPath, { recursive: true, force: true });
5959
}
6060

61-
// If already cloned and version matches, just update
61+
// If already cloned and version matches, skip or update
6262
if (isRepoCloned(config.name)) {
63+
if (config.tag || config.commit) {
64+
log?.(`${config.name}: Already cloned at correct ${config.tag ? "tag" : "commit"}, skipping`, "debug");
65+
return `${config.name} already at ${config.commit || config.tag}`;
66+
}
6367
log?.(`${config.name}: Already cloned, updating`, "debug");
6468
return await updateRepo(config.name, log);
6569
}
@@ -112,27 +116,6 @@ export async function cloneRepo(
112116
await repoGit.fetch(["--depth=1", "origin", `refs/tags/${config.tag}:refs/tags/${config.tag}`]);
113117
log?.(`${config.name}: Checking out tag`, "debug");
114118
await repoGit.checkout(config.tag);
115-
116-
// Apply sparse path overrides from different branches
117-
if (config.sparsePathOverrides) {
118-
for (const override of config.sparsePathOverrides) {
119-
log?.(`${config.name}: Fetching override branch ${override.branch}`, "debug");
120-
await repoGit.fetch(["--depth=1", "origin", override.branch]);
121-
try {
122-
log?.(`${config.name}: Checking out override paths from ${override.branch}: ${override.paths.join(", ")}`, "debug");
123-
await repoGit.checkout([`origin/${override.branch}`, "--", ...override.paths]);
124-
} catch (error) {
125-
const repoBase = config.url.replace(/\.git$/, "");
126-
const parentDirs = [...new Set(override.paths.map((p) => p.split("/").slice(0, -1).join("/")))];
127-
const browseLinks = parentDirs.map((d) => `${repoBase}/tree/${override.branch}/${d}`);
128-
log?.(`${config.name}: sparsePathOverrides failed for branch "${override.branch}"`, "error");
129-
throw new Error(
130-
`sparsePathOverrides failed for branch "${override.branch}": could not checkout paths [${override.paths.join(", ")}]. ` +
131-
`Check the actual folder names at: ${browseLinks.join(" , ")}`,
132-
);
133-
}
134-
}
135-
}
136119
} else {
137120
await git.clone(config.url, repoPath, [
138121
"--filter=blob:none",

‎src/utils/search.ts‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -81,19 +81,26 @@ export function searchDocs(
8181
): SearchResult[] {
8282
const { section, maxResults = 30 } = options;
8383

84-
// Determine search path based on section
84+
// Search versioned docs under developer_versioned_docs, narrowing by section if possible
8585
let repo: string | undefined;
86-
if (section) {
87-
const docsPath = join(REPOS_DIR, "aztec-packages", "docs", "docs", section);
88-
if (existsSync(docsPath)) {
89-
// Search within the specific section by using a narrowed path
90-
repo = `aztec-packages/docs/docs/${section}`;
86+
const versionedDocsGlob = join(REPOS_DIR, "aztec-packages-docs", "docs", "developer_versioned_docs");
87+
const versionDirs = existsSync(versionedDocsGlob) ? globbySync("version-*", { cwd: versionedDocsGlob, onlyDirectories: true }) : [];
88+
const versionDir = versionDirs[0];
89+
90+
if (section && versionDir) {
91+
const sectionPath = join(versionedDocsGlob, versionDir, section);
92+
if (existsSync(sectionPath)) {
93+
repo = `aztec-packages-docs/docs/developer_versioned_docs/${versionDir}/${section}`;
9194
}
9295
}
9396

97+
if (!repo && versionDir) {
98+
repo = `aztec-packages-docs/docs/developer_versioned_docs/${versionDir}`;
99+
}
100+
94101
return searchCode(query, {
95102
filePattern: "*.{md,mdx}",
96-
repo: repo || "aztec-packages",
103+
repo: repo || "aztec-packages-docs",
97104
maxResults,
98105
});
99106
}

‎tests/tools/search.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,11 +97,11 @@ describe("searchAztecCode", () => {
9797
});
9898

9999
describe("searchAztecDocs", () => {
100-
it("returns failure when aztec-packages not cloned", () => {
100+
it("returns failure when aztec-packages-docs not cloned", () => {
101101
mockIsRepoCloned.mockReturnValue(false);
102102
const result = searchAztecDocs({ query: "tutorial" });
103103
expect(result.success).toBe(false);
104-
expect(result.message).toContain("aztec-packages is not cloned");
104+
expect(result.message).toContain("aztec-packages-docs is not cloned");
105105
});
106106

107107
it("delegates to searchDocs with correct options", () => {

‎tests/tools/sync.test.ts‎

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,13 @@ vi.mock("../../src/repos/config.js", () => ({
1010
name: "aztec-packages",
1111
url: "https://github.com/AztecProtocol/aztec-packages",
1212
tag: "v1.0.0",
13+
sparse: ["noir-projects/aztec-nr", "yarn-project"],
14+
sparsePathOverrides: [
15+
{
16+
paths: ["docs/developer_versioned_docs/version-v1.0.0", "docs/static/api"],
17+
branch: "next",
18+
},
19+
],
1320
description: "Main repo",
1421
},
1522
{
@@ -42,6 +49,12 @@ vi.mock("../../src/repos/config.js", () => ({
4249
name: "aztec-packages",
4350
url: "https://github.com/AztecProtocol/aztec-packages",
4451
tag: version,
52+
sparsePathOverrides: [
53+
{
54+
paths: ["docs/developer_versioned_docs/version-" + version],
55+
branch: "next",
56+
},
57+
],
4558
description: "Main repo",
4659
},
4760
{
@@ -88,11 +101,13 @@ describe("syncRepos", () => {
88101

89102
await syncRepos({});
90103

91-
// aztec-packages should be first
104+
// aztec-packages should be first (blocking clone)
92105
expect(callOrder[0]).toBe("aztec-packages");
93106
// noir repos should come after aztec-packages
94107
const noirIndex = callOrder.indexOf("noir");
95108
expect(noirIndex).toBeGreaterThan(0);
109+
// synthetic docs repo should be included
110+
expect(callOrder).toContain("aztec-packages-docs");
96111
});
97112

98113
it("extracts noir commit from aztec-packages and applies it", async () => {
@@ -112,8 +127,8 @@ describe("syncRepos", () => {
112127
it("uses AZTEC_REPOS when no version specified", async () => {
113128
await syncRepos({});
114129

115-
// Should clone repos from AZTEC_REPOS (5 repos)
116-
expect(mockCloneRepo).toHaveBeenCalledTimes(5);
130+
// Should clone repos from AZTEC_REPOS (5 repos + 1 synthetic docs repo)
131+
expect(mockCloneRepo).toHaveBeenCalledTimes(6);
117132
expect(mockGetAztecRepos).not.toHaveBeenCalled();
118133
});
119134

@@ -126,8 +141,13 @@ describe("syncRepos", () => {
126141
it("filters to specific repos when repos option provided", async () => {
127142
await syncRepos({ repos: ["aztec-packages"] });
128143

129-
expect(mockCloneRepo).toHaveBeenCalledTimes(1);
144+
// aztec-packages + synthetic aztec-packages-docs
145+
expect(mockCloneRepo).toHaveBeenCalledTimes(2);
130146
expect(mockCloneRepo.mock.calls[0][0].name).toBe("aztec-packages");
147+
const docsCall = mockCloneRepo.mock.calls.find((c: any[]) => c[0].name === "aztec-packages-docs");
148+
expect(docsCall).toBeDefined();
149+
expect(docsCall![0].branch).toBe("next");
150+
expect(docsCall![0].sparse).toEqual(["docs/developer_versioned_docs/version-v1.0.0", "docs/static/api"]);
131151
});
132152

133153
it("returns success:false when no repos match filter", async () => {

‎tests/utils/git.test.ts‎

Lines changed: 5 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -147,80 +147,6 @@ describe("cloneRepo", () => {
147147
expect(mockGitInstance.checkout).toHaveBeenCalledWith("v1.0.0");
148148
});
149149

150-
it("sparse + tag + sparsePathOverrides: fetches override branch and checks out paths", async () => {
151-
const overrideConfig: RepoConfig = {
152-
...sparseConfig,
153-
sparsePathOverrides: [
154-
{
155-
paths: [
156-
"docs/developer_versioned_docs/version-v1.0.0",
157-
"docs/static/api",
158-
],
159-
branch: "next",
160-
},
161-
],
162-
};
163-
mockExistsSync.mockReturnValue(false);
164-
mockGitInstance.clone.mockResolvedValue(undefined);
165-
mockGitInstance.raw.mockResolvedValue(undefined);
166-
mockGitInstance.fetch.mockResolvedValue(undefined);
167-
mockGitInstance.checkout.mockResolvedValue(undefined);
168-
169-
const result = await cloneRepo(overrideConfig);
170-
expect(result).toContain("Cloned aztec-packages");
171-
172-
// Normal tag checkout happens first
173-
expect(mockGitInstance.checkout).toHaveBeenCalledWith("v1.0.0");
174-
175-
// Then override: fetch the branch and checkout paths from it
176-
expect(mockGitInstance.fetch).toHaveBeenCalledWith([
177-
"--depth=1",
178-
"origin",
179-
"next",
180-
]);
181-
expect(mockGitInstance.checkout).toHaveBeenCalledWith([
182-
"origin/next",
183-
"--",
184-
"docs/developer_versioned_docs/version-v1.0.0",
185-
"docs/static/api",
186-
]);
187-
});
188-
189-
it("sparse + tag + sparsePathOverrides: throws descriptive error with GitHub links on failure", async () => {
190-
const overrideConfig: RepoConfig = {
191-
...sparseConfig,
192-
sparsePathOverrides: [
193-
{
194-
paths: [
195-
"docs/developer_versioned_docs/version-v1.0.0",
196-
"docs/static/aztec-nr-api/devnet",
197-
"docs/static/typescript-api/devnet",
198-
],
199-
branch: "next",
200-
},
201-
],
202-
};
203-
mockExistsSync.mockReturnValue(false);
204-
mockGitInstance.clone.mockResolvedValue(undefined);
205-
mockGitInstance.raw.mockResolvedValue(undefined);
206-
mockGitInstance.fetch.mockResolvedValue(undefined);
207-
// Tag checkout succeeds, override checkout fails
208-
mockGitInstance.checkout
209-
.mockResolvedValueOnce(undefined) // tag checkout
210-
.mockRejectedValueOnce(new Error("pathspec did not match"));
211-
212-
try {
213-
await cloneRepo(overrideConfig);
214-
expect.unreachable("should have thrown");
215-
} catch (e: any) {
216-
expect(e.message).toMatch(/sparsePathOverrides failed for branch "next"/);
217-
expect(e.message).toContain("docs/developer_versioned_docs/version-v1.0.0");
218-
expect(e.message).toContain("https://github.com/AztecProtocol/aztec-packages/tree/next/docs/developer_versioned_docs");
219-
expect(e.message).toContain("https://github.com/AztecProtocol/aztec-packages/tree/next/docs/static/aztec-nr-api");
220-
expect(e.message).toContain("https://github.com/AztecProtocol/aztec-packages/tree/next/docs/static/typescript-api");
221-
}
222-
});
223-
224150
it("sparse + commit: clones with sparse flags, fetches commit", async () => {
225151
const commitConfig: RepoConfig = {
226152
...sparseConfig,
@@ -303,18 +229,17 @@ describe("cloneRepo", () => {
303229
);
304230
});
305231

306-
it("already cloned + version match delegates to updateRepo", async () => {
232+
it("already cloned + version match skips update for tag-pinned repos", async () => {
307233
// needsReclone: isRepoCloned returns true, tag matches
308234
mockExistsSync.mockReturnValue(true);
309235
// getRepoTag needs git.raw to return the tag
310236
mockGitInstance.raw.mockResolvedValue("v1.0.0\n");
311-
// isRepoCloned returns true -> delegates to updateRepo
312-
// updateRepo does fetch + reset
313-
mockGitInstance.fetch.mockResolvedValue(undefined);
314-
mockGitInstance.reset.mockResolvedValue(undefined);
315237

316238
const result = await cloneRepo(sparseConfig);
317-
expect(result).toContain("Updated");
239+
expect(result).toContain("already at v1.0.0");
240+
// Should NOT call fetch/reset (updateRepo not invoked)
241+
expect(mockGitInstance.fetch).not.toHaveBeenCalled();
242+
expect(mockGitInstance.reset).not.toHaveBeenCalled();
318243
});
319244
});
320245

0 commit comments

Comments
 (0)