Skip to content

Commit 34e91c9

Browse files
fboschfbosch
committed
feat: implement unwrapSingleRootDir
1 parent a8f81f2 commit 34e91c9

5 files changed

Lines changed: 164 additions & 24 deletions

File tree

‎src/git/fetch-source.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,12 @@ type FetchParams = {
131131
timeoutMs?: number;
132132
};
133133

134+
type FetchResult = {
135+
repoDir: string;
136+
cleanup: () => Promise<void>;
137+
fromCache: boolean;
138+
};
139+
134140
const runGitArchive = async (
135141
repo: string,
136142
resolvedCommit: string,
@@ -344,7 +350,9 @@ const archiveRepo = async (params: FetchParams) => {
344350
}
345351
};
346352

347-
export const fetchSource = async (params: FetchParams) => {
353+
export const fetchSource = async (
354+
params: FetchParams,
355+
): Promise<FetchResult> => {
348356
assertSafeSourceId(params.sourceId, "sourceId");
349357
try {
350358
const archiveDir = await archiveRepo(params);
@@ -353,6 +361,7 @@ export const fetchSource = async (params: FetchParams) => {
353361
cleanup: async () => {
354362
await removeDir(archiveDir);
355363
},
364+
fromCache: false,
356365
};
357366
} catch {
358367
const tempDir = await mkdtemp(
@@ -365,6 +374,7 @@ export const fetchSource = async (params: FetchParams) => {
365374
cleanup: async () => {
366375
await removeDir(tempDir);
367376
},
377+
fromCache: true,
368378
};
369379
} catch (error) {
370380
await removeDir(tempDir);

‎src/materialize.ts‎

Lines changed: 55 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ type MaterializeParams = {
2727
exclude?: string[];
2828
maxBytes: number;
2929
maxFiles?: number;
30+
unwrapSingleRootDir?: boolean;
3031
};
3132

3233
type ManifestStats = {
@@ -72,6 +73,44 @@ const openFileNoFollow = async (filePath: string) => {
7273
}
7374
};
7475

76+
const resolveUnwrapPrefix = (
77+
entries: Array<{ normalized: string }>,
78+
unwrapSingleRootDir?: boolean,
79+
) => {
80+
if (!unwrapSingleRootDir || entries.length === 0) {
81+
return null;
82+
}
83+
let prefix = "";
84+
while (true) {
85+
let rootDir: string | null = null;
86+
for (const entry of entries) {
87+
const remaining = prefix
88+
? entry.normalized.replace(prefix, "")
89+
: entry.normalized;
90+
const parts = remaining.split("/");
91+
if (parts.length < 2) {
92+
return prefix || null;
93+
}
94+
const nextRoot = parts[0];
95+
if (!rootDir) {
96+
rootDir = nextRoot;
97+
continue;
98+
}
99+
if (rootDir !== nextRoot) {
100+
return prefix || null;
101+
}
102+
}
103+
if (!rootDir) {
104+
return prefix || null;
105+
}
106+
const nextPrefix = `${prefix}${rootDir}/`;
107+
if (nextPrefix === prefix) {
108+
return prefix || null;
109+
}
110+
prefix = nextPrefix;
111+
}
112+
};
113+
75114
const acquireLock = async (lockPath: string, timeoutMs = 5000) => {
76115
const start = Date.now();
77116
while (Date.now() - start < timeoutMs) {
@@ -139,9 +178,16 @@ export const materializeSource = async (params: MaterializeParams) => {
139178
normalized: normalizePath(relativePath),
140179
}))
141180
.sort((left, right) => left.normalized.localeCompare(right.normalized));
181+
const unwrapPrefix = resolveUnwrapPrefix(
182+
entries,
183+
params.unwrapSingleRootDir,
184+
);
142185
const targetDirs = new Set<string>();
143-
for (const { relativePath } of entries) {
144-
targetDirs.add(path.dirname(relativePath));
186+
for (const { relativePath, normalized } of entries) {
187+
const rootPath = unwrapPrefix
188+
? normalized.replace(unwrapPrefix, "")
189+
: relativePath;
190+
targetDirs.add(path.dirname(rootPath));
145191
}
146192
await Promise.all(
147193
Array.from(targetDirs, (dir) =>
@@ -197,7 +243,10 @@ export const materializeSource = async (params: MaterializeParams) => {
197243
if (!stats.isFile()) {
198244
return null;
199245
}
200-
const targetPath = path.join(tempDir, entry.relativePath);
246+
const normalizedPath = unwrapPrefix
247+
? entry.normalized.replace(unwrapPrefix, "")
248+
: entry.relativePath;
249+
const targetPath = path.join(tempDir, normalizedPath);
201250
ensureSafePath(tempDir, targetPath);
202251
if (stats.size >= STREAM_COPY_THRESHOLD_BYTES) {
203252
const reader = createReadStream(filePath, {
@@ -211,7 +260,9 @@ export const materializeSource = async (params: MaterializeParams) => {
211260
await writeFile(targetPath, data);
212261
}
213262
return {
214-
path: entry.normalized,
263+
path: unwrapPrefix
264+
? entry.normalized.replace(unwrapPrefix, "")
265+
: entry.normalized,
215266
size: stats.size,
216267
};
217268
} finally {

‎src/sync.ts‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -368,9 +368,6 @@ export const runSync = async (options: SyncOptions, deps: SyncDeps = {}) => {
368368
index += 1;
369369
const { result, source } = job;
370370
const lockEntry = plan.lockData?.sources?.[source.id];
371-
if (!options.json) {
372-
ui.step("Fetching", source.id);
373-
}
374371
const fetch = await runFetch({
375372
sourceId: source.id,
376373
repo: source.repo,
@@ -380,6 +377,12 @@ export const runSync = async (options: SyncOptions, deps: SyncDeps = {}) => {
380377
include: source.include ?? defaults.include,
381378
timeoutMs: options.timeoutMs,
382379
});
380+
if (!options.json) {
381+
ui.step(
382+
fetch.fromCache ? "Restoring from cache" : "Downloading repo",
383+
source.id,
384+
);
385+
}
383386
try {
384387
const manifestPath = path.join(
385388
plan.cacheDir,
@@ -389,6 +392,7 @@ export const runSync = async (options: SyncOptions, deps: SyncDeps = {}) => {
389392
if (
390393
result.status !== "up-to-date" &&
391394
lockEntry?.manifestSha256 &&
395+
lockEntry?.rulesSha256 === result.rulesSha256 &&
392396
(await exists(manifestPath))
393397
) {
394398
const computed = await computeManifestHash({
@@ -412,6 +416,9 @@ export const runSync = async (options: SyncOptions, deps: SyncDeps = {}) => {
412416
return;
413417
}
414418
}
419+
if (!options.json) {
420+
ui.step("Building cache layout", source.id);
421+
}
415422
const stats = await runMaterialize({
416423
sourceId: source.id,
417424
repoDir: fetch.repoDir,
@@ -420,6 +427,7 @@ export const runSync = async (options: SyncOptions, deps: SyncDeps = {}) => {
420427
exclude: source.exclude,
421428
maxBytes: source.maxBytes ?? defaults.maxBytes,
422429
maxFiles: source.maxFiles ?? defaults.maxFiles,
430+
unwrapSingleRootDir: source.unwrapSingleRootDir,
423431
});
424432
if (source.targetDir) {
425433
const resolvedTarget = resolveTargetDir(

‎src/targets.ts‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import { cp, mkdir, readdir, rm, symlink } from "node:fs/promises";
22
import path from "node:path";
3+
import { MANIFEST_FILENAME } from "./manifest";
4+
import { DEFAULT_TOC_FILENAME } from "./paths";
35

46
type TargetDeps = {
57
cp: typeof cp;
@@ -28,8 +30,16 @@ const resolveSourceDir = async (params: TargetParams, deps: TargetDeps) => {
2830
return params.sourceDir;
2931
}
3032
const entries = await deps.readdir(params.sourceDir, { withFileTypes: true });
31-
const directories = entries.filter((entry) => entry.isDirectory());
32-
if (directories.length !== 1) {
33+
const metaFiles = new Set([MANIFEST_FILENAME, DEFAULT_TOC_FILENAME]);
34+
const nonMeta = entries.filter((entry) => {
35+
if (entry.isFile() && metaFiles.has(entry.name)) {
36+
return false;
37+
}
38+
return true;
39+
});
40+
const directories = nonMeta.filter((entry) => entry.isDirectory());
41+
const nonMetaFiles = nonMeta.filter((entry) => entry.isFile());
42+
if (directories.length !== 1 || nonMetaFiles.length > 0) {
3343
return params.sourceDir;
3444
}
3545
return path.join(params.sourceDir, directories[0].name);

‎tests/sync-materialize.test.js‎

Lines changed: 75 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,6 @@ test("sync target can unwrap single root directory", async () => {
209209
const cacheDir = path.join(tmpRoot, ".docs");
210210
const repoDir = path.join(tmpRoot, "repo");
211211
const configPath = path.join(tmpRoot, "docs.config.json");
212-
const targetDir = path.join(tmpRoot, "target");
213212

214213
const config = {
215214
$schema:
@@ -218,12 +217,15 @@ test("sync target can unwrap single root directory", async () => {
218217
{
219218
id: "local",
220219
repo: "https://example.com/repo.git",
221-
targetDir: "target",
220+
include: ["17/umbraco-forms/**"],
222221
unwrapSingleRootDir: true,
223222
},
224223
],
225224
};
226225
await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, "utf8");
226+
const repoFile = path.join(repoDir, "17", "umbraco-forms", "README.md");
227+
await mkdir(path.dirname(repoFile), { recursive: true });
228+
await writeFile(repoFile, "hello", "utf8");
227229

228230
await runSync(
229231
{
@@ -244,26 +246,85 @@ test("sync target can unwrap single root directory", async () => {
244246
repoDir,
245247
cleanup: async () => undefined,
246248
}),
247-
materializeSource: async ({ cacheDir: cacheRoot, sourceId }) => {
248-
const outDir = path.join(cacheRoot, sourceId, "umbraco-forms");
249-
await mkdir(outDir, { recursive: true });
250-
await writeFile(
251-
path.join(cacheRoot, sourceId, ".manifest.jsonl"),
252-
`${JSON.stringify({ path: "umbraco-forms/README.md", size: 5 })}\n`,
253-
);
254-
await writeFile(path.join(outDir, "README.md"), "hello", "utf8");
255-
return { bytes: 5, fileCount: 1 };
256-
},
257249
},
258250
);
259251

260-
assert.equal(await exists(path.join(targetDir, "README.md")), true);
252+
assert.equal(await exists(path.join(cacheDir, "local", "README.md")), true);
261253
assert.equal(
262-
await exists(path.join(targetDir, "umbraco-forms", "README.md")),
254+
await exists(
255+
path.join(cacheDir, "local", "17", "umbraco-forms", "README.md"),
256+
),
263257
false,
264258
);
265259
});
266260

261+
test("sync re-materializes when unwrapSingleRootDir changes", async () => {
262+
const tmpRoot = path.join(
263+
tmpdir(),
264+
`docs-cache-unwrap-toggle-${Date.now().toString(36)}`,
265+
);
266+
await mkdir(tmpRoot, { recursive: true });
267+
const cacheDir = path.join(tmpRoot, ".docs");
268+
const repoDir = path.join(tmpRoot, "repo");
269+
const configPath = path.join(tmpRoot, "docs.config.json");
270+
271+
const repoFile = path.join(repoDir, "17", "umbraco-forms", "README.md");
272+
await mkdir(path.dirname(repoFile), { recursive: true });
273+
await writeFile(repoFile, "hello", "utf8");
274+
275+
const writeConfigWithUnwrap = async (unwrapSingleRootDir) => {
276+
const config = {
277+
$schema:
278+
"https://raw.githubusercontent.com/fbosch/docs-cache/main/docs.config.schema.json",
279+
sources: [
280+
{
281+
id: "local",
282+
repo: "https://example.com/repo.git",
283+
include: ["17/umbraco-forms/**"],
284+
unwrapSingleRootDir,
285+
},
286+
],
287+
};
288+
await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, "utf8");
289+
};
290+
291+
const run = async () =>
292+
runSync(
293+
{
294+
configPath,
295+
cacheDirOverride: cacheDir,
296+
json: false,
297+
lockOnly: false,
298+
offline: false,
299+
failOnMiss: false,
300+
},
301+
{
302+
resolveRemoteCommit: async () => ({
303+
repo: "https://example.com/repo.git",
304+
ref: "HEAD",
305+
resolvedCommit: "abc123",
306+
}),
307+
fetchSource: async () => ({
308+
repoDir,
309+
cleanup: async () => undefined,
310+
}),
311+
},
312+
);
313+
314+
await writeConfigWithUnwrap(false);
315+
await run();
316+
assert.equal(
317+
await exists(
318+
path.join(cacheDir, "local", "17", "umbraco-forms", "README.md"),
319+
),
320+
true,
321+
);
322+
323+
await writeConfigWithUnwrap(true);
324+
await run();
325+
assert.equal(await exists(path.join(cacheDir, "local", "README.md")), true);
326+
});
327+
267328
test("sync offline allows missing optional sources", async () => {
268329
const tmpRoot = path.join(
269330
tmpdir(),

0 commit comments

Comments
 (0)