Skip to content

Commit ef50cdf

Browse files
fboschfbosch
committed
fix(default-includes): add brace expansion limit
1 parent 21656fa commit ef50cdf

6 files changed

Lines changed: 160 additions & 121 deletions

File tree

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,7 @@ coverage
1212
TODO.md
1313
.docs/
1414
benchmarks/
15+
16+
*.md
17+
!AGENTS.md
18+
!README.md

‎README.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,10 +100,12 @@ These fields can be set in `defaults` and are inherited by every source unless o
100100
| `maxBytes` | Maximum total bytes to materialize. Default: `200000000` (200 MB). |
101101
| `maxFiles` | Maximum total files to materialize. |
102102
| `ignoreHidden` | Skip hidden files and directories (dotfiles). Default: `false`. |
103-
| `allowHosts` | Allowed Git hosts. Default: `["github.com", "gitlab.com", "visualstudio.com"]`. |
103+
| `allowHosts` | Allowed Git hosts. Default: `["github.com", "gitlab.com", "visualstudio.com"]`. |
104104
| `toc` | Generate per-source `TOC.md`. Default: `true`. Supports `true`, `false`, or a format: `"tree"` (human readable), `"compressed"` |
105105
| `unwrapSingleRootDir` | If the materialized output is nested under a single directory, unwrap it (recursively). Default: `true`. |
106106

107+
> Brace expansion in `include` supports comma-separated lists (including multiple groups) like `**/*.{md,mdx}` and is capped at 500 expanded patterns per include entry. It does not support nested braces or numeric ranges.
108+
107109
### Source options
108110

109111
#### Required

‎src/git/fetch-source.ts‎

Lines changed: 36 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -9,40 +9,13 @@ import { execa } from "execa";
99
import { getErrnoCode } from "#core/errors";
1010
import { assertSafeSourceId } from "#core/source-id";
1111
import { exists, resolveGitCacheDir } from "#git/cache-dir";
12+
import { buildGitEnv } from "#git/git-env";
1213

1314
const DEFAULT_TIMEOUT_MS = 120000; // 120 seconds (2 minutes)
1415
const DEFAULT_GIT_DEPTH = 1;
1516
const DEFAULT_RM_RETRIES = 3;
1617
const DEFAULT_RM_BACKOFF_MS = 100;
17-
18-
const buildGitEnv = () => {
19-
const pathValue = process.env.PATH ?? process.env.Path;
20-
const pathExtValue =
21-
process.env.PATHEXT ??
22-
(process.platform === "win32" ? ".COM;.EXE;.BAT;.CMD" : undefined);
23-
return {
24-
...process.env,
25-
...(pathValue ? { PATH: pathValue, Path: pathValue } : {}),
26-
...(pathExtValue ? { PATHEXT: pathExtValue } : {}),
27-
HOME: process.env.HOME,
28-
USER: process.env.USER,
29-
USERPROFILE: process.env.USERPROFILE,
30-
TMPDIR: process.env.TMPDIR,
31-
TMP: process.env.TMP,
32-
TEMP: process.env.TEMP,
33-
SYSTEMROOT: process.env.SYSTEMROOT,
34-
WINDIR: process.env.WINDIR,
35-
SSH_AUTH_SOCK: process.env.SSH_AUTH_SOCK,
36-
SSH_AGENT_PID: process.env.SSH_AGENT_PID,
37-
HTTP_PROXY: process.env.HTTP_PROXY,
38-
HTTPS_PROXY: process.env.HTTPS_PROXY,
39-
NO_PROXY: process.env.NO_PROXY,
40-
GIT_TERMINAL_PROMPT: "0",
41-
GIT_CONFIG_NOSYSTEM: "1",
42-
GIT_CONFIG_NOGLOBAL: "1",
43-
...(process.platform === "win32" ? {} : { GIT_ASKPASS: "/bin/false" }),
44-
};
45-
};
18+
const MAX_BRACE_EXPANSIONS = 500;
4619

4720
const buildGitConfigs = (allowFileProtocol?: boolean) => [
4821
"-c",
@@ -287,14 +260,40 @@ const patternHasGlob = (pattern: string) =>
287260
pattern.includes("*") || pattern.includes("?") || pattern.includes("[");
288261

289262
const expandBracePattern = (pattern: string): string[] => {
290-
// Match patterns like **/*.{md,mdx,txt}
291-
const braceMatch = pattern.match(/^(.*)\.{([^}]+)}(.*)$/);
292-
if (!braceMatch) {
293-
return [pattern];
294-
}
295-
const [, prefix, extensions, suffix] = braceMatch;
296-
const extList = extensions.split(",").map((ext) => ext.trim());
297-
return extList.map((ext) => `${prefix}.${ext}${suffix}`);
263+
const results: string[] = [];
264+
const expand = (value: string) => {
265+
const braceMatch = value.match(/^(.*?){([^}]+)}(.*)$/);
266+
if (!braceMatch) {
267+
results.push(value);
268+
if (results.length > MAX_BRACE_EXPANSIONS) {
269+
throw new Error(
270+
`Brace expansion exceeded ${MAX_BRACE_EXPANSIONS} patterns for '${pattern}'.`,
271+
);
272+
}
273+
return;
274+
}
275+
const [, prefix, values, suffix] = braceMatch;
276+
const valueList = values
277+
.split(",")
278+
.map((entry) => entry.trim())
279+
.filter((entry) => entry.length > 0);
280+
if (valueList.length === 0) {
281+
results.push(value);
282+
if (results.length > MAX_BRACE_EXPANSIONS) {
283+
throw new Error(
284+
`Brace expansion exceeded ${MAX_BRACE_EXPANSIONS} patterns for '${pattern}'.`,
285+
);
286+
}
287+
return;
288+
}
289+
for (const entry of valueList) {
290+
const expandedPattern = `${prefix}${entry}${suffix}`;
291+
expand(expandedPattern);
292+
}
293+
};
294+
295+
expand(pattern);
296+
return results;
298297
};
299298

300299
const normalizeSparsePatterns = (include?: string[]) => {

‎src/git/git-env.ts‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
const buildGitEnv = (): NodeJS.ProcessEnv => {
2+
const pathValue = process.env.PATH ?? process.env.Path;
3+
const pathExtValue =
4+
process.env.PATHEXT ??
5+
(process.platform === "win32" ? ".COM;.EXE;.BAT;.CMD" : undefined);
6+
return {
7+
...process.env,
8+
...(pathValue ? { PATH: pathValue, Path: pathValue } : {}),
9+
...(pathExtValue ? { PATHEXT: pathExtValue } : {}),
10+
HOME: process.env.HOME,
11+
USER: process.env.USER,
12+
USERPROFILE: process.env.USERPROFILE,
13+
TMPDIR: process.env.TMPDIR,
14+
TMP: process.env.TMP,
15+
TEMP: process.env.TEMP,
16+
SYSTEMROOT: process.env.SYSTEMROOT,
17+
WINDIR: process.env.WINDIR,
18+
SSH_AUTH_SOCK: process.env.SSH_AUTH_SOCK,
19+
SSH_AGENT_PID: process.env.SSH_AGENT_PID,
20+
HTTP_PROXY: process.env.HTTP_PROXY,
21+
HTTPS_PROXY: process.env.HTTPS_PROXY,
22+
NO_PROXY: process.env.NO_PROXY,
23+
GIT_TERMINAL_PROMPT: "0",
24+
GIT_CONFIG_NOSYSTEM: "1",
25+
GIT_CONFIG_NOGLOBAL: "1",
26+
...(process.platform === "win32" ? {} : { GIT_ASKPASS: "/bin/false" }),
27+
};
28+
};
29+
30+
export { buildGitEnv };

‎src/git/resolve-remote.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { execFile } from "node:child_process";
22
import { promisify } from "node:util";
3-
3+
import { buildGitEnv } from "#git/git-env";
44
import { redactRepoUrl } from "#git/redact";
55

66
const execFileAsync = promisify(execFile);
@@ -90,6 +90,7 @@ export const resolveRemoteCommit = async (params: ResolveRemoteParams) => {
9090
{
9191
timeout: params.timeoutMs ?? DEFAULT_TIMEOUT_MS,
9292
maxBuffer: 1024 * 1024,
93+
env: buildGitEnv(),
9394
},
9495
);
9596

‎tests/sparse-brace-expansion.test.js‎

Lines changed: 85 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,8 @@ process.exit(0);
5858
await writeFile(cmdPath, cmdPayload, "utf8");
5959
};
6060

61-
test("sync expands brace patterns for git sparse-checkout", async () => {
62-
const tmpRoot = path.join(
63-
tmpdir(),
64-
`docs-cache-brace-${Date.now().toString(36)}`,
65-
);
61+
const createTestContext = async (label) => {
62+
const tmpRoot = path.join(tmpdir(), `${label}-${Date.now().toString(36)}`);
6663
const binDir = path.join(tmpRoot, "bin");
6764
const logPath = path.join(tmpRoot, "git.log");
6865
const cacheDir = path.join(tmpRoot, ".docs");
@@ -77,6 +74,65 @@ test("sync expands brace patterns for git sparse-checkout", async () => {
7774
await writeGitShim(binDir, logPath);
7875
await writeFile(logPath, "", "utf8");
7976

77+
const cleanup = async () => {
78+
await rm(tmpRoot, { recursive: true, force: true });
79+
};
80+
81+
return {
82+
binDir,
83+
logPath,
84+
cacheDir,
85+
configPath,
86+
gitCacheRoot,
87+
repo,
88+
cleanup,
89+
};
90+
};
91+
92+
const withModifiedPath = async (binDir, gitCacheRoot, fn) => {
93+
const saved = {
94+
PATH: process.env.PATH,
95+
Path: process.env.Path,
96+
PATHEXT: process.env.PATHEXT,
97+
DOCS_CACHE_GIT_DIR: process.env.DOCS_CACHE_GIT_DIR,
98+
};
99+
const previousPath = process.env.PATH ?? process.env.Path;
100+
const nextPath = previousPath
101+
? `${binDir}${path.delimiter}${previousPath}`
102+
: binDir;
103+
104+
process.env.PATH = nextPath;
105+
process.env.Path = nextPath;
106+
if (process.platform === "win32") {
107+
process.env.PATHEXT = ".CMD;.BAT;.EXE;.COM";
108+
}
109+
process.env.DOCS_CACHE_GIT_DIR = gitCacheRoot;
110+
111+
try {
112+
return await fn();
113+
} finally {
114+
process.env.PATH = saved.PATH;
115+
process.env.Path = saved.Path;
116+
process.env.PATHEXT = saved.PATHEXT;
117+
process.env.DOCS_CACHE_GIT_DIR = saved.DOCS_CACHE_GIT_DIR;
118+
}
119+
};
120+
121+
const getSparsePatterns = (args) => {
122+
const patternIndex = args.indexOf("set");
123+
if (patternIndex === -1) return [];
124+
const noConeIndex = args.indexOf("--no-cone");
125+
const patternsStart =
126+
noConeIndex !== -1 && noConeIndex > patternIndex
127+
? noConeIndex + 1
128+
: patternIndex + 1;
129+
return args.slice(patternsStart).filter((arg) => !arg.startsWith("--"));
130+
};
131+
132+
test("sync expands brace patterns for git sparse-checkout", async () => {
133+
const { binDir, logPath, cacheDir, configPath, gitCacheRoot, repo, cleanup } =
134+
await createTestContext("docs-cache-brace");
135+
80136
const config = {
81137
$schema:
82138
"https://raw.githubusercontent.com/fbosch/docs-cache/main/docs.config.schema.json",
@@ -93,29 +149,16 @@ test("sync expands brace patterns for git sparse-checkout", async () => {
93149
};
94150
await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, "utf8");
95151

96-
const previousPath = process.env.PATH ?? process.env.Path;
97-
const previousPathExt = process.env.PATHEXT;
98-
const previousGitDir = process.env.DOCS_CACHE_GIT_DIR;
99-
const nextPath =
100-
process.platform === "win32"
101-
? `${binDir};${previousPath ?? ""}`
102-
: `${binDir}:${previousPath ?? ""}`;
103-
const nextPathExt =
104-
process.platform === "win32" ? ".CMD;.BAT;.EXE;.COM" : previousPathExt;
105-
106-
process.env.PATH = nextPath;
107-
process.env.Path = nextPath;
108-
process.env.PATHEXT = nextPathExt;
109-
process.env.DOCS_CACHE_GIT_DIR = gitCacheRoot;
110-
111152
try {
112-
await runSync({
113-
configPath,
114-
cacheDirOverride: cacheDir,
115-
json: false,
116-
lockOnly: false,
117-
offline: false,
118-
failOnMiss: false,
153+
await withModifiedPath(binDir, gitCacheRoot, async () => {
154+
await runSync({
155+
configPath,
156+
cacheDirOverride: cacheDir,
157+
json: false,
158+
lockOnly: false,
159+
offline: false,
160+
failOnMiss: false,
161+
});
119162
});
120163

121164
const logRaw = await readFile(logPath, "utf8");
@@ -139,9 +182,7 @@ test("sync expands brace patterns for git sparse-checkout", async () => {
139182
const hasExpandedPatterns = sparseArgs.some((args) => {
140183
// Should have expanded **/*.{md,mdx,txt} into:
141184
// **/*.md, **/*.mdx, **/*.txt
142-
const patternIndex = args.indexOf("set");
143-
if (patternIndex === -1) return false;
144-
const patterns = args.slice(patternIndex + 2); // skip "set" and "--no-cone"
185+
const patterns = getSparsePatterns(args);
145186
return (
146187
patterns.includes("**/*.md") &&
147188
patterns.includes("**/*.mdx") &&
@@ -154,32 +195,13 @@ test("sync expands brace patterns for git sparse-checkout", async () => {
154195
`Expected brace patterns to be expanded. Got: ${JSON.stringify(sparseArgs, null, 2)}`,
155196
);
156197
} finally {
157-
process.env.PATH = previousPath;
158-
process.env.Path = previousPath;
159-
process.env.PATHEXT = previousPathExt;
160-
process.env.DOCS_CACHE_GIT_DIR = previousGitDir;
161-
await rm(tmpRoot, { recursive: true, force: true });
198+
await cleanup();
162199
}
163200
});
164201

165202
test("sync expands default brace pattern when no include specified", async () => {
166-
const tmpRoot = path.join(
167-
tmpdir(),
168-
`docs-cache-default-brace-${Date.now().toString(36)}`,
169-
);
170-
const binDir = path.join(tmpRoot, "bin");
171-
const logPath = path.join(tmpRoot, "git.log");
172-
const cacheDir = path.join(tmpRoot, ".docs");
173-
const configPath = path.join(tmpRoot, "docs.config.json");
174-
const gitCacheRoot = path.join(tmpRoot, "git-cache");
175-
const repo = "https://example.com/repo.git";
176-
const repoHash = hashRepoUrl(repo);
177-
const cachePath = path.join(gitCacheRoot, repoHash);
178-
179-
await mkdir(binDir, { recursive: true });
180-
await mkdir(cachePath, { recursive: true });
181-
await writeGitShim(binDir, logPath);
182-
await writeFile(logPath, "", "utf8");
203+
const { binDir, logPath, cacheDir, configPath, gitCacheRoot, repo, cleanup } =
204+
await createTestContext("docs-cache-default-brace");
183205

184206
const config = {
185207
$schema:
@@ -197,29 +219,16 @@ test("sync expands default brace pattern when no include specified", async () =>
197219
};
198220
await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, "utf8");
199221

200-
const previousPath = process.env.PATH ?? process.env.Path;
201-
const previousPathExt = process.env.PATHEXT;
202-
const previousGitDir = process.env.DOCS_CACHE_GIT_DIR;
203-
const nextPath =
204-
process.platform === "win32"
205-
? `${binDir};${previousPath ?? ""}`
206-
: `${binDir}:${previousPath ?? ""}`;
207-
const nextPathExt =
208-
process.platform === "win32" ? ".CMD;.BAT;.EXE;.COM" : previousPathExt;
209-
210-
process.env.PATH = nextPath;
211-
process.env.Path = nextPath;
212-
process.env.PATHEXT = nextPathExt;
213-
process.env.DOCS_CACHE_GIT_DIR = gitCacheRoot;
214-
215222
try {
216-
await runSync({
217-
configPath,
218-
cacheDirOverride: cacheDir,
219-
json: false,
220-
lockOnly: false,
221-
offline: false,
222-
failOnMiss: false,
223+
await withModifiedPath(binDir, gitCacheRoot, async () => {
224+
await runSync({
225+
configPath,
226+
cacheDirOverride: cacheDir,
227+
json: false,
228+
lockOnly: false,
229+
offline: false,
230+
failOnMiss: false,
231+
});
223232
});
224233

225234
const logRaw = await readFile(logPath, "utf8");
@@ -241,9 +250,7 @@ test("sync expands default brace pattern when no include specified", async () =>
241250
// Check that default brace pattern was expanded
242251
const sparseArgs = sparseCheckoutCalls.map((call) => JSON.parse(call));
243252
const hasExpandedDefaults = sparseArgs.some((args) => {
244-
const patternIndex = args.indexOf("set");
245-
if (patternIndex === -1) return false;
246-
const patterns = args.slice(patternIndex + 2);
253+
const patterns = getSparsePatterns(args);
247254
// Default is **/*.{md,mdx,markdown,mkd,txt,rst,adoc,asciidoc}
248255
return (
249256
patterns.includes("**/*.md") &&
@@ -258,10 +265,6 @@ test("sync expands default brace pattern when no include specified", async () =>
258265
`Expected default brace patterns to be expanded. Got: ${JSON.stringify(sparseArgs, null, 2)}`,
259266
);
260267
} finally {
261-
process.env.PATH = previousPath;
262-
process.env.Path = previousPath;
263-
process.env.PATHEXT = previousPathExt;
264-
process.env.DOCS_CACHE_GIT_DIR = previousGitDir;
265-
await rm(tmpRoot, { recursive: true, force: true });
268+
await cleanup();
266269
}
267270
});

0 commit comments

Comments
 (0)