Skip to content

Commit 49aef84

Browse files
committed
Include preprocessing file
1 parent 6293ce5 commit 49aef84

8 files changed

Lines changed: 206 additions & 5 deletions

File tree

src/spec-node/dockerCompose.ts

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import { Mount, parseMount } from '../spec-configuration/containerFeaturesConfig
1919
import path from 'path';
2020
import { getDevcontainerMetadata, getImageBuildInfoFromDockerfile, getImageBuildInfoFromImage, getImageMetadataFromContainer, ImageBuildInfo, lifecycleCommandOriginMapFromMetadata, mergeConfiguration, MergedDevContainerConfig } from './imageMetadata';
2121
import { ensureDockerfileHasFinalStageName } from './dockerfileUtils';
22+
import { resolveDockerfileIncludesIfNeeded } from './dockerfilePreprocess';
2223
import { randomUUID } from 'crypto';
2324

2425
const projectLabel = 'com.docker.compose.project';
@@ -163,11 +164,16 @@ export async function buildAndExtendDockerCompose(configWithRaw: SubstitutedConf
163164
let baseName = 'dev_container_auto_added_stage_label';
164165
let dockerfile: string | undefined;
165166
let imageBuildInfo: ImageBuildInfo;
167+
let preprocessedDockerfilePathForComposeBuild: string | undefined;
166168
const serviceInfo = getBuildInfoForService(composeService, cliHost.path, localComposeFiles);
167169
if (serviceInfo.build) {
168170
const { context, dockerfilePath, target } = serviceInfo.build;
169171
const resolvedDockerfilePath = cliHost.path.isAbsolute(dockerfilePath) ? dockerfilePath : path.resolve(context, dockerfilePath);
170-
const originalDockerfile = (await cliHost.readFile(resolvedDockerfilePath)).toString();
172+
const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, resolvedDockerfilePath);
173+
const originalDockerfile = resolvedDockerfile.effectiveDockerfileContent;
174+
if (resolvedDockerfile.preprocessed) {
175+
preprocessedDockerfilePathForComposeBuild = resolvedDockerfile.effectiveDockerfilePath;
176+
}
171177
dockerfile = originalDockerfile;
172178
if (target) {
173179
// Explictly set build target for the dev container build features on that
@@ -194,6 +200,10 @@ export async function buildAndExtendDockerCompose(configWithRaw: SubstitutedConf
194200

195201
let overrideImageName: string | undefined;
196202
let buildOverrideContent = '';
203+
if (preprocessedDockerfilePathForComposeBuild && !extendImageBuildInfo?.featureBuildInfo) {
204+
buildOverrideContent += ' build:\n';
205+
buildOverrideContent += ` dockerfile: ${preprocessedDockerfilePathForComposeBuild}\n`;
206+
}
197207
if (extendImageBuildInfo?.featureBuildInfo) {
198208
// Avoid retagging a previously pulled image.
199209
if (!serviceInfo.build) {
Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
* Licensed under the MIT License. See License.txt in the project root for license information.
4+
*--------------------------------------------------------------------------------------------*/
5+
6+
import { randomUUID } from 'crypto';
7+
import { CLIHost } from '../spec-common/cliHost';
8+
import { ContainerError } from '../spec-common/errors';
9+
10+
const includeLine = /^\s*#include\s+"([^"]+)"\s*$/;
11+
12+
export interface ResolvedDockerfile {
13+
originalDockerfilePath: string;
14+
effectiveDockerfilePath: string;
15+
effectiveDockerfileContent: string;
16+
preprocessed: boolean;
17+
}
18+
19+
export async function resolveDockerfileIncludesIfNeeded(cliHost: CLIHost, dockerfilePath: string): Promise<ResolvedDockerfile> {
20+
const dockerfileText = (await cliHost.readFile(dockerfilePath)).toString();
21+
if (!dockerfilePath.toLowerCase().endsWith('.in')) {
22+
return {
23+
originalDockerfilePath: dockerfilePath,
24+
effectiveDockerfilePath: dockerfilePath,
25+
effectiveDockerfileContent: dockerfileText,
26+
preprocessed: false,
27+
};
28+
}
29+
30+
const effectiveDockerfileContent = await preprocessDockerfileIncludes(cliHost, dockerfilePath, []);
31+
const preprocessedDockerfilePath = await writePreprocessedDockerfile(cliHost, dockerfilePath, effectiveDockerfileContent);
32+
33+
return {
34+
originalDockerfilePath: dockerfilePath,
35+
effectiveDockerfilePath: preprocessedDockerfilePath,
36+
effectiveDockerfileContent,
37+
preprocessed: true,
38+
};
39+
}
40+
41+
async function preprocessDockerfileIncludes(cliHost: CLIHost, currentPath: string, stack: string[]): Promise<string> {
42+
if (stack.includes(currentPath)) {
43+
const chain = [...stack, currentPath].join(' -> ');
44+
throw new ContainerError({ description: `Cyclic #include detected while preprocessing Dockerfile: ${chain}` });
45+
}
46+
if (!(await cliHost.isFile(currentPath))) {
47+
throw new ContainerError({ description: `Included Dockerfile not found: ${currentPath}` });
48+
}
49+
50+
const currentText = (await cliHost.readFile(currentPath)).toString();
51+
const lines = currentText.split(/\r?\n/);
52+
const expanded: string[] = [];
53+
const nextStack = [...stack, currentPath];
54+
for (const line of lines) {
55+
const match = includeLine.exec(line);
56+
if (!match) {
57+
expanded.push(line);
58+
continue;
59+
}
60+
61+
const includePath = match[1];
62+
const resolvedIncludePath = cliHost.path.isAbsolute(includePath)
63+
? includePath
64+
: cliHost.path.resolve(cliHost.path.dirname(currentPath), includePath);
65+
expanded.push(await preprocessDockerfileIncludes(cliHost, resolvedIncludePath, nextStack));
66+
}
67+
68+
return expanded.join('\n');
69+
}
70+
71+
async function writePreprocessedDockerfile(cliHost: CLIHost, sourceDockerfilePath: string, content: string): Promise<string> {
72+
const cacheFolder = cliHost.path.join(
73+
await cliHost.tmpdir(),
74+
cliHost.platform === 'linux' ? `devcontainercli-${await cliHost.getUsername()}` : 'devcontainercli',
75+
'dockerfile-preprocess'
76+
);
77+
await cliHost.mkdirp(cacheFolder);
78+
79+
const sourceBasename = cliHost.path.basename(sourceDockerfilePath);
80+
const targetBasename = sourceBasename.replace(/\.in$/i, '') || 'Dockerfile';
81+
const preprocessedDockerfilePath = cliHost.path.join(cacheFolder, `${Date.now()}-${randomUUID()}-${targetBasename}`);
82+
await cliHost.writeFile(preprocessedDockerfilePath, Buffer.from(content));
83+
return preprocessedDockerfilePath;
84+
}

src/spec-node/imageMetadata.ts

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import { ContainerDetails, DockerCLIParameters, ImageDetails } from '../spec-shu
1212
import { Log, LogLevel } from '../spec-utils/log';
1313
import { getBuildInfoForService, readDockerComposeConfig } from './dockerCompose';
1414
import { Dockerfile, extractDockerfile, findBaseImage, findUserStatement } from './dockerfileUtils';
15+
import { resolveDockerfileIncludesIfNeeded } from './dockerfilePreprocess';
1516
import { SubstituteConfig, SubstitutedConfig, DockerResolverParameters, inspectDockerImage, uriToWSLFsPath, envListToObj } from './utils';
1617

1718
const pickConfigProperties: (keyof DevContainerConfig & keyof ImageMetadataEntry)[] = [
@@ -342,7 +343,8 @@ export async function getImageBuildInfo(params: DockerResolverParameters | Docke
342343
if (!cliHost.isFile(dockerfilePath)) {
343344
throw new ContainerError({ description: `Dockerfile (${dockerfilePath}) not found.` });
344345
}
345-
const dockerfile = (await cliHost.readFile(dockerfilePath)).toString();
346+
const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, dockerfilePath);
347+
const dockerfile = resolvedDockerfile.effectiveDockerfileContent;
346348
return getImageBuildInfoFromDockerfile(params, dockerfile, config.build?.args || {}, config.build?.target, configWithRaw.substitute);
347349

348350
} else if ('dockerComposeFile' in config) {
@@ -363,7 +365,8 @@ export async function getImageBuildInfo(params: DockerResolverParameters | Docke
363365
if (serviceInfo.build) {
364366
const { context, dockerfilePath } = serviceInfo.build;
365367
const resolvedDockerfilePath = cliHost.path.isAbsolute(dockerfilePath) ? dockerfilePath : cliHost.path.resolve(context, dockerfilePath);
366-
const dockerfile = (await cliHost.readFile(resolvedDockerfilePath)).toString();
368+
const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, resolvedDockerfilePath);
369+
const dockerfile = resolvedDockerfile.effectiveDockerfileContent;
367370
return getImageBuildInfoFromDockerfile(params, dockerfile, serviceInfo.build.args || {}, serviceInfo.build.target, configWithRaw.substitute);
368371
} else {
369372
return getImageBuildInfoFromImage(params, composeService.image, configWithRaw.substitute);

src/spec-node/singleContainer.ts

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import { LogLevel, Log, makeLog } from '../spec-utils/log';
1313
import { extendImage, getExtendImageBuildInfo, updateRemoteUserUID } from './containerFeatures';
1414
import { getDevcontainerMetadata, getImageBuildInfoFromDockerfile, getImageMetadataFromContainer, ImageMetadataEntry, lifecycleCommandOriginMapFromMetadata, mergeConfiguration, MergedDevContainerConfig } from './imageMetadata';
1515
import { ensureDockerfileHasFinalStageName, generateMountCommand } from './dockerfileUtils';
16+
import { resolveDockerfileIncludesIfNeeded } from './dockerfilePreprocess';
1617

1718
export const hostFolderLabel = 'devcontainer.local_folder'; // used to label containers created from a workspace/folder
1819
export const configFileLabel = 'devcontainer.config_file';
@@ -130,7 +131,8 @@ async function buildAndExtendImage(buildParams: DockerResolverParameters, config
130131
throw new ContainerError({ description: `Dockerfile (${dockerfilePath}) not found.` });
131132
}
132133

133-
let dockerfile = (await cliHost.readFile(dockerfilePath)).toString();
134+
const resolvedDockerfile = await resolveDockerfileIncludesIfNeeded(cliHost, dockerfilePath);
135+
let dockerfile = resolvedDockerfile.effectiveDockerfileContent;
134136
const originalDockerfile = dockerfile;
135137
let baseName = 'dev_container_auto_added_stage_label';
136138
if (config.build?.target) {
@@ -149,7 +151,7 @@ async function buildAndExtendImage(buildParams: DockerResolverParameters, config
149151
const imageBuildInfo = await getImageBuildInfoFromDockerfile(buildParams, originalDockerfile, config.build?.args || {}, config.build?.target, configWithRaw.substitute);
150152
const extendImageBuildInfo = await getExtendImageBuildInfo(buildParams, configWithRaw, baseName, imageBuildInfo, undefined, additionalFeatures, false);
151153

152-
let finalDockerfilePath = dockerfilePath;
154+
let finalDockerfilePath = resolvedDockerfile.effectiveDockerfilePath;
153155
const additionalBuildArgs: string[] = [];
154156
if (extendImageBuildInfo?.featureBuildInfo) {
155157
const { featureBuildInfo } = extendImageBuildInfo;
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"build": {
3+
"dockerfile": "cpp.Dockerfile.in"
4+
}
5+
}
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
#include "tools.Dockerfile"
2+
RUN apt-get update && apt-get install -y clang
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
FROM docker.io/debian:latest
2+
RUN apt-get update && apt-get install -y vim
Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
*--------------------------------------------------------------------------------------------*/
4+
5+
import { assert } from 'chai';
6+
import * as fs from 'fs';
7+
import * as path from 'path';
8+
import { CLIHost } from '../spec-common/cliHost';
9+
import { resolveDockerfileIncludesIfNeeded } from '../spec-node/dockerfilePreprocess';
10+
11+
function createMockCLIHost(files: Record<string, string>, platform: NodeJS.Platform = 'linux'): CLIHost {
12+
const pathModule = platform === 'win32' ? path.win32 : path.posix;
13+
return {
14+
type: 'local',
15+
platform,
16+
arch: 'x64',
17+
path: pathModule,
18+
cwd: platform === 'win32' ? 'C:\\' : '/',
19+
env: {},
20+
exec: () => { throw new Error('Not implemented'); },
21+
ptyExec: () => { throw new Error('Not implemented'); },
22+
homedir: async () => platform === 'win32' ? 'C:\\Users\\test' : '/home/test',
23+
tmpdir: async () => platform === 'win32' ? 'C:\\tmp' : '/tmp',
24+
isFile: async (filepath: string) => filepath in files,
25+
isFolder: async () => false,
26+
readFile: async (filepath: string) => {
27+
if (!(filepath in files)) {
28+
throw new Error(`File not found: ${filepath}`);
29+
}
30+
return Buffer.from(files[filepath]);
31+
},
32+
writeFile: async (filepath: string, content: Buffer) => {
33+
files[filepath] = content.toString();
34+
},
35+
rename: async () => { },
36+
mkdirp: async () => { },
37+
readDir: async () => [],
38+
getUsername: async () => 'test',
39+
toCommonURI: async () => undefined,
40+
connect: () => { throw new Error('Not implemented'); },
41+
};
42+
}
43+
44+
describe('resolveDockerfileIncludesIfNeeded', () => {
45+
it('returns source Dockerfile unchanged when not using .in extension', async () => {
46+
const files: Record<string, string> = {
47+
'/workspace/Dockerfile': 'FROM debian:latest\nRUN echo ok',
48+
};
49+
const cliHost = createMockCLIHost(files);
50+
const result = await resolveDockerfileIncludesIfNeeded(cliHost, '/workspace/Dockerfile');
51+
assert.isFalse(result.preprocessed);
52+
assert.equal(result.effectiveDockerfilePath, '/workspace/Dockerfile');
53+
assert.equal(result.effectiveDockerfileContent, files['/workspace/Dockerfile']);
54+
});
55+
56+
it('expands #include lines and writes a generated Dockerfile for .in files', async () => {
57+
const podmanTestConfigPath = path.resolve(__dirname, 'configs', 'podman-test');
58+
const sourceDockerfilePath = path.join(podmanTestConfigPath, 'cpp.Dockerfile.in');
59+
const includedDockerfilePath = path.join(podmanTestConfigPath, 'tools.Dockerfile');
60+
const sourceDockerfileContent = fs.readFileSync(sourceDockerfilePath).toString();
61+
const includedDockerfileContent = fs.readFileSync(includedDockerfilePath).toString();
62+
const files: Record<string, string> = {
63+
[sourceDockerfilePath]: sourceDockerfileContent,
64+
[includedDockerfilePath]: includedDockerfileContent,
65+
};
66+
const cliHost = createMockCLIHost(files);
67+
const result = await resolveDockerfileIncludesIfNeeded(cliHost, sourceDockerfilePath);
68+
assert.isTrue(result.preprocessed);
69+
assert.notEqual(result.effectiveDockerfilePath, sourceDockerfilePath);
70+
assert.include(result.effectiveDockerfilePath, '/tmp/devcontainercli-test/dockerfile-preprocess/');
71+
assert.equal(
72+
result.effectiveDockerfileContent,
73+
'FROM docker.io/debian:latest\nRUN apt-get update && apt-get install -y vim\nRUN apt-get update && apt-get install -y clang'
74+
);
75+
assert.equal(files[result.effectiveDockerfilePath], result.effectiveDockerfileContent);
76+
});
77+
78+
it('fails with a clear error when #include has a cycle', async () => {
79+
const files: Record<string, string> = {
80+
'/workspace/a.Dockerfile.in': '#include "b.Dockerfile"\nRUN echo a',
81+
'/workspace/b.Dockerfile': '#include "a.Dockerfile.in"\nRUN echo b',
82+
};
83+
const cliHost = createMockCLIHost(files);
84+
let err: any;
85+
try {
86+
await resolveDockerfileIncludesIfNeeded(cliHost, '/workspace/a.Dockerfile.in');
87+
} catch (e) {
88+
err = e;
89+
}
90+
assert.ok(err);
91+
assert.include(String(err.message || err), 'Cyclic #include detected while preprocessing Dockerfile');
92+
});
93+
});

0 commit comments

Comments
 (0)