Skip to content

Commit b90675b

Browse files
abcxffabcxff
committed
feat(toolchain): build args support for remote builds
1 parent 9782ae6 commit b90675b

12 files changed

Lines changed: 225 additions & 86 deletions

File tree

‎cloud/packages/ci-manager/src/build-store.ts‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,13 @@ export class BuildStore {
2020
await mkdir(this.tempDir, { recursive: true });
2121
}
2222

23-
createBuild(buildName: string, dockerfilePath: string, environmentId: string): string {
23+
createBuild(
24+
buildName: string,
25+
dockerfilePath: string,
26+
environmentId: string,
27+
buildArgs: Record<string, string>,
28+
buildTarget: string | undefined
29+
): string {
2430
const id = randomUUID();
2531
const contextPath = join(this.tempDir, id, "context.tar.gz");
2632
const outputPath = join(this.tempDir, id, "output.tar.gz");
@@ -32,6 +38,8 @@ export class BuildStore {
3238
dockerfilePath,
3339
environmentId,
3440
contextPath,
41+
buildArgs,
42+
buildTarget,
3543
outputPath,
3644
events: [],
3745
createdAt: new Date(),
@@ -86,7 +94,7 @@ export class BuildStore {
8694
}
8795
}
8896

89-
getContextPath(id: string): string | undefined {
97+
getContextPath(id: string): string | undefined{
9098
return this.builds.get(id)?.contextPath;
9199
}
92100

@@ -118,17 +126,15 @@ export class BuildStore {
118126
}
119127

120128
// Remove build directory and all files
121-
if (build.contextPath) {
122-
const buildDir = dirname(build.contextPath);
123-
try {
124-
await rm(buildDir, { recursive: true, force: true });
125-
console.log(`Removed build directory: ${buildDir}`);
126-
} catch (error) {
127-
console.warn(
128-
`Failed to remove build directory ${buildDir}:`,
129-
error,
130-
);
131-
}
129+
const buildDir = dirname(build.contextPath);
130+
try {
131+
await rm(buildDir, { recursive: true, force: true });
132+
console.log(`Removed build directory: ${buildDir}`);
133+
} catch (error) {
134+
console.warn(
135+
`Failed to remove build directory ${buildDir}:`,
136+
error,
137+
);
132138
}
133139

134140
// Remove from memory

‎cloud/packages/ci-manager/src/common.ts‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
// Keep in sync with ci-runner/entry.sh
2+
export const UNIT_SEP_CHAR = "\x1F";
3+
export const NO_SEP_CHAR_REGEX = /^[^\x1F]+$/;
4+
5+
interface KanikoArguments {
6+
contextUrl: string;
7+
outputUrl: string;
8+
destination: string;
9+
dockerfilePath: string;
10+
buildArgs: Record<string, string>;
11+
buildTarget?: string;
12+
}
13+
14+
// SAFETY: buildArgs keys never have equal signs or spaces
15+
function convertBuildArgsToArgs(
16+
buildArgs: Record<string, string>,
17+
): string[] {
18+
return Object.entries(buildArgs).flatMap(([key, value]) => [
19+
`--build-arg`,
20+
`${key}=${value}`,
21+
]);
22+
}
23+
24+
export function serializeKanikoArguments(args: KanikoArguments): string {
25+
// SAFETY: Nothing needed to be escaped, as values are already sanitized,
26+
// and are joined by IFS=UNIT_SEP_CHAR (see entry.sh of ci-runner).
27+
const preparedArgs = [
28+
...convertBuildArgsToArgs(args.buildArgs),
29+
`--context=${args.contextUrl}`,
30+
`--destination=${args.destination}`,
31+
`--upload-tar=${args.outputUrl}`,
32+
`--dockerfile=${args.dockerfilePath}`,
33+
...(args.buildTarget ? [`--target='${args.buildTarget}'`] : []),
34+
"--no-push",
35+
"--single-snapshot",
36+
"--verbosity=info",
37+
].map(arg => {
38+
// Args should never contain UNIT_SEP_CHAR, but we can
39+
// escape it if they do.
40+
return arg.replaceAll(UNIT_SEP_CHAR, "\\" + UNIT_SEP_CHAR)
41+
});
42+
43+
return preparedArgs.join(UNIT_SEP_CHAR);
44+
}

‎cloud/packages/ci-manager/src/executors/docker.ts‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { spawn } from "node:child_process";
22
import { BuildStore } from "../build-store";
3+
import { serializeKanikoArguments, UNIT_SEP_CHAR } from "../common";
34

45
export async function runDockerBuild(
56
buildStore: BuildStore,
@@ -19,13 +20,16 @@ export async function runDockerBuild(
1920
"--rm",
2021
"--network=host",
2122
"-e",
22-
`CONTEXT_URL=${contextUrl}`,
23-
"-e",
24-
`OUTPUT_URL=${outputUrl}`,
25-
"-e",
26-
`DESTINATION=${buildId}:latest`,
27-
"-e",
28-
`DOCKERFILE_PATH=${build.dockerfilePath}`,
23+
`KANIKO_ARGS=${
24+
serializeKanikoArguments({
25+
contextUrl,
26+
outputUrl,
27+
destination: `${buildId}:latest`,
28+
dockerfilePath: build.dockerfilePath,
29+
buildArgs: build.buildArgs,
30+
buildTarget: build.buildTarget,
31+
})
32+
}`,
2933
"ci-runner",
3034
];
3135

@@ -36,7 +40,7 @@ export async function runDockerBuild(
3640

3741
return new Promise<void>((resolve, reject) => {
3842
const dockerProcess = spawn("docker", kanikoArgs, {
39-
stdio: ["pipe", "pipe", "pipe"],
43+
stdio: ["pipe", "pipe", "pipe"]
4044
});
4145

4246
buildStore.setContainerProcess(buildId, dockerProcess);

‎cloud/packages/ci-manager/src/executors/rivet.ts‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { RivetClient } from "@rivet-gg/api";
22
import { BuildStore } from "../build-store";
3+
import { serializeKanikoArguments } from "../common";
34

45
export async function runRivetBuild(
56
buildStore: BuildStore,
@@ -49,10 +50,14 @@ export async function runRivetBuild(
4950
build: kanikoBuildId,
5051
runtime: {
5152
environment: {
52-
CONTEXT_URL: contextUrl,
53-
OUTPUT_URL: outputUrl,
54-
DESTINATION: `${buildId}:latest`,
55-
DOCKERFILE_PATH: build.dockerfilePath!,
53+
KANIKO_ARGS: serializeKanikoArguments({
54+
contextUrl,
55+
outputUrl,
56+
destination: `${buildId}:latest`,
57+
dockerfilePath: build.dockerfilePath,
58+
buildArgs: build.buildArgs,
59+
buildTarget: build.buildTarget,
60+
})
5661
},
5762
},
5863
network: {

‎cloud/packages/ci-manager/src/kaniko-runner.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,10 @@ async function validateBuildUpload(
3737
buildId: string,
3838
): Promise<void> {
3939
const build = buildStore.getBuild(buildId);
40-
if (!build || !build.outputPath) {
40+
if (!build) {
4141
buildStore.updateStatus(buildId, {
4242
type: "failure",
43-
data: { reason: "Build not found or missing output path" },
43+
data: { reason: "Build not found" },
4444
});
4545
return;
4646
}

‎cloud/packages/ci-manager/src/server.ts‎

Lines changed: 28 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,16 @@ import {
1717
uploadOCIBundleToRivet,
1818
type RivetUploadConfig,
1919
} from "./rivet-uploader";
20+
import { UNIT_SEP_CHAR } from "./common";
21+
import { BuildRequestSchema } from "./types";
2022

2123
async function processRivetUpload(
2224
buildStore: BuildStore,
2325
buildId: string,
2426
): Promise<void> {
2527
const build = buildStore.getBuild(buildId);
26-
if (!build || !build.outputPath) {
27-
throw new Error(`Build ${buildId} not found or missing output path`);
28+
if (!build) {
29+
throw new Error(`Build ${buildId} not found`);
2830
}
2931

3032
try {
@@ -63,7 +65,7 @@ async function processRivetUpload(
6365

6466
const uploadResult = await uploadOCIBundleToRivet(
6567
conversionResult.bundleTarPath,
66-
build.buildName!,
68+
build.buildName,
6769
`${buildId}:latest`, // Match kaniko destination format
6870
rivetConfig,
6971
new Date().toISOString(), // Use timestamp as version for now
@@ -100,30 +102,31 @@ export async function createServer(port: number = 3000) {
100102

101103
app.post("/builds", async (c) => {
102104
try {
103-
const formData = await c.req.formData();
104-
const buildName = formData.get("buildName") as string;
105-
const dockerfilePath = formData.get("dockerfilePath") as string;
106-
const environmentId = formData.get("environmentId") as string;
107-
const contextFile = formData.get("context") as File;
108-
109-
if (!buildName) {
110-
return c.json({ error: "buildName is required" }, 400);
111-
}
112-
113-
if (!dockerfilePath) {
114-
return c.json({ error: "dockerfilePath is required" }, 400);
115-
}
116-
117-
if (!environmentId) {
118-
return c.json({ error: "environmentId is required" }, 400);
119-
}
120-
121-
if (!contextFile) {
122-
return c.json({ error: "context file is required" }, 400);
105+
const body = await c.req.parseBody();
106+
const parseResult = BuildRequestSchema.safeParse(body);
107+
if (!parseResult.success) {
108+
return c.json(
109+
{ error: "Invalid build request format" },
110+
400,
111+
);
123112
}
113+
const {
114+
buildName,
115+
dockerfilePath,
116+
environmentId,
117+
buildArgs,
118+
buildTarget,
119+
context: contextFile
120+
} = parseResult.data;
124121

125122
// Create the build
126-
const buildId = buildStore.createBuild(buildName, dockerfilePath, environmentId);
123+
const buildId = buildStore.createBuild(
124+
buildName,
125+
dockerfilePath,
126+
environmentId,
127+
buildArgs,
128+
buildTarget
129+
);
127130
const contextPath = buildStore.getContextPath(buildId);
128131

129132
if (!contextPath) {
@@ -169,6 +172,7 @@ export async function createServer(port: number = 3000) {
169172

170173
return c.json({ buildId });
171174
} catch (error) {
175+
console.error("Error processing build request:", error);
172176
return c.json({ error: "Failed to process build request" }, 500);
173177
}
174178
});

‎cloud/packages/ci-manager/src/types.ts‎

Lines changed: 64 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,43 +1,83 @@
11
import { z } from "zod";
2+
import { NO_SEP_CHAR_REGEX, UNIT_SEP_CHAR } from "./common";
23

34
export const StatusSchema = z.discriminatedUnion("type", [
4-
z.object({ type: z.literal("starting"), data: z.object({}) }),
5-
z.object({ type: z.literal("running"), data: z.object({}) }),
6-
z.object({ type: z.literal("finishing"), data: z.object({}) }),
7-
z.object({ type: z.literal("converting"), data: z.object({}) }),
8-
z.object({ type: z.literal("uploading"), data: z.object({}) }),
9-
z.object({ type: z.literal("failure"), data: z.object({ reason: z.string() }) }),
10-
z.object({ type: z.literal("success"), data: z.object({ buildId: z.string() }) }),
5+
z.object({ type: z.literal("starting"), data: z.object({}) }),
6+
z.object({ type: z.literal("running"), data: z.object({}) }),
7+
z.object({ type: z.literal("finishing"), data: z.object({}) }),
8+
z.object({ type: z.literal("converting"), data: z.object({}) }),
9+
z.object({ type: z.literal("uploading"), data: z.object({}) }),
10+
z.object({ type: z.literal("failure"), data: z.object({ reason: z.string() }) }),
11+
z.object({ type: z.literal("success"), data: z.object({ buildId: z.string() }) }),
1112
]);
1213

1314
export type Status = z.infer<typeof StatusSchema>;
1415

16+
const ILLEGAL_BUILD_ARG_KEY = /[\s'"\\]/g;
17+
const BuildArgsSchema = z.string()
18+
.transform((str) => JSON.parse(str))
19+
.pipe(z.array(z.string()))
20+
.refine((arr) => {
21+
// Check each key=value pair to ensure keys have no spaces
22+
return arr.every(item => {
23+
const [key] = item.split('=');
24+
if (!key) return false;
25+
if (ILLEGAL_BUILD_ARG_KEY.test(key)) return false;
26+
if (item.includes(UNIT_SEP_CHAR)) return false;
27+
return true;
28+
});
29+
}, { message: "Argument key/value contains invalid character" })
30+
.transform((arr) => {
31+
const result: Record<string, string> = Object.create(null);
32+
// Convert array of strings to an object
33+
for (const item of arr) {
34+
const [key, ...valueParts] = item.split('=');
35+
const value = valueParts.join('=');
36+
37+
if (key && value !== undefined) {
38+
result[key] = value;
39+
}
40+
}
41+
42+
return result;
43+
});
44+
1545
export const BuildRequestSchema = z.object({
16-
buildName: z.string(),
17-
dockerfilePath: z.string(),
18-
environmentId: z.string(),
46+
buildName: z.string()
47+
.regex(NO_SEP_CHAR_REGEX, "buildName cannot contain special characters"),
48+
dockerfilePath: z.string()
49+
.regex(NO_SEP_CHAR_REGEX, "dockerfilePath cannot contain special characters"),
50+
environmentId: z.string()
51+
.regex(NO_SEP_CHAR_REGEX, "environmentId cannot contain special characters"),
52+
buildArgs: BuildArgsSchema,
53+
buildTarget: z.string()
54+
.regex(NO_SEP_CHAR_REGEX, "buildTarget cannot contain special characters")
55+
.optional(),
56+
context: z.instanceof(File)
1957
});
2058

2159
export type BuildRequest = z.infer<typeof BuildRequestSchema>;
2260

2361
export const BuildEventSchema = z.discriminatedUnion("type", [
24-
z.object({ type: z.literal("status"), data: StatusSchema }),
25-
z.object({ type: z.literal("log"), data: z.object({ line: z.string() }) }),
62+
z.object({ type: z.literal("status"), data: StatusSchema }),
63+
z.object({ type: z.literal("log"), data: z.object({ line: z.string() }) }),
2664
]);
2765

2866
export type BuildEvent = z.infer<typeof BuildEventSchema>;
2967

3068
export interface BuildInfo {
31-
id: string;
32-
status: Status;
33-
buildName?: string;
34-
dockerfilePath?: string;
35-
environmentId?: string;
36-
contextPath?: string;
37-
outputPath?: string;
38-
events: BuildEvent[];
39-
containerProcess?: any;
40-
createdAt: Date;
41-
downloadedAt?: Date;
42-
cleanupTimeout?: NodeJS.Timeout;
69+
id: string;
70+
status: Status;
71+
buildName: string;
72+
dockerfilePath: string;
73+
environmentId: string;
74+
contextPath: string;
75+
buildArgs: Record<string, string>;
76+
buildTarget?: string;
77+
outputPath: string;
78+
events: BuildEvent[];
79+
containerProcess?: any;
80+
createdAt: Date;
81+
downloadedAt?: Date;
82+
cleanupTimeout?: NodeJS.Timeout;
4383
}

‎cloud/packages/ci-runner/Dockerfile‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,6 @@ FROM ghcr.io/rivet-gg/executor@sha256:439d4dbb0f3f8c1c6c2195e144d29195b4930b8716
55
COPY --from=builder /bin/sh /bin/sh
66
COPY --from=builder /lib/ld-musl-x86_64.so.1 /lib/ld-musl-x86_64.so.1
77

8-
# HACK: Use env vars to interpolate args bc Rivet doesn't support passing args
9-
ENTRYPOINT ["/bin/sh", "-c", "/kaniko/executor --context=${CONTEXT_URL} --destination=${DESTINATION} --upload-tar=${OUTPUT_URL} --dockerfile=${DOCKERFILE_PATH} --no-push --single-snapshot --verbosity=info"]
8+
COPY entry.sh ~/entry.sh
9+
10+
ENTRYPOINT ["/bin/sh", "~/entry.sh"]

0 commit comments

Comments
 (0)