feat(toolchain): remote build method

cloud/packages/ci-manager/Dockerfile

@@ -0,0 +1,25 @@
+# Build stage - includes dev dependencies
+FROM node:22-alpine AS builder
+ENV NODE_ENV=development
+
+WORKDIR /app
+RUN npm i -g corepack && corepack enable
+COPY package.json yarn.lock ./
+RUN yarn install --immutable
+COPY . .
+RUN yarn build
+
+# Production stage - only prod dependencies
+FROM node:22-alpine AS release
+ENV NODE_ENV=production
+
+RUN adduser -s /bin/sh -D rivet
+WORKDIR /app
+RUN apk add --no-cache skopeo umoci && npm i -g corepack && corepack enable
+COPY package.json yarn.lock ./
+RUN yarn config set nodeLinker node-modules && yarn install --immutable
+COPY --from=builder /app/dist ./dist
+RUN chown -R rivet:rivet /app
+USER rivet
+EXPOSE 3000
+CMD ["node", "dist/index.js"]

cloud/packages/ci-manager/package.json

@@ -0,0 +1,32 @@
+{
+	"name": "ci-manager",
+	"private": true,
+	"peerDependencies": {
+		"typescript": "^5"
+	},
+	"scripts": {
+		"prepare-builds": "tsx scripts/prepare-builds.ts",
+		"check-types": "tsc --noEmit",
+		"test": "vitest",
+		"build": "tsc"
+	},
+	"dependencies": {
+		"@rivet-gg/api": "^25.4.2",
+		"hono": "^4.7.11",
+		"nanoevents": "^9.1.0",
+		"zod": "^3.25.56"
+	},
+	"devDependencies": {
+		"@hono/node-server": "^1.14.4",
+		"@types/eventsource": "^1.1.15",
+		"@types/node": "^22.15.30",
+		"eventsource": "^4.0.0",
+		"get-port": "^7.1.0",
+		"tar": "^7.4.3",
+		"tsx": "^4.19.4",
+		"typescript": "^5.7.2",
+		"vitest": "^3.2.2",
+		"zx": "^8.1.9"
+	},
+	"packageManager": "yarn@4.6.0"
+}

cloud/packages/ci-manager/src/build-store.ts

@@ -0,0 +1,146 @@
+import { BuildInfo, BuildEvent, Status } from "./types";
+import { randomUUID } from "crypto";
+import { mkdir, rm } from "fs/promises";
+import { join, dirname } from "path";
+import { createNanoEvents } from "nanoevents";
+
+export class BuildStore {
+	private builds = new Map<string, BuildInfo>();
+	private tempDir: string;
+	public emitter = createNanoEvents<{
+		"build-event": (buildId: string, event: BuildEvent) => void;
+		"status-change": (buildId: string, status: Status) => void;
+	}>();
+
+	constructor(tempDir: string = "/tmp/ci-builds") {
+		this.tempDir = tempDir;
+	}
+
+	async init() {
+		await mkdir(this.tempDir, { recursive: true });
+	}
+
+	createBuild(buildName: string, dockerfilePath: string, environmentId: string): string {
+		const id = randomUUID();
+		const contextPath = join(this.tempDir, id, "context.tar.gz");
+		const outputPath = join(this.tempDir, id, "output.tar.gz");
+
+		const build: BuildInfo = {
+			id,
+			status: { type: "starting", data: {} },
+			buildName,
+			dockerfilePath,
+			environmentId,
+			contextPath,
+			outputPath,
+			events: [],
+			createdAt: new Date(),
+		};
+
+		// Set up 10-minute cleanup timeout
+		build.cleanupTimeout = setTimeout(
+			() => {
+				this.cleanupBuild(id, "timeout");
+			},
+			10 * 60 * 1000,
+		); // 10 minutes
+
+		this.builds.set(id, build);
+		return id;
+	}
+
+	getBuild(id: string): BuildInfo | undefined {
+		return this.builds.get(id);
+	}
+
+	updateStatus(id: string, status: Status) {
+		const build = this.builds.get(id);
+		if (
+			build &&
+			build.status.type !== "success" &&
+			build.status.type !== "failure"
+		) {
+			build.status = status;
+			const event = { type: "status", data: status } as BuildEvent;
+			build.events.push(event);
+			this.emitter.emit("build-event", id, event);
+			this.emitter.emit("status-change", id, status);
+			console.log(`[${id}] status: ${JSON.stringify(status)}`);
+		}
+	}
+
+	addLog(id: string, line: string) {
+		console.log(`[${id}] ${line}`);
+		const build = this.builds.get(id);
+		if (build) {
+			const event: BuildEvent = { type: "log", data: { line } };
+			build.events.push(event);
+			this.emitter.emit("build-event", id, event);
+		}
+	}
+
+	setContainerProcess(id: string, process: any) {
+		const build = this.builds.get(id);
+		if (build) {
+			build.containerProcess = process;
+		}
+	}
+
+	getContextPath(id: string): string | undefined {
+		return this.builds.get(id)?.contextPath;
+	}
+
+	getOutputPath(id: string): string | undefined {
+		return this.builds.get(id)?.outputPath;
+	}
+
+	markDownloaded(id: string) {
+		const build = this.builds.get(id);
+		if (build) {
+			build.downloadedAt = new Date();
+			// Trigger cleanup after download
+			setTimeout(() => {
+				this.cleanupBuild(id, "downloaded");
+			}, 1000); // Small delay to ensure download is complete
+		}
+	}
+
+	private async cleanupBuild(id: string, reason: "timeout" | "downloaded") {
+		const build = this.builds.get(id);
+		if (!build) return;
+
+		console.log(`Cleaning up build ${id} (reason: ${reason})`);
+
+		try {
+			// Clear the timeout if it exists
+			if (build.cleanupTimeout) {
+				clearTimeout(build.cleanupTimeout);
+			}
+
+			// Remove build directory and all files
+			if (build.contextPath) {
+				const buildDir = dirname(build.contextPath);
+				try {
+					await rm(buildDir, { recursive: true, force: true });
+					console.log(`Removed build directory: ${buildDir}`);
+				} catch (error) {
+					console.warn(
+						`Failed to remove build directory ${buildDir}:`,
+						error,
+					);
+				}
+			}
+
+			// Remove from memory
+			this.builds.delete(id);
+			console.log(`Build ${id} cleaned up successfully`);
+		} catch (error) {
+			console.error(`Error cleaning up build ${id}:`, error);
+		}
+	}
+
+	// Manual cleanup method for testing or admin use
+	async manualCleanup(id: string) {
+		await this.cleanupBuild(id, "downloaded");
+	}
+}

cloud/packages/ci-manager/src/executors/docker.ts

@@ -0,0 +1,94 @@
+import { spawn } from "node:child_process";
+import { BuildStore } from "../build-store";
+
+export async function runDockerBuild(
+	buildStore: BuildStore,
+	serverUrl: string,
+	buildId: string,
+): Promise<void> {
+	const build = buildStore.getBuild(buildId);
+	if (!build) {
+		throw new Error(`Build ${buildId} not found`);
+	}
+
+	const contextUrl = `${serverUrl}/builds/${buildId}/kaniko/context.tar.gz`;
+	const outputUrl = `${serverUrl}/builds/${buildId}/kaniko/output.tar.gz`;
+
+	const kanikoArgs = [
+		"run",
+		"--rm",
+		"--network=host",
+		"-e",
+		`CONTEXT_URL=${contextUrl}`,
+		"-e",
+		`OUTPUT_URL=${outputUrl}`,
+		"-e",
+		`DESTINATION=${buildId}:latest`,
+		"-e",
+		`DOCKERFILE_PATH=${build.dockerfilePath}`,
+		"ci-runner",
+	];
+
+	buildStore.addLog(
+		buildId,
+		`Starting kaniko with args: docker ${kanikoArgs.join(" ")}`,
+	);
+
+	return new Promise<void>((resolve, reject) => {
+		const dockerProcess = spawn("docker", kanikoArgs, {
+			stdio: ["pipe", "pipe", "pipe"],
+		});
+
+		buildStore.setContainerProcess(buildId, dockerProcess);
+
+		dockerProcess.stdout?.on("data", (data) => {
+			const lines = data
+				.toString()
+				.split("\n")
+				.filter((line: string) => line.trim());
+			lines.forEach((line: string) => {
+				buildStore.addLog(buildId, `[kaniko] ${line}`);
+			});
+		});
+
+		dockerProcess.stderr?.on("data", (data) => {
+			const lines = data
+				.toString()
+				.split("\n")
+				.filter((line: string) => line.trim());
+			lines.forEach((line: string) => {
+				buildStore.addLog(buildId, `[kaniko-error] ${line}`);
+			});
+		});
+
+		dockerProcess.on("close", (code) => {
+			buildStore.addLog(buildId, `Docker process closed with exit code: ${code}`);
+			buildStore.updateStatus(buildId, { type: "finishing", data: {} });
+
+			if (code === 0) {
+				resolve();
+			} else {
+				buildStore.updateStatus(buildId, {
+					type: "failure",
+					data: { reason: `Container exited with code ${code}` },
+				});
+				reject(new Error(`Container exited with code ${code}`));
+			}
+		});
+
+		dockerProcess.on("spawn", () => {
+			buildStore.addLog(buildId, "Docker process spawned successfully");
+		});
+
+		dockerProcess.on("error", (error) => {
+			buildStore.addLog(buildId, `Docker process error: ${error.message}`);
+			buildStore.updateStatus(buildId, {
+				type: "failure",
+				data: { reason: `Failed to start kaniko: ${error.message}` },
+			});
+			reject(error);
+		});
+	});
+}
+
+