feat: measure remote modeling latency and consider bulk operations

[angeloashmore]

Summary

Each field operation makes a network call to the Custom Types API. Need to test round-trip latency in agent workflows and determine if a field add-bulk command is warranted.

API performance is reportedly strong, but agent workflows that add many fields sequentially could be slow. A bulk command introduces new challenges (input format, discoverability of options) so this should only be pursued if latency is a real problem.

Raised in RFC: Prismic CLI Modeling.

[angeloashmore]

Benchmark Results

Benchmarked a typical modeling workflow (10 iterations each, measured via Vitest bench against a live Prismic repo):

Per-operation latency

Operation	Mean	Min	Max
type create (page)	786ms	654ms	1,652ms
slice create	685ms	646ms	782ms
field add boolean	786ms	730ms	905ms
field add rich-text	809ms	760ms	923ms
field add image	968ms	761ms	2,103ms
slice connect	1,057ms	986ms	1,252ms
full workflow	4,900ms	4,704ms	5,168ms

Full workflow = create page type + create slice + add 3 fields + connect slice.

Overhead breakdown

Layer	Mean	% of field-add
Subprocess spawn (Node startup)	58ms	7%
Network: GET + POST round-trip	602ms	77%
Local work (adapter I/O, type generation)	~126ms	16%

Network round-trip dominates. Each field-add does 1 GET (fetch latest model) + 1 POST (push update), averaging ~300ms each.

Optimization opportunities

1. Batch mutations — the biggest win. Adding 10 fields currently means 20 network round-trips (~6s of network time). Batching into 1 GET + mutate + 1 POST would reduce that to ~600ms. This makes a strong case for a bulk command.
2. Defer type generation — generateTypes() runs after every operation (re-globs all models, runs codegen, writes prismicio-types.d.ts). A --no-types flag would let agents skip this on intermediate operations and run it once at the end.
3. Reduce redundant file reads — within a single CLI invocation, prismic.config.json is read 3 times and slice/type model directories are globbed twice. Memoizing these would trim local work.

[angeloashmore]

Benchmark file (test/modeling.bench.ts)

Run with node --run build && npx vitest bench test/modeling.bench.ts --project concurrent.

Requires the same E2E credentials as the test suite (.env.test.local).

import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { x } from "tinyexec";
import { bench, describe, inject } from "vitest";

import { buildCustomType, buildSlice } from "./it";
import { insertCustomType, insertSlice } from "./prismic";

const BIN = fileURLToPath(new URL("../dist/index.mjs", import.meta.url));

const token = inject("token");
const repo = inject("repo");
const host = process.env.PRISMIC_HOST ?? "prismic.io";

// Create temp directory structure (same as test fixtures)
const dir = await mkdtemp(join(tmpdir(), "prismic-bench-"));
const home = pathToFileURL(dir + "/");
const project = new URL("project/", home);
await mkdir(project, { recursive: true });

// Stub npm
const binDir = new URL("bin/", home);
await mkdir(binDir);
const lockfilePath = fileURLToPath(new URL("package-lock.json", project));
await writeFile(
	new URL("npm", binDir),
	`#!/bin/sh\necho '{}' > "${lockfilePath}"\necho "added 0 packages"\n`,
	{ mode: 0o755 },
);

// Stub Next.js
await writeFile(
	new URL("package.json", project),
	JSON.stringify({ dependencies: { next: "latest" } }),
);
const nextDir = new URL("node_modules/next/", project);
await mkdir(nextDir, { recursive: true });
await writeFile(new URL("package.json", nextDir), JSON.stringify({ version: "16.0.0" }));
await mkdir(new URL("app/", project));

// Write config and credentials
await writeFile(
	new URL("prismic.config.json", project),
	JSON.stringify({ repositoryName: repo }),
);
const configDir = new URL(".config/prismic/", home);
await mkdir(configDir, { recursive: true });
await writeFile(new URL("credentials.json", configDir), JSON.stringify({ token, host }));

// Pre-create a slice for field-add benches
const targetSlice = buildSlice();
await insertSlice(targetSlice, { repo, token, host });

// Pre-create a page type with slice zone for connect bench
const connectTargetType = buildCustomType({
	format: "page",
	json: {
		Main: {
			slices: {
				type: "Slices",
				fieldset: "Slice Zone",
				config: { choices: {} },
			},
		},
	},
});
await insertCustomType(connectTargetType, { repo, token, host });

// Pre-create slices for connect bench (warmup + iterations)
const connectSliceIds: string[] = [];
for (let i = 0; i < 15; i++) {
	const slice = buildSlice();
	await insertSlice(slice, { repo, token, host });
	connectSliceIds.push(slice.id);
}

function prismic(command: string, args: string[] = []) {
	return x("node", [BIN, command, ...args], {
		nodeOptions: {
			cwd: fileURLToPath(project),
			env: {
				...process.env,
				PRISMIC_TYPE_BUILDER_ENABLED: "true",
				NO_UPDATE_NOTIFIER: "1",
				PRISMIC_CONFIG_DIR: fileURLToPath(configDir),
				PATH: `${fileURLToPath(binDir)}:${process.env.PATH}`,
				HOME: fileURLToPath(home),
			},
		},
	});
}

let counter = 0;
const OPTS = { iterations: 10, warmupIterations: 1 };

// Direct API wrappers (no subprocess) to isolate network latency
async function apiGetSlice(sliceId: string) {
	const url = new URL(`slices/${encodeURIComponent(sliceId)}`, `https://customtypes.${host}/`);
	const res = await fetch(url, {
		headers: { Authorization: `Bearer ${token}`, repository: repo },
	});
	if (!res.ok) throw new Error(`GET slice failed: ${res.status}`);
	return await res.json();
}

async function apiUpdateSlice(slice: object) {
	const url = new URL("slices/update", `https://customtypes.${host}/`);
	const res = await fetch(url, {
		method: "POST",
		headers: {
			"Content-Type": "application/json",
			Authorization: `Bearer ${token}`,
			repository: repo,
		},
		body: JSON.stringify(slice),
	});
	if (!res.ok) throw new Error(`POST slice update failed: ${res.status}`);
}

describe("overhead breakdown", () => {
	bench(
		"subprocess spawn (--help)",
		async () => {
			await prismic("--help");
		},
		OPTS,
	);

	bench(
		"API: GET slice",
		async () => {
			await apiGetSlice(targetSlice.id);
		},
		OPTS,
	);

	bench(
		"API: GET + POST slice (round-trip)",
		async () => {
			const slice = await apiGetSlice(targetSlice.id);
			slice.variations[0].primary[`api_field_${++counter}`] = { type: "Boolean" };
			await apiUpdateSlice(slice);
		},
		OPTS,
	);
});

describe("modeling workflow", () => {
	bench(
		"type create (page)",
		async () => {
			const n = ++counter;
			await prismic("type", ["create", `BenchType${n}`, "--format", "page", "--id", `bench-type-${n}`]);
		},
		OPTS,
	);

	bench(
		"slice create",
		async () => {
			const n = ++counter;
			await prismic("slice", ["create", `BenchSlice${n}`, "--id", `bench-slice-${n}`]);
		},
		OPTS,
	);

	bench(
		"field add boolean",
		async () => {
			await prismic("field", ["add", "boolean", `field_${++counter}`, "--to-slice", targetSlice.id]);
		},
		OPTS,
	);

	bench(
		"field add rich-text",
		async () => {
			await prismic("field", ["add", "rich-text", `field_${++counter}`, "--to-slice", targetSlice.id]);
		},
		OPTS,
	);

	bench(
		"field add image",
		async () => {
			await prismic("field", ["add", "image", `field_${++counter}`, "--to-slice", targetSlice.id]);
		},
		OPTS,
	);

	bench(
		"slice connect",
		async () => {
			const sliceId = connectSliceIds.shift()!;
			await prismic("slice", ["connect", sliceId, "--to", connectTargetType.id]);
		},
		OPTS,
	);

	bench(
		"full workflow",
		async () => {
			const n = ++counter;
			const typeId = `bench-wf-type-${n}`;
			const sliceId = `bench-wf-slice-${n}`;

			await prismic("type", ["create", `BenchWF${n}`, "--format", "page", "--id", typeId]);
			await prismic("slice", ["create", `BenchWFSlice${n}`, "--id", sliceId]);
			await prismic("field", ["add", "boolean", `featured_${n}`, "--to-slice", sliceId]);
			await prismic("field", ["add", "rich-text", `body_${n}`, "--to-slice", sliceId]);
			await prismic("field", ["add", "image", `image_${n}`, "--to-slice", sliceId]);
			await prismic("slice", ["connect", sliceId, "--to", typeId]);
		},
		OPTS,
	);
});