Add embedding gateway override for semantic search (#697)

Co-authored-by: Cursor Agent <cursoragent@cursor.com>

Author: kentcdodds

.env.example

@@ -105,6 +105,9 @@ CLOUDFLARE_API_TOKEN=MOCK_CLOUDFLARE_API_TOKEN
 CLOUDFLARE_VECTORIZE_INDEX=MOCK_CLOUDFLARE_VECTORIZE_INDEX
 # Route Workers AI requests through Cloudflare AI Gateway (gateway name/id)
 CLOUDFLARE_AI_GATEWAY_ID=MOCK_CLOUDFLARE_AI_GATEWAY_ID
+# Optional: route embeddings through a different AI Gateway (for example,
+# without guardrails). Falls back to CLOUDFLARE_AI_GATEWAY_ID when omitted.
+CLOUDFLARE_AI_EMBEDDING_GATEWAY_ID=MOCK_CLOUDFLARE_AI_EMBEDDING_GATEWAY_ID
 # AI Gateway Authenticated Gateway token (sent as `cf-aig-authorization`)
 CLOUDFLARE_AI_GATEWAY_AUTH_TOKEN=MOCK_CLOUDFLARE_AI_GATEWAY_AUTH_TOKEN
 # Text model used for general generation (defaults to @cf/meta/llama-3.1-8b-instruct)

app/utils/env.server.ts

@@ -67,6 +67,11 @@ const schemaBase = z.object({
 	CLOUDFLARE_API_TOKEN: nonEmptyString,
 	/** AI Gateway "id" is the gateway name you create in Cloudflare. */
 	CLOUDFLARE_AI_GATEWAY_ID: nonEmptyString,
+	/**
+	 * Optional embedding-specific AI Gateway id.
+	 * Falls back to `CLOUDFLARE_AI_GATEWAY_ID` when omitted.
+	 */
+	CLOUDFLARE_AI_EMBEDDING_GATEWAY_ID: z.string().trim().optional(),
 	/**
 	 * AI Gateway authenticated gateway token (used as `cf-aig-authorization`).
 	 */
@@ -154,7 +159,11 @@ type BaseEnvInput = z.input<typeof schemaBase>
 
 export type Env = Omit<
 	BaseEnv,
-	'PORT' | 'MOCKS' | 'DATABASE_PATH' | 'FLY_MACHINE_ID'
+	| 'PORT'
+	| 'MOCKS'
+	| 'DATABASE_PATH'
+	| 'FLY_MACHINE_ID'
+	| 'CLOUDFLARE_AI_EMBEDDING_GATEWAY_ID'
 > & {
 	PORT: number
 	MOCKS: boolean
@@ -173,6 +182,11 @@ export type Env = Omit<
 	 * Used to format generated Call Kent transcripts into readable paragraphs.
 	 */
 	CLOUDFLARE_AI_CALL_KENT_TRANSCRIPT_FORMAT_MODEL: string
+	/**
+	 * Embeddings can be routed through a separate gateway (for example, with
+	 * guardrails disabled) without affecting other AI routes.
+	 */
+	CLOUDFLARE_AI_EMBEDDING_GATEWAY_ID: string
 	/** Derived from CLOUDFLARE_ACCOUNT_ID when not explicitly set. */
 	R2_ENDPOINT: string
 }
@@ -241,6 +255,9 @@ export function getEnv(): Env {
 		allowedActionOrigins: computeAllowedActionOrigins(values),
 		FLY_MACHINE_ID: values.FLY_MACHINE_ID ?? 'unknown',
 		R2_ENDPOINT: derivedR2Endpoint,
+		CLOUDFLARE_AI_EMBEDDING_GATEWAY_ID:
+			values.CLOUDFLARE_AI_EMBEDDING_GATEWAY_ID ||
+			values.CLOUDFLARE_AI_GATEWAY_ID,
 		CLOUDFLARE_AI_CALL_KENT_METADATA_MODEL:
 			values.CLOUDFLARE_AI_CALL_KENT_METADATA_MODEL ??
 			values.CLOUDFLARE_AI_TEXT_MODEL,

app/utils/semantic-search.server.ts

@@ -178,7 +178,7 @@ function getRequiredSemanticSearchEnv() {
 	return {
 		accountId: env.CLOUDFLARE_ACCOUNT_ID,
 		apiToken: env.CLOUDFLARE_API_TOKEN,
-		gatewayId: env.CLOUDFLARE_AI_GATEWAY_ID,
+		embeddingGatewayId: env.CLOUDFLARE_AI_EMBEDDING_GATEWAY_ID,
 		gatewayAuthToken: env.CLOUDFLARE_AI_GATEWAY_AUTH_TOKEN,
 		indexName: env.CLOUDFLARE_VECTORIZE_INDEX,
 		embeddingModel: env.CLOUDFLARE_AI_EMBEDDING_MODEL,
@@ -221,19 +221,23 @@ async function cloudflareFetch(
 async function getEmbedding({
 	accountId,
 	apiToken,
-	gatewayId,
+	embeddingGatewayId,
 	gatewayAuthToken,
 	model,
 	text,
 }: {
 	accountId: string
 	apiToken: string
-	gatewayId: string
+	embeddingGatewayId: string
 	gatewayAuthToken: string
 	model: string
 	text: string
 }) {
-	const url = getWorkersAiRunUrl({ accountId, gatewayId, model })
+	const url = getWorkersAiRunUrl({
+		accountId,
+		gatewayId: embeddingGatewayId,
+		model,
+	})
 	const res = await fetch(url, {
 		method: 'POST',
 		headers: {
@@ -492,7 +496,7 @@ export async function semanticSearchKCD({
 	const {
 		accountId,
 		apiToken,
-		gatewayId,
+		embeddingGatewayId,
 		gatewayAuthToken,
 		indexName,
 		embeddingModel,
@@ -529,7 +533,7 @@ export async function semanticSearchKCD({
 			const vector = await getEmbedding({
 				accountId,
 				apiToken,
-				gatewayId,
+				embeddingGatewayId,
 				gatewayAuthToken,
 				model: embeddingModel,
 				text: cleanedQuery,

other/semantic-search/cloudflare.ts

@@ -14,10 +14,15 @@ function getRequiredEnv(name: string) {
 }
 
 export function getCloudflareConfig() {
+	const defaultGatewayId = getRequiredEnv('CLOUDFLARE_AI_GATEWAY_ID')
+	const embeddingGatewayId =
+		process.env.CLOUDFLARE_AI_EMBEDDING_GATEWAY_ID?.trim() || defaultGatewayId
+
 	return {
 		accountId: getRequiredEnv('CLOUDFLARE_ACCOUNT_ID'),
 		apiToken: getRequiredEnv('CLOUDFLARE_API_TOKEN'),
-		gatewayId: getRequiredEnv('CLOUDFLARE_AI_GATEWAY_ID'),
+		// Embedding jobs can use a dedicated gateway without guardrails.
+		gatewayId: embeddingGatewayId,
 		gatewayAuthToken: getRequiredEnv('CLOUDFLARE_AI_GATEWAY_AUTH_TOKEN'),
 		vectorizeIndex: getRequiredEnv('CLOUDFLARE_VECTORIZE_INDEX'),
 		embeddingModel:

other/semantic-search/readme.md

@@ -24,6 +24,7 @@ and shared utilities.
 - `CLOUDFLARE_API_TOKEN`
 - `CLOUDFLARE_VECTORIZE_INDEX`
 - `CLOUDFLARE_AI_EMBEDDING_MODEL` (optional; defaults in code)
+- `CLOUDFLARE_AI_EMBEDDING_GATEWAY_ID` (optional; defaults to `CLOUDFLARE_AI_GATEWAY_ID`)
 
 - `R2_BUCKET`