Refactor image handling in OpenAI image generation and update sample script to use new model

Author: pelikhan

packages/core/src/bufferlike.ts

@@ -78,9 +78,9 @@ export async function resolveBufferLikeAndExt(
  * @param mime - Optional MIME type override. If not provided, the MIME type will be inferred from the buffer, or defaults to "application/octet-stream".
  * @returns A Blob object constructed from the input data.
  */
-export async function BufferToBlob(buffer: Buffer | Uint8Array, mime?: string) {
+export async function BufferToBlob(buffer: Buffer | Uint8Array, mime?: string): Promise<Blob> {
   const type = await fileTypeFromBuffer(buffer);
-  return new Blob([buffer], {
+  return new Blob([buffer as any], {
     type: mime || type?.mime || "application/octet-stream",
   });
 }

packages/core/src/fetch.ts

@@ -126,7 +126,9 @@ export async function createFetch(
   const crossFetchWithProxy: typeof fetch = (url, opts) => {
     const requestInit = deleteUndefinedValues({ signal, agent, ...(opts || {}) });
     dbg(`%s %s`, opts?.method || "GET", url);
-    return crossFetch(url, requestInit);
+    dbg(`content-type: %s`, (opts?.headers as any)?.["Content-Type"] as string);
+    if (opts?.body instanceof FormData) return globalThis.fetch(url, requestInit);
+    else return crossFetch(url, requestInit);
   };
 
   // Return the default fetch if no retry status codes are specified

packages/core/src/fetchtext.ts

@@ -183,6 +183,8 @@ export function traceFetchPost(
       }
     });
     httpRequest += `--${boundary}--\n`;
+  } else if (body === "string") {
+    httpRequest += "\n" + body;
   } else {
     httpRequest += "\n" + JSON.stringify(body, null, 2);
   }

packages/core/src/llmsdata.ts

@@ -78,7 +78,7 @@ export default {
         reasoning_small: "o3-mini",
         transcription: "whisper-1",
         speech: "tts-1",
-        image: "dall-e-3",
+        image: "gpt-image-1",
         intent: "gpt-4.1-mini",
       },
       models: {

packages/core/src/openai.ts

@@ -37,7 +37,7 @@ import { traceFetchPost } from "./fetchtext.js";
 import { genaiscriptDebug } from "./debug.js";
 import { OpenAIv2ResponsesChatCompletion } from "./openai-responses.js";
 import type { LanguageModelInfo, RetryOptions, TranscriptionResult } from "./types.js";
-import { resolveBufferLike } from "./bufferlike.js";
+import { BufferToBlob, resolveBufferLike } from "./bufferlike.js";
 import { getConfigHeaders, OpenAIv1ChatCompletion } from "./openai-chatcompletion.js";
 
 const dbg = genaiscriptDebug("openai");
@@ -336,7 +336,7 @@ export async function OpenAIImageGeneration(
 
   if (isMultipart) {
     // Use FormData for image uploads
-    body = new FormData();
+    const form = (body = new FormData());
 
     // Add the image file
     const imageBuffer = await resolveBufferLike(image);
@@ -346,43 +346,43 @@ export async function OpenAIImageGeneration(
         error: serializeError(new Error("Failed to resolve image buffer")),
       };
     }
-    body.append("image", new Blob([imageBuffer], { type: "image/png" }), "image.png");
+    form.append("image", await BufferToBlob(imageBuffer, "image/png"), "image.png");
 
     // Add mask if provided (only for edit mode)
     if (mode === "edit" && mask) {
       const maskBuffer = await resolveBufferLike(mask);
       if (maskBuffer) {
-        body.append("mask", new Blob([maskBuffer], { type: "image/png" }), "mask.png");
+        form.append("mask", await BufferToBlob(maskBuffer, "image/png"), "mask.png");
       }
     }
 
     // Add model
-    body.append("model", model);
+    form.append("model", model);
 
     // Add prompt (required for edit mode)
     if (mode === "edit") {
-      body.append("prompt", prompt);
+      form.append("prompt", prompt);
     }
 
     // Add processed parameters
     if (shouldIncludeSize) {
-      body.append("size", processedParams.size);
+      form.append("size", processedParams.size);
     }
 
     if (shouldIncludeQuality) {
-      body.append("quality", processedParams.quality);
+      form.append("quality", processedParams.quality);
     }
 
     if (shouldIncludeStyle) {
-      body.append("style", processedParams.style);
+      form.append("style", processedParams.style);
     }
 
     if (shouldIncludeOutputFormat) {
-      body.append("output_format", processedParams.outputFormat);
+      form.append("output_format", processedParams.outputFormat);
     }
 
     // Always request b64_json for response format
-    body.append("response_format", "b64_json");
+    if (isDallE) form.append("response_format", "b64_json");
 
     // Don't set Content-Type header for FormData, let the browser set it with boundary
     delete headers["Content-Type"];
@@ -416,9 +416,7 @@ export async function OpenAIImageGeneration(
     }
 
     headers["Content-Type"] = "application/json";
-    body = JSON.stringify(body);
   }
-
   dbg("%o", {
     mode,
     endpoint,
@@ -440,13 +438,12 @@ export async function OpenAIImageGeneration(
     const freq = {
       method: "POST",
       headers,
-      body,
+      body: isMultipart ? body : JSON.stringify(body),
     };
 
     trace?.itemValue(`url`, `[${url}](${url})`);
-    if (!isMultipart) {
-      traceFetchPost(trace, url, freq.headers, JSON.parse(body));
-    }
+
+    traceFetchPost(trace, url, freq.headers, body);
 
     const res = await fetch(url, freq as any);
     dbg(`response: %d %s`, res.status, res.statusText);

samples/sample/genaisrc/image-edit-advanced.genai.mjs

@@ -6,45 +6,17 @@ script({
   description: "Demonstrates advanced image editing capabilities with creative transformations",
   group: "Image Generation"
 })
+const { output } = env
 
 // Edit the robot image with specific instructions
 const { image } = await generateImage(
   "Add a colorful rainbow in the background and make the robot look more cheerful",
   {
     mode: "edit",
     image: "src/robots.jpg",
-    model: "openai:gpt-image-1",
     quality: "high",
     size: "square"
   }
 )
 
-console.log(`🤖 Generated edited robot: ${image.filename}`)
-
-// Edit the mushroom image with artistic style
-const { image: landscapeEdit } = await generateImage(
-  "Transform this into a magical fairy tale scene with glowing lights and enchanted atmosphere",
-  {
-    mode: "edit",
-    image: "src/images/Little Mushroom in the Grass.jpg",
-    model: "openai:gpt-image-1",
-    quality: "medium",
-    size: "landscape"
-  }
-)
-
-console.log(`🍄 Generated magical mushroom scene: ${landscapeEdit.filename}`)
-
-// Edit the galaxy image with sci-fi elements
-const { image: galaxyEdit } = await generateImage(
-  "Add futuristic spaceships and nebula effects to create an epic space battle scene",
-  {
-    mode: "edit",
-    image: "src/vision/galaxy.jpg",
-    model: "openai:gpt-image-1",
-    quality: "high",
-    size: "portrait"
-  }
-)
-
-console.log(`🌌 Generated space battle scene: ${galaxyEdit.filename}`)
+output.item(`🤖 Generated edited robot: ${image.filename}`)