Web-base64 (#1822)

pelikhan · web-flow · commit a12e8a08d5a3 · 2025-08-04T00:19:29.000-07:00
* Add @jsonjoy.com/base64 package and update related imports; improve error logging in various modules

* Remove @jsonjoy.com/base64 package and replace its usage with base64-js; update related imports and dependencies

* Update serve:web script to include -provider github option

* Update serve:cli script to include -provider github option

* Update serve:cli script to use --provider option; enhance typecheck script with --explainFiles flag

* chore: update @modelcontextprotocol/sdk to version 1.17.1
diff --git a/.gitignore b/.gitignore
@@ -77,6 +77,8 @@ dev-proxy-ca.crt
 .clinerules.bak
 .codex/config.toml
 .codex/config.toml.bak
+.crush.json
+.crush.json.bak
 .cursor/mcp.json.bak
 .cursor/rules/ruler_cursor_instructions.mdc
 .cursor/rules/ruler_cursor_instructions.mdc.bak
@@ -104,10 +106,14 @@ dev-proxy-ca.crt
 .vscode/settings.json.bak
 .windsurf/rules/ruler_windsurf_instructions.md
 .windsurf/rules/ruler_windsurf_instructions.md.bak
+AGENT.md
+AGENT.md.bak
 AGENTS.md
 AGENTS.md.bak
 CLAUDE.md
 CLAUDE.md.bak
+CRUSH.md
+CRUSH.md.bak
 GEMINI.md
 GEMINI.md.bak
 opencode.json
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -161,6 +161,7 @@
         "jaegertracing",
         "Jamba",
         "jinja",
+        "jsonjoy",
         "jsonlinecache",
         "JSONLLM",
         "labeli",
diff --git a/docs/package.json b/docs/package.json
@@ -25,7 +25,7 @@
   "dependencies": {
     "@astrojs/check": "^0.9.4",
     "@astrojs/starlight": "0.34.4",
-    "astro": "^5.12.5",
+    "astro": "^5.12.8",
     "astro-embed": "^0.9.0",
     "rehype-mermaid": "^3.0.0",
     "starlight-blog": "^0.24.0",
diff --git a/docs/public/genaiscript.d.ts b/docs/public/genaiscript.d.ts
diff --git a/package.json b/package.json
@@ -84,7 +84,7 @@
     "run:script": "cd samples/sample/ && pnpm run:script",
     "run:script:devproxy": "cd samples/sample/ && HTTPS_PROXY=http://127.0.0.1:8000 NODE_TLS_REJECT_UNAUTHORIZED=0 DEBUG=http,https,net,script,genaiscript:fetch*,proxy-agent pnpm run:script",
     "serve": "pnpm build:cli && run-p serve:*",
-    "serve:cli": "node --watch --watch-path=packages/cli/dist packages/cli/dist/src/index.js serve --dispatch-progress",
+    "serve:cli": "node --watch --watch-path=packages/cli/dist packages/cli/dist/src/index.js serve --dispatch-progress --provider github",
     "serve:web": "pnpm --filter=@genaiscript/web watch",
     "serve:webapi": "node --watch --watch-path=packages/cli/dist packages/cli/dist/src/index.js webapi --network --cors \"*\"",
     "setup": "git submodule update --init --recursive",
@@ -121,11 +121,11 @@
   },
   "devDependencies": {
     "@inquirer/prompts": "catalog:",
-    "@intellectronica/ruler": "^0.2.17",
+    "@intellectronica/ruler": "^0.2.19",
     "@types/node": "catalog:",
     "npm-check-updates": "^18.0.2",
     "npm-run-all": "^4.1.5",
-    "playwright": "^1.54.1",
+    "playwright": "^1.54.2",
     "prettier": "catalog:",
     "prettier-plugin-curly": "^0.3.2",
     "release-it": "^19.0.4",
diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -35,14 +35,14 @@
   "license": "MIT",
   "dependencies": {
     "@actions/core": "^1.11.1",
-    "@fastify/cors": "^11.0.1",
+    "@fastify/cors": "^11.1.0",
     "@fastify/swagger": "^9.5.1",
     "@fastify/swagger-ui": "^5.2.3",
     "@genaiscript/api": "workspace:*",
     "@genaiscript/core": "workspace:*",
     "@genaiscript/runtime": "workspace:*",
     "@inquirer/prompts": "catalog:",
-    "@modelcontextprotocol/sdk": "^1.17.0",
+    "@modelcontextprotocol/sdk": "^1.17.1",
     "chokidar": "^4.0.3",
     "commander": "^14.0.0",
     "debug": "catalog:",
@@ -51,7 +51,7 @@
     "execa": "catalog:",
     "fastify": "^5.4.0",
     "inflection": "catalog:",
-    "openai": "^5.10.2",
+    "openai": "^5.11.0",
     "replace-ext": "^2.0.0",
     "tslib": "catalog:",
     "typescript": "catalog:",
diff --git a/packages/core/package.json b/packages/core/package.json
@@ -54,8 +54,8 @@
     "@azure/identity": "^4.10.2",
     "@azure/search-documents": "^12.1.0",
     "@huggingface/jinja": "^0.5.1",
-    "@modelcontextprotocol/sdk": "^1.17.0",
-    "@napi-rs/canvas": "^0.1.74",
+    "@modelcontextprotocol/sdk": "^1.17.1",
+    "@napi-rs/canvas": "^0.1.76",
     "@octokit/core": "^7.0.3",
     "@octokit/plugin-paginate-rest": "^13.1.1",
     "@octokit/plugin-retry": "^8.0.1",
@@ -89,7 +89,7 @@
     "json5": "^2.2.3",
     "jsonrepair": "^3.13.0",
     "magic-string": "^0.30.17",
-    "mammoth": "^1.9.1",
+    "mammoth": "^1.10.0",
     "mathjs": "^14.6.0",
     "merge-descriptors": "^2.0.0",
     "mime": "^4.0.7",
@@ -100,7 +100,7 @@
     "node-sarif-builder": "^3.2.0",
     "object-inspect": "^1.13.4",
     "octokit": "^5.0.3",
-    "openai": "^5.10.2",
+    "openai": "^5.11.0",
     "p-limit": "^6.2.0",
     "package-manager-detector": "^1.3.0",
     "parse-diff": "^0.11.1",
@@ -121,7 +121,7 @@
     "tsx": "catalog:",
     "turndown": "^7.2.0",
     "turndown-plugin-gfm": "^1.0.2",
-    "undici": "^7.12.0",
+    "undici": "^7.13.0",
     "unified": "^11.0.5",
     "unist-util-visit": "^5.0.0",
     "uuid": "^11.1.0",
diff --git a/packages/core/src/base64.ts b/packages/core/src/base64.ts
@@ -11,7 +11,7 @@ export function fromBase64(base64: string): Uint8Array {
   if (typeof base64 !== "string" || !/^[A-Za-z0-9+/=\s]+$/.test(base64)) {
     throw new Error("Input is not a valid base64 string");
   }
-  return new Uint8Array(Buffer.from(base64, "base64"));
+  return Buffer.from(base64, "base64");
 }
 
 /**
diff --git a/packages/core/src/fetchtext.ts b/packages/core/src/fetchtext.ts
@@ -78,7 +78,7 @@ export async function fetchText(
     try {
       bytes = await runtimeHost.readFile(url);
     } catch (e) {
-      logVerbose(e);
+      dbg(`failed to read file`);
       ok = false;
       status = 404;
     }
diff --git a/packages/core/src/fileedits.ts b/packages/core/src/fileedits.ts
@@ -104,7 +104,7 @@ export async function computeFileEdits(
                   val,
                 )) ?? val;
           } catch (e) {
-            logVerbose(e);
+            logVerbose(errorMessage(e));
             trace?.error(`error custom merging diff in ${fn}`, e);
           }
         } else fileEdit.after = val;
@@ -113,12 +113,12 @@ export async function computeFileEdits(
         try {
           fileEdit.after = applyLLMPatch(fileEdit.after || fileEdit.before, chunks);
         } catch (e) {
-          logVerbose(e);
+          logVerbose(errorMessage(e));
           trace?.error(`error applying patch to ${fn}`, e);
           try {
             fileEdit.after = applyLLMDiff(fileEdit.after || fileEdit.before, chunks);
           } catch (e) {
-            logVerbose(e);
+            logVerbose(errorMessage(e));
             trace?.error(`error merging diff in ${fn}`, e);
           }
         }
diff --git a/packages/core/src/server/messages.ts b/packages/core/src/server/messages.ts
@@ -152,10 +152,10 @@ export interface PromptScriptRunOptions {
   ignoreGitIgnore: boolean;
   runRetry: string;
   out: string;
-  retry: string;
-  retryDelay: string;
-  maxDelay: string;
-  maxRetryAfter: string;
+  retry: string | number;
+  retryDelay: string | number;
+  maxDelay: string | number;
+  maxRetryAfter: string | number;
   json: boolean;
   outTrace: string;
   outOutput: string;
diff --git a/packages/core/src/xlsx.ts b/packages/core/src/xlsx.ts
@@ -2,8 +2,9 @@
 // Licensed under the MIT License.
 
 // Import the logInfo function for logging purposes
-import { logInfo } from "./util.js";
 import type { ParseXLSXOptions, WorkbookSheet } from "./types.js";
+import { genaiscriptDebug } from "./debug.js";
+const dbg = genaiscriptDebug("xlsx")
 
 /**
  * Parses XLSX data into an array of workbook sheets.
@@ -49,7 +50,7 @@ export async function XLSXTryParse(
     return await XLSXParse(data, options);
   } catch (e) {
     // Log any errors encountered during parsing
-    logInfo(e);
+    dbg(`parser error %O`, e);
     // Return an empty array if parsing fails
     return [];
   }
diff --git a/packages/core/src/xml.ts b/packages/core/src/xml.ts
@@ -25,7 +25,7 @@ export async function XMLTryParse(
     return (await XMLParse(text, options)) ?? defaultValue;
   } catch (e) {
     // Return the default value if parsing fails
-    dbg(`error: %s`, e?.message);
+    dbg(`error: %O`, e);
     return defaultValue;
   }
 }
diff --git a/packages/plugin-playwright/package.json b/packages/plugin-playwright/package.json
@@ -16,7 +16,7 @@
   "dependencies": {
     "@genaiscript/core": "workspace:*",
     "@genaiscript/runtime": "workspace:*",
-    "playwright": "^1.54.1"
+    "playwright": "^1.54.2"
   },
   "devDependencies": {
     "@genaiscript/eslint-plugin-genaiscript": "workspace:*",
diff --git a/packages/web/package.json b/packages/web/package.json
@@ -4,15 +4,15 @@
   "license": "MIT",
   "private": true,
   "scripts": {
-    "build": "npm run clean && node build.mjs",
+    "build": "node build.mjs",
     "clean": "rimraf dist",
     "format:check": "prettier --config ../../.prettierrc.json --ignore-path ../../.prettierignore --check  \"src/**/*.{ts,cts,mts,tsx,css}\" \"*.{js,cjs,mjs,json}\" ",
     "format:fix": "prettier --config ../../.prettierrc.json  --ignore-path ../../.prettierignore  --write \"src/**/*.{ts,cts,mts,tsx,css}\" \"*.{js,cjs,mjs,json}\" ",
     "lint:check": "eslint src",
     "lint:fix": "eslint src --fix --fix-type [problem,suggestion]",
     "watch": "node --watch --watch-path=./src --watch-path=../core/src build.mjs --dispatch-progress",
     "serve": "serve -n .",
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsc --noEmit --explainFiles"
   },
   "devDependencies": {
     "@daiji256/rehype-mathml": "^1.2.1",
@@ -24,6 +24,7 @@
     "@vscode-elements/elements": "1.16.0",
     "@vscode-elements/webview-playground": "1.6.0",
     "@vscode/codicons": "^0.0.36",
+    "base64-js": "^1.5.1",
     "clsx": "^2.1.1",
     "dedent": "^1.6.0",
     "es-toolkit": "catalog:",
@@ -35,6 +36,7 @@
     "globals": "^16.3.0",
     "html-escaper": "3.0.3",
     "inflection": "catalog:",
+    "marked": "^16.1.2",
     "mermaid": "^11.9.0",
     "nanoid": "^5.1.5",
     "prettier": "catalog:",
diff --git a/packages/web/src/ApiContext.tsx b/packages/web/src/ApiContext.tsx
@@ -9,6 +9,7 @@ import { fetchEnv, fetchScripts, fetchModels } from "./api";
 import { useUrlSearchParams } from "./useUrlSearchParam";
 import { useScriptId } from "./ScriptContext";
 import type { ImportedFile } from "./types";
+import type { ModelConnectionOptions, ModelOptions, PromptParameters } from "../../core/src/types";
 
 export const ApiContext = createContext<{
   project: Promise<Project | undefined>;
diff --git a/packages/web/src/JSONSchema.tsx b/packages/web/src/JSONSchema.tsx
@@ -7,6 +7,8 @@ import type {
   JSONSchemaSimpleType,
   JSONSchemaString,
   JSONSchemaNumber,
+  JSONSchemaBoolean,
+  JSONSchemaObject,
 } from "../../core/src/types";
 
 import { underscore } from "inflection";
diff --git a/packages/web/src/Results.tsx b/packages/web/src/Results.tsx
@@ -38,6 +38,7 @@ import { useResult, useTrace, useRunner, useOutput } from "./RunnerContext";
 import { useEventListener } from "./useEventListener";
 import dedent from "dedent";
 import { stringify as YAMLStringify } from "yaml";
+import type { Logprob } from "../../core/src/types";
 
 import "@vscode-elements/elements/dist/vscode-tabs";
 import "@vscode-elements/elements/dist/vscode-tab-header";
diff --git a/packages/web/src/RunnerContext.tsx b/packages/web/src/RunnerContext.tsx
@@ -4,7 +4,6 @@
 import React from "react";
 import { createContext, startTransition, use, useCallback, useEffect, useState } from "react";
 import type { GenerationResult, PromptScriptRunOptions } from "../../core/src/server/messages";
-import { toBase64 } from "../../core/src/base64";
 import { isBinaryMimeType } from "../../core/src/binary";
 import { fetchRun } from "./api";
 import { RunClient } from "./RunClient";
@@ -14,6 +13,8 @@ import { useLocationHashValue } from "./useLocationHashValue";
 import { useScriptId } from "./ScriptContext";
 import type { ImportedFile } from "./types";
 import { generateId } from "../../core/src/id";
+import type { PromptParameters, WorkspaceFile } from "../../core/src/types";
+import { fromByteArray } from "base64-js";
 
 export const RunnerContext = createContext<{
   runId: string | undefined;
@@ -92,7 +93,7 @@ export function RunnerProvider({ children }: { children: React.ReactNode }) {
         .map(async (f) => {
           const binary = isBinaryMimeType(f.type);
           const buffer = binary ? new Uint8Array(await f.arrayBuffer()) : undefined;
-          const content = buffer ? toBase64(buffer) : await f.text();
+          const content = buffer ? fromByteArray(buffer) : await f.text();
           return {
             filename: f.path || f.relativePath,
             type: f.type,
diff --git a/packages/web/src/useUrlSearchParam.ts b/packages/web/src/useUrlSearchParam.ts
@@ -3,6 +3,12 @@
 
 import { useState, useEffect } from "react";
 import { urlParams } from "./configuration";
+import type {
+  JSONSchemaString,
+  JSONSchemaNumber,
+  JSONSchemaBoolean,
+  JSONSchemaArray,
+} from "../../core/src/types";
 
 export function useUrlSearchParams<T>(
   initialValues: T,
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/samples/sample/package.json b/samples/sample/package.json

Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,7 @@ export function fromBase64(base64: string): Uint8Array {`
`11`	`11`	`if (typeof base64 !== "string" \|\| !/^[A-Za-z0-9+/=\s]+$/.test(base64)) {`
`12`	`12`	`throw new Error("Input is not a valid base64 string");`
`13`	`13`	`}`
`14`		`- return new Uint8Array(Buffer.from(base64, "base64"));`
	`14`	`+ return Buffer.from(base64, "base64");`
`15`	`15`	`}`
`16`	`16`
`17`	`17`	`/**`
Original file line number	Diff line number	Diff line change
`@@ -78,7 +78,7 @@ export async function fetchText(`
`78`	`78`	`try {`
`79`	`79`	`bytes = await runtimeHost.readFile(url);`
`80`	`80`	`} catch (e) {`
`81`		`- logVerbose(e);`
	`81`	+ dbg(`failed to read file`);
`82`	`82`	`ok = false;`
`83`	`83`	`status = 404;`
`84`	`84`	`}`