refactor: simplify parser creation and remove unused parameters in prompt context

Author: pelikhan

packages/core/src/globals.ts

@@ -26,6 +26,7 @@ import { resolveGlobal } from "./global.js";
 import { MarkdownStringify } from "./markdown.js";
 import { diffCreatePatch, diffFindChunk, tryDiffParse } from "./diff.js";
 import type { PromptContext } from "./types.js";
+import { createParsers } from "./parsers.js";
 
 let _globalsInstalled = false;
 /**
@@ -55,6 +56,8 @@ export function installGlobals() {
   dbg("install");
   const glb = resolveGlobal(); // Get the global context
 
+  glb.parsers = createParsers();
+
   // Freeze YAML utilities to prevent modification
   glb.YAML = createYAML();
 

packages/core/src/parsers.ts

@@ -4,9 +4,7 @@
 import { CSVTryParse } from "./csv.js";
 import { filenameOrFileToContent, filenameOrFileToFilename, unfence } from "./unwrappers.js";
 import { JSON5TryParse, JSONLLMTryParse } from "./json5.js";
-import { estimateTokens } from "./tokens.js";
 import { TOMLTryParse } from "./toml.js";
-import { TraceOptions } from "./trace.js";
 import { YAMLTryParse } from "./yaml.js";
 import { DOCXTryParse } from "./docx.js";
 import { frontmatterTryParse } from "./frontmatter.js";
@@ -24,15 +22,13 @@ import { host } from "./host.js";
 import { unzip } from "./zip.js";
 import { JSONLTryParse } from "./jsonl.js";
 import { resolveFileContent } from "./file.js";
-import { resolveTokenEncoder } from "./encoders.js";
 import { mustacheRender } from "./mustache.js";
 import { jinjaRender } from "./jinja.js";
 import { llmifyDiff } from "./llmdiff.js";
 import { tidyData } from "./tidy.js";
 import { hash } from "./crypto.js";
 import { GROQEvaluate } from "./groq.js";
 import { unthink } from "./think.js";
-import { CancellationOptions } from "./cancellation.js";
 import { dedent } from "./indent.js";
 import { vttSrtParse } from "./transcription.js";
 import { encodeIDs } from "./cleaners.js";
@@ -84,14 +80,7 @@ import type { Parsers, WorkspaceFile } from "./types.js";
  *   - dedent: Dedents indented text content.
  *   - encodeIDs: Encodes identifiers for use in various operations.
  */
-export async function createParsers(
-  options: {
-    model: string;
-  } & TraceOptions &
-    CancellationOptions,
-): Promise<Parsers> {
-  const { trace, model, cancellationToken } = options;
-  const { encode: encoder } = await resolveTokenEncoder(model);
+export function createParsers(): Parsers {
   return Object.freeze<Parsers>({
     JSON5: (text, options) =>
       tryValidateJSONWithSchema(
@@ -131,31 +120,15 @@ export async function createParsers(
       ),
     transcription: (text) => vttSrtParse(filenameOrFileToContent(text)),
     unzip: async (file, options) => await unzip(await host.readFile(file.filename), options),
-    tokens: (text) => estimateTokens(filenameOrFileToContent(text), encoder),
     fences: (text) => extractFenced(filenameOrFileToContent(text)),
     annotations: (text) => parseAnnotations(filenameOrFileToContent(text)),
-    HTMLToText: (text, options) =>
-      HTMLToText(filenameOrFileToContent(text), {
-        ...(options || {}),
-        trace,
-        cancellationToken,
-      }),
-    HTMLToMarkdown: (text, options) =>
-      HTMLToMarkdown(filenameOrFileToContent(text), {
-        ...(options || {}),
-        trace,
-        cancellationToken,
-      }),
+    HTMLToText: (text, options) => HTMLToText(filenameOrFileToContent(text), options),
+    HTMLToMarkdown: (text, options) => HTMLToMarkdown(filenameOrFileToContent(text), options),
     DOCX: async (file, options) => await DOCXTryParse(file, options),
     PDF: async (file, options) => {
       if (!file) return { file: undefined, pages: [], data: [] };
-      const opts = {
-        ...(options || {}),
-        trace,
-        cancellationToken,
-      };
       const filename = typeof file === "string" ? file : file.filename;
-      const { pages, content } = (await parsePdf(filename, opts)) || {};
+      const { pages, content } = (await parsePdf(filename, options)) || {};
       return {
         file: <WorkspaceFile>{
           filename,
@@ -171,8 +144,8 @@ export async function createParsers(
       const res = await mermaidParse(f);
       return res;
     },
-    math: async (expression, scope) => await MathTryEvaluate(expression, { scope, trace }),
-    validateJSON: (schema, content) => validateJSONWithSchema(content, schema, { trace }),
+    math: async (expression, scope) => await MathTryEvaluate(expression, { scope }),
+    validateJSON: (schema, content) => validateJSONWithSchema(content, schema),
     mustache: (file, args) => {
       const f = filenameOrFileToContent(file);
       return mustacheRender(f, args);
@@ -190,7 +163,7 @@ export async function createParsers(
     dedent: dedent,
     encodeIDs: encodeIDs,
     prompty: async (file) => {
-      await resolveFileContent(file, { trace });
+      await resolveFileContent(file);
       return promptyParse(file.filename, file.content);
     },
   });

packages/core/src/promptcontext.ts

@@ -72,7 +72,6 @@ export async function createPromptContext(
   };
   assert(!!output, "missing output");
   // Create parsers for the given trace and model
-  const parsers = await createParsers({ trace, cancellationToken, model });
   const path = runtimeHost.path;
   const runDir = ev.runDir;
   assert(!!runDir, "missing run directory");
@@ -382,7 +381,6 @@ export async function createPromptContext(
     env: undefined, // set later
     path,
     workspace,
-    parsers,
     retrieval,
     host: promptHost,
   };

packages/core/src/schema.ts

@@ -246,9 +246,10 @@ export function tryValidateJSONWithSchema<T = unknown>(
  * @returns Validation result indicating success status and error details if validation fails.
  */
 export function validateJSONWithSchema(
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
   object: any,
   schema: JSONSchema,
-  options?: { trace: MarkdownTrace },
+  options?: TraceOptions,
 ): FileEditValidation {
   const { trace } = options || {};
   if (!schema)

packages/core/src/types.ts

@@ -2977,12 +2977,6 @@ export interface Parsers {
    */
   unzip(file: WorkspaceFile, options?: ParseZipOptions): Promise<WorkspaceFile[]>;
 
-  /**
-   * Estimates the number of tokens in the content.
-   * @param content content to tokenize
-   */
-  tokens(content: string | WorkspaceFile): number;
-
   /**
    * Parses fenced code sections in a markdown text
    */
@@ -6346,7 +6340,6 @@ export interface PromptContext extends ChatGenerationContext {
   script(options: PromptArgs): void;
   system(options: PromptSystemArgs): void;
   path: Path;
-  parsers: Parsers;
   retrieval: Retrieval;
   workspace: WorkspaceFileSystem;
   host: PromptHost;
@@ -6355,7 +6348,6 @@ export interface PromptContext extends ChatGenerationContext {
 export type RuntimePromptContext = Pick<
   PromptContext,
   | "host"
-  | "parsers"
   | "env"
   | "workspace"
   | "retrieval"

packages/core/src/types/prompt_template.d.ts

@@ -2967,12 +2967,6 @@ interface Parsers {
    */
   unzip(file: WorkspaceFile, options?: ParseZipOptions): Promise<WorkspaceFile[]>;
 
-  /**
-   * Estimates the number of tokens in the content.
-   * @param content content to tokenize
-   */
-  tokens(content: string | WorkspaceFile): number;
-
   /**
    * Parses fenced code sections in a markdown text
    */
@@ -6330,7 +6324,6 @@ interface PromptContext extends ChatGenerationContext {
   script(options: PromptArgs): void;
   system(options: PromptSystemArgs): void;
   path: Path;
-  parsers: Parsers;
   retrieval: Retrieval;
   workspace: WorkspaceFileSystem;
   host: PromptHost;

packages/core/test/parsers.test.ts

@@ -3,22 +3,17 @@
 
 import { describe, test, assert, beforeEach } from "vitest";
 import { createParsers } from "../src/parsers.js";
-import { MarkdownTrace } from "../src/trace.js";
 import { XLSXParse } from "../src/xlsx.js";
 import { readFile } from "fs/promises";
 import { resolve } from "path";
 import { TestHost } from "../src/testhost.js";
 import { writeFile } from "fs/promises";
 
 describe("parsers", async () => {
-  let trace: MarkdownTrace;
-  let model: string;
-  let parsers: Awaited<ReturnType<typeof createParsers>>;
+  let parsers: Parsers;
 
   beforeEach(async () => {
-    trace = new MarkdownTrace({});
-    model = "test model";
-    parsers = await createParsers({ trace, model });
+    parsers = createParsers();
     TestHost.install();
   });
 

packages/runtime/src/runtime.ts

@@ -31,12 +31,6 @@ const dbg = genaiscriptDebug("runtime");
 
 let _nodeHost: NodeHost | undefined;
 
-export function resolveRuntime(): RuntimePromptContext {
-  if (!_nodeHost) throw new Error("Runtime not configured. Call `config` first.");
-  const globalPromptContext: RuntimePromptContext = globalThis as unknown as RuntimePromptContext;
-  return globalPromptContext;
-}
-
 export function resolveChatGenerationContext(
   options?: ChatGenerationContextOptions,
 ): ChatGenerationContext {