chore: developing the package

Author: jortsupetterson

embedder/index.js

@@ -0,0 +1,45 @@
+import { mkdir, readFile, writeFile } from 'node:fs/promises'
+import { dirname } from 'node:path'
+
+const js = String.raw
+
+const modelPath = './models/quantized_models/eng/model.int4.onnx'
+const modelDataPath = './models/quantized_models/eng/model.int4.onnx.data'
+const tokenizerModelPath = './models/quantized_models/eng/tokenizer.model'
+
+const outPath = './src/Model/class.ts'
+
+function toUint8ArraySource(bytes) {
+  return `new Uint8Array([${Array.from(bytes).join(',')}])`
+}
+
+const [model, modelData, tokenizerModel] = await Promise.all([
+  readFile(modelPath),
+  readFile(modelDataPath),
+  readFile(tokenizerModelPath),
+])
+
+const ts = js`
+import * as ort from 'onnxruntime-web'
+import { SentencePieceProcessor } from '@agnai/sentencepiece-js'
+
+export async function createInferenceSession(): Promise<ort.InferenceSession> {
+  return ort.InferenceSession.create(${toUint8ArraySource(model)}, {
+    externalData: [
+      {
+        path: 'model.int4.onnx.data',
+        data: ${toUint8ArraySource(modelData)},
+      },
+    ],
+  })
+}
+
+export async function createTokenProcessor():Promise<SentencePieceProcessor> {
+  const tokenProcessor = new SentencePieceProcessor()
+  await tokenProcessor.load(${toUint8ArraySource(tokenizerModel)})
+  return tokenProcessor
+}
+`.trimStart()
+
+await mkdir(dirname(outPath), { recursive: true })
+await writeFile(outPath, ts, 'utf8')

src/.types/index.d.ts

@@ -0,0 +1,58 @@
+declare module '@agnai/sentencepiece-js' {
+  export interface SentencePieceStatus {
+    delete(): void
+  }
+
+  export interface SentencePieceVector<T> {
+    size(): number
+    get(index: number): T
+    delete(): void
+  }
+
+  export interface SentencePieceStringViewHandle {
+    delete(): void
+  }
+
+  export interface SentencePieceStringView {
+    getView(): SentencePieceStringViewHandle
+    delete(): void
+  }
+
+  export interface SentencePieceProcessorBinding {
+    Load(model: SentencePieceStringViewHandle): SentencePieceStatus
+    EncodeAsIds(text: SentencePieceStringViewHandle): SentencePieceVector<number>
+    EncodeAsPieces(text: SentencePieceStringViewHandle): SentencePieceVector<string>
+    DecodeIds(ids: SentencePieceVector<number>): string
+    LoadVocabulary(vocab: SentencePieceStringViewHandle, threshold: number): SentencePieceStatus
+  }
+
+  export interface SentencePieceModule {
+    FS: {
+      writeFile(path: string, data: string | Uint8Array | ArrayBufferLike): void
+    }
+    StringView: new (value: string) => SentencePieceStringView
+    SentencePieceProcessor: new () => SentencePieceProcessorBinding
+    vecFromJSArray(values: readonly number[]): SentencePieceVector<number>
+  }
+
+  export class SentencePieceProcessor {
+    sentencepiece?: SentencePieceModule
+    processor?: SentencePieceProcessorBinding
+
+    constructor()
+    load(url: string): Promise<void>
+    encodeIds(text: string): number[]
+    encodePieces(text: string): string[]
+    decodeIds(ids: readonly number[]): string
+    loadVocabulary(url: string): void
+  }
+
+  export function cleanText(text: string): string
+
+  const defaultExport: {
+    SentencePieceProcessor: typeof SentencePieceProcessor
+    cleanText: typeof cleanText
+  }
+
+  export default defaultExport
+}

src/.types/index.ts

@@ -1,5 +1 @@
-import { createWorker } from 'tesseract.js'
-
-const ocrWorker = await createWorker(['fin', 'swe', 'eng'])
-
-ocrWorker.readText()
+export { createInferenceSession, createTokenProcessor } from './Model/class.js'

src/Model/class.ts

@@ -12,5 +12,5 @@
     "skipLibCheck": true,
     "lib": ["es2024", "dom", "dom.iterable", "webworker"]
   },
-  "include": ["src", "node_modules/scribe.js-ocr/js/global.d.ts"]
+  "include": ["src", "src/.types/index.d.ts"]
 }