fixes compatibility with Angular, Vite, Rollup, etc

Author: dpfens

src/static/tokenizer.d.ts src/static/tokenizer.tssrc/static/tokenizer.d.ts renamed to src/static/tokenizer.ts

@@ -279,7 +279,7 @@ export type TokenizerConfigPreTokenizer =
 // PostProcessor
 // ----------------------------------------------------------------------------
 
-interface SpecialToken {
+export interface SpecialToken {
   id: string;
   ids: number[];
   tokens: string[];

tsconfig.json

@@ -15,7 +15,7 @@
     "types": ["node", "jest"],
     "baseUrl": ".",
     "paths": {
-      "@utils": ["src/utils/index.ts"],
+      "@utils": ["src/utils/index"],
       "@utils/*": ["src/utils/*"],
       "@core/*": ["src/core/*"],
       "@static/*": ["src/static/*"]