Reworked Tokenizer

Author: IHateYourCode

Source/Tokenizer.js

@@ -2,43 +2,38 @@
 const { iterator } = Symbol
 
 
-const tokens = prepare({
-    '\'\'\'' : 'Multiline' ,
-    '[^\\s,:\\[\\]\\{\\}]+' : 'Word' ,
-    '[^\\S\\n]+' : 'Space' ,
-    '(\\r)?\\n' : 'Newline' ,
-    '\\{' : 'ObjectStart' ,
-    '\\}' : 'ObjectEnd' ,
-    '\\[' : 'ArrayStart' ,
-    '\\]' : 'ArrayEnd' ,
-    ',' : 'Comma' ,
-    ':' : 'Colon'
-})
-
-
-function prepare(tokens){
-    return Object
-        .entries(tokens)
-        .map(toToken);
-}
-
-function toToken([ pattern , token ]){
-    return [ toRegex(pattern) , token ];
-}
-
-function toRegex(pattern){
-    return new RegExp(`^(${ pattern })`,'u');
-}
+const tokens = [
+    
+    [ 'ObjectStart'  , /^\{/ ] ,
+    [ 'ObjectEnd'    , /^\}/ ] ,
+    
+    [ 'ArrayStart'   , /^\[/ ] ,
+    [ 'ArrayEnd'     , /^\]/ ] ,
+    
+    [ 'Comma'        , /^,/ ] ,
+    [ 'Colon'        , /^:/ ] ,
+    
+    [ 'Newline'      , /^\r?\n/ ] ,
+    [ 'Space'        , /^[\s]+/ , 0 ] ,
+        
+    [ 'MultiString'  , /^'''([\s\S]*?)'''/u , 1 ] ,
+    [ 'MultiComment' , /^\/\*([\s\S]*?)\*\//u ] ,
+    [ 'SingleString' , /^'(([^\\]|(\\["'\\/bfnrt])|(\\u\d{4}))*?)'/ , 1 ] ,
+    [ 'DoubleString' , /^"(([^\\]|(\\["'\\/bfnrt])|(\\u\d{4}))*?)"/ , 1 ] ,
+    [ 'Comment'      , /^(#)|(\/\/)([^\n]*)/u ] ,
+    [ 'Member'       , /^[^\s,:\[\]\{\}][^\s]*/ , 0 ] ,
+    [ 'Quoteless'    , /^[^\s,:\[\]\{\}][^\n]*/ , 0 ]
+]
 
 
 export default class Tokenizer {
 
-    #position;
+    #position = 0;
     #string;
 
     constructor(string){
-        this.#position = 0;
-        this.#string = string;
+        this.#string = string
+            .trim();
     }
 
 
@@ -61,16 +56,21 @@ export default class Tokenizer {
 
         const string = this.#string
 
-        for(const [ regex , type ] of tokens)
+        for(const [ type , regex , match ] of tokens)
             if(regex.test(string)){
 
-                const match = string.match(regex);
+                const found = string.match(regex);
+                
+                console.log(type,found);
+                
+                this.#string = string.substring(found[0].length);
 
-                this.#string = string.substring(match[0].length);
+                const token = { type };
 
-                const [ _ , value ] = match;
+                if(match != null)
+                    token.value = found[match];
 
-                return { type , value }
+                return token;
             }
 
         throw `No Token Found for '${ string }'`;

Tests/Tokenize.sh

@@ -0,0 +1,16 @@
+
+clear
+
+echo ""
+echo ""
+echo ""
+echo ""
+echo ""
+echo ""
+echo ""
+echo ""
+echo ""
+
+deno test                           \
+    --importmap=Tests/Imports.json  \
+    Tests/Tokens

Tests/Tokens/Brackets.test.js

@@ -6,11 +6,9 @@ const hjson = '{}';
 
 
 const tokens = [{
-    type : 'ObjectStart' ,
-    value : '{'
+    type : 'ObjectStart'
 },{
-    type : 'ObjectEnd' ,
-    value : '}'
+    type : 'ObjectEnd'
 }];
 
 

Tests/Tokens/Member.test.js

@@ -9,41 +9,30 @@ const hjson =
 
 
 const tokens = [{
-    type : 'ObjectStart' ,
-    value : '{'
+    type : 'ObjectStart'
 },{
-    type : 'Newline' ,
-    value : '\n'
+    type : 'Newline'
 },{
     type : 'Space' ,
     value : '    '
 },{
-    type : 'Word' ,
+    type : 'Member' ,
     value : 'member'
 },{
     type : 'Space' ,
     value : ' '
 },{
-    type : 'Colon' ,
-    value : ':'
+    type : 'Colon'
 },{
     type : 'Space' ,
     value : ' '
 },{
-    type : 'Word' ,
-    value : '\'String'
+    type : 'SingleString' ,
+    value : 'String Value'
 },{
-    type : 'Space' ,
-    value : ' '
-},{
-    type : 'Word' ,
-    value : 'Value\''
-},{
-    type : 'Newline' ,
-    value : '\n'
+    type : 'Newline'
 },{
-    type : 'ObjectEnd' ,
-    value : '}'
+    type : 'ObjectEnd'
 }];
 
 

Tests/Tokens/Space.test.js

@@ -26,10 +26,7 @@ const hjson = Object
     .join('');
 
 
-const tokens = [{
-    type : 'Space' ,
-    value : hjson
-}];
+const tokens = [];
 
 test('String made of whitespace',() => {
 

Tests/Tokens/Test.js

@@ -2,12 +2,19 @@
 import { assertEquals } from 'Assert'
 import Tokenizer from 'Tokenizer'
 
+const { log } = console;
+
 
 export const test = Deno.test;
 
 
 export function assertSameTokens(hjson,tokens){
-    assertEquals(tokenize(hjson),tokens);
+    
+    const parsed = tokenize(hjson);
+    
+    log(parsed);
+    
+    assertEquals(parsed,tokens);
 }
 
 function tokenize(string){