Fixed Tokenizer + Added Test

Author: IHateYourCode

Source/Tokenizer.js

@@ -14,9 +14,10 @@ const tokens = [
     [ 'Colon'        , /^:/ ] ,
 
     [ 'Newline'      , /^\r?\n/ ] ,
-    [ 'Space'        , /^[\s]+/ , 0 ] ,
+    
+    [ 'MultiString'  , /^[^\S\n]*'''[\s\S]*?'''/u , 0 ] ,
+    [ 'Space'        , /^[^\S\n]+/ , 0 ] ,
 
-    [ 'MultiString'  , /^'''([\s\S]*?)'''/u , 1 ] ,
     [ 'MultiComment' , /^\/\*([\s\S]*?)\*\//u ] ,
     [ 'SingleString' , /^'(([^\\]|(\\["'\\/bfnrt])|(\\u\d{4}))*?)'/ , 1 ] ,
     [ 'DoubleString' , /^"(([^\\]|(\\["'\\/bfnrt])|(\\u\d{4}))*?)"/ , 1 ] ,
@@ -61,8 +62,6 @@ export default class Tokenizer {
 
                 const found = string.match(regex);
 
-                console.log(type,found);
-                
                 this.#string = string.substring(found[0].length);
 
                 const token = { type };

Tests/Normalize.sh

@@ -0,0 +1,16 @@
+
+clear
+
+echo ""
+echo ""
+echo ""
+echo ""
+echo ""
+echo ""
+echo ""
+echo ""
+echo ""
+
+deno test                           \
+    --importmap=Tests/Imports.json  \
+    Tests/Normalize

Tests/Normalizer/2Members.test.js Tests/Normalize/2Members.test.jsTests/Normalizer/2Members.test.js renamed to Tests/Normalize/2Members.test.js

@@ -0,0 +1,56 @@
+
+import { assertSameTokens , test } from 'TokenTest';
+
+
+const hjson = 
+`{
+    multiline-value : 
+        
+        '''
+        Tes\nting
+        
+        this
+        '''
+}`;
+
+
+const tokens = [{
+    type : 'ObjectStart'
+},{
+    type : 'Newline'
+},{
+    type : 'Space' ,
+    value : '    '
+},{
+    type : 'Member' ,
+    value : 'multiline-value'
+},{
+    type : 'Space' ,
+    value : ' '
+},{
+    type : 'Colon'
+},{
+    type : 'Space' ,
+    value : ' '
+},{
+    type : 'Newline'
+},{
+    type : 'Space' ,
+    value : '        '
+},{
+    type : 'Newline'
+},{
+    type : 'MultiString' ,
+    value : `        '''\n        Tes\nting\n        \n        this\n        '''`
+},{
+    type : 'Newline'
+},{
+    type : 'ObjectEnd'
+}];
+
+
+test('Multiline Member',() => {
+    
+    assertSameTokens(hjson,tokens);
+
+})