lexer accept floating point decimals and exponents

Author: ghewgill

lexer.cpp

@@ -144,7 +144,7 @@ std::vector<Token> tokenize(const std::string &source)
             i = j;
         } else if (isdigit(c)) {
             t.type = NUMBER;
-            if (c == '0' && not isdigit(source.at(i+1))) {
+            if (c == '0' && source.at(i+1) != '.' && tolower(source.at(i+1)) != 'e' && not isdigit(source.at(i+1))) {
                 c = tolower(source.at(i+1));
                 if (isalpha(c) || c == '#') {
                     long base;
@@ -195,10 +195,24 @@ std::vector<Token> tokenize(const std::string &source)
                 }
             } else {
                 auto j = i;
-                // TODO: decimal and exponent
                 while (j < source.length() && isdigit(source.at(j))) {
                     j++;
                 }
+                if (j < source.length() && source.at(j) == '.') {
+                    j++;
+                    while (j < source.length() && isdigit(source.at(j))) {
+                        j++;
+                    }
+                }
+                if (j < source.length() && tolower(source.at(j)) == 'e') {
+                    j++;
+                    if (j < source.length() && (source.at(j) == '+' || source.at(j) == '-')) {
+                        j++;
+                    }
+                    while (j < source.length() && isdigit(source.at(j))) {
+                        j++;
+                    }
+                }
                 t.value = number_from_string(source.substr(i, j-i));
                 i = j;
             }

t/decimal.simple

@@ -1,10 +1,14 @@
-% TODO
-VAR a: number
+print(str(0.3))
+%= 3E-1
 
-a := 0.3
-print(a)
-%= 0.3
+print(str(3.3e33))
+%= 33E+32
 
-a := 3.3e33
-print(a)
-%= 3.3e33
+print(str(3.3e-33))
+%= 33E-34
+
+print(str(0e33))
+%= 0E+33
+
+print(str(1.e33))
+%= 1E+33