WIP on main: 3c57b73 add check for test case using strinf literal

Author: martcpp

src/lexericals/lexer.rs

@@ -33,6 +33,7 @@ impl Lexer {
 
     pub fn next_token(&mut self) -> Token {
             //"=+(){},;";
+        self.skip_whitespace(); // Skip whitespace characters
         let token = match self.ch {
             '=' => Token::new(Tokenkind::Assign, self.ch.to_string()),
             '+' => Token::new(Tokenkind::Plus, self.ch.to_string()),
@@ -49,6 +50,11 @@ impl Lexer {
                 let literal = self.read_identifier();
                 let kind = Tokenkind::lookup_ident(&literal);
                 Token::new(kind, literal)
+            }else if Lexer::is_digit(self.ch) {
+                let kind = Tokenkind::Int;
+                let literal = self.read_number();
+                Token::new(kind, literal)
+
             } else {
                 Token::new(Tokenkind::Illegal, self.ch.to_string())
             },
@@ -58,10 +64,30 @@ impl Lexer {
         self.read_token(); // Move to the next character
         token
     }
+
+    fn skip_whitespace(&mut self) {
+        while self.ch.is_whitespace() {
+            self.read_token();
+        }
+    }
+
+    fn is_digit(ch: char) -> bool {
+        ch.is_numeric()
+    }
+
     fn is_letter(ch: char) -> bool {
         ch.is_alphabetic() || ch == '_'
     }
 
+    fn read_number(&mut self) -> String {
+        let mut literal = String::new();
+        while Lexer::is_digit(self.ch) {
+            literal.push(self.ch);
+            self.read_token();
+        }
+        literal
+    }
+
     fn read_identifier(&mut self) -> String {
         let mut literal =  String::new();
         while Lexer::is_letter(self.ch){
@@ -79,7 +105,8 @@ mod test {
     use super::*;
     #[test]
     fn read_token() {
-        let input =r#"let five = 5;
+        let input =r#"
+        let five = 5;
         let ten = 10;
         let add = fn(x, y) {
             x + y;

src/main.rs

@@ -1,5 +1,4 @@
 mod lexericals;
-use crate::lexericals::tokenizer::{Token, Tokenkind};
 use crate::lexericals::lexer::Lexer;
 
 
@@ -11,17 +10,44 @@ fn main() {
     // println!("Token kind: {:?}, literal: {}", token.kind, token.literal);
     // println!("Token {:?}", token);
 
-    let input = "+=(){};";
+        let input =r#"
+        let five = 5;
+        let ten = 10;
+        let add = fn(x, y) {
+            x + y;
+        };
+        let result = add(five, ten);
+        "#;
+
     let mut lexer = Lexer::new(input);
     println!("Lexer initialized with input: {}", input);
-    // println!("Current character: {:?}", lexer);
-
-    for (idx,token) in lexer.input.clone().into_iter().enumerate() {
-        println!("Token at index {}: {}", idx, token);
-        println!("Current character: {:?}", lexer.next_token());
+    let mut tokens = Vec::new();
+    let mut token = lexer.next_token();
+    while token.kind != lexericals::tokenizer::Tokenkind::Eof {
+        tokens.push(token.literal);
+        token = lexer.next_token();
     }
 
+    // loop {
+    //     let token = lexer.next_token();
+    //     if token.kind == lexericals::tokenizer::Tokenkind::Eof {
+    //         break;
+    //     }
+    //     tokens.push(token.literal);
+    // }
+    println!("Tokens: {:?}", tokens);
+     
+    // for (idx,token) in lexer.input.clone().into_iter().enumerate() {
+    //     println!("charater at index:{} token:{}", idx, token);
+    //     let token = lexer.next_token();
+    //     println!("Tokenkind at index:{} is:{}", idx, token.kind);
+    //     println!("literal at index:{} is:{} ", idx, token.literal);
+    // }
+
     // println!(" last Current character: {:?}", lexer.next_token());
 
 }
 
+fn read_string(ch:char) -> String {
+    todo!("Implement string reading logic");
+}