Try stemming

Author: SamarthPyati

.gitignore

@@ -1,6 +1,6 @@
 target/ 
 docs.gl/
 
-index.json
-index.db
-index-big.json
+index*
+
+.[dD]S_Store

Cargo.lock

@@ -5,6 +5,7 @@ edition = "2021"
 
 [dependencies]
 colored = "3.0.0"
+rust-stemmers = "1.2.0"
 serde = { version = "1.0.219" , features = ["derive"] }
 serde_json = "1.0.140"
 sqlite = "0.30.3"

Cargo.toml

@@ -1,3 +1,5 @@
+use rust_stemmers::{Algorithm, Stemmer};
+
 #[derive(Debug)]
 pub struct Lexer<'a> {
     // Lifetimes implemented as content is not owned
@@ -38,6 +40,7 @@ impl<'a> Lexer<'a> {
     }
 
     fn next_token(&mut self) -> Option<String> {
+        let stemmer = Stemmer::create(Algorithm::English);
         self.trim_left();
 
         if self.content.len() == 0 {
@@ -53,7 +56,10 @@ impl<'a> Lexer<'a> {
 
         if self.content[0].is_alphabetic() {
             let result = self.chop_while(|x| x.is_alphanumeric());
-            return Some(result.iter().map(|x| x.to_ascii_uppercase()).collect());
+            let token = result.iter().collect::<String>();
+            let stemmed_token = stemmer.stem(&token).into_owned().to_uppercase();
+            println!("Before: {}, After: {}", token, stemmed_token);
+            return Some(stemmed_token);
         }
 
         // // Ignore single-character unwanted punctuation

src/lexer.rs

@@ -19,6 +19,7 @@ fn parse_xml_file(file_path: &Path) -> Result<String, ()> {
     })?;
     let er = EventReader::new(BufReader::new(file));
     let mut content = String::new();
+
     for event in er.into_iter() {
         let event = event.map_err(|err| {
             let TextPosition {row, column} = err.position();
@@ -166,7 +167,7 @@ fn fetch_model(index_path: &str) -> Result<InMemoryModel, ()> {
 
     return Ok(model);
 }
-
+    
 fn usage(program: &String) {
     eprintln!("{}: {program} [SUBCOMMAND] [OPTIONS]", "USAGE".bold().cyan(), program = program.bright_blue());
     eprintln!("Subcommands:");
@@ -195,7 +196,7 @@ fn entry() -> Result<(), ()> {
 
     let subcommand = subcommand.ok_or_else(|| {
         usage(&program);
-        eprintln!("ERROR: no subcommand is provided");
+        eprintln!("{}: no subcommand is provided", "ERROR".red().bold());
     })?;
 
     match subcommand.as_str() {