Add stop words removal

Author: SamarthPyati

src/lexer.rs

@@ -1,5 +1,7 @@
 use rust_stemmers::{Algorithm, Stemmer};
 
+const STOP_WORDS: [&str; 4] = ["is", "as", "the", "a"];
+
 #[derive(Debug)]
 pub struct Lexer<'a> {
     content: &'a [char]
@@ -55,6 +57,13 @@ impl<'a> Lexer<'a> {
         if self.content[0].is_alphabetic() {
             let result = self.chop_while(|x| x.is_alphanumeric());
             let token = result.iter().collect::<String>();
+            
+            // Remove stop words
+            if STOP_WORDS.contains(&token.as_str()) {
+                return None;
+            }
+
+            // Stemming
             let stemmed_token = stemmer.stem(&token).into_owned().to_uppercase();
             return Some(stemmed_token);
         }

src/main.rs

@@ -207,8 +207,6 @@ fn append_folder_to_model(dir_path: &Path, model: Arc<Mutex<InMemoryModel>>, pro
 
 /* Check the amount of files present in the main frequency table index */
 fn check_index(index_path: &str) -> Result<(), ()> {
-    let a: f64 = 0.1;
-
     let index_file = fs::File::open(index_path).map_err(|err| {
         eprintln!("{}: Could not open file {file} as \"{err}\"", "ERROR".bold().red(), file = index_path.bright_blue(), err = err.to_string().red());
         exit(1);

src/model.rs

@@ -26,7 +26,7 @@ pub type GlobalTermFreq = HashMap::<String, usize>;
 #[derive(Serialize, Deserialize)]
 pub struct Doc {
     count: usize,                   // Total number of terms (tokens) present in this document.
-    ft: FreqTable,                  //  Frequency table mapping each term to the number of times it appears within this document
+    ft: FreqTable,                  // Frequency table mapping each term to the number of times it appears within this document
     last_modified: SystemTime       // The last time this document was modified on disk. Used to detect outdated indexes and trigger reindexing when needed.
 }
 
@@ -81,8 +81,8 @@ fn tf(term: &str, doc: &Doc) -> f32 {
 }
 // For TF-IDF Ranking
 fn idf(term: &str, model: &InMemoryModel) -> f32 {
-    let n = model.docs.len() as f32;
-    let d  = model.gtf.get(term).cloned().unwrap_or(1) as f32;
+    let n = model.docs.len() as f32;                                    // total docs in corpus 
+    let d = model.gtf.get(term).cloned().unwrap_or(1) as f32;   // number of time the terms has appeared in the entire corpus
     f32::log10(n / d)
 }