From 363996c9a2a0595f4a28014f8a755b44161b0b2b Mon Sep 17 00:00:00 2001
From: Samarth Sanjay Pyati <samarthpyati@gmail.com>
Date: Sun, 22 Jun 2025 15:39:46 +0530
Subject: [PATCH] Try stemming

---
 .gitignore   |  6 +++---
 Cargo.lock   | 11 +++++++++++
 Cargo.toml   |  1 +
 src/lexer.rs |  8 +++++++-
 src/main.rs  |  5 +++--
 5 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/.gitignore b/.gitignore
index 47fcb6b..a3defd9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,6 @@
 target/ 
 docs.gl/
 
-index.json
-index.db
-index-big.json
+index*
+
+.[dD]S_Store
\ No newline at end of file
diff --git a/Cargo.lock b/Cargo.lock
index d887407..14340ae 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -7,6 +7,7 @@ name = "DocSense"
 version = "0.1.0"
 dependencies = [
  "colored",
+ "rust-stemmers",
  "serde",
  "serde_json",
  "sqlite",
@@ -99,6 +100,16 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "rust-stemmers"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54"
+dependencies = [
+ "serde",
+ "serde_derive",
+]
+
 [[package]]
 name = "ryu"
 version = "1.0.20"
diff --git a/Cargo.toml b/Cargo.toml
index 4ceb49d..91a21fe 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -5,6 +5,7 @@ edition = "2021"
 
 [dependencies]
 colored = "3.0.0"
+rust-stemmers = "1.2.0"
 serde = { version = "1.0.219" , features = ["derive"] }
 serde_json = "1.0.140"
 sqlite = "0.30.3"
diff --git a/src/lexer.rs b/src/lexer.rs
index 07b0bc4..f24fae5 100644
--- a/src/lexer.rs
+++ b/src/lexer.rs
@@ -1,3 +1,5 @@
+use rust_stemmers::{Algorithm, Stemmer};
+
 #[derive(Debug)]
 pub struct Lexer<'a> {
     // Lifetimes implemented as content is not owned
@@ -38,6 +40,7 @@ impl<'a> Lexer<'a> {
     }
 
     fn next_token(&mut self) -> Option<String> {
+        let stemmer = Stemmer::create(Algorithm::English);
         self.trim_left();
 
         if self.content.len() == 0 {
@@ -53,7 +56,10 @@ impl<'a> Lexer<'a> {
 
         if self.content[0].is_alphabetic() {
             let result = self.chop_while(|x| x.is_alphanumeric());
-            return Some(result.iter().map(|x| x.to_ascii_uppercase()).collect());
+            let token = result.iter().collect::<String>();
+            let stemmed_token = stemmer.stem(&token).into_owned().to_uppercase();
+            println!("Before: {}, After: {}", token, stemmed_token);
+            return Some(stemmed_token);
         }
         
         // // Ignore single-character unwanted punctuation
diff --git a/src/main.rs b/src/main.rs
index 13be991..44469dd 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -19,6 +19,7 @@ fn parse_xml_file(file_path: &Path) -> Result<String, ()> {
     })?;
     let er = EventReader::new(BufReader::new(file));
     let mut content = String::new();
+
     for event in er.into_iter() {
         let event = event.map_err(|err| {
             let TextPosition {row, column} = err.position();
@@ -166,7 +167,7 @@ fn fetch_model(index_path: &str) -> Result<InMemoryModel, ()> {
 
     return Ok(model);
 }
-
+    
 fn usage(program: &String) {
     eprintln!("{}: {program} [SUBCOMMAND] [OPTIONS]", "USAGE".bold().cyan(), program = program.bright_blue());
     eprintln!("Subcommands:");
@@ -195,7 +196,7 @@ fn entry() -> Result<(), ()> {
 
     let subcommand = subcommand.ok_or_else(|| {
         usage(&program);
-        eprintln!("ERROR: no subcommand is provided");
+        eprintln!("{}: no subcommand is provided", "ERROR".red().bold());
     })?;
 
     match subcommand.as_str() {