Skip to content

Commit 68e3c8f

Browse files
michaelmiormichaelmior
committed
feat: make index threshold configurable
Added index_threshold configuration parameter (default 1024 bytes). Updated DB, Collection, MemTable, and JSTable to propagate and use this threshold for building the sparse index. Updated tests.
1 parent c26d0be commit 68e3c8f

5 files changed

Lines changed: 61 additions & 11 deletions

File tree

‎CONFIGURATION.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,10 @@
11
# Configuration parameters
22

33
All configuration parameters for ArgusDB are listed here.
4+
5+
* `host`: The host to bind the server to (default: "127.0.0.1")
6+
* `port`: The port to bind the server to (default: 5432)
7+
* `memtable_threshold`: The maximum number of documents in memory before flushing (default: 10)
8+
* `jstable_threshold`: The maximum number of JSTables before compaction (default: 5)
9+
* `jstable_dir`: The directory to store JSTables (default: "argus_data")
10+
* `index_threshold`: The number of bytes of data between index entries (default: 1024)

‎src/bin/argusdb.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ struct Settings {
5050
jstable_threshold: u64,
5151
#[serde(default = "default_jstable_dir")]
5252
jstable_dir: String,
53+
#[serde(default = "default_index_threshold")]
54+
index_threshold: u64,
5355
}
5456

5557
fn default_host() -> String {
@@ -72,6 +74,10 @@ fn default_jstable_dir() -> String {
7274
"argus_data".to_string()
7375
}
7476

77+
fn default_index_threshold() -> u64 {
78+
1024
79+
}
80+
7581
pub struct ArgusHandler {
7682
db: Arc<Mutex<DB>>,
7783
}
@@ -264,6 +270,7 @@ async fn main() {
264270
&settings.jstable_dir,
265271
settings.memtable_threshold,
266272
settings.jstable_threshold,
273+
settings.index_threshold,
267274
Some(1024 * 1024),
268275
)));
269276
let handler = Arc::new(ArgusHandler::new(db));

‎src/db.rs‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ struct Collection {
9292
logger: Box<dyn Log>,
9393
memtable_threshold: usize,
9494
jstable_threshold: u64,
95+
index_threshold: u64,
9596
tables: Vec<LoadedTable>,
9697
}
9798

@@ -101,6 +102,7 @@ impl Collection {
101102
dir: PathBuf,
102103
memtable_threshold: usize,
103104
jstable_threshold: u64,
105+
index_threshold: u64,
104106
log_rotation_threshold: Option<u64>,
105107
) -> Self {
106108
fs::create_dir_all(&dir).unwrap();
@@ -137,6 +139,7 @@ impl Collection {
137139
logger,
138140
memtable_threshold,
139141
jstable_threshold,
142+
index_threshold,
140143
tables,
141144
}
142145
}
@@ -179,7 +182,11 @@ impl Collection {
179182
fn flush(&mut self) {
180183
let jstable_path = self.dir.join(format!("jstable-{}", self.jstable_count));
181184
self.memtable
182-
.flush(jstable_path.to_str().unwrap(), self.name.clone())
185+
.flush(
186+
jstable_path.to_str().unwrap(),
187+
self.name.clone(),
188+
self.index_threshold,
189+
)
183190
.unwrap();
184191

185192
// Load the new filter and index
@@ -216,7 +223,9 @@ impl Collection {
216223
}
217224

218225
let new_path = self.dir.join("jstable-0");
219-
merged_table.write(new_path.to_str().unwrap()).unwrap();
226+
merged_table
227+
.write(new_path.to_str().unwrap(), self.index_threshold)
228+
.unwrap();
220229

221230
// Reset tables
222231
self.tables.clear();
@@ -319,6 +328,7 @@ pub struct DB {
319328
collections: HashMap<String, Collection>,
320329
memtable_threshold: usize,
321330
jstable_threshold: u64,
331+
index_threshold: u64,
322332
log_rotation_threshold: Option<u64>,
323333
}
324334

@@ -327,6 +337,7 @@ impl DB {
327337
root_dir: &str,
328338
memtable_threshold: usize,
329339
jstable_threshold: u64,
340+
index_threshold: u64,
330341
log_rotation_threshold: Option<u64>,
331342
) -> Self {
332343
fs::create_dir_all(root_dir).unwrap();
@@ -360,6 +371,7 @@ impl DB {
360371
dir_path.clone(), // Clone dir_path for collection
361372
memtable_threshold,
362373
jstable_threshold,
374+
index_threshold,
363375
log_rotation_threshold,
364376
);
365377

@@ -399,6 +411,7 @@ impl DB {
399411
collections,
400412
memtable_threshold,
401413
jstable_threshold,
414+
index_threshold,
402415
log_rotation_threshold,
403416
}
404417
}
@@ -426,6 +439,7 @@ impl DB {
426439
col_dir,
427440
self.memtable_threshold,
428441
self.jstable_threshold,
442+
self.index_threshold,
429443
self.log_rotation_threshold,
430444
);
431445
self.collections.insert(name.to_string(), collection);
@@ -478,6 +492,7 @@ mod tests {
478492

479493
const MEMTABLE_THRESHOLD: usize = 10;
480494
const JSTABLE_THRESHOLD: u64 = 5;
495+
const INDEX_THRESHOLD: u64 = 1024;
481496

482497
#[test]
483498
fn test_db_flush() {
@@ -486,6 +501,7 @@ mod tests {
486501
dir.path().to_str().unwrap(),
487502
MEMTABLE_THRESHOLD,
488503
JSTABLE_THRESHOLD,
504+
INDEX_THRESHOLD,
489505
Some(1024 * 1024),
490506
);
491507
db.create_collection("test").unwrap();
@@ -515,6 +531,7 @@ mod tests {
515531
dir.path().to_str().unwrap(),
516532
MEMTABLE_THRESHOLD,
517533
JSTABLE_THRESHOLD,
534+
INDEX_THRESHOLD,
518535
Some(1024 * 1024),
519536
);
520537
db.create_collection("test").unwrap();
@@ -563,6 +580,7 @@ mod tests {
563580
dir.path().to_str().unwrap(),
564581
MEMTABLE_THRESHOLD,
565582
JSTABLE_THRESHOLD,
583+
INDEX_THRESHOLD,
566584
Some(1024 * 1024),
567585
);
568586
db.create_collection("test").unwrap();
@@ -578,6 +596,7 @@ mod tests {
578596
dir.path().to_str().unwrap(),
579597
MEMTABLE_THRESHOLD,
580598
JSTABLE_THRESHOLD,
599+
INDEX_THRESHOLD,
581600
Some(1024 * 1024),
582601
);
583602
// "test" should be loaded if it persisted JSTable or fallback to dir name
@@ -595,6 +614,7 @@ mod tests {
595614
dir.path().to_str().unwrap(),
596615
MEMTABLE_THRESHOLD,
597616
JSTABLE_THRESHOLD,
617+
INDEX_THRESHOLD,
598618
Some(1024 * 1024),
599619
);
600620
db.create_collection("test").unwrap();
@@ -618,6 +638,7 @@ mod tests {
618638
dir.path().to_str().unwrap(),
619639
MEMTABLE_THRESHOLD,
620640
JSTABLE_THRESHOLD,
641+
INDEX_THRESHOLD,
621642
Some(1024 * 1024),
622643
);
623644
db.create_collection("test").unwrap();
@@ -665,6 +686,7 @@ mod tests {
665686
dir.path().to_str().unwrap(),
666687
MEMTABLE_THRESHOLD,
667688
JSTABLE_THRESHOLD,
689+
INDEX_THRESHOLD,
668690
Some(1024 * 1024),
669691
);
670692
db.create_collection("test").unwrap();
@@ -692,6 +714,7 @@ mod tests {
692714
dir.path().to_str().unwrap(),
693715
MEMTABLE_THRESHOLD,
694716
JSTABLE_THRESHOLD,
717+
INDEX_THRESHOLD,
695718
Some(1024 * 1024),
696719
);
697720
db.create_collection("test").unwrap();
@@ -705,6 +728,7 @@ mod tests {
705728
dir.path().to_str().unwrap(),
706729
MEMTABLE_THRESHOLD,
707730
JSTABLE_THRESHOLD,
731+
INDEX_THRESHOLD,
708732
Some(1024 * 1024),
709733
);
710734
db.create_collection("test").unwrap();
@@ -719,6 +743,7 @@ mod tests {
719743
dir.path().to_str().unwrap(),
720744
MEMTABLE_THRESHOLD,
721745
JSTABLE_THRESHOLD,
746+
INDEX_THRESHOLD,
722747
Some(1024 * 1024),
723748
);
724749
db.create_collection("test").unwrap();
@@ -734,6 +759,7 @@ mod tests {
734759
dir.path().to_str().unwrap(),
735760
MEMTABLE_THRESHOLD,
736761
JSTABLE_THRESHOLD,
762+
INDEX_THRESHOLD,
737763
Some(1024 * 1024),
738764
);
739765
let res = db.drop_collection("test");
@@ -747,6 +773,7 @@ mod tests {
747773
dir.path().to_str().unwrap(),
748774
MEMTABLE_THRESHOLD,
749775
JSTABLE_THRESHOLD,
776+
INDEX_THRESHOLD,
750777
Some(1024 * 1024),
751778
);
752779
db.create_collection("test1").unwrap();
@@ -764,6 +791,7 @@ mod tests {
764791
dir.path().to_str().unwrap(),
765792
MEMTABLE_THRESHOLD,
766793
JSTABLE_THRESHOLD,
794+
INDEX_THRESHOLD,
767795
Some(1024 * 1024),
768796
);
769797
let res = db.insert("test", json!({ "a": 1 }));
@@ -777,6 +805,7 @@ mod tests {
777805
dir.path().to_str().unwrap(),
778806
MEMTABLE_THRESHOLD,
779807
JSTABLE_THRESHOLD,
808+
INDEX_THRESHOLD,
780809
Some(1024 * 1024),
781810
);
782811
db.create_collection("test").unwrap();
@@ -785,6 +814,7 @@ mod tests {
785814
dir.path().to_str().unwrap(),
786815
MEMTABLE_THRESHOLD,
787816
JSTABLE_THRESHOLD,
817+
INDEX_THRESHOLD,
788818
Some(1024 * 1024),
789819
);
790820
assert!(db2.collections.contains_key("test"));
@@ -797,6 +827,7 @@ mod tests {
797827
dir.path().to_str().unwrap(),
798828
MEMTABLE_THRESHOLD,
799829
JSTABLE_THRESHOLD,
830+
INDEX_THRESHOLD,
800831
Some(1024 * 1024),
801832
);
802833
db.create_collection("test").unwrap();

‎src/jstable.rs‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ impl JSTable {
3535
}
3636
}
3737

38-
pub fn write(&self, path: &str) -> io::Result<()> {
38+
pub fn write(&self, path: &str, index_threshold: u64) -> io::Result<()> {
3939
let summary_path = format!("{}.summary", path);
4040
let data_path = format!("{}.data", path);
4141

@@ -85,7 +85,7 @@ impl JSTable {
8585

8686
for (id, doc) in &self.documents {
8787
// Add index entry if needed
88-
if first || bytes_since_last_index >= 1024 {
88+
if first || bytes_since_last_index >= index_threshold {
8989
index.push((id.clone(), current_offset));
9090
bytes_since_last_index = 0;
9191
first = false;
@@ -355,7 +355,7 @@ mod tests {
355355

356356
let dir = tempdir()?;
357357
let file_path = dir.path().join("test_table");
358-
jstable.write(file_path.to_str().unwrap()).unwrap();
358+
jstable.write(file_path.to_str().unwrap(), 1024).unwrap();
359359

360360
let read_table = read_jstable(file_path.to_str().unwrap()).unwrap();
361361

@@ -390,7 +390,7 @@ mod tests {
390390

391391
let dir = tempdir()?;
392392
let file_path = dir.path().join("test_table");
393-
jstable.write(file_path.to_str().unwrap()).unwrap();
393+
jstable.write(file_path.to_str().unwrap(), 1024).unwrap();
394394

395395
let iterator = JSTableIterator::new(file_path.to_str().unwrap())?;
396396
assert_eq!(iterator.timestamp, 12345);
@@ -433,7 +433,7 @@ mod tests {
433433

434434
let dir = tempdir()?;
435435
let file_path = dir.path().join("test_table");
436-
jstable.write(file_path.to_str().unwrap()).unwrap();
436+
jstable.write(file_path.to_str().unwrap(), 1024).unwrap();
437437

438438
let filter = read_filter(file_path.to_str().unwrap())?;
439439

@@ -500,7 +500,7 @@ mod tests {
500500

501501
let dir = tempdir()?;
502502
let file_path = dir.path().join("test_table");
503-
jstable.write(file_path.to_str().unwrap())?;
503+
jstable.write(file_path.to_str().unwrap(), 1024)?;
504504

505505
let iterator = JSTableIterator::new(file_path.to_str().unwrap())?;
506506
let keys: Vec<String> = iterator.map(|r| r.unwrap().0).collect();
@@ -533,7 +533,7 @@ mod tests {
533533
let jstable = JSTable::new(123, "idx_test".to_string(), schema, documents);
534534
let dir = tempdir()?;
535535
let path = dir.path().join("idx_table");
536-
jstable.write(path.to_str().unwrap())?;
536+
jstable.write(path.to_str().unwrap(), 1024)?;
537537

538538
let index = read_index(path.to_str().unwrap())?;
539539

‎src/storage.rs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,12 @@ impl MemTable {
3434
self.documents.is_empty()
3535
}
3636

37-
pub fn flush(&self, path: &str, collection: String) -> std::io::Result<()> {
37+
pub fn flush(
38+
&self,
39+
path: &str,
40+
collection: String,
41+
index_threshold: u64,
42+
) -> std::io::Result<()> {
3843
let timestamp = std::time::SystemTime::now()
3944
.duration_since(std::time::UNIX_EPOCH)
4045
.unwrap()
@@ -45,7 +50,7 @@ impl MemTable {
4550
self.schema.clone(),
4651
self.documents.clone(),
4752
);
48-
jstable.write(path)
53+
jstable.write(path, index_threshold)
4954
}
5055

5156
pub fn insert(&mut self, id: String, doc: Value) {

0 commit comments

Comments
 (0)