Skip to content

Commit 92a768d

Browse files
TinittoTinitto
authored
Merge pull request #2 from sopherapps/ft-optional-searching
Make searching optional
2 parents a60ce71 + 1edf88a commit 92a768d

12 files changed

Lines changed: 181 additions & 79 deletions

‎CHANGELOG.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
77

88
## [Unreleased]
99

10+
## [0.2.0] - 2023-01-16
11+
12+
### Added
13+
14+
### Changed
15+
16+
- Changed the `Store()` and `AsyncStore()` signatures, replacing `max_index_key_len` option with `is_search_enabled`.
17+
- Permanently set the maximum index key length to 3
18+
- Changed benchmarks to compare operations when search is enabled to when search is disabled.
19+
1020
## [0.1.1] - 2023-01-14
1121

1222
### Added

‎Cargo.lock‎

Lines changed: 3 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "py_scdb"
3-
version = "0.1.1"
3+
version = "0.2.0"
44
edition = "2021"
55

66
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -10,6 +10,6 @@ crate-type = ["cdylib"]
1010

1111
[dependencies]
1212
pyo3 = { version = "0.17", features = ["extension-module"] }
13-
scdb = "0.1"
13+
scdb = "0.2"
1414
pyo3-asyncio = { version = "0.17", features = ["attributes", "async-std-runtime"] }
1515
async-std = "1.12"

‎README.md‎

Lines changed: 50 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ Of course to make it a little more appealing, it has some extra features like:
2525
- Time-to-live (TTL) where a key-value pair expires after a given time
2626
- Non-blocking reads from separate processes, and threads.
2727
- Fast Sequential writes to the store, queueing any writes from multiple processes and threads.
28+
- Optional searching of keys that begin with a given subsequence. This option is turned on when `scdb::new()` is called.
29+
Note: **When searching is enabled, `delete`, `get`, `compact`, `clear` become considerably slower.**
2830

2931
## Dependencies
3032

@@ -62,7 +64,7 @@ if __name__ == "__main__":
6264
redundant_blocks=1,
6365
pool_capacity=10,
6466
compaction_interval=1800,
65-
max_index_key_len=3,
67+
is_search_enabled=True,
6668
)
6769

6870
# inserting without ttl
@@ -148,7 +150,7 @@ async def run_async_example():
148150
redundant_blocks=1,
149151
pool_capacity=10,
150152
compaction_interval=1800,
151-
max_index_key_len=3,
153+
is_search_enabled=True,
152154
)
153155

154156
# inserting without ttl
@@ -273,7 +275,7 @@ OR the summary
273275

274276
```shell
275277
# synchronous API
276-
pytest test/test_benchmarks.py --benchmark-columns=mean,min,max --benchmark-name=short
278+
pytest test/test_benchmarks.py --benchmark-columns=mean,min,max --benchmark-name=short --benchmark-sort=NAME
277279
```
278280

279281
## Benchmarks
@@ -283,57 +285,51 @@ On an average PC (17Core, 16 GB RAM)
283285
### Synchronous
284286

285287
```
286-
------------------------------------------------------ benchmark: 47 tests -------------------------------------------------------
287-
Name (time in ns) Mean Min Max
288-
----------------------------------------------------------------------------------------------------------------------------------
289-
benchmark_get[sync_store-hey] 800.4672 (1.03) 698.0000 (1.0) 59,185.0000 (1.48)
290-
benchmark_get[sync_store-oi] 776.5698 (1.0) 699.0000 (1.00) 39,861.0000 (1.0)
291-
benchmark_get[sync_store-salut] 785.0628 (1.01) 702.0000 (1.01) 49,960.0000 (1.25)
292-
benchmark_get[sync_store-bonjour] 780.4937 (1.01) 703.0000 (1.01) 71,929.0000 (1.80)
293-
benchmark_get[sync_store-mulimuta] 783.5682 (1.01) 705.0000 (1.01) 50,234.0000 (1.26)
294-
benchmark_get[sync_store-hi] 790.7354 (1.02) 709.0000 (1.02) 70,327.0000 (1.76)
295-
benchmark_get[sync_store-hola] 802.1952 (1.03) 721.0000 (1.03) 61,532.0000 (1.54)
296-
benchmark_paginated_search[sync_store-for] 6,962.4849 (8.97) 6,547.0000 (9.38) 56,507.0000 (1.42)
297-
benchmark_paginated_search[sync_store-pigg] 6,919.7214 (8.91) 6,556.0000 (9.39) 130,593.0000 (3.28)
298-
benchmark_paginated_search[sync_store-bar] 7,079.0014 (9.12) 6,556.0000 (9.39) 86,028.0000 (2.16)
299-
benchmark_paginated_search[sync_store-pi] 6,936.8665 (8.93) 6,556.0000 (9.39) 167,335.0000 (4.20)
300-
benchmark_search[sync_store-pigg] 6,902.7711 (8.89) 6,613.0000 (9.47) 61,975.0000 (1.55)
301-
benchmark_paginated_search[sync_store-pig] 7,028.6412 (9.05) 6,643.0000 (9.52) 89,304.0000 (2.24)
302-
benchmark_paginated_search[sync_store-ban] 7,234.7422 (9.32) 6,653.0000 (9.53) 82,575.0000 (2.07)
303-
benchmark_paginated_search[sync_store-p] 7,027.5200 (9.05) 6,688.0000 (9.58) 58,940.0000 (1.48)
304-
benchmark_search[sync_store-for] 10,294.3237 (13.26) 9,821.0000 (14.07) 71,856.0000 (1.80)
305-
benchmark_search[sync_store-p] 10,391.7488 (13.38) 9,822.0000 (14.07) 111,787.0000 (2.80)
306-
benchmark_search[sync_store-ban] 10,403.6132 (13.40) 9,831.0000 (14.08) 89,734.0000 (2.25)
307-
benchmark_search[sync_store-pig] 10,374.8678 (13.36) 9,841.0000 (14.10) 60,577.0000 (1.52)
308-
benchmark_search[sync_store-pi] 10,380.7388 (13.37) 9,941.0000 (14.24) 108,986.0000 (2.73)
309-
benchmark_search[sync_store-bar] 10,449.2485 (13.46) 9,983.0000 (14.30) 71,575.0000 (1.80)
310-
benchmark_paginated_search[sync_store-fo] 12,587.1956 (16.21) 12,075.0000 (17.30) 69,118.0000 (1.73)
311-
benchmark_paginated_search[sync_store-ba] 13,841.9919 (17.82) 12,089.0000 (17.32) 22,140,639.0000 (555.45)
312-
benchmark_paginated_search[sync_store-f] 12,743.9164 (16.41) 12,150.0000 (17.41) 167,326.0000 (4.20)
313-
benchmark_paginated_search[sync_store-b] 12,762.3290 (16.43) 12,185.0000 (17.46) 141,532.0000 (3.55)
314-
benchmark_paginated_search[sync_store-foo] 12,808.0472 (16.49) 12,268.0000 (17.58) 160,061.0000 (4.02)
315-
benchmark_delete[sync_store-oi] 21,704.5624 (27.95) 13,880.0000 (19.89) 112,458.0000 (2.82)
316-
benchmark_delete[sync_store-hi] 20,088.1860 (25.87) 14,064.0000 (20.15) 93,398.0000 (2.34)
317-
benchmark_delete[sync_store-mulimuta] 21,373.7075 (27.52) 14,105.0000 (20.21) 64,670.0000 (1.62)
318-
benchmark_delete[sync_store-salut] 21,230.0633 (27.34) 14,236.0000 (20.40) 111,376.0000 (2.79)
319-
benchmark_delete[sync_store-hey] 19,834.3945 (25.54) 14,298.0000 (20.48) 81,824.0000 (2.05)
320-
benchmark_delete[sync_store-hola] 21,709.5685 (27.96) 14,315.0000 (20.51) 9,488,572.0000 (238.04)
321-
benchmark_delete[sync_store-bonjour] 21,215.8223 (27.32) 14,406.0000 (20.64) 76,129.0000 (1.91)
322-
benchmark_search[sync_store-ba] 15,359.4292 (19.78) 14,754.0000 (21.14) 127,285.0000 (3.19)
323-
benchmark_search[sync_store-b] 15,462.2251 (19.91) 14,840.0000 (21.26) 74,957.0000 (1.88)
324-
benchmark_search[sync_store-foo] 15,673.4764 (20.18) 14,889.0000 (21.33) 77,673.0000 (1.95)
325-
benchmark_search[sync_store-f] 20,645.8552 (26.59) 19,583.0000 (28.06) 83,335.0000 (2.09)
326-
benchmark_search[sync_store-fo] 20,593.4482 (26.52) 19,716.0000 (28.25) 114,361.0000 (2.87)
327-
benchmark_set[sync_store-hi-English] 26,811.3686 (34.53) 24,591.0000 (35.23) 102,322.0000 (2.57)
328-
benchmark_set[sync_store-oi-Portuguese] 26,792.9735 (34.50) 24,691.0000 (35.37) 105,970.0000 (2.66)
329-
benchmark_set[sync_store-salut-French] 34,000.6656 (43.78) 31,847.0000 (45.63) 114,291.0000 (2.87)
330-
benchmark_set[sync_store-hola-Spanish] 34,219.5426 (44.06) 31,857.0000 (45.64) 103,374.0000 (2.59)
331-
benchmark_set[sync_store-bonjour-French] 34,491.8102 (44.42) 32,046.0000 (45.91) 123,111.0000 (3.09)
332-
benchmark_set[sync_store-mulimuta-Runyoro] 33,995.6793 (43.78) 32,332.0000 (46.32) 117,633.0000 (2.95)
333-
benchmark_set[sync_store-hey-English] 34,103.0846 (43.92) 32,513.0000 (46.58) 107,059.0000 (2.69)
334-
benchmark_clear[sync_store] 168,770.7111 (217.33) 128,008.0000 (183.39) 471,408.0000 (11.83)
335-
benchmark_compact[sync_store] 106,328,709.5000 (>1000.0) 103,646,207.0000 (>1000.0) 108,741,183.0000 (>1000.0)
336-
----------------------------------------------------------------------------------------------------------------------------------
288+
289+
--------------------------------------------------------- benchmark: 40 tests ---------------------------------------------------------
290+
Name (time in us) Mean Min Max
291+
---------------------------------------------------------------------------------------------------------------------------------------
292+
benchmark_clear[sync_store] 100.3807 (127.70) 83.0300 (116.78) 311.0190 (6.71)
293+
benchmark_clear_with_search[sync_searchable_store] 171.0275 (217.57) 133.6630 (187.99) 433.9330 (9.36)
294+
benchmark_compact[sync_store] 110,236.5678 (>1000.0) 106,076.8240 (>1000.0) 117,694.6450 (>1000.0)
295+
benchmark_compact_with_search[sync_searchable_store] 111,180.5895 (>1000.0) 102,213.0760 (>1000.0) 127,501.7640 (>1000.0)
296+
benchmark_delete[sync_store-hey] 3.7683 (4.79) 3.4310 (4.83) 82.3730 (1.78)
297+
benchmark_delete[sync_store-hi] 3.7664 (4.79) 3.4440 (4.84) 54.3430 (1.17)
298+
benchmark_delete_with_search[sync_searchable_store-hey] 20.7839 (26.44) 15.0590 (21.18) 122.8020 (2.65)
299+
benchmark_delete_with_search[sync_searchable_store-hi] 20.7891 (26.45) 14.7560 (20.75) 123.6690 (2.67)
300+
benchmark_get[sync_store-hey] 0.7861 (1.0) 0.7110 (1.0) 64.0260 (1.38)
301+
benchmark_get[sync_store-hi] 0.7877 (1.00) 0.7190 (1.01) 55.0760 (1.19)
302+
benchmark_get_with_search[sync_searchable_store-hey] 0.7991 (1.02) 0.7220 (1.02) 49.2670 (1.06)
303+
benchmark_get_with_search[sync_searchable_store-hi] 0.7899 (1.00) 0.7230 (1.02) 46.3370 (1.0)
304+
benchmark_paginated_search[sync_searchable_store-b] 12.9258 (16.44) 12.1430 (17.08) 156.1050 (3.37)
305+
benchmark_paginated_search[sync_searchable_store-ba] 12.9307 (16.45) 12.1640 (17.11) 115.5400 (2.49)
306+
benchmark_paginated_search[sync_searchable_store-ban] 7.0885 (9.02) 6.6530 (9.36) 71.0410 (1.53)
307+
benchmark_paginated_search[sync_searchable_store-bar] 6.9927 (8.90) 6.6160 (9.31) 82.8730 (1.79)
308+
benchmark_paginated_search[sync_searchable_store-f] 13.2767 (16.89) 12.0420 (16.94) 21,129.0600 (455.99)
309+
benchmark_paginated_search[sync_searchable_store-fo] 12.9289 (16.45) 12.1860 (17.14) 439.6100 (9.49)
310+
benchmark_paginated_search[sync_searchable_store-foo] 12.9970 (16.53) 12.3170 (17.32) 102.5070 (2.21)
311+
benchmark_paginated_search[sync_searchable_store-for] 6.9771 (8.88) 6.6010 (9.28) 94.3090 (2.04)
312+
benchmark_paginated_search[sync_searchable_store-p] 6.9895 (8.89) 6.5870 (9.26) 149.4620 (3.23)
313+
benchmark_paginated_search[sync_searchable_store-pi] 6.9171 (8.80) 6.6080 (9.29) 69.2660 (1.49)
314+
benchmark_paginated_search[sync_searchable_store-pig] 6.9475 (8.84) 6.5980 (9.28) 67.4790 (1.46)
315+
benchmark_paginated_search[sync_searchable_store-pigg] 6.9717 (8.87) 6.6070 (9.29) 111.5230 (2.41)
316+
benchmark_search[sync_searchable_store-b] 15.5810 (19.82) 14.6940 (20.67) 105.9520 (2.29)
317+
benchmark_search[sync_searchable_store-ba] 15.6253 (19.88) 14.7780 (20.78) 107.5030 (2.32)
318+
benchmark_search[sync_searchable_store-ban] 10.5073 (13.37) 9.9340 (13.97) 118.7900 (2.56)
319+
benchmark_search[sync_searchable_store-bar] 10.8698 (13.83) 9.8550 (13.86) 97.3030 (2.10)
320+
benchmark_search[sync_searchable_store-f] 20.7594 (26.41) 19.5820 (27.54) 116.9510 (2.52)
321+
benchmark_search[sync_searchable_store-fo] 20.8279 (26.50) 19.6790 (27.68) 196.4210 (4.24)
322+
benchmark_search[sync_searchable_store-foo] 15.7794 (20.07) 14.8380 (20.87) 94.2340 (2.03)
323+
benchmark_search[sync_searchable_store-for] 10.4470 (13.29) 9.8640 (13.87) 128.4710 (2.77)
324+
benchmark_search[sync_searchable_store-p] 10.3812 (13.21) 9.7870 (13.77) 102.3630 (2.21)
325+
benchmark_search[sync_searchable_store-pi] 10.3460 (13.16) 9.8380 (13.84) 84.1260 (1.82)
326+
benchmark_search[sync_searchable_store-pig] 10.4069 (13.24) 9.8400 (13.84) 78.9080 (1.70)
327+
benchmark_search[sync_searchable_store-pigg] 6.9492 (8.84) 6.6000 (9.28) 86.0650 (1.86)
328+
benchmark_set[sync_store-hey-English] 10.6412 (13.54) 9.2990 (13.08) 110.4370 (2.38)
329+
benchmark_set[sync_store-hi-English] 10.7570 (13.68) 9.1760 (12.91) 100.9780 (2.18)
330+
benchmark_set_with_search[sync_searchable_store-hey-English] 37.2746 (47.42) 32.7920 (46.12) 8,647.2350 (186.62)
331+
benchmark_set_with_search[sync_searchable_store-hi-English] 28.0105 (35.63) 25.5050 (35.87) 105.3320 (2.27)
332+
---------------------------------------------------------------------------------------------------------------------------------------
337333
```
338334

339335
## Acknowledgement

‎py_scdb/py_scdb.pyi‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ class Store:
3434
Compaction is important because it reclaims this space and reduces the size
3535
of the database file.
3636
Default: 3600s (1 hour)
37+
:param is_search_enabled: Whether the search capability of the store is enabled.
38+
Note that when search is enabled, `set`, `delete`, `clear`, `compact`
39+
operations become slower.
40+
Default: False
3741
"""
3842

3943
def __init__(
@@ -43,6 +47,7 @@ class Store:
4347
redundant_blocks: Optional[int] = None,
4448
pool_capacity: Optional[int] = None,
4549
compaction_interval: Optional[int] = None,
50+
is_search_enabled: bool = False,
4651
) -> None: ...
4752
def set(self, k: str, v: str, ttl: Optional[int] = None) -> None:
4853
"""
@@ -135,6 +140,10 @@ class AsyncStore:
135140
Compaction is important because it reclaims this space and reduces the size
136141
of the database file.
137142
Default: 3600s (1 hour)
143+
:param is_search_enabled: Whether the search capability of the store is enabled.
144+
Note that when search is enabled, `set`, `delete`, `clear`, `compact`
145+
operations become slower.
146+
Default: False
138147
"""
139148

140149
def __init__(
@@ -144,6 +153,7 @@ class AsyncStore:
144153
redundant_blocks: Optional[int] = None,
145154
pool_capacity: Optional[int] = None,
146155
compaction_interval: Optional[int] = None,
156+
is_search_enabled: bool = False,
147157
) -> None: ...
148158
async def set(self, k: str, v: str, ttl: Optional[int] = None) -> None:
149159
"""

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ build-backend = "maturin"
44

55
[project]
66
name = "py_scdb"
7-
version = "0.1.1"
7+
version = "0.2.0"
88
requires-python = ">=3.7"
99
classifiers = [
1010
"Programming Language :: Rust",

‎src/async_store.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ impl AsyncStore {
1616
redundant_blocks = "None",
1717
pool_capacity = "None",
1818
compaction_interval = "None",
19-
max_index_key_len = "None"
19+
is_search_enabled = "false"
2020
)]
2121
#[new]
2222
pub fn new(
@@ -25,15 +25,15 @@ impl AsyncStore {
2525
redundant_blocks: Option<u16>,
2626
pool_capacity: Option<usize>,
2727
compaction_interval: Option<u32>,
28-
max_index_key_len: Option<u32>,
28+
is_search_enabled: bool,
2929
) -> PyResult<Self> {
3030
let db = io_to_py_result!(scdb::Store::new(
3131
store_path,
3232
max_keys,
3333
redundant_blocks,
3434
pool_capacity,
3535
compaction_interval,
36-
max_index_key_len,
36+
is_search_enabled,
3737
))?;
3838
Ok(Self {
3939
db: Arc::new(Mutex::new(db)),

‎src/store.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ impl Store {
1515
redundant_blocks = "None",
1616
pool_capacity = "None",
1717
compaction_interval = "None",
18-
max_index_key_len = "None"
18+
is_search_enabled = "false"
1919
)]
2020
#[new]
2121
pub fn new(
@@ -24,15 +24,15 @@ impl Store {
2424
redundant_blocks: Option<u16>,
2525
pool_capacity: Option<usize>,
2626
compaction_interval: Option<u32>,
27-
max_index_key_len: Option<u32>,
27+
is_search_enabled: bool,
2828
) -> PyResult<Self> {
2929
let db = io_to_py_result!(scdb::Store::new(
3030
store_path,
3131
max_keys,
3232
redundant_blocks,
3333
pool_capacity,
3434
compaction_interval,
35-
max_index_key_len,
35+
is_search_enabled,
3636
))?;
3737
Ok(Self { db })
3838
}

0 commit comments

Comments
 (0)