Merge pull request #53 from Paulanerus/dev

Dev

Author: Paulanerus

api/src/main/kotlin/dev/paulee/api/data/IDataService.kt

@@ -32,14 +32,14 @@ interface IDataService : Closeable {
 
     suspend fun exportPool(dataInfo: DataInfo, path: Path): Boolean
 
-    fun getPage(
+    suspend fun getPage(
         query: String,
-        isSemantic: Boolean,
+        similarityScore: Float,
         order: QueryOrder?,
         pageCount: Int,
     ): Pair<List<Map<String, String>>, Map<String, List<Map<String, String>>>>
 
-    fun getPageCount(query: String, isSemantic: Boolean): Triple<Long, Long, Set<String>>
+    suspend fun getPageCount(query: String, similarityScore: Float): Triple<Long, Long, Set<String>>
 
     fun createStorageProvider(infoName: String, path: Path): IStorageProvider?
 

core/src/main/kotlin/dev/paulee/core/data/DataServiceImpl.kt

@@ -26,7 +26,7 @@ import kotlin.coroutines.cancellation.CancellationException
 import kotlin.io.path.*
 import kotlin.math.ceil
 
-typealias QueryKey = Triple<Int, String, QueryOrder?>
+private data class QueryKey(val pageCount: Int, val query: String, val order: QueryOrder?, val similarity: Float)
 
 typealias PageResult = Pair<List<Map<String, String>>, Map<String, List<Map<String, String>>>>
 
@@ -537,13 +537,18 @@ object DataServiceImpl : IDataService {
         return result
     }
 
-    override fun getPage(query: String, isSemantic: Boolean, order: QueryOrder?, pageCount: Int): PageResult {
+    override suspend fun getPage(
+        query: String,
+        similarityScore: Float,
+        order: QueryOrder?,
+        pageCount: Int,
+    ): PageResult {
 
         if (this.currentPool == null || this.currentField == null) return Pair(emptyList(), emptyMap())
 
-        logger.info("Query (${order ?: "None"} | Semantic: $isSemantic): $query")
+        logger.info("Query (${order ?: "None"} | Similarity: $similarityScore): $query")
 
-        val key = Triple(pageCount, query, order)
+        val key = QueryKey(pageCount, query, order, similarityScore)
 
         pageCache.getIfPresent(key)?.let { return it }
 
@@ -553,7 +558,8 @@ object DataServiceImpl : IDataService {
 
         val (filterQuery, filter) = this.getPreFilter(query)
 
-        val indexResult = dataPool.search(this.handleReplacements(dataPool.metadata, filterQuery), isSemantic)
+        val indexResult =
+            dataPool.search(this.handleReplacements(dataPool.metadata, filterQuery), similarityScore)
 
         if (filter.isEmpty() && indexResult.isEmpty()) return Pair(emptyList(), emptyMap())
 
@@ -589,7 +595,7 @@ object DataServiceImpl : IDataService {
         return result
     }
 
-    override fun getPageCount(query: String, isSemantic: Boolean): Triple<Long, Long, Set<String>> {
+    override suspend fun getPageCount(query: String, similarityScore: Float): Triple<Long, Long, Set<String>> {
         if (this.currentPool == null || this.currentField == null) return Triple(-1, -1, emptySet())
 
         val dataPool = this.dataPools[this.currentPool] ?: return Triple(-1, -1, emptySet())
@@ -598,7 +604,8 @@ object DataServiceImpl : IDataService {
 
         val (filterQuery, filter) = this.getPreFilter(query)
 
-        val indexResult = dataPool.search(handleReplacements(dataPool.metadata, filterQuery), isSemantic)
+        val indexResult =
+            dataPool.search(handleReplacements(dataPool.metadata, filterQuery), similarityScore)
 
         if (filter.isEmpty() && indexResult.isEmpty()) return Triple(0, 0, emptySet())
 
@@ -717,5 +724,4 @@ object DataServiceImpl : IDataService {
         ) null
         else token.trim()
     }
-}
-    
+}

core/src/main/kotlin/dev/paulee/core/data/model/DataPool.kt

@@ -90,7 +90,7 @@ internal class DataPool(val indexer: Indexer?, val dataInfo: DataInfo, val stora
         }
     }
 
-    fun search(query: String, semantic: Boolean): IndexSearchResult {
+    fun search(query: String, similarityThreshold: Float): IndexSearchResult {
         if (indexer == null) return IndexSearchResult()
 
         val ids = LinkedHashSet<Long>()
@@ -102,7 +102,7 @@ internal class DataPool(val indexer: Indexer?, val dataInfo: DataInfo, val stora
             docs.forEach { it.getField(idFieldName)?.numericValue()?.toLong()?.let(ids::add) }
         }
 
-        val similarityThreshold = 0.85f
+        val isSemantic = similarityThreshold > 0 && similarityThreshold <= 1.0
 
         val indexedValues = mutableSetOf<String>()
 
@@ -139,7 +139,7 @@ internal class DataPool(val indexer: Indexer?, val dataInfo: DataInfo, val stora
                     indexedValues.add(value)
 
                     val docs =
-                        if (semantic) indexer.searchMatchingVec(field, value, similarityThreshold)
+                        if (isSemantic) indexer.searchMatchingVec(field, value, similarityThreshold)
                         else indexer.searchFieldIndex(field, value)
 
                     addIdsFrom(docs, fieldClass)
@@ -153,7 +153,7 @@ internal class DataPool(val indexer: Indexer?, val dataInfo: DataInfo, val stora
 
                 defaultIndexField?.let { defaultField ->
                     val docs =
-                        if (semantic) indexer.searchMatchingVec(defaultField, joined, similarityThreshold)
+                        if (isSemantic) indexer.searchMatchingVec(defaultField, joined, similarityThreshold)
                         else indexer.searchFieldIndex(defaultField, joined)
 
                     addIdsFrom(docs, defaultClass)
@@ -162,7 +162,7 @@ internal class DataPool(val indexer: Indexer?, val dataInfo: DataInfo, val stora
         } else {
             defaultIndexField?.let {
                 val docs =
-                    if (semantic) indexer.searchMatchingVec(it, query, similarityThreshold)
+                    if (isSemantic) indexer.searchMatchingVec(it, query, similarityThreshold)
                     else indexer.searchFieldIndex(it, query)
 
                 addIdsFrom(docs, defaultClass)

gradle.properties

@@ -2,7 +2,7 @@ org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8
 org.gradle.configuration-cache=true
 kotlin.code.style=official
 kotlin.version=2.2.20
-compose.version=1.8.2
+compose.version=1.9.2
 icons.version=1.7.3
 lucene.version=10.3.1
 log4j.version=2.25.2
@@ -14,7 +14,7 @@ onnx.version=1.22.0
 duckdb.version=1.4.1.0
 caffeine.version=3.2.2
 
-api.version=1.16.2
-core.version=1.18.7
-ui.version=1.18.5
-app.version=1.6.4
+api.version=1.16.4
+core.version=1.18.12
+ui.version=1.19.5
+app.version=1.6.5

ui/build.gradle.kts

@@ -20,4 +20,5 @@ dependencies {
 
     implementation(compose.desktop.currentOs)
     implementation("org.jetbrains.compose.material:material-icons-core:${rootProject.extra["icons.version"]}")
+    implementation("org.jetbrains.compose.material:material-icons-extended:${rootProject.extra["icons.version"]}")
 }

ui/src/main/kotlin/dev/paulee/ui/Config.kt

@@ -36,6 +36,8 @@ object Config {
 
     var searchExpanded by mutableStateOf(true)
 
+    var queryEmbSimilarity by mutableStateOf(0.8f)
+
     private var configFile = "config"
 
     private var configPath = Path(configFile)