Merge pull request #48 from Paulanerus/dev

Dev

Author: Paulanerus

api/src/main/kotlin/dev/paulee/api/data/Data.kt

@@ -128,4 +128,4 @@ data class Source(
     val preFilter: PreFilter? = null,
 )
 
-data class DataInfo(val name: String, val sources: List<Source>, val storageType: StorageType = StorageType.SQLITE)
+data class DataInfo(val name: String, val sources: List<Source>, val storageType: StorageType = StorageType.Default)

api/src/main/kotlin/dev/paulee/api/data/provider/StorageProvider.kt

@@ -3,24 +3,26 @@ package dev.paulee.api.data.provider
 import dev.paulee.api.data.DataInfo
 import java.io.Closeable
 import java.nio.file.Path
+import java.util.LinkedHashMap
 
 typealias QueryOrder = Pair<String, Boolean>
 
 enum class StorageType {
+    Default,
+
+    @Deprecated("SQLITE support has been removed. Kept only to detect legacy data pools; use StorageType.Default.")
     SQLITE,
 }
 
 enum class ProviderStatus {
-    SUCCESS,
-    FAILED,
-    EXISTS,
+    Success,
+    Failed,
+    Exists,
 }
 
 interface IStorageProvider : Closeable {
 
-    fun init(dataInfo: DataInfo, path: Path, lock: Boolean = false): ProviderStatus
-
-    fun insert(name: String, entries: List<Map<String, String>>)
+    fun init(dataInfo: DataInfo, path: Path): ProviderStatus
 
     fun get(
         name: String,
@@ -40,4 +42,6 @@ interface IStorageProvider : Closeable {
     ): Long
 
     fun suggestions(name: String, field: String, value: String, amount: Int): List<String>
+
+    fun streamData(name: String): Sequence<LinkedHashMap<String, String>>
 }

api/src/main/kotlin/dev/paulee/api/internal/Embedding.kt

@@ -11,16 +11,16 @@ object Embedding {
     ) {
         EmbeddingGemma(
             "onnx-community/embeddinggemma-300m-ONNX",
-            "A lightweight open embedding model from Google, built on Gemma 3 and trained on 100+ spoken languages.",
+            "model_management.embedding_gemma.desc",
             "Google DeepMind",
             "300M",
             "https://huggingface.co/google/embeddinggemma-300m",
             ModelData()
         ),
         AncientGreekBert(
             "onnx-community/Ancient-Greek-BERT-ONNX",
-            "A BERT model specialized for Greek and Ancient Greek texts.",
-            "Pranaydeep Singh, Gorik Rutten and Els Lefever",
+            "model_management.ancient_greek_bert.desc",
+            "Pranaydeep Singh, Gorik Rutten, Els Lefever",
             "110M",
             "https://huggingface.co/pranaydeeps/Ancient-Greek-BERT",
             ModelData(

build.gradle.kts

@@ -64,7 +64,7 @@ compose.desktop {
             }
 
             macOS {
-                // iconFile.set(project.file("ui/src/main/resources/icon.icns"))
+                iconFile.set(project.file("ui/src/main/resources/icon.icns"))
 
                 appCategory = "public.app-category.utilities"
             }

core/build.gradle.kts

@@ -18,7 +18,7 @@ dependencies {
 
     implementation(kotlin("reflect"))
 
-    implementation("org.xerial:sqlite-jdbc:${rootProject.extra["sqlite-jdbc.version"]}")
+    implementation("org.duckdb:duckdb_jdbc:${rootProject.extra["duckdb.version"]}")
 
     implementation("io.github.java-diff-utils:java-diff-utils:${rootProject.extra["jdu.version"]}")
 
@@ -45,6 +45,8 @@ dependencies {
     if(os.isMacOsX) implementation("com.microsoft.onnxruntime:onnxruntime:${rootProject.extra["onnx.version"]}")
     else implementation("com.microsoft.onnxruntime:onnxruntime_gpu:${rootProject.extra["onnx.version"]}")
 
+    implementation("com.github.ben-manes.caffeine:caffeine:${rootProject.extra["caffeine.version"]}")
+
     testImplementation(kotlin("test"))
 }
 

core/src/main/kotlin/dev/paulee/core/data/DataServiceImpl.kt

@@ -1,15 +1,13 @@
 package dev.paulee.core.data
 
-import dev.paulee.api.data.DataInfo
-import dev.paulee.api.data.IDataService
-import dev.paulee.api.data.PreFilter
-import dev.paulee.api.data.VariantMapping
+import com.github.benmanes.caffeine.cache.Cache
+import com.github.benmanes.caffeine.cache.Caffeine
+import dev.paulee.api.data.*
 import dev.paulee.api.data.provider.IStorageProvider
 import dev.paulee.api.data.provider.ProviderStatus
 import dev.paulee.api.data.provider.QueryOrder
 import dev.paulee.api.internal.Embedding
 import dev.paulee.core.data.analysis.Indexer
-import dev.paulee.core.data.io.BufferedCSVReader
 import dev.paulee.core.data.model.DataPool
 import dev.paulee.core.data.provider.EmbeddingProvider
 import dev.paulee.core.data.provider.StorageProvider
@@ -19,19 +17,22 @@ import kotlinx.coroutines.withContext
 import org.slf4j.LoggerFactory.getLogger
 import java.io.IOException
 import java.nio.file.Path
+import java.time.Duration
 import kotlin.coroutines.cancellation.CancellationException
 import kotlin.io.path.*
 import kotlin.math.ceil
 
+typealias QueryKey = Triple<Int, String, QueryOrder?>
+
 typealias PageResult = Pair<List<Map<String, String>>, Map<String, List<Map<String, String>>>>
 
 object DataServiceImpl : IDataService {
 
     private val logger = getLogger(DataServiceImpl::class.java)
 
-    private const val PAGE_SIZE = 50
+    private const val PAGE_SIZE = 100
 
-    private const val CSV_READER_BATCH_SIZE = 300
+    private const val BATCH_SIZE = 1000
 
     private val variantPattern = Regex("@([^:]+):(\\S+)")
 
@@ -41,11 +42,10 @@ object DataServiceImpl : IDataService {
 
     private var currentField: String? = null
 
-    private val pageCache = object : LinkedHashMap<Triple<Int, String, QueryOrder?>, PageResult>(6, 0.75f, true) {
-        override fun removeEldestEntry(eldest: MutableMap.MutableEntry<Triple<Int, String, QueryOrder?>, PageResult>?): Boolean {
-            return size > 6
-        }
-    }
+    private val pageCache: Cache<QueryKey, PageResult> = Caffeine.newBuilder()
+        .maximumSize(32)
+        .expireAfterAccess(Duration.ofDays(3))
+        .build()
 
     private val storageProvider = mutableMapOf<String, IStorageProvider>()
 
@@ -60,11 +60,13 @@ object DataServiceImpl : IDataService {
             try {
                 val poolPath = FileService.dataDir.resolve(dataInfo.name)
 
-                val storageProvider = StorageProvider.of(dataInfo.storageType)
+                val currentProvider = StorageProvider.of(dataInfo.storageType)
 
-                val initStatus = storageProvider.init(dataInfo, poolPath)
-
-                if (initStatus != ProviderStatus.SUCCESS) return@withContext initStatus == ProviderStatus.EXISTS
+                when (currentProvider.init(dataInfo, poolPath)) {
+                    ProviderStatus.Success -> Unit
+                    ProviderStatus.Exists -> return@withContext true
+                    else -> return@withContext false
+                }
 
                 dataInfoToString(dataInfo)?.let { json ->
                     poolPath.resolve("info.json").writeText(json)
@@ -75,62 +77,45 @@ object DataServiceImpl : IDataService {
                     DataPool(
                         indexer = Indexer(poolPath.resolve("index"), dataInfo),
                         dataInfo = dataInfo,
-                        storageProvider = storageProvider
+                        storageProvider = currentProvider
                     )
                 }.getOrElse { e ->
                     logger.error("Exception: Failed to create data pool.", e)
                     return@withContext false
                 }
 
-                val totalBatches = dataInfo.sources.sumOf { source ->
-                    val sourcePath =
-                        FileService.dataDir.resolve(source.name.let { if (it.endsWith(".csv")) it else "$it.csv" })
+                val sourcesWithIndex = dataInfo.sources.filter { it.fields.any { field -> field is IndexField } }
 
-                    if (sourcePath.exists()) BufferedCSVReader.estimateBatches(sourcePath, CSV_READER_BATCH_SIZE)
-                    else 0
-                }
+                val totalBatches =
+                    ((sourcesWithIndex.sumOf { currentProvider.count(it.name) } + BATCH_SIZE - 1) / BATCH_SIZE).toInt()
 
                 var processedBatches = 0
                 var lastPercentage = 0
                 onProgress(0)
 
-                dataInfo.sources.forEach { source ->
-                    val file = source.name
-
-                    if (file.isEmpty()) {
-                        logger.warn("No data source provided for ${file}.")
-                        return@forEach
-                    }
+                sourcesWithIndex
+                    .forEach { source ->
+                        val name = source.name
 
-                    val sourcePath =
-                        FileService.dataDir.resolve(file.let { if (it.endsWith(".csv")) it else "$it.csv" })
+                        currentProvider.streamData(name)
+                            .chunked(BATCH_SIZE)
+                            .forEach { entries ->
+                                dataPool.indexer.indexEntries(name, entries)
 
-                    if (!sourcePath.exists()) {
-                        logger.warn("Source file '$sourcePath' not found.")
-                        return@forEach
-                    }
+                                processedBatches++
 
-                    val idGenerator = generateSequence(1L) { it + 1 }.iterator()
+                                val percentage =
+                                    (if (totalBatches > 0) (processedBatches * 100) / totalBatches else 0)
 
-                    BufferedCSVReader(sourcePath, batchSize = CSV_READER_BATCH_SIZE).readLines { lines ->
-                        val entries = lines.map { line ->
-                            if (dataPool.hasIdentifier(file, line)) line
-                            else line + ("${file}_ag_id" to idGenerator.next().toString())
-                        }
-
-                        dataPool.indexer.indexEntries(file, entries)
-                        storageProvider.insert(file, entries)
-
-                        processedBatches++
-
-                        val percentage = if (totalBatches > 0) (processedBatches * 100) / totalBatches else 0
-
-                        if (percentage > lastPercentage) {
-                            onProgress(percentage)
-                            lastPercentage = percentage
-                        }
+                                if (percentage > lastPercentage) {
+                                    onProgress(percentage)
+                                    lastPercentage = percentage
+                                }
+                            }
                     }
-                }
+
+                dataPool.indexer.finish()
+                EmbeddingProvider.finish()
 
                 dataPools[dataInfo.name] = dataPool
 
@@ -170,7 +155,7 @@ object DataServiceImpl : IDataService {
 
             val storageProvider = StorageProvider.of(dataInfo.storageType)
 
-            if (storageProvider.init(dataInfo, child) == ProviderStatus.EXISTS) {
+            if (storageProvider.init(dataInfo, child) == ProviderStatus.Exists) {
                 dataPools[dataInfo.name] =
                     DataPool(Indexer(child.resolve("index"), dataInfo), dataInfo, storageProvider)
 
@@ -247,7 +232,7 @@ object DataServiceImpl : IDataService {
 
         val key = Triple(pageCount, query, order)
 
-        pageCache[key]?.let { return it }
+        pageCache.getIfPresent(key)?.let { return it }
 
         val dataPool = this.dataPools[this.currentPool] ?: return Pair(emptyList(), emptyMap())
 
@@ -284,7 +269,7 @@ object DataServiceImpl : IDataService {
 
         val result = PageResult(entries, links)
 
-        pageCache[key] = result
+        pageCache.put(key, result)
 
         return result
     }
@@ -323,7 +308,7 @@ object DataServiceImpl : IDataService {
             return null
         }
 
-        provider.init(dataInfo, path, true)
+        provider.init(dataInfo, path)
 
         return provider
     }

core/src/main/kotlin/dev/paulee/core/data/FileService.kt

@@ -37,6 +37,31 @@ internal object FileService {
         }
     }
 
+    val isCuda12xInstalled: Boolean by lazy {
+        if (OperatingSystem.isMacOS) return@lazy false
+
+        runCatching {
+            val process = ProcessBuilder().apply {
+                command("nvcc", "--version")
+                redirectErrorStream(true)
+            }.start()
+
+            val output = process.inputStream.bufferedReader().use { it.readText() }
+            process.waitFor()
+
+            val versionRegex = Regex("""V(\d+)\.(\d+)\.(\d+)""")
+            val matchResult = versionRegex.find(output)
+
+            if (matchResult != null) {
+                val majorVersion = matchResult.groupValues[1].toIntOrNull() ?: 0
+
+                majorVersion == 12
+            } else {
+                false
+            }
+        }.getOrDefault(false)
+    }
+
     val appDir: Path get() = ensureDir(".textexplorer", true)
 
     val pluginsDir: Path get() = ensureDir("plugins")
@@ -50,7 +75,7 @@ internal object FileService {
     private val mapper = jacksonObjectMapper().apply { enable(SerializationFeature.INDENT_OUTPUT) }
 
     init {
-        logger.info("Operating system: ${OperatingSystem.current}")
+        logger.info("Operating system: ${OperatingSystem.current} | CUDA: $isCuda12xInstalled")
     }
 
     fun toJson(dataInfo: DataInfo): String? = runCatching { this.mapper.writeValueAsString(dataInfo) }