Merge pull request #43 from Paulanerus/dev

Semantic Search (1.5.0)

Author: Paulanerus

.github/workflows/build-and-publish.yml

@@ -41,7 +41,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up Java
-        uses: actions/setup-java@v4.5.0
+        uses: actions/setup-java@v5.0.0
         with:
           java-version: '21'
           distribution: 'corretto'
@@ -70,7 +70,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up Java
-        uses: actions/setup-java@v4.5.0
+        uses: actions/setup-java@v5.0.0
         with:
           java-version: '21'
           distribution: 'corretto'
@@ -93,17 +93,23 @@ jobs:
 
     strategy:
       matrix:
-        os: [windows-latest, macos-latest, ubuntu-latest]
+        include:
+          - os: windows-latest
+            distribution: microsoft
+          - os: macos-latest
+            distribution: corretto
+          - os: ubuntu-latest
+            distribution: corretto
 
     steps:
       - name: Check out code
         uses: actions/checkout@v4
 
-      - name: Set up Java
-        uses: actions/setup-java@v4.5.0
+      - name: Set up Java (${{ matrix.distribution }})
+        uses: actions/setup-java@v5.0.0
         with:
           java-version: '21'
-          distribution: 'corretto'
+          distribution: ${{ matrix.distribution }}
           cache: 'gradle'
 
       - name: Build ${{ matrix.os }}
@@ -113,10 +119,10 @@ jobs:
         if: matrix.os == 'windows-latest'
         run: gh release upload v${{ env.VERSION }} build/compose/binaries/main-release/msi/TextVariantExplorer-${{ env.VERSION }}.msi --clobber
 
-      - name: Upload Linux  Artifact
+      - name: Upload Linux Artifact
         if: matrix.os == 'ubuntu-latest'
         run: gh release upload v${{ env.VERSION }} build/compose/binaries/main-release/deb/textvariantexplorer_${{ env.VERSION }}_amd64.deb --clobber
 
       - name: Upload macOS Artifact
         if: matrix.os == 'macos-latest'
-        run: gh release upload v${{ env.VERSION }} build/compose/binaries/main-release/dmg/TextVariantExplorer-${{ env.VERSION }}.Dmg --clobber
+        run: gh release upload v${{ env.VERSION }} build/compose/binaries/main-release/dmg/TextVariantExplorer-${{ env.VERSION }}.dmg --clobber

api/src/main/kotlin/dev/paulee/api/data/Data.kt

@@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonInclude
 import com.fasterxml.jackson.annotation.JsonSubTypes
 import com.fasterxml.jackson.annotation.JsonTypeInfo
 import dev.paulee.api.data.provider.StorageType
+import dev.paulee.api.internal.Embedding
 
 enum class Language {
     ARABIC,
@@ -62,7 +63,7 @@ annotation class ViewFilter(
     val name: String,
     val fields: Array<String> = [],
     val alwaysShow: Array<String> = [],
-    val global: Boolean = true
+    val global: Boolean = true,
 )
 
 enum class FieldType {
@@ -93,7 +94,7 @@ sealed interface SourceField {
 data class BasicField(
     override val name: String,
     override val fieldType: FieldType,
-    @param:JsonInclude(JsonInclude.Include.NON_EMPTY) override val sourceLink: String = ""
+    @param:JsonInclude(JsonInclude.Include.NON_EMPTY) override val sourceLink: String = "",
 ) :
     SourceField
 
@@ -103,15 +104,16 @@ data class IndexField(
     @param:JsonInclude(JsonInclude.Include.NON_EMPTY)
     override val sourceLink: String = "",
     val lang: Language,
-    val default: Boolean = false
+    val default: Boolean = false,
+    val embeddingModel: Embedding.Model? = null,
 ) : SourceField
 
 data class UniqueField(
     override val name: String,
     override val fieldType: FieldType,
     @param:JsonInclude(JsonInclude.Include.NON_EMPTY)
     override val sourceLink: String = "",
-    val identify: Boolean = false
+    val identify: Boolean = false,
 ) : SourceField
 
 data class VariantMapping(val base: String, val variants: List<String>)
@@ -123,7 +125,7 @@ data class Source(
     val name: String,
     val fields: List<SourceField>,
     val variantMapping: VariantMapping? = null,
-    val preFilter: PreFilter? = null
+    val preFilter: PreFilter? = null,
 )
 
 data class DataInfo(val name: String, val sources: List<Source>, val storageType: StorageType = StorageType.SQLITE)

api/src/main/kotlin/dev/paulee/api/data/IDataService.kt

@@ -2,14 +2,13 @@ package dev.paulee.api.data
 
 import dev.paulee.api.data.provider.IStorageProvider
 import dev.paulee.api.data.provider.QueryOrder
+import dev.paulee.api.internal.Embedding
 import java.io.Closeable
 import java.nio.file.Path
 
 interface IDataService : Closeable {
 
-    suspend fun createDataPool(dataInfo: DataInfo, path: Path, onProgress: (progress: Int) -> Unit): Boolean
-
-    fun loadDataPools(path: Path): Int
+    suspend fun createDataPool(dataInfo: DataInfo, onProgress: (progress: Int) -> Unit): Boolean
 
     fun selectDataPool(selection: String)
 
@@ -23,13 +22,26 @@ interface IDataService : Closeable {
 
     fun getSuggestions(field: String, value: String): List<String>
 
-    fun getPage(query: String, order: QueryOrder?, pageCount: Int): Pair<List<Map<String, String>>, Map<String, List<Map<String, String>>>>
+    suspend fun downloadModel(model: Embedding.Model, path: Path, onProgress: (progress: Int) -> Unit)
+
+    fun getPage(
+        query: String,
+        isSemantic: Boolean,
+        order: QueryOrder?,
+        pageCount: Int,
+    ): Pair<List<Map<String, String>>, Map<String, List<Map<String, String>>>>
 
-    fun getPageCount(query: String): Triple<Long, Long, Set<String>>
+    fun getPageCount(query: String, isSemantic: Boolean): Triple<Long, Long, Set<String>>
 
     fun createStorageProvider(infoName: String, path: Path): IStorageProvider?
 
     fun dataInfoToString(dataInfo: DataInfo): String?
 
     fun dataInfoFromString(dataInfo: String): DataInfo?
+
+    fun appDir(): Path
+
+    fun dataDir(): Path
+
+    fun modelDir(): Path
 }

api/src/main/kotlin/dev/paulee/api/data/provider/StorageProvider.kt

@@ -24,7 +24,7 @@ interface IStorageProvider : Closeable {
 
     fun get(
         name: String,
-        ids: Set<Long> = emptySet(),
+        ids: List<Long> = emptyList(),
         whereClause: List<String> = emptyList(),
         filter: List<String> = emptyList(),
         order: QueryOrder? = null,
@@ -34,7 +34,7 @@ interface IStorageProvider : Closeable {
 
     fun count(
         name: String,
-        ids: Set<Long> = emptySet(),
+        ids: List<Long> = emptyList(),
         whereClause: List<String> = emptyList(),
         filter: List<String> = emptyList(),
     ): Long

api/src/main/kotlin/dev/paulee/api/internal/Embedding.kt

@@ -0,0 +1,41 @@
+package dev.paulee.api.internal
+
+object Embedding {
+    enum class Model(
+        val id: String,
+        val description: String,
+        val author: String,
+        val parameter: String,
+        val link: String,
+        val modelData: ModelData,
+    ) {
+        EmbeddingGemma(
+            "onnx-community/embeddinggemma-300m-ONNX",
+            "A lightweight open embedding model from Google, built on Gemma 3 and trained on 100+ spoken languages.",
+            "Google DeepMind",
+            "300M",
+            "https://huggingface.co/google/embeddinggemma-300m",
+            ModelData()
+        ),
+        AncientGreekBert(
+            "onnx-community/Ancient-Greek-BERT-ONNX",
+            "A BERT model specialized for Greek and Ancient Greek texts.",
+            "Pranaydeep Singh, Gorik Rutten and Els Lefever",
+            "110M",
+            "https://huggingface.co/pranaydeeps/Ancient-Greek-BERT",
+            ModelData(
+                maxLength = 512,
+                modelData = ""
+            )
+        )
+    }
+
+    data class ModelData(
+        val dimension: Int = 768,
+        val maxLength: Int = 2048,
+        val model: String = "onnx/model.onnx",
+        val modelData: String = "onnx/model.onnx_data",
+        val tokenizer: String = "tokenizer.json",
+        val tokenizerConfig: String = "tokenizer_config.json",
+    )
+}

api/src/main/kotlin/dev/paulee/api/plugin/IPluginService.kt

@@ -6,17 +6,17 @@ import java.nio.file.Path
 
 interface IPluginService {
 
-    fun loadFromDirectory(path: Path): Int
-
     fun loadPlugin(path: Path): IPlugin?
 
     fun getPluginMetadata(plugin: IPlugin): PluginMetadata?
 
     fun getDataInfo(plugin: IPlugin): String?
 
-    fun initAll(dataService: IDataService, path: Path)
+    fun initAll(dataService: IDataService)
 
     fun getPlugins(): List<IPlugin>
 
     fun getViewFilter(plugin: IPlugin): ViewFilter?
+
+    fun pluginDir(): Path
 }

build.gradle.kts

@@ -37,6 +37,7 @@ compose.desktop {
 
         jvmArgs += listOf(
             "--add-modules", "java.sql",
+            "--add-modules", "jdk.incubator.vector",
             "-Dapi.version=${property("api.version")}",
             "-Dcore.version=${property("core.version")}",
             "-Dui.version=${property("ui.version")}",
@@ -47,7 +48,26 @@ compose.desktop {
             targetFormats(TargetFormat.Dmg, TargetFormat.Msi, TargetFormat.Deb)
             packageName = "TextVariantExplorer"
 
-            modules += listOf("java.sql")
+            licenseFile.set(project.file("LICENSE.md"))
+
+            modules += listOf("java.sql", "jdk.incubator.vector")
+
+            linux {
+                iconFile.set(project.file("ui/src/main/resources/icon.png"))
+            }
+
+            windows {
+                iconFile.set(project.file("ui/src/main/resources/icon.ico"))
+
+                menu = true
+                upgradeUuid = "C4AF61D5-8472-482D-B2A0-F92E32D7A18C"
+            }
+
+            macOS {
+                iconFile.set(project.file("ui/src/main/resources/icon.icns"))
+
+                appCategory = "public.app-category.utilities"
+            }
         }
     }
 }

core/build.gradle.kts

@@ -1,10 +1,14 @@
+import org.gradle.internal.os.OperatingSystem
+
 plugins {
     kotlin("jvm")
 }
 
 group = "dev.paulee"
 version = rootProject.extra["core.version"] as String
 
+val os = OperatingSystem.current()
+
 repositories {
     mavenCentral()
 }
@@ -36,6 +40,11 @@ dependencies {
 
     implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:${rootProject.extra["coroutines.version"]}")
 
+    implementation("ai.djl.huggingface:tokenizers:${rootProject.extra["tokenizers.version"]}")
+
+    if(os.isMacOsX) implementation("com.microsoft.onnxruntime:onnxruntime:${rootProject.extra["onnx.version"]}")
+    else implementation("com.microsoft.onnxruntime:onnxruntime_gpu:${rootProject.extra["onnx.version"]}")
+
     testImplementation(kotlin("test"))
 }