From abe1480923dfd78ebc00bada2ffa9b29c72c283e Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Sun, 23 Nov 2025 17:08:05 +0100 Subject: [PATCH 001/243] Refactor `PaperExcerpt` to `org.jabref.model.ai` and clean up unused localization code in `AiTemplate`. --- .../jabref/logic/ai/chatting/AiChatLogic.java | 2 +- .../logic/ai/chatting/model/Gpt4AllModel.java | 2 +- .../jabref/logic/ai/templates/AiTemplate.java | 29 ++----------------- .../ai/templates/AiTemplatesService.java | 1 + .../templates => model/ai}/PaperExcerpt.java | 2 +- 5 files changed, 7 insertions(+), 29 deletions(-) rename jablib/src/main/java/org/jabref/{logic/ai/templates => model/ai}/PaperExcerpt.java (62%) diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/AiChatLogic.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/AiChatLogic.java index 128dd442cf51..f220ba222ce6 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/AiChatLogic.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/AiChatLogic.java @@ -11,8 +11,8 @@ import org.jabref.logic.ai.ingestion.FileEmbeddingsManager; import org.jabref.logic.ai.templates.AiTemplate; import org.jabref.logic.ai.templates.AiTemplatesService; -import org.jabref.logic.ai.templates.PaperExcerpt; import org.jabref.logic.ai.util.ErrorMessage; +import org.jabref.model.ai.PaperExcerpt; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/model/Gpt4AllModel.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/model/Gpt4AllModel.java index 7b1eccf9943c..df21fb15868e 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/model/Gpt4AllModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/model/Gpt4AllModel.java @@ -87,7 +87,7 @@ public ChatResponse chat(List list) { throw new IllegalArgumentException("Generated text is null or empty"); } - // Note: We do not check the token usage and finish reason here. + // Note: We do not check the token usage and the finish reason here. // This class is not a complete implementation of langchain4j's ChatLanguageModel. // We only implemented the functionality we specifically need. return new ChatResponse.Builder().aiMessage(new AiMessage(generatedText)) diff --git a/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplate.java b/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplate.java index d7d0665d20b5..9b0d322e961f 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplate.java +++ b/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplate.java @@ -1,42 +1,19 @@ package org.jabref.logic.ai.templates; -import org.jabref.logic.l10n.Localization; - public enum AiTemplate { - // Templates that are used in AI chats, + // Templates that are used in AI chats. CHATTING_SYSTEM_MESSAGE, CHATTING_USER_MESSAGE, - // Templates that are used for summarization of text chunks, + // Templates that are used for summarization of text chunks. SUMMARIZATION_CHUNK_SYSTEM_MESSAGE, SUMMARIZATION_CHUNK_USER_MESSAGE, - // Templates that are used for combining summaries of several chunks, + // Templates that are used for combining summaries of several chunks. SUMMARIZATION_COMBINE_SYSTEM_MESSAGE, SUMMARIZATION_COMBINE_USER_MESSAGE, // Templates that are used to convert a raw citation into a {@link BibEntry}. CITATION_PARSING_SYSTEM_MESSAGE, CITATION_PARSING_USER_MESSAGE; - - public String getLocalizedName() { - return switch (this) { - case CHATTING_SYSTEM_MESSAGE -> - Localization.lang("System message for chatting"); - case CHATTING_USER_MESSAGE -> - Localization.lang("User message for chatting"); - case SUMMARIZATION_CHUNK_SYSTEM_MESSAGE -> - Localization.lang("System message for summarization of a chunk"); - case SUMMARIZATION_CHUNK_USER_MESSAGE -> - Localization.lang("User message for summarization of a chunk"); - case SUMMARIZATION_COMBINE_SYSTEM_MESSAGE -> - Localization.lang("System message for summarization of several chunks"); - case SUMMARIZATION_COMBINE_USER_MESSAGE -> - Localization.lang("User message for summarization of several chunks"); - case CITATION_PARSING_SYSTEM_MESSAGE -> - Localization.lang("System message for citation parsing"); - case CITATION_PARSING_USER_MESSAGE -> - Localization.lang("User message for citation parsing"); - }; - } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java b/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java index 3ad8b570a99d..2f1aa4e6c0bd 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java @@ -4,6 +4,7 @@ import java.util.List; import org.jabref.logic.ai.AiPreferences; +import org.jabref.model.ai.PaperExcerpt; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.CanonicalBibEntry; diff --git a/jablib/src/main/java/org/jabref/logic/ai/templates/PaperExcerpt.java b/jablib/src/main/java/org/jabref/model/ai/PaperExcerpt.java similarity index 62% rename from jablib/src/main/java/org/jabref/logic/ai/templates/PaperExcerpt.java rename to jablib/src/main/java/org/jabref/model/ai/PaperExcerpt.java index ac4743c5c9bc..7c76ed940d35 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/templates/PaperExcerpt.java +++ b/jablib/src/main/java/org/jabref/model/ai/PaperExcerpt.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.templates; +package org.jabref.model.ai; public record PaperExcerpt(String citationKey, String text) { } From 088037a720cce7a7b2cd799dd891c2ef1231c3ea Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Sun, 23 Nov 2025 17:13:36 +0100 Subject: [PATCH 002/243] Refactor AI-related classes to `org.jabref.model.ai`, adjust imports, and restructure package hierarchy. --- .../jabref/gui/ai/components/summary/SummaryComponent.java | 4 ++-- .../gui/ai/components/summary/SummaryShowingComponent.java | 2 +- .../ai/components/util/EmbeddingModelGuardedComponent.java | 2 +- .../src/main/java/org/jabref/gui/preferences/ai/AiTab.java | 2 +- .../java/org/jabref/gui/preferences/ai/AiTabViewModel.java | 2 +- jablib/src/main/java/module-info.java | 3 ++- .../main/java/org/jabref/logic/ai/AiDefaultPreferences.java | 2 +- jablib/src/main/java/org/jabref/logic/ai/AiPreferences.java | 2 +- jablib/src/main/java/org/jabref/logic/ai/AiService.java | 4 ++-- .../main/java/org/jabref/logic/ai/chatting/AiChatLogic.java | 2 +- .../ai/chatting/{model => }/JabRefChatLanguageModel.java | 5 +++-- .../embeddingmodels}/DeepJavaEmbeddingModel.java | 2 +- .../model => customimplementations/llms}/Gpt4AllModel.java | 2 +- .../llms}/JvmOpenAiChatLanguageModel.java | 2 +- .../logic/ai/ingestion/GenerateEmbeddingsForSeveralTask.java | 4 ++-- .../java/org/jabref/logic/ai/ingestion/IngestionService.java | 4 ++-- .../logic/ai/ingestion/{model => }/JabRefEmbeddingModel.java | 3 ++- .../ai/ingestion/{model => }/UpdateEmbeddingModelTask.java | 3 ++- .../ai/summarization/GenerateSummaryForSeveralTask.java | 5 +++-- .../jabref/logic/ai/summarization/GenerateSummaryTask.java | 3 ++- .../org/jabref/logic/ai/summarization/SummariesService.java | 5 +++-- .../org/jabref/logic/ai/summarization/SummariesStorage.java | 2 ++ .../ai/summarization/storages/MVStoreSummariesStorage.java | 2 +- .../org/jabref/logic/ai/templates/AiTemplatesService.java | 1 + .../org/jabref/logic/preferences/JabRefCliPreferences.java | 2 +- .../jabref/{logic/ai/templates => model/ai}/AiTemplate.java | 2 +- .../jabref/{logic/ai/summarization => model/ai}/Summary.java | 4 +--- .../{logic => model}/ai/processingstatus/ProcessingInfo.java | 2 +- .../ai/processingstatus/ProcessingState.java | 2 +- .../jabref/logic/ai/summarization/SummariesStorageTest.java | 1 + 30 files changed, 45 insertions(+), 36 deletions(-) rename jablib/src/main/java/org/jabref/logic/ai/chatting/{model => }/JabRefChatLanguageModel.java (97%) rename jablib/src/main/java/org/jabref/logic/ai/{ingestion/model => customimplementations/embeddingmodels}/DeepJavaEmbeddingModel.java (96%) rename jablib/src/main/java/org/jabref/logic/ai/{chatting/model => customimplementations/llms}/Gpt4AllModel.java (99%) rename jablib/src/main/java/org/jabref/logic/ai/{chatting/model => customimplementations/llms}/JvmOpenAiChatLanguageModel.java (98%) rename jablib/src/main/java/org/jabref/logic/ai/ingestion/{model => }/JabRefEmbeddingModel.java (97%) rename jablib/src/main/java/org/jabref/logic/ai/ingestion/{model => }/UpdateEmbeddingModelTask.java (96%) rename jablib/src/main/java/org/jabref/{logic/ai/templates => model/ai}/AiTemplate.java (93%) rename jablib/src/main/java/org/jabref/{logic/ai/summarization => model/ai}/Summary.java (69%) rename jablib/src/main/java/org/jabref/{logic => model}/ai/processingstatus/ProcessingInfo.java (97%) rename jablib/src/main/java/org/jabref/{logic => model}/ai/processingstatus/ProcessingState.java (64%) diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java index 8e1c6089ca1e..372e4b9a3ee4 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java @@ -11,13 +11,13 @@ import org.jabref.gui.frame.ExternalApplicationsPreferences; import org.jabref.logic.ai.AiPreferences; import org.jabref.logic.ai.AiService; -import org.jabref.logic.ai.processingstatus.ProcessingInfo; -import org.jabref.logic.ai.summarization.Summary; import org.jabref.logic.ai.util.CitationKeyCheck; import org.jabref.logic.citationkeypattern.CitationKeyGenerator; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.io.FileUtil; +import org.jabref.model.ai.Summary; +import org.jabref.model.ai.processingstatus.ProcessingInfo; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryShowingComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryShowingComponent.java index 74386d50cd13..ed6a1198a443 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryShowingComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryShowingComponent.java @@ -13,8 +13,8 @@ import javafx.scene.web.WebView; import org.jabref.gui.util.WebViewStore; -import org.jabref.logic.ai.summarization.Summary; import org.jabref.logic.layout.format.MarkdownFormatter; +import org.jabref.model.ai.Summary; import com.airhacks.afterburner.views.ViewLoader; diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java index 89426aabeb20..90354f320c77 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java @@ -10,7 +10,7 @@ import org.jabref.gui.util.UiTaskExecutor; import org.jabref.logic.ai.AiPreferences; import org.jabref.logic.ai.AiService; -import org.jabref.logic.ai.ingestion.model.JabRefEmbeddingModel; +import org.jabref.logic.ai.ingestion.JabRefEmbeddingModel; import org.jabref.logic.l10n.Localization; import com.google.common.eventbus.Subscribe; diff --git a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java index 1f708435c3f8..21eae8c557a6 100644 --- a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java +++ b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java @@ -21,10 +21,10 @@ import org.jabref.gui.preferences.AbstractPreferenceTabView; import org.jabref.gui.preferences.PreferencesTab; import org.jabref.gui.util.ViewModelListCellFactory; -import org.jabref.logic.ai.templates.AiTemplate; import org.jabref.logic.help.HelpFile; import org.jabref.logic.l10n.Localization; import org.jabref.model.ai.AiProvider; +import org.jabref.model.ai.AiTemplate; import org.jabref.model.ai.EmbeddingModel; import com.airhacks.afterburner.views.ViewLoader; diff --git a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java index c6308a77f292..1ec030fca5f3 100644 --- a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java +++ b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java @@ -22,13 +22,13 @@ import org.jabref.gui.preferences.PreferenceTabViewModel; import org.jabref.logic.ai.AiDefaultPreferences; import org.jabref.logic.ai.AiPreferences; -import org.jabref.logic.ai.templates.AiTemplate; import org.jabref.logic.l10n.Localization; import org.jabref.logic.preferences.CliPreferences; import org.jabref.logic.util.LocalizedNumbers; import org.jabref.logic.util.OptionalObjectProperty; import org.jabref.logic.util.strings.StringUtil; import org.jabref.model.ai.AiProvider; +import org.jabref.model.ai.AiTemplate; import org.jabref.model.ai.EmbeddingModel; import de.saxsys.mvvmfx.utils.validation.FunctionBasedValidator; diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index 67112a8e227a..475da89f3f60 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -62,7 +62,7 @@ exports org.jabref.logic.ai.ingestion; exports org.jabref.logic.ai.ingestion.model; exports org.jabref.model.ai; - exports org.jabref.logic.ai.processingstatus; + exports org.jabref.model.ai.processingstatus; exports org.jabref.logic.ai.summarization; exports org.jabref.logic.layout.format; exports org.jabref.logic.auxparser; @@ -120,6 +120,7 @@ exports org.jabref.logic.git.merge.planning; exports org.jabref.logic.git.merge.execution; exports org.jabref.model.sciteTallies; + exports org.jabref.logic.ai.customimplementations.embeddingmodels; requires java.base; diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiDefaultPreferences.java b/jablib/src/main/java/org/jabref/logic/ai/AiDefaultPreferences.java index 1acbdf3eed2d..7d96e724826d 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiDefaultPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiDefaultPreferences.java @@ -4,8 +4,8 @@ import java.util.List; import java.util.Map; -import org.jabref.logic.ai.templates.AiTemplate; import org.jabref.model.ai.AiProvider; +import org.jabref.model.ai.AiTemplate; import org.jabref.model.ai.EmbeddingModel; public class AiDefaultPreferences { diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiPreferences.java b/jablib/src/main/java/org/jabref/logic/ai/AiPreferences.java index 9356407be718..6db7d743585d 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiPreferences.java @@ -16,9 +16,9 @@ import javafx.beans.property.SimpleStringProperty; import javafx.beans.property.StringProperty; -import org.jabref.logic.ai.templates.AiTemplate; import org.jabref.logic.util.strings.StringUtil; import org.jabref.model.ai.AiProvider; +import org.jabref.model.ai.AiTemplate; import org.jabref.model.ai.EmbeddingModel; import com.github.javakeyring.Keyring; diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiService.java b/jablib/src/main/java/org/jabref/logic/ai/AiService.java index 9e8d864275ff..033fabaa2e25 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiService.java @@ -9,11 +9,11 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.chatting.AiChatService; import org.jabref.logic.ai.chatting.ChatHistoryService; +import org.jabref.logic.ai.chatting.JabRefChatLanguageModel; import org.jabref.logic.ai.chatting.chathistory.storages.MVStoreChatHistoryStorage; -import org.jabref.logic.ai.chatting.model.JabRefChatLanguageModel; import org.jabref.logic.ai.ingestion.IngestionService; +import org.jabref.logic.ai.ingestion.JabRefEmbeddingModel; import org.jabref.logic.ai.ingestion.MVStoreEmbeddingStore; -import org.jabref.logic.ai.ingestion.model.JabRefEmbeddingModel; import org.jabref.logic.ai.ingestion.storages.MVStoreFullyIngestedDocumentsTracker; import org.jabref.logic.ai.summarization.SummariesService; import org.jabref.logic.ai.summarization.storages.MVStoreSummariesStorage; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/AiChatLogic.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/AiChatLogic.java index f220ba222ce6..a7202ef61d67 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/AiChatLogic.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/AiChatLogic.java @@ -9,9 +9,9 @@ import org.jabref.logic.ai.AiPreferences; import org.jabref.logic.ai.ingestion.FileEmbeddingsManager; -import org.jabref.logic.ai.templates.AiTemplate; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.ai.util.ErrorMessage; +import org.jabref.model.ai.AiTemplate; import org.jabref.model.ai.PaperExcerpt; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/model/JabRefChatLanguageModel.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/JabRefChatLanguageModel.java similarity index 97% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/model/JabRefChatLanguageModel.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/JabRefChatLanguageModel.java index f809b8f73693..7c53a8e94dd6 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/model/JabRefChatLanguageModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/JabRefChatLanguageModel.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.chatting.model; +package org.jabref.logic.ai.chatting; import java.net.http.HttpClient; import java.time.Duration; @@ -8,7 +8,8 @@ import java.util.concurrent.Executors; import org.jabref.logic.ai.AiPreferences; -import org.jabref.logic.ai.chatting.AiChatLogic; +import org.jabref.logic.ai.customimplementations.llms.Gpt4AllModel; +import org.jabref.logic.ai.customimplementations.llms.JvmOpenAiChatLanguageModel; import org.jabref.logic.l10n.Localization; import org.jabref.model.ai.AiProvider; diff --git a/jablib/src/main/java/org/jabref/logic/ai/ingestion/model/DeepJavaEmbeddingModel.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingmodels/DeepJavaEmbeddingModel.java similarity index 96% rename from jablib/src/main/java/org/jabref/logic/ai/ingestion/model/DeepJavaEmbeddingModel.java rename to jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingmodels/DeepJavaEmbeddingModel.java index 560a953a6e36..b8a4e457967b 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/ingestion/model/DeepJavaEmbeddingModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingmodels/DeepJavaEmbeddingModel.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.ingestion.model; +package org.jabref.logic.ai.customimplementations.embeddingmodels; import java.io.IOException; import java.util.ArrayList; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/model/Gpt4AllModel.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/Gpt4AllModel.java similarity index 99% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/model/Gpt4AllModel.java rename to jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/Gpt4AllModel.java index df21fb15868e..05da6eb9fe79 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/model/Gpt4AllModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/Gpt4AllModel.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.chatting.model; +package org.jabref.logic.ai.customimplementations.llms; import java.io.IOException; import java.net.http.HttpClient; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/model/JvmOpenAiChatLanguageModel.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/JvmOpenAiChatLanguageModel.java similarity index 98% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/model/JvmOpenAiChatLanguageModel.java rename to jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/JvmOpenAiChatLanguageModel.java index eac9c29b118b..f7b571937500 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/model/JvmOpenAiChatLanguageModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/JvmOpenAiChatLanguageModel.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.chatting.model; +package org.jabref.logic.ai.customimplementations.llms; import java.net.http.HttpClient; import java.util.List; diff --git a/jablib/src/main/java/org/jabref/logic/ai/ingestion/GenerateEmbeddingsForSeveralTask.java b/jablib/src/main/java/org/jabref/logic/ai/ingestion/GenerateEmbeddingsForSeveralTask.java index 9d74dd8fa3c4..6aadaf8e6dc6 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/ingestion/GenerateEmbeddingsForSeveralTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/ingestion/GenerateEmbeddingsForSeveralTask.java @@ -10,12 +10,12 @@ import javafx.util.Pair; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.processingstatus.ProcessingInfo; -import org.jabref.logic.ai.processingstatus.ProcessingState; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.ProgressCounter; import org.jabref.logic.util.TaskExecutor; +import org.jabref.model.ai.processingstatus.ProcessingInfo; +import org.jabref.model.ai.processingstatus.ProcessingState; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.LinkedFile; diff --git a/jablib/src/main/java/org/jabref/logic/ai/ingestion/IngestionService.java b/jablib/src/main/java/org/jabref/logic/ai/ingestion/IngestionService.java index fd5812cfe7ef..ae782a1d10bb 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/ingestion/IngestionService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/ingestion/IngestionService.java @@ -10,9 +10,9 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.AiPreferences; -import org.jabref.logic.ai.processingstatus.ProcessingInfo; -import org.jabref.logic.ai.processingstatus.ProcessingState; import org.jabref.logic.util.TaskExecutor; +import org.jabref.model.ai.processingstatus.ProcessingInfo; +import org.jabref.model.ai.processingstatus.ProcessingState; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.event.EntriesAddedEvent; import org.jabref.model.entry.LinkedFile; diff --git a/jablib/src/main/java/org/jabref/logic/ai/ingestion/model/JabRefEmbeddingModel.java b/jablib/src/main/java/org/jabref/logic/ai/ingestion/JabRefEmbeddingModel.java similarity index 97% rename from jablib/src/main/java/org/jabref/logic/ai/ingestion/model/JabRefEmbeddingModel.java rename to jablib/src/main/java/org/jabref/logic/ai/ingestion/JabRefEmbeddingModel.java index 487743d2701e..6260e2852b63 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/ingestion/model/JabRefEmbeddingModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/ingestion/JabRefEmbeddingModel.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.ingestion.model; +package org.jabref.logic.ai.ingestion; import java.util.List; import java.util.Optional; @@ -9,6 +9,7 @@ import javafx.beans.property.SimpleObjectProperty; import org.jabref.logic.ai.AiPreferences; +import org.jabref.logic.ai.customimplementations.embeddingmodels.DeepJavaEmbeddingModel; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.NotificationService; import org.jabref.logic.util.TaskExecutor; diff --git a/jablib/src/main/java/org/jabref/logic/ai/ingestion/model/UpdateEmbeddingModelTask.java b/jablib/src/main/java/org/jabref/logic/ai/ingestion/UpdateEmbeddingModelTask.java similarity index 96% rename from jablib/src/main/java/org/jabref/logic/ai/ingestion/model/UpdateEmbeddingModelTask.java rename to jablib/src/main/java/org/jabref/logic/ai/ingestion/UpdateEmbeddingModelTask.java index da236b69c342..fd90b17cb25b 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/ingestion/model/UpdateEmbeddingModelTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/ingestion/UpdateEmbeddingModelTask.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.ingestion.model; +package org.jabref.logic.ai.ingestion; import java.io.IOException; import java.util.Optional; @@ -6,6 +6,7 @@ import javafx.beans.property.ObjectProperty; import org.jabref.logic.ai.AiPreferences; +import org.jabref.logic.ai.customimplementations.embeddingmodels.DeepJavaEmbeddingModel; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.ProgressCounter; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/GenerateSummaryForSeveralTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/GenerateSummaryForSeveralTask.java index 63f7377e80f7..675f6963c992 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/GenerateSummaryForSeveralTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/GenerateSummaryForSeveralTask.java @@ -11,13 +11,14 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.AiPreferences; -import org.jabref.logic.ai.processingstatus.ProcessingInfo; -import org.jabref.logic.ai.processingstatus.ProcessingState; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.ProgressCounter; import org.jabref.logic.util.TaskExecutor; +import org.jabref.model.ai.Summary; +import org.jabref.model.ai.processingstatus.ProcessingInfo; +import org.jabref.model.ai.processingstatus.ProcessingState; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/GenerateSummaryTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/GenerateSummaryTask.java index 72a8e4b94fce..41048b6821e6 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/GenerateSummaryTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/GenerateSummaryTask.java @@ -13,12 +13,13 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.AiPreferences; import org.jabref.logic.ai.ingestion.FileToDocument; -import org.jabref.logic.ai.templates.AiTemplate; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.ai.util.CitationKeyCheck; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.ProgressCounter; +import org.jabref.model.ai.AiTemplate; +import org.jabref.model.ai.Summary; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java index 370a12a35bf6..2ec81b012b69 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java @@ -10,11 +10,12 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.AiPreferences; -import org.jabref.logic.ai.processingstatus.ProcessingInfo; -import org.jabref.logic.ai.processingstatus.ProcessingState; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.ai.util.CitationKeyCheck; import org.jabref.logic.util.TaskExecutor; +import org.jabref.model.ai.Summary; +import org.jabref.model.ai.processingstatus.ProcessingInfo; +import org.jabref.model.ai.processingstatus.ProcessingState; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.event.EntriesAddedEvent; import org.jabref.model.entry.BibEntry; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesStorage.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesStorage.java index 70e8d9598138..7ac27d8083a7 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesStorage.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesStorage.java @@ -3,6 +3,8 @@ import java.nio.file.Path; import java.util.Optional; +import org.jabref.model.ai.Summary; + public interface SummariesStorage { void set(Path bibDatabasePath, String citationKey, Summary summary); diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/MVStoreSummariesStorage.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/MVStoreSummariesStorage.java index 96ad5ef21a48..c929d15b2ace 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/MVStoreSummariesStorage.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/MVStoreSummariesStorage.java @@ -5,10 +5,10 @@ import java.util.Optional; import org.jabref.logic.ai.summarization.SummariesStorage; -import org.jabref.logic.ai.summarization.Summary; import org.jabref.logic.ai.util.MVStoreBase; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.NotificationService; +import org.jabref.model.ai.Summary; public class MVStoreSummariesStorage extends MVStoreBase implements SummariesStorage { private static final String SUMMARIES_MAP_PREFIX = "summaries"; diff --git a/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java b/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java index 2f1aa4e6c0bd..de85c987ea79 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java @@ -4,6 +4,7 @@ import java.util.List; import org.jabref.logic.ai.AiPreferences; +import org.jabref.model.ai.AiTemplate; import org.jabref.model.ai.PaperExcerpt; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.CanonicalBibEntry; diff --git a/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java b/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java index e69f31c93be7..edfc08a0c0f3 100644 --- a/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java @@ -35,7 +35,6 @@ import org.jabref.logic.LibraryPreferences; import org.jabref.logic.ai.AiDefaultPreferences; import org.jabref.logic.ai.AiPreferences; -import org.jabref.logic.ai.templates.AiTemplate; import org.jabref.logic.bibtex.FieldPreferences; import org.jabref.logic.citationkeypattern.CitationKeyPattern; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; @@ -97,6 +96,7 @@ import org.jabref.logic.util.strings.StringUtil; import org.jabref.logic.xmp.XmpPreferences; import org.jabref.model.ai.AiProvider; +import org.jabref.model.ai.AiTemplate; import org.jabref.model.ai.EmbeddingModel; import org.jabref.model.database.BibDatabaseMode; import org.jabref.model.entry.BibEntryPreferences; diff --git a/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplate.java b/jablib/src/main/java/org/jabref/model/ai/AiTemplate.java similarity index 93% rename from jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplate.java rename to jablib/src/main/java/org/jabref/model/ai/AiTemplate.java index 9b0d322e961f..15ba5fc992f3 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplate.java +++ b/jablib/src/main/java/org/jabref/model/ai/AiTemplate.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.templates; +package org.jabref.model.ai; public enum AiTemplate { // Templates that are used in AI chats. diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/Summary.java b/jablib/src/main/java/org/jabref/model/ai/Summary.java similarity index 69% rename from jablib/src/main/java/org/jabref/logic/ai/summarization/Summary.java rename to jablib/src/main/java/org/jabref/model/ai/Summary.java index 4177af34c71b..b05f5038b9dd 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/Summary.java +++ b/jablib/src/main/java/org/jabref/model/ai/Summary.java @@ -1,9 +1,7 @@ -package org.jabref.logic.ai.summarization; +package org.jabref.model.ai; import java.io.Serializable; import java.time.LocalDateTime; -import org.jabref.model.ai.AiProvider; - public record Summary(LocalDateTime timestamp, AiProvider aiProvider, String model, String content) implements Serializable { } diff --git a/jablib/src/main/java/org/jabref/logic/ai/processingstatus/ProcessingInfo.java b/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingInfo.java similarity index 97% rename from jablib/src/main/java/org/jabref/logic/ai/processingstatus/ProcessingInfo.java rename to jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingInfo.java index cd838f0ba2ae..9810a695c321 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/processingstatus/ProcessingInfo.java +++ b/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingInfo.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.processingstatus; +package org.jabref.model.ai.processingstatus; import java.util.Optional; diff --git a/jablib/src/main/java/org/jabref/logic/ai/processingstatus/ProcessingState.java b/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingState.java similarity index 64% rename from jablib/src/main/java/org/jabref/logic/ai/processingstatus/ProcessingState.java rename to jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingState.java index 466685bcc78a..9c0ad6c11234 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/processingstatus/ProcessingState.java +++ b/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingState.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.processingstatus; +package org.jabref.model.ai.processingstatus; public enum ProcessingState { PROCESSING, diff --git a/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesStorageTest.java b/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesStorageTest.java index 45f1d9bdb051..231e0d67257b 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesStorageTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesStorageTest.java @@ -5,6 +5,7 @@ import java.util.Optional; import org.jabref.model.ai.AiProvider; +import org.jabref.model.ai.Summary; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; From 2984c01f615057c3fbd6061af18117f96857b078 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Sun, 23 Nov 2025 17:31:33 +0100 Subject: [PATCH 003/243] Refactor AI-related classes to `org.jabref.logic.ai.rag` and `org.jabref.logic.ai.chatting`, remove `AiChatService`, and introduce `GenerateAiResponseTask`. --- .../ai/components/aichat/AiChatComponent.java | 59 +++++++++++-------- .../util/EmbeddingModelGuardedComponent.java | 2 +- .../aichat/AiChatComponentTest.java | 8 +-- jablib/src/main/java/module-info.java | 12 +++- .../java/org/jabref/logic/ai/AiService.java | 24 ++++---- .../logic/ai/chatting/AiChatService.java | 55 ----------------- .../ai/chatting/JabRefChatLanguageModel.java | 1 + .../{ => algorithms}/AiChatLogic.java | 4 +- .../{ => chathistory}/ChatHistoryService.java | 3 +- .../tasks/GenerateAiResponseTask.java | 26 ++++++++ .../{ingestion => rag}/IngestionService.java | 6 +- .../JabRefContentInjector.java | 4 +- .../JabRefEmbeddingModel.java | 3 +- .../algorithms}/FileToDocument.java | 2 +- .../algorithms}/LowLevelIngestor.java | 2 +- .../storages}/FileEmbeddingsManager.java | 3 +- .../FullyIngestedDocumentsTracker.java | 2 +- .../storages}/MVStoreEmbeddingStore.java | 4 +- .../MVStoreFullyIngestedDocumentsTracker.java | 3 +- .../GenerateEmbeddingsForSeveralTask.java | 3 +- .../tasks}/GenerateEmbeddingsTask.java | 6 +- .../tasks}/UpdateEmbeddingModelTask.java | 2 +- .../ai/summarization/SummariesService.java | 3 + .../storages/MVStoreSummariesStorage.java | 1 - .../{ => storages}/SummariesStorage.java | 2 +- .../GenerateSummaryForSeveralTask.java | 3 +- .../{ => tasks}/GenerateSummaryTask.java | 6 +- .../FullyIngestedDocumentsTrackerTest.java | 2 + ...toreFullyIngestedDocumentsTrackerTest.java | 3 +- .../MVStoreSummariesStorageTest.java | 1 + .../summarization/SummariesStorageTest.java | 1 + 31 files changed, 128 insertions(+), 128 deletions(-) delete mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/AiChatService.java rename jablib/src/main/java/org/jabref/logic/ai/chatting/{ => algorithms}/AiChatLogic.java (98%) rename jablib/src/main/java/org/jabref/logic/ai/chatting/{ => chathistory}/ChatHistoryService.java (99%) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/tasks/GenerateAiResponseTask.java rename jablib/src/main/java/org/jabref/logic/ai/{ingestion => rag}/IngestionService.java (96%) rename jablib/src/main/java/org/jabref/logic/ai/{chatting => rag}/JabRefContentInjector.java (95%) rename jablib/src/main/java/org/jabref/logic/ai/{ingestion => rag}/JabRefEmbeddingModel.java (98%) rename jablib/src/main/java/org/jabref/logic/ai/{ingestion => rag/algorithms}/FileToDocument.java (98%) rename jablib/src/main/java/org/jabref/logic/ai/{ingestion => rag/algorithms}/LowLevelIngestor.java (98%) rename jablib/src/main/java/org/jabref/logic/ai/{ingestion => rag/storages}/FileEmbeddingsManager.java (97%) rename jablib/src/main/java/org/jabref/logic/ai/{ingestion => rag/storages}/FullyIngestedDocumentsTracker.java (92%) rename jablib/src/main/java/org/jabref/logic/ai/{ingestion => rag/storages}/MVStoreEmbeddingStore.java (98%) rename jablib/src/main/java/org/jabref/logic/ai/{ingestion => rag}/storages/MVStoreFullyIngestedDocumentsTracker.java (95%) rename jablib/src/main/java/org/jabref/logic/ai/{ingestion => rag/tasks}/GenerateEmbeddingsForSeveralTask.java (97%) rename jablib/src/main/java/org/jabref/logic/ai/{ingestion => rag/tasks}/GenerateEmbeddingsTask.java (95%) rename jablib/src/main/java/org/jabref/logic/ai/{ingestion => rag/tasks}/UpdateEmbeddingModelTask.java (98%) rename jablib/src/main/java/org/jabref/logic/ai/summarization/{ => storages}/SummariesStorage.java (86%) rename jablib/src/main/java/org/jabref/logic/ai/summarization/{ => tasks}/GenerateSummaryForSeveralTask.java (97%) rename jablib/src/main/java/org/jabref/logic/ai/summarization/{ => tasks}/GenerateSummaryTask.java (98%) diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java index ea518de18ae3..0b32435593ac 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java @@ -29,7 +29,8 @@ import org.jabref.gui.util.UiTaskExecutor; import org.jabref.logic.ai.AiPreferences; import org.jabref.logic.ai.AiService; -import org.jabref.logic.ai.chatting.AiChatLogic; +import org.jabref.logic.ai.chatting.algorithms.AiChatLogic; +import org.jabref.logic.ai.chatting.tasks.GenerateAiResponseTask; import org.jabref.logic.ai.util.CitationKeyCheck; import org.jabref.logic.ai.util.ErrorMessage; import org.jabref.logic.l10n.Localization; @@ -97,7 +98,17 @@ public AiChatComponent(AiService aiService, this.dialogService = dialogService; this.taskExecutor = taskExecutor; - this.aiChatLogic = aiService.getAiChatService().makeChat(name, chatHistory, entries, bibDatabaseContext); + this.aiChatLogic = new AiChatLogic( + aiPreferences, + aiService.getChatLanguageModel(), + aiService.getEmbeddingModel(), + aiService.getEmbeddingStore(), + aiService.getTemplatesService(), + name, + chatHistory, + entries, + bibDatabaseContext + ); aiService.getIngestionService().ingest(name, ListUtil.getLinkedFiles(entries).toList(), bibDatabaseContext); @@ -282,30 +293,26 @@ private void onSendMessage(String userPrompt) { updatePromptHistory(); setLoading(true); - BackgroundTask task = - BackgroundTask - .wrap(() -> aiChatLogic.execute(userMessage)) - .showToUser(true) - .onSuccess(aiMessage -> { - setLoading(false); - chatPrompt.requestPromptFocus(); - }) - .onFailure(e -> { - LOGGER.error("Got an error while sending a message to AI", e); - setLoading(false); - - // Typically, if user has entered an invalid API base URL, we get either "401 - null" or "404 - null" strings. - // Since there might be other strings returned from other API endpoints, we use startsWith() here. - if (e.getMessage().startsWith("404") || e.getMessage().startsWith("401")) { - addError(Localization.lang("API base URL setting appears to be incorrect. Please check it in AI expert settings.")); - } else { - addError(e.getMessage()); - } - - chatPrompt.switchToErrorState(userPrompt); - }); - - task.titleProperty().set(Localization.lang("Waiting for AI reply...")); + BackgroundTask task = new GenerateAiResponseTask(userMessage, aiChatLogic) + .onSuccess(aiMessage -> { + setLoading(false); + chatPrompt.requestPromptFocus(); + }) + .onFailure(e -> { + LOGGER.error("Got an error while sending a message to AI", e); + setLoading(false); + + // Typically, if a user has entered an invalid API base URL, we get either "401 - null" or "404 - null" strings. + // Since there might be other strings returned from other API endpoints, we use startsWith() here. + if (e.getMessage().startsWith("404") || e.getMessage().startsWith("401")) { + addError(Localization.lang("API base URL setting appears to be incorrect. Please check it in AI expert settings.")); + } else { + addError(e.getMessage()); + } + + chatPrompt.switchToErrorState(userPrompt); + }); + ; task.executeWith(taskExecutor); } diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java index 90354f320c77..85f15bcaa33c 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java @@ -10,7 +10,7 @@ import org.jabref.gui.util.UiTaskExecutor; import org.jabref.logic.ai.AiPreferences; import org.jabref.logic.ai.AiService; -import org.jabref.logic.ai.ingestion.JabRefEmbeddingModel; +import org.jabref.logic.ai.rag.JabRefEmbeddingModel; import org.jabref.logic.l10n.Localization; import com.google.common.eventbus.Subscribe; diff --git a/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java b/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java index 809eba4e5114..cb69736e7173 100644 --- a/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java +++ b/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java @@ -11,9 +11,8 @@ import org.jabref.gui.DialogService; import org.jabref.logic.ai.AiPreferences; import org.jabref.logic.ai.AiService; -import org.jabref.logic.ai.chatting.AiChatLogic; -import org.jabref.logic.ai.chatting.AiChatService; -import org.jabref.logic.ai.ingestion.IngestionService; +import org.jabref.logic.ai.chatting.algorithms.AiChatLogic; +import org.jabref.logic.ai.rag.IngestionService; import org.jabref.logic.l10n.Language; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.TaskExecutor; @@ -75,11 +74,8 @@ void setUp() { }); aiService = mock(AiService.class, Mockito.RETURNS_DEEP_STUBS); - AiChatService chatService = mock(AiChatService.class); AiChatLogic chatLogic = mock(AiChatLogic.class, Mockito.RETURNS_DEEP_STUBS); when(chatLogic.getChatHistory()).thenReturn(FXCollections.observableArrayList()); - when(chatService.makeChat(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(chatLogic); - when(aiService.getAiChatService()).thenReturn(chatService); IngestionService ingestionService = mock(IngestionService.class, Mockito.RETURNS_DEEP_STUBS); when(aiService.getIngestionService()).thenReturn(ingestionService); diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index 475da89f3f60..516ffa94cac2 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -59,7 +59,7 @@ exports org.jabref.model.metadata.event; exports org.jabref.logic.ai.chatting; exports org.jabref.logic.ai.util; - exports org.jabref.logic.ai.ingestion; + exports org.jabref.logic.ai.rag.ingestion; exports org.jabref.logic.ai.ingestion.model; exports org.jabref.model.ai; exports org.jabref.model.ai.processingstatus; @@ -121,6 +121,15 @@ exports org.jabref.logic.git.merge.execution; exports org.jabref.model.sciteTallies; exports org.jabref.logic.ai.customimplementations.embeddingmodels; + exports org.jabref.logic.ai.rag; + exports org.jabref.logic.ai.chatting.chathistory; + exports org.jabref.logic.ai.summarization.tasks; + exports org.jabref.logic.ai.summarization.storages; + exports org.jabref.logic.ai.rag.tasks; + exports org.jabref.logic.ai.rag.storages; + exports org.jabref.logic.ai.rag.algorithms; + exports org.jabref.logic.ai.chatting.algorithms; + exports org.jabref.logic.ai.chatting.tasks; requires java.base; @@ -267,5 +276,6 @@ requires org.jooq.jool; requires org.libreoffice.uno; requires transitive org.jspecify; + requires org.jabref.jablib; // endregion } diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiService.java b/jablib/src/main/java/org/jabref/logic/ai/AiService.java index 033fabaa2e25..8659eed5f00a 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiService.java @@ -7,14 +7,13 @@ import javafx.beans.property.SimpleBooleanProperty; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.chatting.AiChatService; -import org.jabref.logic.ai.chatting.ChatHistoryService; import org.jabref.logic.ai.chatting.JabRefChatLanguageModel; +import org.jabref.logic.ai.chatting.chathistory.ChatHistoryService; import org.jabref.logic.ai.chatting.chathistory.storages.MVStoreChatHistoryStorage; -import org.jabref.logic.ai.ingestion.IngestionService; -import org.jabref.logic.ai.ingestion.JabRefEmbeddingModel; -import org.jabref.logic.ai.ingestion.MVStoreEmbeddingStore; -import org.jabref.logic.ai.ingestion.storages.MVStoreFullyIngestedDocumentsTracker; +import org.jabref.logic.ai.rag.IngestionService; +import org.jabref.logic.ai.rag.JabRefEmbeddingModel; +import org.jabref.logic.ai.rag.storages.MVStoreEmbeddingStore; +import org.jabref.logic.ai.rag.storages.MVStoreFullyIngestedDocumentsTracker; import org.jabref.logic.ai.summarization.SummariesService; import org.jabref.logic.ai.summarization.storages.MVStoreSummariesStorage; import org.jabref.logic.ai.templates.AiTemplatesService; @@ -25,6 +24,8 @@ import org.jabref.model.database.BibDatabaseContext; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.store.embedding.EmbeddingStore; /** * The main class for the AI functionality. @@ -57,7 +58,6 @@ public class AiService implements AutoCloseable { private final ChatHistoryService chatHistoryService; private final JabRefChatLanguageModel jabRefChatLanguageModel; private final JabRefEmbeddingModel jabRefEmbeddingModel; - private final AiChatService aiChatService; private final IngestionService ingestionService; private final SummariesService summariesService; @@ -78,8 +78,6 @@ public AiService(AiPreferences aiPreferences, this.jabRefChatLanguageModel = new JabRefChatLanguageModel(aiPreferences); this.jabRefEmbeddingModel = new JabRefEmbeddingModel(aiPreferences, notificationService, taskExecutor); - this.aiChatService = new AiChatService(aiPreferences, jabRefChatLanguageModel, jabRefEmbeddingModel, mvStoreEmbeddingStore, templatesService); - this.ingestionService = new IngestionService( aiPreferences, shutdownSignal, @@ -113,10 +111,6 @@ public ChatHistoryService getChatHistoryService() { return chatHistoryService; } - public AiChatService getAiChatService() { - return aiChatService; - } - public IngestionService getIngestionService() { return ingestionService; } @@ -129,6 +123,10 @@ public AiTemplatesService getTemplatesService() { return templatesService; } + public EmbeddingStore getEmbeddingStore() { + return mvStoreEmbeddingStore; + } + public void setupDatabase(BibDatabaseContext context) { chatHistoryService.setupDatabase(context); ingestionService.setupDatabase(context); diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/AiChatService.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/AiChatService.java deleted file mode 100644 index 5506fe4a975f..000000000000 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/AiChatService.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.jabref.logic.ai.chatting; - -import javafx.beans.property.StringProperty; -import javafx.collections.ObservableList; - -import org.jabref.logic.ai.AiPreferences; -import org.jabref.logic.ai.templates.AiTemplatesService; -import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.entry.BibEntry; - -import dev.langchain4j.data.message.ChatMessage; -import dev.langchain4j.data.segment.TextSegment; -import dev.langchain4j.model.chat.ChatModel; -import dev.langchain4j.model.embedding.EmbeddingModel; -import dev.langchain4j.store.embedding.EmbeddingStore; - -public class AiChatService { - private final AiPreferences aiPreferences; - private final ChatModel chatLanguageModel; - private final EmbeddingModel embeddingModel; - private final EmbeddingStore embeddingStore; - private final AiTemplatesService aiTemplatesService; - - public AiChatService(AiPreferences aiPreferences, - ChatModel chatLanguageModel, - EmbeddingModel embeddingModel, - EmbeddingStore embeddingStore, - AiTemplatesService aiTemplatesService - ) { - this.aiPreferences = aiPreferences; - this.chatLanguageModel = chatLanguageModel; - this.embeddingModel = embeddingModel; - this.embeddingStore = embeddingStore; - this.aiTemplatesService = aiTemplatesService; - } - - public AiChatLogic makeChat( - StringProperty name, - ObservableList chatHistory, - ObservableList entries, - BibDatabaseContext bibDatabaseContext - ) { - return new AiChatLogic( - aiPreferences, - chatLanguageModel, - embeddingModel, - embeddingStore, - aiTemplatesService, - name, - chatHistory, - entries, - bibDatabaseContext - ); - } -} diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/JabRefChatLanguageModel.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/JabRefChatLanguageModel.java index 7c53a8e94dd6..bff1d995355a 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/JabRefChatLanguageModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/JabRefChatLanguageModel.java @@ -8,6 +8,7 @@ import java.util.concurrent.Executors; import org.jabref.logic.ai.AiPreferences; +import org.jabref.logic.ai.chatting.algorithms.AiChatLogic; import org.jabref.logic.ai.customimplementations.llms.Gpt4AllModel; import org.jabref.logic.ai.customimplementations.llms.JvmOpenAiChatLanguageModel; import org.jabref.logic.l10n.Localization; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/AiChatLogic.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/algorithms/AiChatLogic.java similarity index 98% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/AiChatLogic.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/algorithms/AiChatLogic.java index a7202ef61d67..e9c0203fe0d9 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/AiChatLogic.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/algorithms/AiChatLogic.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.chatting; +package org.jabref.logic.ai.chatting.algorithms; import java.util.List; import java.util.Optional; @@ -8,7 +8,7 @@ import javafx.collections.ObservableList; import org.jabref.logic.ai.AiPreferences; -import org.jabref.logic.ai.ingestion.FileEmbeddingsManager; +import org.jabref.logic.ai.rag.storages.FileEmbeddingsManager; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.ai.util.ErrorMessage; import org.jabref.model.ai.AiTemplate; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryService.java similarity index 99% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryService.java index 0a1de1274a1d..1a1652fcb1f7 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryService.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.chatting; +package org.jabref.logic.ai.chatting.chathistory; import java.util.Comparator; import java.util.HashSet; @@ -9,7 +9,6 @@ import javafx.collections.FXCollections; import javafx.collections.ObservableList; -import org.jabref.logic.ai.chatting.chathistory.ChatHistoryStorage; import org.jabref.logic.ai.util.CitationKeyCheck; import org.jabref.logic.citationkeypattern.CitationKeyGenerator; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/tasks/GenerateAiResponseTask.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/tasks/GenerateAiResponseTask.java new file mode 100644 index 000000000000..8d4809ea7e50 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/tasks/GenerateAiResponseTask.java @@ -0,0 +1,26 @@ +package org.jabref.logic.ai.chatting.tasks; + +import org.jabref.logic.ai.chatting.algorithms.AiChatLogic; +import org.jabref.logic.l10n.Localization; +import org.jabref.logic.util.BackgroundTask; + +import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.data.message.UserMessage; + +public class GenerateAiResponseTask extends BackgroundTask { + private final UserMessage userMessage; + private final AiChatLogic aiChatLogic; + + public GenerateAiResponseTask(UserMessage userMessage, AiChatLogic aiChatLogic) { + this.userMessage = userMessage; + this.aiChatLogic = aiChatLogic; + + showToUser(true); + titleProperty().set(Localization.lang("Waiting for AI reply...")); + } + + @Override + public AiMessage call() throws Exception { + return aiChatLogic.execute(userMessage); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/ingestion/IngestionService.java b/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java similarity index 96% rename from jablib/src/main/java/org/jabref/logic/ai/ingestion/IngestionService.java rename to jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java index ae782a1d10bb..48c298094c81 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/ingestion/IngestionService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.ingestion; +package org.jabref.logic.ai.rag; import java.util.ArrayList; import java.util.Comparator; @@ -10,6 +10,10 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.AiPreferences; +import org.jabref.logic.ai.rag.storages.FileEmbeddingsManager; +import org.jabref.logic.ai.rag.storages.FullyIngestedDocumentsTracker; +import org.jabref.logic.ai.rag.tasks.GenerateEmbeddingsForSeveralTask; +import org.jabref.logic.ai.rag.tasks.GenerateEmbeddingsTask; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.ai.processingstatus.ProcessingInfo; import org.jabref.model.ai.processingstatus.ProcessingState; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/JabRefContentInjector.java b/jablib/src/main/java/org/jabref/logic/ai/rag/JabRefContentInjector.java similarity index 95% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/JabRefContentInjector.java rename to jablib/src/main/java/org/jabref/logic/ai/rag/JabRefContentInjector.java index fae7710baa33..5eaf0cd7a0c3 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/JabRefContentInjector.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/JabRefContentInjector.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.chatting; +package org.jabref.logic.ai.rag; import java.util.HashMap; import java.util.List; @@ -15,7 +15,7 @@ import dev.langchain4j.rag.content.Content; import dev.langchain4j.rag.content.injector.ContentInjector; -import static org.jabref.logic.ai.ingestion.FileEmbeddingsManager.LINK_METADATA_KEY; +import static org.jabref.logic.ai.rag.storages.FileEmbeddingsManager.LINK_METADATA_KEY; public class JabRefContentInjector implements ContentInjector { public static final PromptTemplate DEFAULT_PROMPT_TEMPLATE = PromptTemplate.from("{{userMessage}}\n\nAnswer using the following information:\n{{contents}}"); diff --git a/jablib/src/main/java/org/jabref/logic/ai/ingestion/JabRefEmbeddingModel.java b/jablib/src/main/java/org/jabref/logic/ai/rag/JabRefEmbeddingModel.java similarity index 98% rename from jablib/src/main/java/org/jabref/logic/ai/ingestion/JabRefEmbeddingModel.java rename to jablib/src/main/java/org/jabref/logic/ai/rag/JabRefEmbeddingModel.java index 6260e2852b63..90b9eed9352f 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/ingestion/JabRefEmbeddingModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/JabRefEmbeddingModel.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.ingestion; +package org.jabref.logic.ai.rag; import java.util.List; import java.util.Optional; @@ -10,6 +10,7 @@ import org.jabref.logic.ai.AiPreferences; import org.jabref.logic.ai.customimplementations.embeddingmodels.DeepJavaEmbeddingModel; +import org.jabref.logic.ai.rag.tasks.UpdateEmbeddingModelTask; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.NotificationService; import org.jabref.logic.util.TaskExecutor; diff --git a/jablib/src/main/java/org/jabref/logic/ai/ingestion/FileToDocument.java b/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/FileToDocument.java similarity index 98% rename from jablib/src/main/java/org/jabref/logic/ai/ingestion/FileToDocument.java rename to jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/FileToDocument.java index bf76329148cc..8400183f5044 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/ingestion/FileToDocument.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/FileToDocument.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.ingestion; +package org.jabref.logic.ai.rag.algorithms; import java.io.IOException; import java.io.StringWriter; diff --git a/jablib/src/main/java/org/jabref/logic/ai/ingestion/LowLevelIngestor.java b/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/LowLevelIngestor.java similarity index 98% rename from jablib/src/main/java/org/jabref/logic/ai/ingestion/LowLevelIngestor.java rename to jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/LowLevelIngestor.java index 3138dcf424b7..35a2e4eaae16 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/ingestion/LowLevelIngestor.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/LowLevelIngestor.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.ingestion; +package org.jabref.logic.ai.rag.algorithms; import java.util.List; diff --git a/jablib/src/main/java/org/jabref/logic/ai/ingestion/FileEmbeddingsManager.java b/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FileEmbeddingsManager.java similarity index 97% rename from jablib/src/main/java/org/jabref/logic/ai/ingestion/FileEmbeddingsManager.java rename to jablib/src/main/java/org/jabref/logic/ai/rag/storages/FileEmbeddingsManager.java index a9a4063a80db..78654508cb92 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/ingestion/FileEmbeddingsManager.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FileEmbeddingsManager.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.ingestion; +package org.jabref.logic.ai.rag.storages; import java.util.List; import java.util.Optional; @@ -7,6 +7,7 @@ import javafx.beans.property.ReadOnlyBooleanProperty; import org.jabref.logic.ai.AiPreferences; +import org.jabref.logic.ai.rag.algorithms.LowLevelIngestor; import org.jabref.model.entry.LinkedFile; import dev.langchain4j.data.document.Document; diff --git a/jablib/src/main/java/org/jabref/logic/ai/ingestion/FullyIngestedDocumentsTracker.java b/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FullyIngestedDocumentsTracker.java similarity index 92% rename from jablib/src/main/java/org/jabref/logic/ai/ingestion/FullyIngestedDocumentsTracker.java rename to jablib/src/main/java/org/jabref/logic/ai/rag/storages/FullyIngestedDocumentsTracker.java index 9ba066e7900f..9b4522c3debe 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/ingestion/FullyIngestedDocumentsTracker.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FullyIngestedDocumentsTracker.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.ingestion; +package org.jabref.logic.ai.rag.storages; import java.util.Optional; diff --git a/jablib/src/main/java/org/jabref/logic/ai/ingestion/MVStoreEmbeddingStore.java b/jablib/src/main/java/org/jabref/logic/ai/rag/storages/MVStoreEmbeddingStore.java similarity index 98% rename from jablib/src/main/java/org/jabref/logic/ai/ingestion/MVStoreEmbeddingStore.java rename to jablib/src/main/java/org/jabref/logic/ai/rag/storages/MVStoreEmbeddingStore.java index dcfedca606d3..234cb13208b0 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/ingestion/MVStoreEmbeddingStore.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/storages/MVStoreEmbeddingStore.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.ingestion; +package org.jabref.logic.ai.rag.storages; import java.io.Serializable; import java.nio.file.Path; @@ -34,7 +34,7 @@ import org.jspecify.annotations.Nullable; import static java.util.Comparator.comparingDouble; -import static org.jabref.logic.ai.ingestion.FileEmbeddingsManager.LINK_METADATA_KEY; +import static org.jabref.logic.ai.rag.storages.FileEmbeddingsManager.LINK_METADATA_KEY; /** * A custom implementation of langchain4j's {@link EmbeddingStore} that uses a {@link MVStore} as an embedded database. diff --git a/jablib/src/main/java/org/jabref/logic/ai/ingestion/storages/MVStoreFullyIngestedDocumentsTracker.java b/jablib/src/main/java/org/jabref/logic/ai/rag/storages/MVStoreFullyIngestedDocumentsTracker.java similarity index 95% rename from jablib/src/main/java/org/jabref/logic/ai/ingestion/storages/MVStoreFullyIngestedDocumentsTracker.java rename to jablib/src/main/java/org/jabref/logic/ai/rag/storages/MVStoreFullyIngestedDocumentsTracker.java index 1eb9809e0079..fe96c3f0496e 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/ingestion/storages/MVStoreFullyIngestedDocumentsTracker.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/storages/MVStoreFullyIngestedDocumentsTracker.java @@ -1,10 +1,9 @@ -package org.jabref.logic.ai.ingestion.storages; +package org.jabref.logic.ai.rag.storages; import java.nio.file.Path; import java.util.Map; import java.util.Optional; -import org.jabref.logic.ai.ingestion.FullyIngestedDocumentsTracker; import org.jabref.logic.ai.util.MVStoreBase; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.NotificationService; diff --git a/jablib/src/main/java/org/jabref/logic/ai/ingestion/GenerateEmbeddingsForSeveralTask.java b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsForSeveralTask.java similarity index 97% rename from jablib/src/main/java/org/jabref/logic/ai/ingestion/GenerateEmbeddingsForSeveralTask.java rename to jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsForSeveralTask.java index 6aadaf8e6dc6..bd51b15e8f70 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/ingestion/GenerateEmbeddingsForSeveralTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsForSeveralTask.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.ingestion; +package org.jabref.logic.ai.rag.tasks; import java.util.ArrayList; import java.util.List; @@ -10,6 +10,7 @@ import javafx.util.Pair; import org.jabref.logic.FilePreferences; +import org.jabref.logic.ai.rag.storages.FileEmbeddingsManager; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.ProgressCounter; diff --git a/jablib/src/main/java/org/jabref/logic/ai/ingestion/GenerateEmbeddingsTask.java b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java similarity index 95% rename from jablib/src/main/java/org/jabref/logic/ai/ingestion/GenerateEmbeddingsTask.java rename to jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java index 499e5e90ae5c..c2769b052a10 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/ingestion/GenerateEmbeddingsTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.ingestion; +package org.jabref.logic.ai.rag.tasks; import java.io.IOException; import java.nio.file.Files; @@ -10,6 +10,8 @@ import javafx.beans.property.ReadOnlyBooleanProperty; import org.jabref.logic.FilePreferences; +import org.jabref.logic.ai.rag.algorithms.FileToDocument; +import org.jabref.logic.ai.rag.storages.FileEmbeddingsManager; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.ProgressCounter; @@ -75,7 +77,7 @@ public Void call() { private void ingestLinkedFile(LinkedFile linkedFile) throws InterruptedException { // Rationale for RuntimeException here: - // See org.jabref.logic.ai.summarization.GenerateSummaryTask.summarizeAll + // See org.jabref.logic.ai.summarization.tasks.GenerateSummaryTask.summarizeAll LOGGER.debug("Generating embeddings for file \"{}\"", linkedFile.getLink()); diff --git a/jablib/src/main/java/org/jabref/logic/ai/ingestion/UpdateEmbeddingModelTask.java b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/UpdateEmbeddingModelTask.java similarity index 98% rename from jablib/src/main/java/org/jabref/logic/ai/ingestion/UpdateEmbeddingModelTask.java rename to jablib/src/main/java/org/jabref/logic/ai/rag/tasks/UpdateEmbeddingModelTask.java index fd90b17cb25b..8fc30cafa3cb 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/ingestion/UpdateEmbeddingModelTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/UpdateEmbeddingModelTask.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.ingestion; +package org.jabref.logic.ai.rag.tasks; import java.io.IOException; import java.util.Optional; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java index 2ec81b012b69..f2fbc1053e3e 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java @@ -10,6 +10,9 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.AiPreferences; +import org.jabref.logic.ai.summarization.storages.SummariesStorage; +import org.jabref.logic.ai.summarization.tasks.GenerateSummaryForSeveralTask; +import org.jabref.logic.ai.summarization.tasks.GenerateSummaryTask; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.ai.util.CitationKeyCheck; import org.jabref.logic.util.TaskExecutor; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/MVStoreSummariesStorage.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/MVStoreSummariesStorage.java index c929d15b2ace..55e9519bcb7d 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/MVStoreSummariesStorage.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/MVStoreSummariesStorage.java @@ -4,7 +4,6 @@ import java.util.Map; import java.util.Optional; -import org.jabref.logic.ai.summarization.SummariesStorage; import org.jabref.logic.ai.util.MVStoreBase; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.NotificationService; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesStorage.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/SummariesStorage.java similarity index 86% rename from jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesStorage.java rename to jablib/src/main/java/org/jabref/logic/ai/summarization/storages/SummariesStorage.java index 7ac27d8083a7..8d9a7e481243 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesStorage.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/SummariesStorage.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.summarization; +package org.jabref.logic.ai.summarization.storages; import java.nio.file.Path; import java.util.Optional; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/GenerateSummaryForSeveralTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java similarity index 97% rename from jablib/src/main/java/org/jabref/logic/ai/summarization/GenerateSummaryForSeveralTask.java rename to jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java index 675f6963c992..ff4f9fb169f4 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/GenerateSummaryForSeveralTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.summarization; +package org.jabref.logic.ai.summarization.tasks; import java.util.ArrayList; import java.util.List; @@ -11,6 +11,7 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.AiPreferences; +import org.jabref.logic.ai.summarization.storages.SummariesStorage; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/GenerateSummaryTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java similarity index 98% rename from jablib/src/main/java/org/jabref/logic/ai/summarization/GenerateSummaryTask.java rename to jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java index 41048b6821e6..a1b2a355e075 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/GenerateSummaryTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.summarization; +package org.jabref.logic.ai.summarization.tasks; import java.nio.file.Path; import java.time.LocalDateTime; @@ -12,7 +12,9 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.AiPreferences; -import org.jabref.logic.ai.ingestion.FileToDocument; +import org.jabref.logic.ai.rag.algorithms.FileToDocument; +import org.jabref.logic.ai.summarization.SummariesService; +import org.jabref.logic.ai.summarization.storages.SummariesStorage; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.ai.util.CitationKeyCheck; import org.jabref.logic.l10n.Localization; diff --git a/jablib/src/test/java/org/jabref/logic/ai/ingestion/FullyIngestedDocumentsTrackerTest.java b/jablib/src/test/java/org/jabref/logic/ai/ingestion/FullyIngestedDocumentsTrackerTest.java index 09dddc716648..403ed903c21b 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/ingestion/FullyIngestedDocumentsTrackerTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/ingestion/FullyIngestedDocumentsTrackerTest.java @@ -3,6 +3,8 @@ import java.nio.file.Path; import java.util.Optional; +import org.jabref.logic.ai.rag.storages.FullyIngestedDocumentsTracker; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsTrackerTest.java b/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsTrackerTest.java index 1626557c274d..c8af73f8dfd2 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsTrackerTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsTrackerTest.java @@ -2,7 +2,8 @@ import java.nio.file.Path; -import org.jabref.logic.ai.ingestion.storages.MVStoreFullyIngestedDocumentsTracker; +import org.jabref.logic.ai.rag.storages.FullyIngestedDocumentsTracker; +import org.jabref.logic.ai.rag.storages.MVStoreFullyIngestedDocumentsTracker; import org.jabref.logic.util.NotificationService; import static org.mockito.Mockito.mock; diff --git a/jablib/src/test/java/org/jabref/logic/ai/summarization/MVStoreSummariesStorageTest.java b/jablib/src/test/java/org/jabref/logic/ai/summarization/MVStoreSummariesStorageTest.java index 0beedc7e8bf7..005efc4d33cf 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/summarization/MVStoreSummariesStorageTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/summarization/MVStoreSummariesStorageTest.java @@ -3,6 +3,7 @@ import java.nio.file.Path; import org.jabref.logic.ai.summarization.storages.MVStoreSummariesStorage; +import org.jabref.logic.ai.summarization.storages.SummariesStorage; import org.jabref.logic.util.NotificationService; import static org.mockito.Mockito.mock; diff --git a/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesStorageTest.java b/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesStorageTest.java index 231e0d67257b..ca1dc2858169 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesStorageTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesStorageTest.java @@ -4,6 +4,7 @@ import java.time.LocalDateTime; import java.util.Optional; +import org.jabref.logic.ai.summarization.storages.SummariesStorage; import org.jabref.model.ai.AiProvider; import org.jabref.model.ai.Summary; From e4b44ff586c9b2aea4abebef370c2d1bfcd646e3 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Sun, 23 Nov 2025 17:33:02 +0100 Subject: [PATCH 004/243] Remove unused AI ingestion exports and dependencies from module-info.java. --- jablib/src/main/java/module-info.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index 516ffa94cac2..ab6dbbaf6c43 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -59,8 +59,6 @@ exports org.jabref.model.metadata.event; exports org.jabref.logic.ai.chatting; exports org.jabref.logic.ai.util; - exports org.jabref.logic.ai.rag.ingestion; - exports org.jabref.logic.ai.ingestion.model; exports org.jabref.model.ai; exports org.jabref.model.ai.processingstatus; exports org.jabref.logic.ai.summarization; @@ -276,6 +274,5 @@ requires org.jooq.jool; requires org.libreoffice.uno; requires transitive org.jspecify; - requires org.jabref.jablib; // endregion } From 39c2b612c5387cc1fb61969d0d1e22d4a93c119e Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Sun, 23 Nov 2025 17:34:51 +0100 Subject: [PATCH 005/243] Refactor `JabRefEmbeddingModel` and `JabRefChatLanguageModel` to `CurrentlySelectedEmbeddingModel` and `CurrentlySelectedChatLanguageModel`; update references and adjust imports. --- .../util/EmbeddingModelGuardedComponent.java | 6 ++-- .../java/org/jabref/logic/ai/AiService.java | 28 +++++++++---------- ...> CurrentlySelectedChatLanguageModel.java} | 4 +-- ...a => CurrentlySelectedEmbeddingModel.java} | 6 ++-- 4 files changed, 22 insertions(+), 22 deletions(-) rename jablib/src/main/java/org/jabref/logic/ai/chatting/{JabRefChatLanguageModel.java => CurrentlySelectedChatLanguageModel.java} (97%) rename jablib/src/main/java/org/jabref/logic/ai/rag/{JabRefEmbeddingModel.java => CurrentlySelectedEmbeddingModel.java} (95%) diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java index 85f15bcaa33c..fd508e72f528 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java @@ -10,7 +10,7 @@ import org.jabref.gui.util.UiTaskExecutor; import org.jabref.logic.ai.AiPreferences; import org.jabref.logic.ai.AiService; -import org.jabref.logic.ai.rag.JabRefEmbeddingModel; +import org.jabref.logic.ai.rag.CurrentlySelectedEmbeddingModel; import org.jabref.logic.l10n.Localization; import com.google.common.eventbus.Subscribe; @@ -68,12 +68,12 @@ public Node showBuildingEmbeddingModel() { } @Subscribe - public void listen(JabRefEmbeddingModel.EmbeddingModelBuiltEvent event) { + public void listen(CurrentlySelectedEmbeddingModel.EmbeddingModelBuiltEvent event) { UiTaskExecutor.runInJavaFXThread(EmbeddingModelGuardedComponent.this::rebuildUi); } @Subscribe - public void listen(JabRefEmbeddingModel.EmbeddingModelBuildingErrorEvent event) { + public void listen(CurrentlySelectedEmbeddingModel.EmbeddingModelBuildingErrorEvent event) { UiTaskExecutor.runInJavaFXThread(EmbeddingModelGuardedComponent.this::rebuildUi); } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiService.java b/jablib/src/main/java/org/jabref/logic/ai/AiService.java index 8659eed5f00a..f35b21b94be5 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiService.java @@ -7,11 +7,11 @@ import javafx.beans.property.SimpleBooleanProperty; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.chatting.JabRefChatLanguageModel; +import org.jabref.logic.ai.chatting.CurrentlySelectedChatLanguageModel; import org.jabref.logic.ai.chatting.chathistory.ChatHistoryService; import org.jabref.logic.ai.chatting.chathistory.storages.MVStoreChatHistoryStorage; +import org.jabref.logic.ai.rag.CurrentlySelectedEmbeddingModel; import org.jabref.logic.ai.rag.IngestionService; -import org.jabref.logic.ai.rag.JabRefEmbeddingModel; import org.jabref.logic.ai.rag.storages.MVStoreEmbeddingStore; import org.jabref.logic.ai.rag.storages.MVStoreFullyIngestedDocumentsTracker; import org.jabref.logic.ai.summarization.SummariesService; @@ -56,8 +56,8 @@ public class AiService implements AutoCloseable { private final AiTemplatesService templatesService; private final ChatHistoryService chatHistoryService; - private final JabRefChatLanguageModel jabRefChatLanguageModel; - private final JabRefEmbeddingModel jabRefEmbeddingModel; + private final CurrentlySelectedChatLanguageModel currentlySelectedChatLanguageModel; + private final CurrentlySelectedEmbeddingModel currentlySelectedEmbeddingModel; private final IngestionService ingestionService; private final SummariesService summariesService; @@ -75,13 +75,13 @@ public AiService(AiPreferences aiPreferences, this.templatesService = new AiTemplatesService(aiPreferences); this.chatHistoryService = new ChatHistoryService(citationKeyPatternPreferences, mvStoreChatHistoryStorage); - this.jabRefChatLanguageModel = new JabRefChatLanguageModel(aiPreferences); - this.jabRefEmbeddingModel = new JabRefEmbeddingModel(aiPreferences, notificationService, taskExecutor); + this.currentlySelectedChatLanguageModel = new CurrentlySelectedChatLanguageModel(aiPreferences); + this.currentlySelectedEmbeddingModel = new CurrentlySelectedEmbeddingModel(aiPreferences, notificationService, taskExecutor); this.ingestionService = new IngestionService( aiPreferences, shutdownSignal, - jabRefEmbeddingModel, + currentlySelectedEmbeddingModel, mvStoreEmbeddingStore, mvStoreFullyIngestedDocumentsTracker, filePreferences, @@ -91,7 +91,7 @@ public AiService(AiPreferences aiPreferences, this.summariesService = new SummariesService( aiPreferences, mvStoreSummariesStorage, - jabRefChatLanguageModel, + currentlySelectedChatLanguageModel, templatesService, shutdownSignal, filePreferences, @@ -99,12 +99,12 @@ public AiService(AiPreferences aiPreferences, ); } - public JabRefChatLanguageModel getChatLanguageModel() { - return jabRefChatLanguageModel; + public CurrentlySelectedChatLanguageModel getChatLanguageModel() { + return currentlySelectedChatLanguageModel; } - public JabRefEmbeddingModel getEmbeddingModel() { - return jabRefEmbeddingModel; + public CurrentlySelectedEmbeddingModel getEmbeddingModel() { + return currentlySelectedEmbeddingModel; } public ChatHistoryService getChatHistoryService() { @@ -138,8 +138,8 @@ public void close() { shutdownSignal.set(true); cachedThreadPool.shutdownNow(); - jabRefChatLanguageModel.close(); - jabRefEmbeddingModel.close(); + currentlySelectedChatLanguageModel.close(); + currentlySelectedEmbeddingModel.close(); mvStoreFullyIngestedDocumentsTracker.close(); mvStoreEmbeddingStore.close(); diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/JabRefChatLanguageModel.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java similarity index 97% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/JabRefChatLanguageModel.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java index bff1d995355a..4711767b4beb 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/JabRefChatLanguageModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java @@ -29,7 +29,7 @@ * Notice, that the real chat model is created lazily, when it's needed. This is done, so API key is fetched only, * when user wants to chat with AI. */ -public class JabRefChatLanguageModel implements ChatModel, AutoCloseable { +public class CurrentlySelectedChatLanguageModel implements ChatModel, AutoCloseable { private static final Duration CONNECTION_TIMEOUT = Duration.ofSeconds(5); private final AiPreferences aiPreferences; @@ -41,7 +41,7 @@ public class JabRefChatLanguageModel implements ChatModel, AutoCloseable { private Optional langchainChatModel = Optional.empty(); - public JabRefChatLanguageModel(AiPreferences aiPreferences) { + public CurrentlySelectedChatLanguageModel(AiPreferences aiPreferences) { this.aiPreferences = aiPreferences; this.httpClient = HttpClient.newBuilder().connectTimeout(CONNECTION_TIMEOUT).executor(executorService).build(); diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/JabRefEmbeddingModel.java b/jablib/src/main/java/org/jabref/logic/ai/rag/CurrentlySelectedEmbeddingModel.java similarity index 95% rename from jablib/src/main/java/org/jabref/logic/ai/rag/JabRefEmbeddingModel.java rename to jablib/src/main/java/org/jabref/logic/ai/rag/CurrentlySelectedEmbeddingModel.java index 90b9eed9352f..bc89ca0ce014 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/JabRefEmbeddingModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/CurrentlySelectedEmbeddingModel.java @@ -29,8 +29,8 @@ *

* This class listens to preferences changes. */ -public class JabRefEmbeddingModel implements EmbeddingModel, AutoCloseable { - private static final Logger LOGGER = LoggerFactory.getLogger(JabRefEmbeddingModel.class); +public class CurrentlySelectedEmbeddingModel implements EmbeddingModel, AutoCloseable { + private static final Logger LOGGER = LoggerFactory.getLogger(CurrentlySelectedEmbeddingModel.class); private final AiPreferences aiPreferences; private final NotificationService notificationService; @@ -54,7 +54,7 @@ public static class EmbeddingModelBuildingErrorEvent { // Empty if there is no error. private String errorWhileBuildingModel = ""; - public JabRefEmbeddingModel(AiPreferences aiPreferences, NotificationService notificationService, TaskExecutor taskExecutor) { + public CurrentlySelectedEmbeddingModel(AiPreferences aiPreferences, NotificationService notificationService, TaskExecutor taskExecutor) { this.aiPreferences = aiPreferences; this.notificationService = notificationService; this.taskExecutor = taskExecutor; From be13639afa8437212ec37f139673ad289c0a4675 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Sun, 23 Nov 2025 17:36:44 +0100 Subject: [PATCH 006/243] Refactor `AiPreferences` and `AiDefaultPreferences` to `org.jabref.logic.ai.preferences`; update package structure and adjust imports across the codebase. --- .../org/jabref/gui/ai/components/aichat/AiChatComponent.java | 2 +- .../gui/ai/components/aichat/AiChatGuardedComponent.java | 2 +- .../org/jabref/gui/ai/components/aichat/AiChatWindow.java | 2 +- .../privacynotice/AiPrivacyNoticeGuardedComponent.java | 2 +- .../ai/components/privacynotice/PrivacyNoticeComponent.java | 2 +- .../jabref/gui/ai/components/summary/SummaryComponent.java | 2 +- .../ai/components/util/EmbeddingModelGuardedComponent.java | 2 +- .../src/main/java/org/jabref/gui/entryeditor/AiChatTab.java | 2 +- .../main/java/org/jabref/gui/entryeditor/AiSummaryTab.java | 2 +- .../java/org/jabref/gui/preferences/ai/AiTabViewModel.java | 4 ++-- .../jabref/gui/ai/components/aichat/AiChatComponentTest.java | 2 +- jablib/src/main/java/module-info.java | 1 + jablib/src/main/java/org/jabref/logic/ai/AiService.java | 1 + .../logic/ai/chatting/CurrentlySelectedChatLanguageModel.java | 2 +- .../org/jabref/logic/ai/chatting/algorithms/AiChatLogic.java | 2 +- .../logic/ai/customimplementations/llms/Gpt4AllModel.java | 2 +- .../llms/JvmOpenAiChatLanguageModel.java | 2 +- .../logic/ai/{ => preferences}/AiDefaultPreferences.java | 2 +- .../org/jabref/logic/ai/{ => preferences}/AiPreferences.java | 2 +- .../jabref/logic/ai/rag/CurrentlySelectedEmbeddingModel.java | 2 +- .../main/java/org/jabref/logic/ai/rag/IngestionService.java | 2 +- .../org/jabref/logic/ai/rag/algorithms/LowLevelIngestor.java | 2 +- .../jabref/logic/ai/rag/storages/FileEmbeddingsManager.java | 2 +- .../jabref/logic/ai/rag/tasks/UpdateEmbeddingModelTask.java | 2 +- .../org/jabref/logic/ai/summarization/SummariesService.java | 2 +- .../ai/summarization/tasks/GenerateSummaryForSeveralTask.java | 2 +- .../logic/ai/summarization/tasks/GenerateSummaryTask.java | 2 +- .../org/jabref/logic/ai/templates/AiTemplatesService.java | 2 +- .../java/org/jabref/logic/preferences/CliPreferences.java | 2 +- .../org/jabref/logic/preferences/JabRefCliPreferences.java | 4 ++-- 30 files changed, 32 insertions(+), 30 deletions(-) rename jablib/src/main/java/org/jabref/logic/ai/{ => preferences}/AiDefaultPreferences.java (99%) rename jablib/src/main/java/org/jabref/logic/ai/{ => preferences}/AiPreferences.java (99%) diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java index 0b32435593ac..4a339cf523ce 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java @@ -27,10 +27,10 @@ import org.jabref.gui.ai.components.util.notifications.NotificationsComponent; import org.jabref.gui.icon.IconTheme; import org.jabref.gui.util.UiTaskExecutor; -import org.jabref.logic.ai.AiPreferences; import org.jabref.logic.ai.AiService; import org.jabref.logic.ai.chatting.algorithms.AiChatLogic; import org.jabref.logic.ai.chatting.tasks.GenerateAiResponseTask; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.util.CitationKeyCheck; import org.jabref.logic.ai.util.ErrorMessage; import org.jabref.logic.l10n.Localization; diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatGuardedComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatGuardedComponent.java index 4a22e6d15192..d813c339496f 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatGuardedComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatGuardedComponent.java @@ -8,8 +8,8 @@ import org.jabref.gui.ai.components.util.EmbeddingModelGuardedComponent; import org.jabref.gui.entryeditor.AdaptVisibleTabs; import org.jabref.gui.frame.ExternalApplicationsPreferences; -import org.jabref.logic.ai.AiPreferences; import org.jabref.logic.ai.AiService; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatWindow.java b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatWindow.java index 8e1fd36bea76..f0151f889a66 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatWindow.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatWindow.java @@ -8,8 +8,8 @@ import org.jabref.gui.entryeditor.AdaptVisibleTabs; import org.jabref.gui.frame.ExternalApplicationsPreferences; import org.jabref.gui.util.BaseWindow; -import org.jabref.logic.ai.AiPreferences; import org.jabref.logic.ai.AiService; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/AiPrivacyNoticeGuardedComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/AiPrivacyNoticeGuardedComponent.java index bb5d2124350c..09e309fa2c24 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/AiPrivacyNoticeGuardedComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/AiPrivacyNoticeGuardedComponent.java @@ -7,7 +7,7 @@ import org.jabref.gui.entryeditor.AdaptVisibleTabs; import org.jabref.gui.frame.ExternalApplicationsPreferences; import org.jabref.gui.util.DynamicallyChangeableNode; -import org.jabref.logic.ai.AiPreferences; +import org.jabref.logic.ai.preferences.AiPreferences; /** * A class that guards a component, before AI privacy policy is accepted. diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/PrivacyNoticeComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/PrivacyNoticeComponent.java index 015f1b4a088f..bbf566811841 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/PrivacyNoticeComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/PrivacyNoticeComponent.java @@ -18,7 +18,7 @@ import org.jabref.gui.entryeditor.EntryEditorPreferences; import org.jabref.gui.frame.ExternalApplicationsPreferences; import org.jabref.gui.preferences.GuiPreferences; -import org.jabref.logic.ai.AiPreferences; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.model.ai.AiProvider; import com.airhacks.afterburner.views.ViewLoader; diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java index 372e4b9a3ee4..06fa76fdf8be 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java @@ -9,8 +9,8 @@ import org.jabref.gui.ai.components.util.errorstate.ErrorStateComponent; import org.jabref.gui.entryeditor.AdaptVisibleTabs; import org.jabref.gui.frame.ExternalApplicationsPreferences; -import org.jabref.logic.ai.AiPreferences; import org.jabref.logic.ai.AiService; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.util.CitationKeyCheck; import org.jabref.logic.citationkeypattern.CitationKeyGenerator; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java index fd508e72f528..077f1f8f8a6c 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java @@ -8,8 +8,8 @@ import org.jabref.gui.entryeditor.AdaptVisibleTabs; import org.jabref.gui.frame.ExternalApplicationsPreferences; import org.jabref.gui.util.UiTaskExecutor; -import org.jabref.logic.ai.AiPreferences; import org.jabref.logic.ai.AiService; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.rag.CurrentlySelectedEmbeddingModel; import org.jabref.logic.l10n.Localization; diff --git a/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java b/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java index 07df4aa94222..f64cdc533f76 100644 --- a/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java +++ b/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java @@ -17,8 +17,8 @@ import org.jabref.gui.ai.components.util.errorstate.ErrorStateComponent; import org.jabref.gui.frame.ExternalApplicationsPreferences; import org.jabref.gui.preferences.GuiPreferences; -import org.jabref.logic.ai.AiPreferences; import org.jabref.logic.ai.AiService; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.util.CitationKeyCheck; import org.jabref.logic.citationkeypattern.CitationKeyGenerator; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; diff --git a/jabgui/src/main/java/org/jabref/gui/entryeditor/AiSummaryTab.java b/jabgui/src/main/java/org/jabref/gui/entryeditor/AiSummaryTab.java index c47f91fb0738..310a76c00564 100644 --- a/jabgui/src/main/java/org/jabref/gui/entryeditor/AiSummaryTab.java +++ b/jabgui/src/main/java/org/jabref/gui/entryeditor/AiSummaryTab.java @@ -7,8 +7,8 @@ import org.jabref.gui.ai.components.summary.SummaryComponent; import org.jabref.gui.frame.ExternalApplicationsPreferences; import org.jabref.gui.preferences.GuiPreferences; -import org.jabref.logic.ai.AiPreferences; import org.jabref.logic.ai.AiService; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; import org.jabref.logic.l10n.Localization; import org.jabref.model.database.BibDatabaseContext; diff --git a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java index 1ec030fca5f3..239aaf2e6fe6 100644 --- a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java +++ b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java @@ -20,8 +20,8 @@ import javafx.collections.FXCollections; import org.jabref.gui.preferences.PreferenceTabViewModel; -import org.jabref.logic.ai.AiDefaultPreferences; -import org.jabref.logic.ai.AiPreferences; +import org.jabref.logic.ai.preferences.AiDefaultPreferences; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.l10n.Localization; import org.jabref.logic.preferences.CliPreferences; import org.jabref.logic.util.LocalizedNumbers; diff --git a/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java b/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java index cb69736e7173..6544faba03b0 100644 --- a/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java +++ b/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java @@ -9,9 +9,9 @@ import javafx.collections.FXCollections; import org.jabref.gui.DialogService; -import org.jabref.logic.ai.AiPreferences; import org.jabref.logic.ai.AiService; import org.jabref.logic.ai.chatting.algorithms.AiChatLogic; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.rag.IngestionService; import org.jabref.logic.l10n.Language; import org.jabref.logic.l10n.Localization; diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index ab6dbbaf6c43..d9641b563700 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -128,6 +128,7 @@ exports org.jabref.logic.ai.rag.algorithms; exports org.jabref.logic.ai.chatting.algorithms; exports org.jabref.logic.ai.chatting.tasks; + exports org.jabref.logic.ai.preferences; requires java.base; diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiService.java b/jablib/src/main/java/org/jabref/logic/ai/AiService.java index f35b21b94be5..dfe8d25166b9 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiService.java @@ -10,6 +10,7 @@ import org.jabref.logic.ai.chatting.CurrentlySelectedChatLanguageModel; import org.jabref.logic.ai.chatting.chathistory.ChatHistoryService; import org.jabref.logic.ai.chatting.chathistory.storages.MVStoreChatHistoryStorage; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.rag.CurrentlySelectedEmbeddingModel; import org.jabref.logic.ai.rag.IngestionService; import org.jabref.logic.ai.rag.storages.MVStoreEmbeddingStore; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java index 4711767b4beb..4b9172840968 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java @@ -7,10 +7,10 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import org.jabref.logic.ai.AiPreferences; import org.jabref.logic.ai.chatting.algorithms.AiChatLogic; import org.jabref.logic.ai.customimplementations.llms.Gpt4AllModel; import org.jabref.logic.ai.customimplementations.llms.JvmOpenAiChatLanguageModel; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.l10n.Localization; import org.jabref.model.ai.AiProvider; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/algorithms/AiChatLogic.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/algorithms/AiChatLogic.java index e9c0203fe0d9..97a4f624a425 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/algorithms/AiChatLogic.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/algorithms/AiChatLogic.java @@ -7,7 +7,7 @@ import javafx.collections.ListChangeListener; import javafx.collections.ObservableList; -import org.jabref.logic.ai.AiPreferences; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.rag.storages.FileEmbeddingsManager; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.ai.util.ErrorMessage; diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/Gpt4AllModel.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/Gpt4AllModel.java index 05da6eb9fe79..7d3c8fb8245d 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/Gpt4AllModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/Gpt4AllModel.java @@ -8,7 +8,7 @@ import java.util.List; import java.util.stream.Collectors; -import org.jabref.logic.ai.AiPreferences; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.util.URLUtil; import com.google.gson.Gson; diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/JvmOpenAiChatLanguageModel.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/JvmOpenAiChatLanguageModel.java index f7b571937500..c19efe6939e4 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/JvmOpenAiChatLanguageModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/JvmOpenAiChatLanguageModel.java @@ -3,7 +3,7 @@ import java.net.http.HttpClient; import java.util.List; -import org.jabref.logic.ai.AiPreferences; +import org.jabref.logic.ai.preferences.AiPreferences; import dev.langchain4j.data.message.AiMessage; import dev.langchain4j.data.message.ChatMessage; diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiDefaultPreferences.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultPreferences.java similarity index 99% rename from jablib/src/main/java/org/jabref/logic/ai/AiDefaultPreferences.java rename to jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultPreferences.java index 7d96e724826d..c05f901abacf 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiDefaultPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultPreferences.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai; +package org.jabref.logic.ai.preferences; import java.util.Arrays; import java.util.List; diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiPreferences.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java similarity index 99% rename from jablib/src/main/java/org/jabref/logic/ai/AiPreferences.java rename to jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java index 6db7d743585d..b7e5c5775cb9 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai; +package org.jabref.logic.ai.preferences; import java.util.List; import java.util.Map; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/CurrentlySelectedEmbeddingModel.java b/jablib/src/main/java/org/jabref/logic/ai/rag/CurrentlySelectedEmbeddingModel.java index bc89ca0ce014..924a2767601b 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/CurrentlySelectedEmbeddingModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/CurrentlySelectedEmbeddingModel.java @@ -8,8 +8,8 @@ import javafx.beans.property.ObjectProperty; import javafx.beans.property.SimpleObjectProperty; -import org.jabref.logic.ai.AiPreferences; import org.jabref.logic.ai.customimplementations.embeddingmodels.DeepJavaEmbeddingModel; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.rag.tasks.UpdateEmbeddingModelTask; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.NotificationService; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java b/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java index 48c298094c81..d2f94cb0c6f8 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java @@ -9,7 +9,7 @@ import javafx.beans.property.StringProperty; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.AiPreferences; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.rag.storages.FileEmbeddingsManager; import org.jabref.logic.ai.rag.storages.FullyIngestedDocumentsTracker; import org.jabref.logic.ai.rag.tasks.GenerateEmbeddingsForSeveralTask; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/LowLevelIngestor.java b/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/LowLevelIngestor.java index 35a2e4eaae16..0d783a4c4e1c 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/LowLevelIngestor.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/LowLevelIngestor.java @@ -5,7 +5,7 @@ import javafx.beans.property.IntegerProperty; import javafx.beans.property.ReadOnlyBooleanProperty; -import org.jabref.logic.ai.AiPreferences; +import org.jabref.logic.ai.preferences.AiPreferences; import dev.langchain4j.data.document.DefaultDocument; import dev.langchain4j.data.document.Document; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FileEmbeddingsManager.java b/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FileEmbeddingsManager.java index 78654508cb92..dc1fc6c2a278 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FileEmbeddingsManager.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FileEmbeddingsManager.java @@ -6,7 +6,7 @@ import javafx.beans.property.IntegerProperty; import javafx.beans.property.ReadOnlyBooleanProperty; -import org.jabref.logic.ai.AiPreferences; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.rag.algorithms.LowLevelIngestor; import org.jabref.model.entry.LinkedFile; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/UpdateEmbeddingModelTask.java b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/UpdateEmbeddingModelTask.java index 8fc30cafa3cb..fcd6e3102d02 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/UpdateEmbeddingModelTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/UpdateEmbeddingModelTask.java @@ -5,8 +5,8 @@ import javafx.beans.property.ObjectProperty; -import org.jabref.logic.ai.AiPreferences; import org.jabref.logic.ai.customimplementations.embeddingmodels.DeepJavaEmbeddingModel; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.ProgressCounter; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java index f2fbc1053e3e..327f16005496 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java @@ -9,7 +9,7 @@ import javafx.beans.property.StringProperty; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.AiPreferences; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.summarization.storages.SummariesStorage; import org.jabref.logic.ai.summarization.tasks.GenerateSummaryForSeveralTask; import org.jabref.logic.ai.summarization.tasks.GenerateSummaryTask; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java index ff4f9fb169f4..865e52c19114 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java @@ -10,7 +10,7 @@ import javafx.util.Pair; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.AiPreferences; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.summarization.storages.SummariesStorage; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.l10n.Localization; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java index a1b2a355e075..d047c47d8c52 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java @@ -11,7 +11,7 @@ import javafx.beans.property.ReadOnlyBooleanProperty; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.AiPreferences; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.rag.algorithms.FileToDocument; import org.jabref.logic.ai.summarization.SummariesService; import org.jabref.logic.ai.summarization.storages.SummariesStorage; diff --git a/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java b/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java index de85c987ea79..f11a8b348b50 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java @@ -3,7 +3,7 @@ import java.io.StringWriter; import java.util.List; -import org.jabref.logic.ai.AiPreferences; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.model.ai.AiTemplate; import org.jabref.model.ai.PaperExcerpt; import org.jabref.model.entry.BibEntry; diff --git a/jablib/src/main/java/org/jabref/logic/preferences/CliPreferences.java b/jablib/src/main/java/org/jabref/logic/preferences/CliPreferences.java index f1a404195221..2927789846bd 100644 --- a/jablib/src/main/java/org/jabref/logic/preferences/CliPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/preferences/CliPreferences.java @@ -8,7 +8,7 @@ import org.jabref.logic.InternalPreferences; import org.jabref.logic.JabRefException; import org.jabref.logic.LibraryPreferences; -import org.jabref.logic.ai.AiPreferences; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.bibtex.FieldPreferences; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; import org.jabref.logic.cleanup.CleanupPreferences; diff --git a/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java b/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java index edfc08a0c0f3..341c87bc0614 100644 --- a/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java @@ -33,8 +33,8 @@ import org.jabref.logic.InternalPreferences; import org.jabref.logic.JabRefException; import org.jabref.logic.LibraryPreferences; -import org.jabref.logic.ai.AiDefaultPreferences; -import org.jabref.logic.ai.AiPreferences; +import org.jabref.logic.ai.preferences.AiDefaultPreferences; +import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.bibtex.FieldPreferences; import org.jabref.logic.citationkeypattern.CitationKeyPattern; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; From add618df7ee5970fe5afdeb88c4e443deeb8125f Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Sun, 23 Nov 2025 17:40:27 +0100 Subject: [PATCH 007/243] Refactor AI-related classes in `org.jabref.model.ai` to a more specific package structure; update imports and restructure hierarchy accordingly. --- .../jabref/gui/ai/components/aichat/AiChatComponent.java | 2 +- .../components/aichat/chatmessage/ChatMessageComponent.java | 2 +- .../ai/components/privacynotice/PrivacyNoticeComponent.java | 2 +- .../jabref/gui/ai/components/summary/SummaryComponent.java | 2 +- .../gui/ai/components/summary/SummaryShowingComponent.java | 2 +- .../src/main/java/org/jabref/gui/preferences/ai/AiTab.java | 6 +++--- .../java/org/jabref/gui/preferences/ai/AiTabViewModel.java | 6 +++--- .../gui/ai/components/aichat/AiChatComponentTest.java | 2 +- jablib/src/main/java/module-info.java | 6 ++++++ jablib/src/main/java/org/jabref/logic/ai/AiService.java | 4 ++-- .../ai/chatting/{chathistory => }/ChatHistoryService.java | 3 ++- .../ai/chatting/CurrentlySelectedChatLanguageModel.java | 5 +++-- .../jabref/logic/ai/chatting/algorithms/AiChatLogic.java | 6 +++--- .../{chathistory => storages}/ChatHistoryStorage.java | 2 +- .../storages/MVStoreChatHistoryStorage.java | 5 ++--- .../jabref/logic/ai/preferences/AiDefaultPreferences.java | 6 +++--- .../java/org/jabref/logic/ai/preferences/AiPreferences.java | 6 +++--- .../org/jabref/logic/ai/summarization/SummariesService.java | 2 +- .../ai/summarization/storages/MVStoreSummariesStorage.java | 2 +- .../logic/ai/summarization/storages/SummariesStorage.java | 2 +- .../summarization/tasks/GenerateSummaryForSeveralTask.java | 2 +- .../logic/ai/summarization/tasks/GenerateSummaryTask.java | 4 ++-- .../org/jabref/logic/ai/templates/AiTemplatesService.java | 4 ++-- .../org/jabref/logic/preferences/JabRefCliPreferences.java | 6 +++--- .../java/org/jabref/model/ai/{ => chatting}/AiProvider.java | 2 +- .../{logic/ai/util => model/ai/chatting}/ErrorMessage.java | 2 +- .../jabref/model/ai/{ => embeddings}/EmbeddingModel.java | 2 +- .../java/org/jabref/model/ai/{ => rag}/PaperExcerpt.java | 2 +- .../org/jabref/model/ai/{ => summarization}/Summary.java | 4 +++- .../org/jabref/model/ai/{ => templating}/AiTemplate.java | 2 +- .../ai/chatting/chathistory/ChatHistoryStorageTest.java | 2 ++ .../chatting/chathistory/MVStoreChatHistoryStorageTest.java | 3 ++- .../jabref/logic/ai/summarization/SummariesStorageTest.java | 4 ++-- 33 files changed, 62 insertions(+), 50 deletions(-) rename jablib/src/main/java/org/jabref/logic/ai/chatting/{chathistory => }/ChatHistoryService.java (99%) rename jablib/src/main/java/org/jabref/logic/ai/chatting/{chathistory => storages}/ChatHistoryStorage.java (91%) rename jablib/src/main/java/org/jabref/logic/ai/chatting/{chathistory => }/storages/MVStoreChatHistoryStorage.java (96%) rename jablib/src/main/java/org/jabref/model/ai/{ => chatting}/AiProvider.java (96%) rename jablib/src/main/java/org/jabref/{logic/ai/util => model/ai/chatting}/ErrorMessage.java (96%) rename jablib/src/main/java/org/jabref/model/ai/{ => embeddings}/EmbeddingModel.java (99%) rename jablib/src/main/java/org/jabref/model/ai/{ => rag}/PaperExcerpt.java (66%) rename jablib/src/main/java/org/jabref/model/ai/{ => summarization}/Summary.java (67%) rename jablib/src/main/java/org/jabref/model/ai/{ => templating}/AiTemplate.java (93%) diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java index 4a339cf523ce..c4f15779de2a 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java @@ -32,11 +32,11 @@ import org.jabref.logic.ai.chatting.tasks.GenerateAiResponseTask; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.util.CitationKeyCheck; -import org.jabref.logic.ai.util.ErrorMessage; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.TaskExecutor; import org.jabref.logic.util.io.FileUtil; +import org.jabref.model.ai.chatting.ErrorMessage; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.util.ListUtil; diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/chatmessage/ChatMessageComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/chatmessage/ChatMessageComponent.java index f620938dea34..57ebe8916397 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/chatmessage/ChatMessageComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/chatmessage/ChatMessageComponent.java @@ -14,8 +14,8 @@ import javafx.scene.layout.VBox; import org.jabref.gui.util.MarkdownTextFlow; -import org.jabref.logic.ai.util.ErrorMessage; import org.jabref.logic.l10n.Localization; +import org.jabref.model.ai.chatting.ErrorMessage; import com.airhacks.afterburner.views.ViewLoader; import dev.langchain4j.data.message.AiMessage; diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/PrivacyNoticeComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/PrivacyNoticeComponent.java index bbf566811841..a5c0fe0b0cdd 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/PrivacyNoticeComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/PrivacyNoticeComponent.java @@ -19,7 +19,7 @@ import org.jabref.gui.frame.ExternalApplicationsPreferences; import org.jabref.gui.preferences.GuiPreferences; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.model.ai.AiProvider; +import org.jabref.model.ai.chatting.AiProvider; import com.airhacks.afterburner.views.ViewLoader; import jakarta.inject.Inject; diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java index 06fa76fdf8be..6f5a78dfa812 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java @@ -16,7 +16,7 @@ import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.io.FileUtil; -import org.jabref.model.ai.Summary; +import org.jabref.model.ai.summarization.Summary; import org.jabref.model.ai.processingstatus.ProcessingInfo; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryShowingComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryShowingComponent.java index ed6a1198a443..ec08d85e4764 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryShowingComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryShowingComponent.java @@ -14,7 +14,7 @@ import org.jabref.gui.util.WebViewStore; import org.jabref.logic.layout.format.MarkdownFormatter; -import org.jabref.model.ai.Summary; +import org.jabref.model.ai.summarization.Summary; import com.airhacks.afterburner.views.ViewLoader; diff --git a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java index 21eae8c557a6..6f388ea854b7 100644 --- a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java +++ b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java @@ -23,9 +23,9 @@ import org.jabref.gui.util.ViewModelListCellFactory; import org.jabref.logic.help.HelpFile; import org.jabref.logic.l10n.Localization; -import org.jabref.model.ai.AiProvider; -import org.jabref.model.ai.AiTemplate; -import org.jabref.model.ai.EmbeddingModel; +import org.jabref.model.ai.chatting.AiProvider; +import org.jabref.model.ai.templating.AiTemplate; +import org.jabref.model.ai.embeddings.EmbeddingModel; import com.airhacks.afterburner.views.ViewLoader; import com.dlsc.unitfx.IntegerInputField; diff --git a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java index 239aaf2e6fe6..612ad772b54f 100644 --- a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java +++ b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java @@ -27,9 +27,9 @@ import org.jabref.logic.util.LocalizedNumbers; import org.jabref.logic.util.OptionalObjectProperty; import org.jabref.logic.util.strings.StringUtil; -import org.jabref.model.ai.AiProvider; -import org.jabref.model.ai.AiTemplate; -import org.jabref.model.ai.EmbeddingModel; +import org.jabref.model.ai.chatting.AiProvider; +import org.jabref.model.ai.templating.AiTemplate; +import org.jabref.model.ai.embeddings.EmbeddingModel; import de.saxsys.mvvmfx.utils.validation.FunctionBasedValidator; import de.saxsys.mvvmfx.utils.validation.ValidationMessage; diff --git a/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java b/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java index 6544faba03b0..9226bffce365 100644 --- a/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java +++ b/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java @@ -16,7 +16,7 @@ import org.jabref.logic.l10n.Language; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.TaskExecutor; -import org.jabref.model.ai.AiProvider; +import org.jabref.model.ai.chatting.AiProvider; import org.jabref.model.database.BibDatabaseContext; import org.junit.jupiter.api.BeforeEach; diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index d9641b563700..50fe9b74f71d 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -129,6 +129,12 @@ exports org.jabref.logic.ai.chatting.algorithms; exports org.jabref.logic.ai.chatting.tasks; exports org.jabref.logic.ai.preferences; + exports org.jabref.model.ai.chatting; + exports org.jabref.model.ai.templating; + exports org.jabref.model.ai.rag; + exports org.jabref.model.ai.embeddings; + exports org.jabref.model.ai.summarization; + exports org.jabref.logic.ai.chatting.storages; requires java.base; diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiService.java b/jablib/src/main/java/org/jabref/logic/ai/AiService.java index dfe8d25166b9..45537c3e3c36 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiService.java @@ -7,9 +7,9 @@ import javafx.beans.property.SimpleBooleanProperty; import org.jabref.logic.FilePreferences; +import org.jabref.logic.ai.chatting.ChatHistoryService; import org.jabref.logic.ai.chatting.CurrentlySelectedChatLanguageModel; -import org.jabref.logic.ai.chatting.chathistory.ChatHistoryService; -import org.jabref.logic.ai.chatting.chathistory.storages.MVStoreChatHistoryStorage; +import org.jabref.logic.ai.chatting.storages.MVStoreChatHistoryStorage; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.rag.CurrentlySelectedEmbeddingModel; import org.jabref.logic.ai.rag.IngestionService; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryService.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java similarity index 99% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryService.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java index 1a1652fcb1f7..c38895fa766c 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.chatting.chathistory; +package org.jabref.logic.ai.chatting; import java.util.Comparator; import java.util.HashSet; @@ -9,6 +9,7 @@ import javafx.collections.FXCollections; import javafx.collections.ObservableList; +import org.jabref.logic.ai.chatting.storages.ChatHistoryStorage; import org.jabref.logic.ai.util.CitationKeyCheck; import org.jabref.logic.citationkeypattern.CitationKeyGenerator; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java index 4b9172840968..796f38e48aa9 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java @@ -8,11 +8,12 @@ import java.util.concurrent.Executors; import org.jabref.logic.ai.chatting.algorithms.AiChatLogic; +import org.jabref.logic.ai.chatting.storages.ChatHistoryStorage; import org.jabref.logic.ai.customimplementations.llms.Gpt4AllModel; import org.jabref.logic.ai.customimplementations.llms.JvmOpenAiChatLanguageModel; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.l10n.Localization; -import org.jabref.model.ai.AiProvider; +import org.jabref.model.ai.chatting.AiProvider; import com.google.common.util.concurrent.ThreadFactoryBuilder; import dev.langchain4j.data.message.ChatMessage; @@ -52,7 +53,7 @@ public CurrentlySelectedChatLanguageModel(AiPreferences aiPreferences) { * Update the underlying {@link dev.langchain4j.model.chat.ChatModel} by current {@link AiPreferences} parameters. * When the model is updated, the chat messages are not lost. * See {@link AiChatLogic}, where messages are stored in {@link ChatMemory}, - * and see {@link org.jabref.logic.ai.chatting.chathistory.ChatHistoryStorage}. + * and see {@link ChatHistoryStorage}. */ private void rebuild() { String apiKey = aiPreferences.getApiKeyForAiProvider(aiPreferences.getAiProvider()); diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/algorithms/AiChatLogic.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/algorithms/AiChatLogic.java index 97a4f624a425..de6656f93129 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/algorithms/AiChatLogic.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/algorithms/AiChatLogic.java @@ -10,9 +10,9 @@ import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.rag.storages.FileEmbeddingsManager; import org.jabref.logic.ai.templates.AiTemplatesService; -import org.jabref.logic.ai.util.ErrorMessage; -import org.jabref.model.ai.AiTemplate; -import org.jabref.model.ai.PaperExcerpt; +import org.jabref.model.ai.chatting.ErrorMessage; +import org.jabref.model.ai.rag.PaperExcerpt; +import org.jabref.model.ai.templating.AiTemplate; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryStorage.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/storages/ChatHistoryStorage.java similarity index 91% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryStorage.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/storages/ChatHistoryStorage.java index 1a30a4332e71..6ce6cf3343a1 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryStorage.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/storages/ChatHistoryStorage.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.chatting.chathistory; +package org.jabref.logic.ai.chatting.storages; import java.nio.file.Path; import java.util.List; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/chathistory/storages/MVStoreChatHistoryStorage.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/storages/MVStoreChatHistoryStorage.java similarity index 96% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/chathistory/storages/MVStoreChatHistoryStorage.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/storages/MVStoreChatHistoryStorage.java index 31e13930b3e9..e62f6c16610e 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/chathistory/storages/MVStoreChatHistoryStorage.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/storages/MVStoreChatHistoryStorage.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.chatting.chathistory.storages; +package org.jabref.logic.ai.chatting.storages; import java.io.Serializable; import java.nio.file.Path; @@ -6,11 +6,10 @@ import java.util.List; import java.util.Map; -import org.jabref.logic.ai.chatting.chathistory.ChatHistoryStorage; -import org.jabref.logic.ai.util.ErrorMessage; import org.jabref.logic.ai.util.MVStoreBase; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.NotificationService; +import org.jabref.model.ai.chatting.ErrorMessage; import dev.langchain4j.data.message.AiMessage; import dev.langchain4j.data.message.ChatMessage; diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultPreferences.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultPreferences.java index c05f901abacf..5e5fcaee907b 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultPreferences.java @@ -4,9 +4,9 @@ import java.util.List; import java.util.Map; -import org.jabref.model.ai.AiProvider; -import org.jabref.model.ai.AiTemplate; -import org.jabref.model.ai.EmbeddingModel; +import org.jabref.model.ai.chatting.AiProvider; +import org.jabref.model.ai.templating.AiTemplate; +import org.jabref.model.ai.embeddings.EmbeddingModel; public class AiDefaultPreferences { public enum PredefinedChatModel { diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java index b7e5c5775cb9..8e25cbab7601 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java @@ -17,9 +17,9 @@ import javafx.beans.property.StringProperty; import org.jabref.logic.util.strings.StringUtil; -import org.jabref.model.ai.AiProvider; -import org.jabref.model.ai.AiTemplate; -import org.jabref.model.ai.EmbeddingModel; +import org.jabref.model.ai.chatting.AiProvider; +import org.jabref.model.ai.templating.AiTemplate; +import org.jabref.model.ai.embeddings.EmbeddingModel; import com.github.javakeyring.Keyring; import com.github.javakeyring.PasswordAccessException; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java index 327f16005496..da9d536b3944 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java @@ -16,7 +16,7 @@ import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.ai.util.CitationKeyCheck; import org.jabref.logic.util.TaskExecutor; -import org.jabref.model.ai.Summary; +import org.jabref.model.ai.summarization.Summary; import org.jabref.model.ai.processingstatus.ProcessingInfo; import org.jabref.model.ai.processingstatus.ProcessingState; import org.jabref.model.database.BibDatabaseContext; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/MVStoreSummariesStorage.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/MVStoreSummariesStorage.java index 55e9519bcb7d..05b502d43430 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/MVStoreSummariesStorage.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/MVStoreSummariesStorage.java @@ -7,7 +7,7 @@ import org.jabref.logic.ai.util.MVStoreBase; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.NotificationService; -import org.jabref.model.ai.Summary; +import org.jabref.model.ai.summarization.Summary; public class MVStoreSummariesStorage extends MVStoreBase implements SummariesStorage { private static final String SUMMARIES_MAP_PREFIX = "summaries"; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/SummariesStorage.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/SummariesStorage.java index 8d9a7e481243..cbee116abf47 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/SummariesStorage.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/SummariesStorage.java @@ -3,7 +3,7 @@ import java.nio.file.Path; import java.util.Optional; -import org.jabref.model.ai.Summary; +import org.jabref.model.ai.summarization.Summary; public interface SummariesStorage { void set(Path bibDatabasePath, String citationKey, Summary summary); diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java index 865e52c19114..8ee2f78d8fb4 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java @@ -17,7 +17,7 @@ import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.ProgressCounter; import org.jabref.logic.util.TaskExecutor; -import org.jabref.model.ai.Summary; +import org.jabref.model.ai.summarization.Summary; import org.jabref.model.ai.processingstatus.ProcessingInfo; import org.jabref.model.ai.processingstatus.ProcessingState; import org.jabref.model.database.BibDatabaseContext; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java index d047c47d8c52..bc228b6feb4a 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java @@ -20,8 +20,8 @@ import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.ProgressCounter; -import org.jabref.model.ai.AiTemplate; -import org.jabref.model.ai.Summary; +import org.jabref.model.ai.templating.AiTemplate; +import org.jabref.model.ai.summarization.Summary; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; diff --git a/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java b/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java index f11a8b348b50..ccb1552a2937 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java @@ -4,8 +4,8 @@ import java.util.List; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.model.ai.AiTemplate; -import org.jabref.model.ai.PaperExcerpt; +import org.jabref.model.ai.templating.AiTemplate; +import org.jabref.model.ai.rag.PaperExcerpt; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.CanonicalBibEntry; diff --git a/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java b/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java index 341c87bc0614..6d4eceab24b6 100644 --- a/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java @@ -95,9 +95,9 @@ import org.jabref.logic.util.io.FileHistory; import org.jabref.logic.util.strings.StringUtil; import org.jabref.logic.xmp.XmpPreferences; -import org.jabref.model.ai.AiProvider; -import org.jabref.model.ai.AiTemplate; -import org.jabref.model.ai.EmbeddingModel; +import org.jabref.model.ai.chatting.AiProvider; +import org.jabref.model.ai.templating.AiTemplate; +import org.jabref.model.ai.embeddings.EmbeddingModel; import org.jabref.model.database.BibDatabaseMode; import org.jabref.model.entry.BibEntryPreferences; import org.jabref.model.entry.BibEntryType; diff --git a/jablib/src/main/java/org/jabref/model/ai/AiProvider.java b/jablib/src/main/java/org/jabref/model/ai/chatting/AiProvider.java similarity index 96% rename from jablib/src/main/java/org/jabref/model/ai/AiProvider.java rename to jablib/src/main/java/org/jabref/model/ai/chatting/AiProvider.java index cd37d85d396d..01c76eb2b56d 100644 --- a/jablib/src/main/java/org/jabref/model/ai/AiProvider.java +++ b/jablib/src/main/java/org/jabref/model/ai/chatting/AiProvider.java @@ -1,4 +1,4 @@ -package org.jabref.model.ai; +package org.jabref.model.ai.chatting; import java.io.Serializable; diff --git a/jablib/src/main/java/org/jabref/logic/ai/util/ErrorMessage.java b/jablib/src/main/java/org/jabref/model/ai/chatting/ErrorMessage.java similarity index 96% rename from jablib/src/main/java/org/jabref/logic/ai/util/ErrorMessage.java rename to jablib/src/main/java/org/jabref/model/ai/chatting/ErrorMessage.java index cbbaa3e833c0..cf13b2413838 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/util/ErrorMessage.java +++ b/jablib/src/main/java/org/jabref/model/ai/chatting/ErrorMessage.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.util; +package org.jabref.model.ai.chatting; import dev.langchain4j.data.message.ChatMessage; import dev.langchain4j.data.message.ChatMessageType; diff --git a/jablib/src/main/java/org/jabref/model/ai/EmbeddingModel.java b/jablib/src/main/java/org/jabref/model/ai/embeddings/EmbeddingModel.java similarity index 99% rename from jablib/src/main/java/org/jabref/model/ai/EmbeddingModel.java rename to jablib/src/main/java/org/jabref/model/ai/embeddings/EmbeddingModel.java index 262aea878473..1447992948df 100644 --- a/jablib/src/main/java/org/jabref/model/ai/EmbeddingModel.java +++ b/jablib/src/main/java/org/jabref/model/ai/embeddings/EmbeddingModel.java @@ -1,4 +1,4 @@ -package org.jabref.model.ai; +package org.jabref.model.ai.embeddings; import org.apache.commons.io.FileUtils; diff --git a/jablib/src/main/java/org/jabref/model/ai/PaperExcerpt.java b/jablib/src/main/java/org/jabref/model/ai/rag/PaperExcerpt.java similarity index 66% rename from jablib/src/main/java/org/jabref/model/ai/PaperExcerpt.java rename to jablib/src/main/java/org/jabref/model/ai/rag/PaperExcerpt.java index 7c76ed940d35..3f054da0a834 100644 --- a/jablib/src/main/java/org/jabref/model/ai/PaperExcerpt.java +++ b/jablib/src/main/java/org/jabref/model/ai/rag/PaperExcerpt.java @@ -1,4 +1,4 @@ -package org.jabref.model.ai; +package org.jabref.model.ai.rag; public record PaperExcerpt(String citationKey, String text) { } diff --git a/jablib/src/main/java/org/jabref/model/ai/Summary.java b/jablib/src/main/java/org/jabref/model/ai/summarization/Summary.java similarity index 67% rename from jablib/src/main/java/org/jabref/model/ai/Summary.java rename to jablib/src/main/java/org/jabref/model/ai/summarization/Summary.java index b05f5038b9dd..fdba2fa7db8c 100644 --- a/jablib/src/main/java/org/jabref/model/ai/Summary.java +++ b/jablib/src/main/java/org/jabref/model/ai/summarization/Summary.java @@ -1,7 +1,9 @@ -package org.jabref.model.ai; +package org.jabref.model.ai.summarization; import java.io.Serializable; import java.time.LocalDateTime; +import org.jabref.model.ai.chatting.AiProvider; + public record Summary(LocalDateTime timestamp, AiProvider aiProvider, String model, String content) implements Serializable { } diff --git a/jablib/src/main/java/org/jabref/model/ai/AiTemplate.java b/jablib/src/main/java/org/jabref/model/ai/templating/AiTemplate.java similarity index 93% rename from jablib/src/main/java/org/jabref/model/ai/AiTemplate.java rename to jablib/src/main/java/org/jabref/model/ai/templating/AiTemplate.java index 15ba5fc992f3..077ab31795a9 100644 --- a/jablib/src/main/java/org/jabref/model/ai/AiTemplate.java +++ b/jablib/src/main/java/org/jabref/model/ai/templating/AiTemplate.java @@ -1,4 +1,4 @@ -package org.jabref.model.ai; +package org.jabref.model.ai.templating; public enum AiTemplate { // Templates that are used in AI chats. diff --git a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryStorageTest.java b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryStorageTest.java index 49528b4abb89..21f1e5193e70 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryStorageTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryStorageTest.java @@ -3,6 +3,8 @@ import java.nio.file.Path; import java.util.List; +import org.jabref.logic.ai.chatting.storages.ChatHistoryStorage; + import dev.langchain4j.data.message.AiMessage; import dev.langchain4j.data.message.ChatMessage; import dev.langchain4j.data.message.UserMessage; diff --git a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryStorageTest.java b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryStorageTest.java index fc8a6be2ccdb..2a318d227bb4 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryStorageTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryStorageTest.java @@ -2,7 +2,8 @@ import java.nio.file.Path; -import org.jabref.logic.ai.chatting.chathistory.storages.MVStoreChatHistoryStorage; +import org.jabref.logic.ai.chatting.storages.ChatHistoryStorage; +import org.jabref.logic.ai.chatting.storages.MVStoreChatHistoryStorage; import org.jabref.logic.util.NotificationService; import static org.mockito.Mockito.mock; diff --git a/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesStorageTest.java b/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesStorageTest.java index ca1dc2858169..d030a7fe7087 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesStorageTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesStorageTest.java @@ -5,8 +5,8 @@ import java.util.Optional; import org.jabref.logic.ai.summarization.storages.SummariesStorage; -import org.jabref.model.ai.AiProvider; -import org.jabref.model.ai.Summary; +import org.jabref.model.ai.chatting.AiProvider; +import org.jabref.model.ai.summarization.Summary; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; From b5b8b4e64ba7abf9dbd71f5465225b93e13c4dc1 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Sun, 23 Nov 2025 22:02:12 +0100 Subject: [PATCH 008/243] Introduce modular summarization algorithms and refactor AI summarization components into dedicated classes and interfaces. --- jablib/src/main/java/module-info.java | 1 + .../algorithms/BibEntrySummarizer.java | 118 ++++++++++ .../ChunkedSummarizationAlgorithm.java | 128 +++++++++++ .../PersistentBibEntrySummarizer.java | 79 +++++++ .../algorithms/SummarizationAlgorithm.java | 9 + .../tasks/GenerateSummaryTask.java | 212 ------------------ 6 files changed, 335 insertions(+), 212 deletions(-) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/BibEntrySummarizer.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/ChunkedSummarizationAlgorithm.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/PersistentBibEntrySummarizer.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/SummarizationAlgorithm.java diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index 50fe9b74f71d..21dc1bc734a0 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -281,5 +281,6 @@ requires org.jooq.jool; requires org.libreoffice.uno; requires transitive org.jspecify; + requires org.jabref.jablib; // endregion } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/BibEntrySummarizer.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/BibEntrySummarizer.java new file mode 100644 index 000000000000..519bf6699af2 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/BibEntrySummarizer.java @@ -0,0 +1,118 @@ +package org.jabref.logic.ai.summarization.algorithms; + +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javafx.beans.property.ReadOnlyBooleanProperty; + +import org.jabref.logic.FilePreferences; +import org.jabref.logic.ai.preferences.AiPreferences; +import org.jabref.logic.ai.rag.algorithms.FileToDocument; +import org.jabref.logic.ai.templates.AiTemplatesService; +import org.jabref.logic.l10n.Localization; +import org.jabref.logic.util.ProgressCounter; +import org.jabref.model.ai.summarization.Summary; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.LinkedFile; + +import dev.langchain4j.data.document.Document; +import dev.langchain4j.model.chat.ChatModel; +import org.slf4j.Logger; + +public class BibEntrySummarizer { + private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(BibEntrySummarizer.class); + + // TODO: Same thing. + private final AiPreferences aiPreferences; + private final FilePreferences filePreferences; + + private final ChunkedSummarizationAlgorithm chunkedSummarizationAlgorithm; + + public BibEntrySummarizer( + AiPreferences aiPreferences, + FilePreferences filePreferences, + AiTemplatesService aiTemplatesService, + ChatModel chatModel + ) { + this.aiPreferences = aiPreferences; + this.filePreferences = filePreferences; + + this.chunkedSummarizationAlgorithm = new ChunkedSummarizationAlgorithm(aiPreferences, aiTemplatesService, chatModel); + } + + public Summary summarize(BibEntry entry, BibDatabaseContext bibDatabaseContext, ProgressCounter progressCounter, ReadOnlyBooleanProperty shutdownSignal) throws InterruptedException { + String citationKey = entry.getCitationKey().orElse(""); + + // Rationale for RuntimeException here: + // It follows the same idiom as in langchain4j. See {@link JabRefChatLanguageModel.generate}, this method + // is used internally in the summarization, and it also throws RuntimeExceptions. + + // Stream API would look better here, but we need to catch InterruptedException. + List linkedFilesSummary = new ArrayList<>(); + for (LinkedFile linkedFile : entry.getFiles()) { + generateSummary(linkedFile, bibDatabaseContext, citationKey, progressCounter, shutdownSignal) + .ifPresent(linkedFilesSummary::add); + } + + if (linkedFilesSummary.isEmpty()) { + progressCounter.increaseWorkDone(1); // Skipped generation of final summary. + throw new RuntimeException(Localization.lang("No summary can be generated for entry '%0'. Could not find attached linked files.", citationKey)); + } + + LOGGER.debug("All summaries for attached files of entry {} are generated. Generating final summary.", citationKey); + + String finalSummary; + + progressCounter.increaseWorkMax(1); // For generating final summary. + + if (linkedFilesSummary.size() == 1) { + finalSummary = linkedFilesSummary.getFirst(); + } else { + finalSummary = summarizeSeveralDocuments(linkedFilesSummary.stream(), progressCounter, shutdownSignal); + } + + progressCounter.increaseWorkDone(1); + + return new Summary( + LocalDateTime.now(), + aiPreferences.getAiProvider(), + aiPreferences.getSelectedChatModel(), + finalSummary + ); + } + + private Optional generateSummary(LinkedFile linkedFile, BibDatabaseContext bibDatabaseContext, String citationKey, ProgressCounter progressCounter, ReadOnlyBooleanProperty shutdownSignal) throws InterruptedException { + LOGGER.debug("Generating summary for file \"{}\" of entry {}", linkedFile.getLink(), citationKey); + + Optional path = linkedFile.findIn(bibDatabaseContext, filePreferences); + + if (path.isEmpty()) { + LOGGER.error("Could not find path for a linked file \"{}\" of entry {}", linkedFile.getLink(), citationKey); + LOGGER.debug("Unable to generate summary for file \"{}\" of entry {}, because it was not found", linkedFile.getLink(), citationKey); + return Optional.empty(); + } + + Optional document = new FileToDocument(shutdownSignal).fromFile(path.get()); + + if (document.isEmpty()) { + LOGGER.warn("Could not extract text from a linked file \"{}\" of entry {}. It will be skipped when generating a summary.", linkedFile.getLink(), citationKey); + LOGGER.debug("Unable to generate summary for file \"{}\" of entry {}, because it was not found", linkedFile.getLink(), citationKey); + return Optional.empty(); + } + + String linkedFileSummary = chunkedSummarizationAlgorithm.summarize(document.get().text(), progressCounter, shutdownSignal); + + LOGGER.debug("Summary for file \"{}\" of entry {} was generated successfully", linkedFile.getLink(), citationKey); + return Optional.of(linkedFileSummary); + } + + public String summarizeSeveralDocuments(Stream documents, ProgressCounter progressCounter, ReadOnlyBooleanProperty shutdownSignal) throws InterruptedException { + return chunkedSummarizationAlgorithm.summarize(documents.collect(Collectors.joining("\n\n")), progressCounter, shutdownSignal); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/ChunkedSummarizationAlgorithm.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/ChunkedSummarizationAlgorithm.java new file mode 100644 index 000000000000..cb07e382cfe0 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/ChunkedSummarizationAlgorithm.java @@ -0,0 +1,128 @@ +package org.jabref.logic.ai.summarization.algorithms; + +import java.util.ArrayList; +import java.util.List; + +import javafx.beans.property.ReadOnlyBooleanProperty; + +import org.jabref.logic.ai.preferences.AiPreferences; +import org.jabref.logic.ai.templates.AiTemplatesService; +import org.jabref.logic.util.ProgressCounter; +import org.jabref.model.ai.templating.AiTemplate; + +import dev.langchain4j.data.document.DefaultDocument; +import dev.langchain4j.data.document.DocumentSplitter; +import dev.langchain4j.data.document.splitter.DocumentSplitters; +import dev.langchain4j.data.message.SystemMessage; +import dev.langchain4j.data.message.UserMessage; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.chat.ChatModel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ChunkedSummarizationAlgorithm implements SummarizationAlgorithm { + private static final Logger LOGGER = LoggerFactory.getLogger(ChunkedSummarizationAlgorithm.class); + + private static final int MAX_OVERLAP_SIZE_IN_CHARS = 100; + private static final int CHAR_TOKEN_FACTOR = 4; // Means, every token is roughly 4 characters. + + // TODO: AiPreferences are used here to determine the context window size of the chat model, but this is wrong, + // because AiPreferences determine the size based on the selected model, but this model can be supplied with other model. + private final AiPreferences aiPreferences; + private final AiTemplatesService aiTemplatesService; + private final ChatModel chatModel; + + public ChunkedSummarizationAlgorithm( + AiPreferences aiPreferences, + AiTemplatesService aiTemplatesService, + ChatModel chatModel + ) { + this.aiPreferences = aiPreferences; + this.aiTemplatesService = aiTemplatesService; + this.chatModel = chatModel; + } + + @Override + public String summarize(String text, ProgressCounter progressCounter, ReadOnlyBooleanProperty shutdownSignal) throws InterruptedException { + LOGGER.debug("Summarizing text ({} chars)", text.length()); + + progressCounter.increaseWorkMax(1); // For the combination of summary chunks. + + DocumentSplitter documentSplitter = DocumentSplitters.recursive( + aiPreferences.getContextWindowSize() - MAX_OVERLAP_SIZE_IN_CHARS * 2 - estimateTokenCount(aiPreferences.getTemplate(AiTemplate.SUMMARIZATION_CHUNK_SYSTEM_MESSAGE)), + MAX_OVERLAP_SIZE_IN_CHARS + ); + + List chunkSummaries = documentSplitter.split(new DefaultDocument(text)).stream().map(TextSegment::text).toList(); + + LOGGER.debug("Text was split into {} chunks", chunkSummaries.size()); + + int passes = 0; + + do + { + passes++; + LOGGER.debug("Summarizing {} chunk (of {}", passes, chunkSummaries.size()); + + progressCounter.increaseWorkMax(chunkSummaries.size()); + + List list = new ArrayList<>(); + + for (String chunkSummary : chunkSummaries) { + if (shutdownSignal.get()) { + throw new InterruptedException(); + } + + String systemMessage = aiTemplatesService.makeSummarizationChunkSystemMessage(); + String userMessage = aiTemplatesService.makeSummarizationChunkUserMessage(chunkSummary); + + LOGGER.debug("Sending request to AI provider to summarize a chunk"); + String chunk = chatModel.chat(List.of( + new SystemMessage(systemMessage), + new UserMessage(userMessage) + )).aiMessage().text(); + LOGGER.debug("Chunk {} summary was generated successfully", passes); + + list.add(chunk); + progressCounter.increaseWorkDone(1); + } + + chunkSummaries = list; + } while (estimateTokenCount(chunkSummaries) > aiPreferences.getContextWindowSize() - estimateTokenCount(aiPreferences.getTemplate(AiTemplate.SUMMARIZATION_COMBINE_SYSTEM_MESSAGE))); + + if (chunkSummaries.size() == 1) { + progressCounter.increaseWorkDone(1); // No need to call LLM for combination of summary chunks. + LOGGER.debug("Summary of the text was generated successfully"); + return chunkSummaries.getFirst(); + } + + String systemMessage = aiTemplatesService.makeSummarizationCombineSystemMessage(); + String userMessage = aiTemplatesService.makeSummarizationCombineUserMessage(chunkSummaries); + + if (shutdownSignal.get()) { + throw new InterruptedException(); + } + + LOGGER.debug("Sending request to AI provider to combine summary chunks"); + String result = chatModel.chat(List.of( + new SystemMessage(systemMessage), + new UserMessage(userMessage) + )).aiMessage().text(); + LOGGER.debug("Summary of the text was generated successfully"); + + progressCounter.increaseWorkDone(1); + return result; + } + + private static int estimateTokenCount(List chunkSummaries) { + return chunkSummaries.stream().mapToInt(ChunkedSummarizationAlgorithm::estimateTokenCount).sum(); + } + + private static int estimateTokenCount(String string) { + return estimateTokenCount(string.length()); + } + + private static int estimateTokenCount(int numOfChars) { + return numOfChars / CHAR_TOKEN_FACTOR; + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/PersistentBibEntrySummarizer.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/PersistentBibEntrySummarizer.java new file mode 100644 index 000000000000..ea648f074e90 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/PersistentBibEntrySummarizer.java @@ -0,0 +1,79 @@ +package org.jabref.logic.ai.summarization.algorithms; + +import java.util.Optional; + +import javafx.beans.property.ReadOnlyBooleanProperty; + +import org.jabref.logic.FilePreferences; +import org.jabref.logic.ai.preferences.AiPreferences; +import org.jabref.logic.ai.summarization.storages.SummariesStorage; +import org.jabref.logic.ai.templates.AiTemplatesService; +import org.jabref.logic.ai.util.CitationKeyCheck; +import org.jabref.logic.util.ProgressCounter; +import org.jabref.model.ai.summarization.Summary; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; + +import dev.langchain4j.model.chat.ChatModel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PersistentBibEntrySummarizer { + private static final Logger LOGGER = LoggerFactory.getLogger(PersistentBibEntrySummarizer.class); + + private final SummariesStorage summariesStorage; + + private final BibEntrySummarizer bibEntrySummarizer; + + public PersistentBibEntrySummarizer( + AiPreferences aiPreferences, + FilePreferences filePreferences, + SummariesStorage summariesStorage, + AiTemplatesService aiTemplatesService, + ChatModel chatModel + ) { + this.summariesStorage = summariesStorage; + + this.bibEntrySummarizer = new BibEntrySummarizer( + aiPreferences, + filePreferences, + aiTemplatesService, + chatModel + ); + } + + public Summary summarize(BibEntry entry, BibDatabaseContext bibDatabaseContext, ProgressCounter progressCounter, ReadOnlyBooleanProperty shutdownSignal) { + Optional

savedSummary = Optional.empty(); + + if (bibDatabaseContext.getDatabasePath().isEmpty()) { + LOGGER.info("No database path is present. Summary will not be stored in the next sessions"); + } else if (entry.getCitationKey().isEmpty()) { + LOGGER.info("No citation key is present. Summary will not be stored in the next sessions"); + } else { + savedSummary = summariesStorage.get(bibDatabaseContext.getDatabasePath().get(), entry.getCitationKey().get()); + } + + Summary summary; + + if (savedSummary.isPresent()) { + summary = savedSummary.get(); + } else { + try { + summary = bibEntrySummarizer.summarize(entry, bibDatabaseContext, progressCounter, shutdownSignal); + } catch (InterruptedException e) { + LOGGER.debug("There was a summarization task for {}. It will be canceled, because user quits JabRef.", entry.getCitationKey().orElse("")); + return null; + } + } + + if (bibDatabaseContext.getDatabasePath().isEmpty()) { + LOGGER.info("No database path is present. Summary will not be stored in the next sessions"); + } else if (CitationKeyCheck.citationKeyIsPresentAndUnique(bibDatabaseContext, entry)) { + LOGGER.info("No valid citation key is present. Summary will not be stored in the next sessions"); + } else { + summariesStorage.set(bibDatabaseContext.getDatabasePath().get(), entry.getCitationKey().get(), summary); + } + + return summary; + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/SummarizationAlgorithm.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/SummarizationAlgorithm.java new file mode 100644 index 000000000000..cb34d2ea8ab6 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/SummarizationAlgorithm.java @@ -0,0 +1,9 @@ +package org.jabref.logic.ai.summarization.algorithms; + +import javafx.beans.property.ReadOnlyBooleanProperty; + +import org.jabref.logic.util.ProgressCounter; + +public interface SummarizationAlgorithm { + String summarize(String text, ProgressCounter progressCounter, ReadOnlyBooleanProperty shutdownSignal) throws InterruptedException; +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java index bc228b6feb4a..3bac2d78de0c 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java @@ -1,38 +1,19 @@ package org.jabref.logic.ai.summarization.tasks; -import java.nio.file.Path; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; - import javafx.beans.property.ReadOnlyBooleanProperty; import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.rag.algorithms.FileToDocument; import org.jabref.logic.ai.summarization.SummariesService; import org.jabref.logic.ai.summarization.storages.SummariesStorage; import org.jabref.logic.ai.templates.AiTemplatesService; -import org.jabref.logic.ai.util.CitationKeyCheck; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.ProgressCounter; -import org.jabref.model.ai.templating.AiTemplate; import org.jabref.model.ai.summarization.Summary; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.LinkedFile; -import dev.langchain4j.data.document.DefaultDocument; -import dev.langchain4j.data.document.Document; -import dev.langchain4j.data.document.DocumentSplitter; -import dev.langchain4j.data.document.splitter.DocumentSplitters; -import dev.langchain4j.data.message.SystemMessage; -import dev.langchain4j.data.message.UserMessage; -import dev.langchain4j.data.segment.TextSegment; import dev.langchain4j.model.chat.ChatModel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,9 +28,6 @@ public class GenerateSummaryTask extends BackgroundTask { private static final Logger LOGGER = LoggerFactory.getLogger(GenerateSummaryTask.class); - private static final int MAX_OVERLAP_SIZE_IN_CHARS = 100; - private static final int CHAR_TOKEN_FACTOR = 4; // Means, every token is roughly 4 characters. - private final BibDatabaseContext bibDatabaseContext; private final BibEntry entry; private final String citationKey; @@ -95,43 +73,7 @@ private void configure() { public Summary call() { LOGGER.debug("Starting summarization task for entry {}", citationKey); - Optional savedSummary = Optional.empty(); - - if (bibDatabaseContext.getDatabasePath().isEmpty()) { - LOGGER.info("No database path is present. Summary will not be stored in the next sessions"); - } else if (entry.getCitationKey().isEmpty()) { - LOGGER.info("No citation key is present. Summary will not be stored in the next sessions"); - } else { - savedSummary = summariesStorage.get(bibDatabaseContext.getDatabasePath().get(), entry.getCitationKey().get()); - } - - Summary summary; - if (savedSummary.isPresent()) { - summary = savedSummary.get(); - } else { - try { - String result = summarizeAll(); - - summary = new Summary( - LocalDateTime.now(), - aiPreferences.getAiProvider(), - aiPreferences.getSelectedChatModel(), - result - ); - } catch (InterruptedException e) { - LOGGER.debug("There was a summarization task for {}. It will be canceled, because user quits JabRef.", citationKey); - return null; - } - } - - if (bibDatabaseContext.getDatabasePath().isEmpty()) { - LOGGER.info("No database path is present. Summary will not be stored in the next sessions"); - } else if (CitationKeyCheck.citationKeyIsPresentAndUnique(bibDatabaseContext, entry)) { - LOGGER.info("No valid citation key is present. Summary will not be stored in the next sessions"); - } else { - summariesStorage.set(bibDatabaseContext.getDatabasePath().get(), entry.getCitationKey().get(), summary); - } LOGGER.debug("Finished summarization task for entry {}", citationKey); progressCounter.stop(); @@ -139,162 +81,8 @@ public Summary call() { return summary; } - private String summarizeAll() throws InterruptedException { - // Rationale for RuntimeException here: - // It follows the same idiom as in langchain4j. See {@link JabRefChatLanguageModel.generate}, this method - // is used internally in the summarization, and it also throws RuntimeExceptions. - - // Stream API would look better here, but we need to catch InterruptedException. - List linkedFilesSummary = new ArrayList<>(); - for (LinkedFile linkedFile : entry.getFiles()) { - generateSummary(linkedFile).ifPresent(linkedFilesSummary::add); - } - - if (linkedFilesSummary.isEmpty()) { - doneOneWork(); // Skipped generation of final summary. - throw new RuntimeException(Localization.lang("No summary can be generated for entry '%0'. Could not find attached linked files.", citationKey)); - } - - LOGGER.debug("All summaries for attached files of entry {} are generated. Generating final summary.", citationKey); - - String finalSummary; - - addMoreWork(1); // For generating final summary. - - if (linkedFilesSummary.size() == 1) { - finalSummary = linkedFilesSummary.getFirst(); - } else { - finalSummary = summarizeSeveralDocuments(linkedFilesSummary.stream()); - } - - doneOneWork(); - - return finalSummary; - } - - private Optional generateSummary(LinkedFile linkedFile) throws InterruptedException { - LOGGER.debug("Generating summary for file \"{}\" of entry {}", linkedFile.getLink(), citationKey); - - Optional path = linkedFile.findIn(bibDatabaseContext, filePreferences); - - if (path.isEmpty()) { - LOGGER.error("Could not find path for a linked file \"{}\" of entry {}", linkedFile.getLink(), citationKey); - LOGGER.debug("Unable to generate summary for file \"{}\" of entry {}, because it was not found", linkedFile.getLink(), citationKey); - return Optional.empty(); - } - - Optional document = new FileToDocument(shutdownSignal).fromFile(path.get()); - - if (document.isEmpty()) { - LOGGER.warn("Could not extract text from a linked file \"{}\" of entry {}. It will be skipped when generating a summary.", linkedFile.getLink(), citationKey); - LOGGER.debug("Unable to generate summary for file \"{}\" of entry {}, because it was not found", linkedFile.getLink(), citationKey); - return Optional.empty(); - } - - String linkedFileSummary = summarizeOneDocument(path.get().toString(), document.get().text()); - - LOGGER.debug("Summary for file \"{}\" of entry {} was generated successfully", linkedFile.getLink(), citationKey); - return Optional.of(linkedFileSummary); - } - - public String summarizeOneDocument(String filePath, String document) throws InterruptedException { - addMoreWork(1); // For the combination of summary chunks. - - DocumentSplitter documentSplitter = DocumentSplitters.recursive( - aiPreferences.getContextWindowSize() - MAX_OVERLAP_SIZE_IN_CHARS * 2 - estimateTokenCount(aiPreferences.getTemplate(AiTemplate.SUMMARIZATION_CHUNK_SYSTEM_MESSAGE)), - MAX_OVERLAP_SIZE_IN_CHARS - ); - - List chunkSummaries = documentSplitter.split(new DefaultDocument(document)).stream().map(TextSegment::text).toList(); - - LOGGER.debug("The file \"{}\" of entry {} was split into {} chunk(s)", filePath, citationKey, chunkSummaries.size()); - - int passes = 0; - - // @formatter:off - do { - // @formatter:on - passes++; - LOGGER.debug("Summarizing chunk(s) for file \"{}\" of entry {} ({} pass)", filePath, citationKey, passes); - - addMoreWork(chunkSummaries.size()); - - List list = new ArrayList<>(); - - for (String chunkSummary : chunkSummaries) { - if (shutdownSignal.get()) { - throw new InterruptedException(); - } - - String systemMessage = aiTemplatesService.makeSummarizationChunkSystemMessage(); - String userMessage = aiTemplatesService.makeSummarizationChunkUserMessage(chunkSummary); - - LOGGER.debug("Sending request to AI provider to summarize a chunk from file \"{}\" of entry {}", filePath, citationKey); - String chunk = chatLanguageModel.chat(List.of( - new SystemMessage(systemMessage), - new UserMessage(userMessage) - )).aiMessage().text(); - LOGGER.debug("Chunk summary for file \"{}\" of entry {} was generated successfully", filePath, citationKey); - - list.add(chunk); - doneOneWork(); - } - - chunkSummaries = list; - } while (estimateTokenCount(chunkSummaries) > aiPreferences.getContextWindowSize() - estimateTokenCount(aiPreferences.getTemplate(AiTemplate.SUMMARIZATION_COMBINE_SYSTEM_MESSAGE))); - - if (chunkSummaries.size() == 1) { - doneOneWork(); // No need to call LLM for combination of summary chunks. - LOGGER.debug("Summary of the file \"{}\" of entry {} was generated successfully", filePath, citationKey); - return chunkSummaries.getFirst(); - } - - String systemMessage = aiTemplatesService.makeSummarizationCombineSystemMessage(); - String userMessage = aiTemplatesService.makeSummarizationCombineUserMessage(chunkSummaries); - - if (shutdownSignal.get()) { - throw new InterruptedException(); - } - - LOGGER.debug("Sending request to AI provider to combine summary chunk(s) for file \"{}\" of entry {}", filePath, citationKey); - String result = chatLanguageModel.chat(List.of( - new SystemMessage(systemMessage), - new UserMessage(userMessage) - )).aiMessage().text(); - LOGGER.debug("Summary of the file \"{}\" of entry {} was generated successfully", filePath, citationKey); - - doneOneWork(); - return result; - } - - public String summarizeSeveralDocuments(Stream documents) throws InterruptedException { - return summarizeOneDocument(citationKey, documents.collect(Collectors.joining("\n\n"))); - } - - private static int estimateTokenCount(List chunkSummaries) { - return chunkSummaries.stream().mapToInt(GenerateSummaryTask::estimateTokenCount).sum(); - } - - private static int estimateTokenCount(String string) { - return estimateTokenCount(string.length()); - } - - private static int estimateTokenCount(int numOfChars) { - return numOfChars / CHAR_TOKEN_FACTOR; - } - private void updateProgress() { updateProgress(progressCounter.getWorkDone(), progressCounter.getWorkMax()); updateMessage(progressCounter.getMessage()); } - - private void addMoreWork(int moreWork) { - progressCounter.increaseWorkMax(moreWork); - updateProgress(); - } - - private void doneOneWork() { - progressCounter.increaseWorkDone(1); - updateProgress(); - } } From 655811e448374f8548ae23b704877aecb41b5689 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Mon, 24 Nov 2025 19:05:23 +0100 Subject: [PATCH 009/243] Finish refactoring GenerateSummaryTask.java --- .../tasks/GenerateSummaryTask.java | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java index 3bac2d78de0c..64c7cbc9376c 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java @@ -5,6 +5,7 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.summarization.SummariesService; +import org.jabref.logic.ai.summarization.algorithms.PersistentBibEntrySummarizer; import org.jabref.logic.ai.summarization.storages.SummariesStorage; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.l10n.Localization; @@ -31,12 +32,9 @@ public class GenerateSummaryTask extends BackgroundTask { private final BibDatabaseContext bibDatabaseContext; private final BibEntry entry; private final String citationKey; - private final ChatModel chatLanguageModel; - private final SummariesStorage summariesStorage; - private final AiTemplatesService aiTemplatesService; private final ReadOnlyBooleanProperty shutdownSignal; - private final AiPreferences aiPreferences; - private final FilePreferences filePreferences; + + private final PersistentBibEntrySummarizer persistentBibEntrySummarizer; private final ProgressCounter progressCounter = new ProgressCounter(); @@ -52,12 +50,15 @@ public GenerateSummaryTask(BibEntry entry, this.bibDatabaseContext = bibDatabaseContext; this.entry = entry; this.citationKey = entry.getCitationKey().orElse(""); - this.chatLanguageModel = chatLanguageModel; - this.summariesStorage = summariesStorage; - this.aiTemplatesService = aiTemplatesService; this.shutdownSignal = shutdownSignal; - this.aiPreferences = aiPreferences; - this.filePreferences = filePreferences; + + this.persistentBibEntrySummarizer = new PersistentBibEntrySummarizer( + aiPreferences, + filePreferences, + summariesStorage, + aiTemplatesService, + chatLanguageModel + ); configure(); } @@ -73,7 +74,7 @@ private void configure() { public Summary call() { LOGGER.debug("Starting summarization task for entry {}", citationKey); - + Summary summary = persistentBibEntrySummarizer.summarize(entry, bibDatabaseContext, progressCounter, shutdownSignal); LOGGER.debug("Finished summarization task for entry {}", citationKey); progressCounter.stop(); From 77ee8c8ca5ece9165d26df49e70e5fc746a5bf87 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Mon, 24 Nov 2025 19:08:46 +0100 Subject: [PATCH 010/243] Rename storages to repositories --- .../java/org/jabref/logic/ai/AiService.java | 18 +++--- .../logic/ai/chatting/ChatHistoryService.java | 12 ++-- .../CurrentlySelectedChatLanguageModel.java | 4 +- ...torage.java => ChatHistoryRepository.java} | 2 +- ...java => MVStoreChatHistoryRepository.java} | 4 +- .../jabref/logic/ai/rag/IngestionService.java | 6 +- .../rag/storages/FileEmbeddingsManager.java | 12 ++-- ... => FullyIngestedDocumentsRepository.java} | 2 +- ...toreFullyIngestedDocumentsRepository.java} | 4 +- .../ai/summarization/SummariesService.java | 4 +- .../PersistentBibEntrySummarizer.java | 12 ++-- ...e.java => MVStoreSummariesRepository.java} | 4 +- ...sStorage.java => SummariesRepository.java} | 2 +- .../tasks/GenerateSummaryForSeveralTask.java | 4 +- .../tasks/GenerateSummaryTask.java | 8 +-- ...st.java => ChatHistoryRepositoryTest.java} | 10 ++-- .../MVStoreChatHistoryRepositoryTest.java | 21 +++++++ .../MVStoreChatHistoryStorageTest.java | 21 ------- ...FullyIngestedDocumentsRepositoryTest.java} | 10 ++-- ...eFullyIngestedDocumentsRepositoryTest.java | 21 +++++++ ...toreFullyIngestedDocumentsTrackerTest.java | 21 ------- .../MVStoreSummariesRepositoryTest.java | 21 +++++++ .../MVStoreSummariesStorageTest.java | 21 ------- .../SummariesRepositoryTest.java | 59 +++++++++++++++++++ .../summarization/SummariesStorageTest.java | 59 ------------------- 25 files changed, 181 insertions(+), 181 deletions(-) rename jablib/src/main/java/org/jabref/logic/ai/chatting/storages/{ChatHistoryStorage.java => ChatHistoryRepository.java} (93%) rename jablib/src/main/java/org/jabref/logic/ai/chatting/storages/{MVStoreChatHistoryStorage.java => MVStoreChatHistoryRepository.java} (96%) rename jablib/src/main/java/org/jabref/logic/ai/rag/storages/{FullyIngestedDocumentsTracker.java => FullyIngestedDocumentsRepository.java} (90%) rename jablib/src/main/java/org/jabref/logic/ai/rag/storages/{MVStoreFullyIngestedDocumentsTracker.java => MVStoreFullyIngestedDocumentsRepository.java} (90%) rename jablib/src/main/java/org/jabref/logic/ai/summarization/storages/{MVStoreSummariesStorage.java => MVStoreSummariesRepository.java} (89%) rename jablib/src/main/java/org/jabref/logic/ai/summarization/storages/{SummariesStorage.java => SummariesRepository.java} (90%) rename jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/{ChatHistoryStorageTest.java => ChatHistoryRepositoryTest.java} (85%) create mode 100644 jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryTest.java delete mode 100644 jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryStorageTest.java rename jablib/src/test/java/org/jabref/logic/ai/ingestion/{FullyIngestedDocumentsTrackerTest.java => FullyIngestedDocumentsRepositoryTest.java} (78%) create mode 100644 jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsRepositoryTest.java delete mode 100644 jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsTrackerTest.java create mode 100644 jablib/src/test/java/org/jabref/logic/ai/summarization/MVStoreSummariesRepositoryTest.java delete mode 100644 jablib/src/test/java/org/jabref/logic/ai/summarization/MVStoreSummariesStorageTest.java create mode 100644 jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesRepositoryTest.java delete mode 100644 jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesStorageTest.java diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiService.java b/jablib/src/main/java/org/jabref/logic/ai/AiService.java index 45537c3e3c36..98f303ea7dfb 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiService.java @@ -9,14 +9,14 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.chatting.ChatHistoryService; import org.jabref.logic.ai.chatting.CurrentlySelectedChatLanguageModel; -import org.jabref.logic.ai.chatting.storages.MVStoreChatHistoryStorage; +import org.jabref.logic.ai.chatting.storages.MVStoreChatHistoryRepository; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.rag.CurrentlySelectedEmbeddingModel; import org.jabref.logic.ai.rag.IngestionService; import org.jabref.logic.ai.rag.storages.MVStoreEmbeddingStore; -import org.jabref.logic.ai.rag.storages.MVStoreFullyIngestedDocumentsTracker; +import org.jabref.logic.ai.rag.storages.MVStoreFullyIngestedDocumentsRepository; import org.jabref.logic.ai.summarization.SummariesService; -import org.jabref.logic.ai.summarization.storages.MVStoreSummariesStorage; +import org.jabref.logic.ai.summarization.storages.MVStoreSummariesRepository; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; import org.jabref.logic.util.Directories; @@ -50,10 +50,10 @@ public class AiService implements AutoCloseable { new ThreadFactoryBuilder().setNameFormat("ai-retrieval-pool-%d").build() ); - private final MVStoreChatHistoryStorage mvStoreChatHistoryStorage; + private final MVStoreChatHistoryRepository mvStoreChatHistoryStorage; private final MVStoreEmbeddingStore mvStoreEmbeddingStore; - private final MVStoreFullyIngestedDocumentsTracker mvStoreFullyIngestedDocumentsTracker; - private final MVStoreSummariesStorage mvStoreSummariesStorage; + private final MVStoreFullyIngestedDocumentsRepository mvStoreFullyIngestedDocumentsTracker; + private final MVStoreSummariesRepository mvStoreSummariesStorage; private final AiTemplatesService templatesService; private final ChatHistoryService chatHistoryService; @@ -69,10 +69,10 @@ public AiService(AiPreferences aiPreferences, TaskExecutor taskExecutor ) { - this.mvStoreChatHistoryStorage = new MVStoreChatHistoryStorage(Directories.getAiFilesDirectory().resolve(CHAT_HISTORY_FILE_NAME), notificationService); + this.mvStoreChatHistoryStorage = new MVStoreChatHistoryRepository(Directories.getAiFilesDirectory().resolve(CHAT_HISTORY_FILE_NAME), notificationService); this.mvStoreEmbeddingStore = new MVStoreEmbeddingStore(Directories.getAiFilesDirectory().resolve(EMBEDDINGS_FILE_NAME), notificationService); - this.mvStoreFullyIngestedDocumentsTracker = new MVStoreFullyIngestedDocumentsTracker(Directories.getAiFilesDirectory().resolve(FULLY_INGESTED_FILE_NAME), notificationService); - this.mvStoreSummariesStorage = new MVStoreSummariesStorage(Directories.getAiFilesDirectory().resolve(SUMMARIES_FILE_NAME), notificationService); + this.mvStoreFullyIngestedDocumentsTracker = new MVStoreFullyIngestedDocumentsRepository(Directories.getAiFilesDirectory().resolve(FULLY_INGESTED_FILE_NAME), notificationService); + this.mvStoreSummariesStorage = new MVStoreSummariesRepository(Directories.getAiFilesDirectory().resolve(SUMMARIES_FILE_NAME), notificationService); this.templatesService = new AiTemplatesService(aiPreferences); this.chatHistoryService = new ChatHistoryService(citationKeyPatternPreferences, mvStoreChatHistoryStorage); diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java index c38895fa766c..2096150d39bc 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java @@ -9,7 +9,7 @@ import javafx.collections.FXCollections; import javafx.collections.ObservableList; -import org.jabref.logic.ai.chatting.storages.ChatHistoryStorage; +import org.jabref.logic.ai.chatting.storages.ChatHistoryRepository; import org.jabref.logic.ai.util.CitationKeyCheck; import org.jabref.logic.citationkeypattern.CitationKeyGenerator; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; @@ -32,7 +32,7 @@ /// [BibEntry] and [org.jabref.model.groups.AbstractGroup]. The chat history is stored in runtime. /// /// To save and load chat history, [BibEntry] and [org.jabref.model.groups.AbstractGroup] must satisfy several constraints. -/// Serialization and deserialization is handled in [ChatHistoryStorage]. +/// Serialization and deserialization is handled in [ChatHistoryRepository]. /// /// Constraints for serialization and deserialization of a chat history of a [BibEntry]: /// 1. There should exist an associated [BibDatabaseContext] for the [BibEntry]. @@ -48,7 +48,7 @@ public class ChatHistoryService implements AutoCloseable { private static final Logger LOGGER = LoggerFactory.getLogger(ChatHistoryService.class); private final CitationKeyPatternPreferences citationKeyPatternPreferences; - private final ChatHistoryStorage implementation; + private final ChatHistoryRepository implementation; // Note about `Optional`: it was necessary in previous version, but currently we never save an `Optional.empty()`. // However, we decided to left it here: to reduce migrations and to make possible to chat with a {@link BibEntry} without {@link BibDatabaseContext} @@ -65,7 +65,7 @@ private record ChatHistoryManagementRecord(Optional bibDatab // We use {@link TreeMap} for group chat history for the same reason as for {@link BibEntry}ies. private final TreeMap groupsChatHistory = new TreeMap<>(Comparator.comparing(GroupTreeNode::getName)); - public ChatHistoryService(CitationKeyPatternPreferences citationKeyPatternPreferences, ChatHistoryStorage implementation) { + public ChatHistoryService(CitationKeyPatternPreferences citationKeyPatternPreferences, ChatHistoryRepository implementation) { this.citationKeyPatternPreferences = citationKeyPatternPreferences; this.implementation = implementation; } @@ -108,7 +108,7 @@ public ObservableList getChatHistoryForEntry(BibDatabaseContext bib * Removes the chat history for the given {@link BibEntry} from the internal RAM map. * If the {@link BibEntry} satisfies requirements for serialization and deserialization of chat history (see * the docstring for the {@link ChatHistoryService}), then the chat history will be stored via the - * {@link ChatHistoryStorage}. + * {@link ChatHistoryRepository}. *

* It is not necessary to call this method (everything will be stored in {@link ChatHistoryService#close()}, * but it's best to call it when the chat history {@link BibEntry} is no longer needed. @@ -158,7 +158,7 @@ public ObservableList getChatHistoryForGroup(BibDatabaseContext bib * Removes the chat history for the given {@link GroupTreeNode} from the internal RAM map. * If the {@link GroupTreeNode} satisfies requirements for serialization and deserialization of chat history (see * the docstring for the {@link ChatHistoryService}), then the chat history will be stored via the - * {@link ChatHistoryStorage}. + * {@link ChatHistoryRepository}. *

* It is not necessary to call this method (everything will be stored in {@link ChatHistoryService#close()}, * but it's best to call it when the chat history {@link GroupTreeNode} is no longer needed. diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java index 796f38e48aa9..239a9e4cf6df 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java @@ -8,7 +8,7 @@ import java.util.concurrent.Executors; import org.jabref.logic.ai.chatting.algorithms.AiChatLogic; -import org.jabref.logic.ai.chatting.storages.ChatHistoryStorage; +import org.jabref.logic.ai.chatting.storages.ChatHistoryRepository; import org.jabref.logic.ai.customimplementations.llms.Gpt4AllModel; import org.jabref.logic.ai.customimplementations.llms.JvmOpenAiChatLanguageModel; import org.jabref.logic.ai.preferences.AiPreferences; @@ -53,7 +53,7 @@ public CurrentlySelectedChatLanguageModel(AiPreferences aiPreferences) { * Update the underlying {@link dev.langchain4j.model.chat.ChatModel} by current {@link AiPreferences} parameters. * When the model is updated, the chat messages are not lost. * See {@link AiChatLogic}, where messages are stored in {@link ChatMemory}, - * and see {@link ChatHistoryStorage}. + * and see {@link ChatHistoryRepository}. */ private void rebuild() { String apiKey = aiPreferences.getApiKeyForAiProvider(aiPreferences.getAiProvider()); diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/storages/ChatHistoryStorage.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/storages/ChatHistoryRepository.java similarity index 93% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/storages/ChatHistoryStorage.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/storages/ChatHistoryRepository.java index 6ce6cf3343a1..1166c037a279 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/storages/ChatHistoryStorage.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/storages/ChatHistoryRepository.java @@ -5,7 +5,7 @@ import dev.langchain4j.data.message.ChatMessage; -public interface ChatHistoryStorage { +public interface ChatHistoryRepository { List loadMessagesForEntry(Path bibDatabasePath, String citationKey); void storeMessagesForEntry(Path bibDatabasePath, String citationKey, List messages); diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/storages/MVStoreChatHistoryStorage.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/storages/MVStoreChatHistoryRepository.java similarity index 96% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/storages/MVStoreChatHistoryStorage.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/storages/MVStoreChatHistoryRepository.java index e62f6c16610e..eff1c6b39019 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/storages/MVStoreChatHistoryStorage.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/storages/MVStoreChatHistoryRepository.java @@ -18,7 +18,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class MVStoreChatHistoryStorage extends MVStoreBase implements ChatHistoryStorage { +public class MVStoreChatHistoryRepository extends MVStoreBase implements ChatHistoryRepository { private static final String ENTRY_CHAT_HISTORY_PREFIX = "entry"; private static final String GROUP_CHAT_HISTORY_PREFIX = "group"; @@ -64,7 +64,7 @@ public ChatMessage toLangchainMessage() { } } - public MVStoreChatHistoryStorage(Path path, NotificationService dialogService) { + public MVStoreChatHistoryRepository(Path path, NotificationService dialogService) { super(path, dialogService); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java b/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java index d2f94cb0c6f8..1aab214cc2d7 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java @@ -11,7 +11,7 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.rag.storages.FileEmbeddingsManager; -import org.jabref.logic.ai.rag.storages.FullyIngestedDocumentsTracker; +import org.jabref.logic.ai.rag.storages.FullyIngestedDocumentsRepository; import org.jabref.logic.ai.rag.tasks.GenerateEmbeddingsForSeveralTask; import org.jabref.logic.ai.rag.tasks.GenerateEmbeddingsTask; import org.jabref.logic.util.TaskExecutor; @@ -50,7 +50,7 @@ public IngestionService(AiPreferences aiPreferences, ReadOnlyBooleanProperty shutdownSignal, EmbeddingModel embeddingModel, EmbeddingStore embeddingStore, - FullyIngestedDocumentsTracker fullyIngestedDocumentsTracker, + FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository, FilePreferences filePreferences, TaskExecutor taskExecutor ) { @@ -63,7 +63,7 @@ public IngestionService(AiPreferences aiPreferences, shutdownSignal, embeddingModel, embeddingStore, - fullyIngestedDocumentsTracker + fullyIngestedDocumentsRepository ); this.shutdownSignal = shutdownSignal; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FileEmbeddingsManager.java b/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FileEmbeddingsManager.java index dc1fc6c2a278..e646624b15d6 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FileEmbeddingsManager.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FileEmbeddingsManager.java @@ -35,19 +35,19 @@ public class FileEmbeddingsManager { private final ReadOnlyBooleanProperty shutdownSignal; private final EmbeddingStore embeddingStore; - private final FullyIngestedDocumentsTracker fullyIngestedDocumentsTracker; + private final FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository; private final LowLevelIngestor lowLevelIngestor; public FileEmbeddingsManager(AiPreferences aiPreferences, ReadOnlyBooleanProperty shutdownSignal, EmbeddingModel embeddingModel, EmbeddingStore embeddingStore, - FullyIngestedDocumentsTracker fullyIngestedDocumentsTracker + FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository ) { this.aiPreferences = aiPreferences; this.shutdownSignal = shutdownSignal; this.embeddingStore = embeddingStore; - this.fullyIngestedDocumentsTracker = fullyIngestedDocumentsTracker; + this.fullyIngestedDocumentsRepository = fullyIngestedDocumentsRepository; this.lowLevelIngestor = new LowLevelIngestor(aiPreferences, embeddingStore, embeddingModel); setupListeningToPreferencesChanges(); @@ -62,13 +62,13 @@ public void addDocument(String link, Document document, long modificationTimeInS lowLevelIngestor.ingestDocument(document, shutdownSignal, workDone, workMax); if (!shutdownSignal.get()) { - fullyIngestedDocumentsTracker.markDocumentAsFullyIngested(link, modificationTimeInSeconds); + fullyIngestedDocumentsRepository.markDocumentAsFullyIngested(link, modificationTimeInSeconds); } } public void removeDocument(String link) { embeddingStore.removeAll(MetadataFilterBuilder.metadataKey(LINK_METADATA_KEY).isEqualTo(link)); - fullyIngestedDocumentsTracker.unmarkDocumentAsFullyIngested(link); + fullyIngestedDocumentsRepository.unmarkDocumentAsFullyIngested(link); } public EmbeddingStore getEmbeddingsStore() { @@ -76,7 +76,7 @@ public EmbeddingStore getEmbeddingsStore() { } public Optional getIngestedDocumentModificationTimeInSeconds(String link) { - return fullyIngestedDocumentsTracker.getIngestedDocumentModificationTimeInSeconds(link); + return fullyIngestedDocumentsRepository.getIngestedDocumentModificationTimeInSeconds(link); } public void clearEmbeddingsFor(List linkedFiles) { diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FullyIngestedDocumentsTracker.java b/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FullyIngestedDocumentsRepository.java similarity index 90% rename from jablib/src/main/java/org/jabref/logic/ai/rag/storages/FullyIngestedDocumentsTracker.java rename to jablib/src/main/java/org/jabref/logic/ai/rag/storages/FullyIngestedDocumentsRepository.java index 9b4522c3debe..36e75eac6604 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FullyIngestedDocumentsTracker.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FullyIngestedDocumentsRepository.java @@ -7,7 +7,7 @@ *

* The class also records the document modification time. */ -public interface FullyIngestedDocumentsTracker { +public interface FullyIngestedDocumentsRepository { void markDocumentAsFullyIngested(String link, long modificationTimeInSeconds); Optional getIngestedDocumentModificationTimeInSeconds(String link); diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/storages/MVStoreFullyIngestedDocumentsTracker.java b/jablib/src/main/java/org/jabref/logic/ai/rag/storages/MVStoreFullyIngestedDocumentsRepository.java similarity index 90% rename from jablib/src/main/java/org/jabref/logic/ai/rag/storages/MVStoreFullyIngestedDocumentsTracker.java rename to jablib/src/main/java/org/jabref/logic/ai/rag/storages/MVStoreFullyIngestedDocumentsRepository.java index fe96c3f0496e..ddabb60410f1 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/storages/MVStoreFullyIngestedDocumentsTracker.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/storages/MVStoreFullyIngestedDocumentsRepository.java @@ -13,7 +13,7 @@ *

* The class also records the document modification time. */ -public class MVStoreFullyIngestedDocumentsTracker extends MVStoreBase implements FullyIngestedDocumentsTracker { +public class MVStoreFullyIngestedDocumentsRepository extends MVStoreBase implements FullyIngestedDocumentsRepository { private static final String INGESTED_MAP_NAME = "ingested"; // This map stores the ingested documents. The key is LinkedDocument.getLink(), and the value is the modification time in seconds. @@ -23,7 +23,7 @@ public class MVStoreFullyIngestedDocumentsTracker extends MVStoreBase implements // it doesn't mean the document is fully ingested. private final Map ingestedMap; - public MVStoreFullyIngestedDocumentsTracker(Path path, NotificationService dialogService) { + public MVStoreFullyIngestedDocumentsRepository(Path path, NotificationService dialogService) { super(path, dialogService); this.ingestedMap = this.mvStore.openMap(INGESTED_MAP_NAME); diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java index da9d536b3944..0207c74a7d00 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java @@ -10,15 +10,15 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.summarization.storages.SummariesStorage; +import org.jabref.logic.ai.summarization.storages.SummariesRepository; import org.jabref.logic.ai.summarization.tasks.GenerateSummaryForSeveralTask; import org.jabref.logic.ai.summarization.tasks.GenerateSummaryTask; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.ai.util.CitationKeyCheck; import org.jabref.logic.util.TaskExecutor; -import org.jabref.model.ai.summarization.Summary; import org.jabref.model.ai.processingstatus.ProcessingInfo; import org.jabref.model.ai.processingstatus.ProcessingState; +import org.jabref.model.ai.summarization.Summary; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.event.EntriesAddedEvent; import org.jabref.model.entry.BibEntry; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/PersistentBibEntrySummarizer.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/PersistentBibEntrySummarizer.java index ea648f074e90..43ce45613f62 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/PersistentBibEntrySummarizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/PersistentBibEntrySummarizer.java @@ -6,7 +6,7 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.summarization.storages.SummariesStorage; +import org.jabref.logic.ai.summarization.storages.SummariesRepository; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.ai.util.CitationKeyCheck; import org.jabref.logic.util.ProgressCounter; @@ -21,18 +21,18 @@ public class PersistentBibEntrySummarizer { private static final Logger LOGGER = LoggerFactory.getLogger(PersistentBibEntrySummarizer.class); - private final SummariesStorage summariesStorage; + private final SummariesRepository summariesRepository; private final BibEntrySummarizer bibEntrySummarizer; public PersistentBibEntrySummarizer( AiPreferences aiPreferences, FilePreferences filePreferences, - SummariesStorage summariesStorage, + SummariesRepository summariesRepository, AiTemplatesService aiTemplatesService, ChatModel chatModel ) { - this.summariesStorage = summariesStorage; + this.summariesRepository = summariesRepository; this.bibEntrySummarizer = new BibEntrySummarizer( aiPreferences, @@ -50,7 +50,7 @@ public Summary summarize(BibEntry entry, BibDatabaseContext bibDatabaseContext, } else if (entry.getCitationKey().isEmpty()) { LOGGER.info("No citation key is present. Summary will not be stored in the next sessions"); } else { - savedSummary = summariesStorage.get(bibDatabaseContext.getDatabasePath().get(), entry.getCitationKey().get()); + savedSummary = summariesRepository.get(bibDatabaseContext.getDatabasePath().get(), entry.getCitationKey().get()); } Summary summary; @@ -71,7 +71,7 @@ public Summary summarize(BibEntry entry, BibDatabaseContext bibDatabaseContext, } else if (CitationKeyCheck.citationKeyIsPresentAndUnique(bibDatabaseContext, entry)) { LOGGER.info("No valid citation key is present. Summary will not be stored in the next sessions"); } else { - summariesStorage.set(bibDatabaseContext.getDatabasePath().get(), entry.getCitationKey().get(), summary); + summariesRepository.set(bibDatabaseContext.getDatabasePath().get(), entry.getCitationKey().get(), summary); } return summary; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/MVStoreSummariesStorage.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/MVStoreSummariesRepository.java similarity index 89% rename from jablib/src/main/java/org/jabref/logic/ai/summarization/storages/MVStoreSummariesStorage.java rename to jablib/src/main/java/org/jabref/logic/ai/summarization/storages/MVStoreSummariesRepository.java index 05b502d43430..280cce3a7775 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/MVStoreSummariesStorage.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/MVStoreSummariesRepository.java @@ -9,10 +9,10 @@ import org.jabref.logic.util.NotificationService; import org.jabref.model.ai.summarization.Summary; -public class MVStoreSummariesStorage extends MVStoreBase implements SummariesStorage { +public class MVStoreSummariesRepository extends MVStoreBase implements SummariesRepository { private static final String SUMMARIES_MAP_PREFIX = "summaries"; - public MVStoreSummariesStorage(Path path, NotificationService dialogService) { + public MVStoreSummariesRepository(Path path, NotificationService dialogService) { super(path, dialogService); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/SummariesStorage.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/SummariesRepository.java similarity index 90% rename from jablib/src/main/java/org/jabref/logic/ai/summarization/storages/SummariesStorage.java rename to jablib/src/main/java/org/jabref/logic/ai/summarization/storages/SummariesRepository.java index cbee116abf47..cf332c139618 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/SummariesStorage.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/SummariesRepository.java @@ -5,7 +5,7 @@ import org.jabref.model.ai.summarization.Summary; -public interface SummariesStorage { +public interface SummariesRepository { void set(Path bibDatabasePath, String citationKey, Summary summary); Optional

get(Path bibDatabasePath, String citationKey); diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java index 8ee2f78d8fb4..0e6764280e6e 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java @@ -11,15 +11,15 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.summarization.storages.SummariesStorage; +import org.jabref.logic.ai.summarization.storages.SummariesRepository; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.ProgressCounter; import org.jabref.logic.util.TaskExecutor; -import org.jabref.model.ai.summarization.Summary; import org.jabref.model.ai.processingstatus.ProcessingInfo; import org.jabref.model.ai.processingstatus.ProcessingState; +import org.jabref.model.ai.summarization.Summary; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java index 64c7cbc9376c..5b93223ac9a7 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java @@ -6,7 +6,7 @@ import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.summarization.SummariesService; import org.jabref.logic.ai.summarization.algorithms.PersistentBibEntrySummarizer; -import org.jabref.logic.ai.summarization.storages.SummariesStorage; +import org.jabref.logic.ai.summarization.storages.SummariesRepository; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; @@ -24,7 +24,7 @@ * It will check if summary was already generated. * And it also will store the summary. *

- * This task is created in the {@link SummariesService}, and stored then in a {@link SummariesStorage}. + * This task is created in the {@link SummariesService}, and stored then in a {@link SummariesRepository}. */ public class GenerateSummaryTask extends BackgroundTask

{ private static final Logger LOGGER = LoggerFactory.getLogger(GenerateSummaryTask.class); @@ -40,7 +40,7 @@ public class GenerateSummaryTask extends BackgroundTask { public GenerateSummaryTask(BibEntry entry, BibDatabaseContext bibDatabaseContext, - SummariesStorage summariesStorage, + SummariesRepository summariesRepository, ChatModel chatLanguageModel, AiTemplatesService aiTemplatesService, ReadOnlyBooleanProperty shutdownSignal, @@ -55,7 +55,7 @@ public GenerateSummaryTask(BibEntry entry, this.persistentBibEntrySummarizer = new PersistentBibEntrySummarizer( aiPreferences, filePreferences, - summariesStorage, + summariesRepository, aiTemplatesService, chatLanguageModel ); diff --git a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryStorageTest.java b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryRepositoryTest.java similarity index 85% rename from jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryStorageTest.java rename to jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryRepositoryTest.java index 21f1e5193e70..3c3206a8664f 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryStorageTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryRepositoryTest.java @@ -3,7 +3,7 @@ import java.nio.file.Path; import java.util.List; -import org.jabref.logic.ai.chatting.storages.ChatHistoryStorage; +import org.jabref.logic.ai.chatting.storages.ChatHistoryRepository; import dev.langchain4j.data.message.AiMessage; import dev.langchain4j.data.message.ChatMessage; @@ -15,14 +15,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -abstract class ChatHistoryStorageTest { +abstract class ChatHistoryRepositoryTest { @TempDir Path tempDir; - private ChatHistoryStorage storage; + private ChatHistoryRepository storage; - abstract ChatHistoryStorage makeStorage(Path path); + abstract ChatHistoryRepository makeStorage(Path path); - abstract void close(ChatHistoryStorage storage); + abstract void close(ChatHistoryRepository storage); @BeforeEach void setUp() { diff --git a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryTest.java b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryTest.java new file mode 100644 index 000000000000..396b32a7bc1d --- /dev/null +++ b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryTest.java @@ -0,0 +1,21 @@ +package org.jabref.logic.ai.chatting.chathistory; + +import java.nio.file.Path; + +import org.jabref.logic.ai.chatting.storages.ChatHistoryRepository; +import org.jabref.logic.ai.chatting.storages.MVStoreChatHistoryRepository; +import org.jabref.logic.util.NotificationService; + +import static org.mockito.Mockito.mock; + +class MVStoreChatHistoryRepositoryTest extends ChatHistoryRepositoryTest { + @Override + ChatHistoryRepository makeStorage(Path path) { + return new MVStoreChatHistoryRepository(path, mock(NotificationService.class)); + } + + @Override + void close(ChatHistoryRepository storage) { + ((MVStoreChatHistoryRepository) storage).close(); + } +} diff --git a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryStorageTest.java b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryStorageTest.java deleted file mode 100644 index 2a318d227bb4..000000000000 --- a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryStorageTest.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.jabref.logic.ai.chatting.chathistory; - -import java.nio.file.Path; - -import org.jabref.logic.ai.chatting.storages.ChatHistoryStorage; -import org.jabref.logic.ai.chatting.storages.MVStoreChatHistoryStorage; -import org.jabref.logic.util.NotificationService; - -import static org.mockito.Mockito.mock; - -class MVStoreChatHistoryStorageTest extends ChatHistoryStorageTest { - @Override - ChatHistoryStorage makeStorage(Path path) { - return new MVStoreChatHistoryStorage(path, mock(NotificationService.class)); - } - - @Override - void close(ChatHistoryStorage storage) { - ((MVStoreChatHistoryStorage) storage).close(); - } -} diff --git a/jablib/src/test/java/org/jabref/logic/ai/ingestion/FullyIngestedDocumentsTrackerTest.java b/jablib/src/test/java/org/jabref/logic/ai/ingestion/FullyIngestedDocumentsRepositoryTest.java similarity index 78% rename from jablib/src/test/java/org/jabref/logic/ai/ingestion/FullyIngestedDocumentsTrackerTest.java rename to jablib/src/test/java/org/jabref/logic/ai/ingestion/FullyIngestedDocumentsRepositoryTest.java index 403ed903c21b..00ddca3d66e9 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/ingestion/FullyIngestedDocumentsTrackerTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/ingestion/FullyIngestedDocumentsRepositoryTest.java @@ -3,7 +3,7 @@ import java.nio.file.Path; import java.util.Optional; -import org.jabref.logic.ai.rag.storages.FullyIngestedDocumentsTracker; +import org.jabref.logic.ai.rag.storages.FullyIngestedDocumentsRepository; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -12,14 +12,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -abstract class FullyIngestedDocumentsTrackerTest { +abstract class FullyIngestedDocumentsRepositoryTest { @TempDir Path tempDir; - private FullyIngestedDocumentsTracker tracker; + private FullyIngestedDocumentsRepository tracker; - abstract FullyIngestedDocumentsTracker makeTracker(Path path); + abstract FullyIngestedDocumentsRepository makeTracker(Path path); - abstract void close(FullyIngestedDocumentsTracker tracker); + abstract void close(FullyIngestedDocumentsRepository tracker); @BeforeEach void setUp() { diff --git a/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsRepositoryTest.java b/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsRepositoryTest.java new file mode 100644 index 000000000000..459e5c837bd0 --- /dev/null +++ b/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsRepositoryTest.java @@ -0,0 +1,21 @@ +package org.jabref.logic.ai.ingestion; + +import java.nio.file.Path; + +import org.jabref.logic.ai.rag.storages.FullyIngestedDocumentsRepository; +import org.jabref.logic.ai.rag.storages.MVStoreFullyIngestedDocumentsRepository; +import org.jabref.logic.util.NotificationService; + +import static org.mockito.Mockito.mock; + +class MVStoreFullyIngestedDocumentsRepositoryTest extends FullyIngestedDocumentsRepositoryTest { + @Override + FullyIngestedDocumentsRepository makeTracker(Path path) { + return new MVStoreFullyIngestedDocumentsRepository(path, mock(NotificationService.class)); + } + + @Override + void close(FullyIngestedDocumentsRepository tracker) { + ((MVStoreFullyIngestedDocumentsRepository) tracker).close(); + } +} diff --git a/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsTrackerTest.java b/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsTrackerTest.java deleted file mode 100644 index c8af73f8dfd2..000000000000 --- a/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsTrackerTest.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.jabref.logic.ai.ingestion; - -import java.nio.file.Path; - -import org.jabref.logic.ai.rag.storages.FullyIngestedDocumentsTracker; -import org.jabref.logic.ai.rag.storages.MVStoreFullyIngestedDocumentsTracker; -import org.jabref.logic.util.NotificationService; - -import static org.mockito.Mockito.mock; - -class MVStoreFullyIngestedDocumentsTrackerTest extends FullyIngestedDocumentsTrackerTest { - @Override - FullyIngestedDocumentsTracker makeTracker(Path path) { - return new MVStoreFullyIngestedDocumentsTracker(path, mock(NotificationService.class)); - } - - @Override - void close(FullyIngestedDocumentsTracker tracker) { - ((MVStoreFullyIngestedDocumentsTracker) tracker).close(); - } -} diff --git a/jablib/src/test/java/org/jabref/logic/ai/summarization/MVStoreSummariesRepositoryTest.java b/jablib/src/test/java/org/jabref/logic/ai/summarization/MVStoreSummariesRepositoryTest.java new file mode 100644 index 000000000000..01b655560e47 --- /dev/null +++ b/jablib/src/test/java/org/jabref/logic/ai/summarization/MVStoreSummariesRepositoryTest.java @@ -0,0 +1,21 @@ +package org.jabref.logic.ai.summarization; + +import java.nio.file.Path; + +import org.jabref.logic.ai.summarization.storages.MVStoreSummariesRepository; +import org.jabref.logic.ai.summarization.storages.SummariesRepository; +import org.jabref.logic.util.NotificationService; + +import static org.mockito.Mockito.mock; + +class MVStoreSummariesRepositoryTest extends SummariesRepositoryTest { + @Override + SummariesRepository makeSummariesStorage(Path path) { + return new MVStoreSummariesRepository(path, mock(NotificationService.class)); + } + + @Override + void close(SummariesRepository summariesRepository) { + ((MVStoreSummariesRepository) summariesRepository).close(); + } +} diff --git a/jablib/src/test/java/org/jabref/logic/ai/summarization/MVStoreSummariesStorageTest.java b/jablib/src/test/java/org/jabref/logic/ai/summarization/MVStoreSummariesStorageTest.java deleted file mode 100644 index 005efc4d33cf..000000000000 --- a/jablib/src/test/java/org/jabref/logic/ai/summarization/MVStoreSummariesStorageTest.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.jabref.logic.ai.summarization; - -import java.nio.file.Path; - -import org.jabref.logic.ai.summarization.storages.MVStoreSummariesStorage; -import org.jabref.logic.ai.summarization.storages.SummariesStorage; -import org.jabref.logic.util.NotificationService; - -import static org.mockito.Mockito.mock; - -class MVStoreSummariesStorageTest extends SummariesStorageTest { - @Override - SummariesStorage makeSummariesStorage(Path path) { - return new MVStoreSummariesStorage(path, mock(NotificationService.class)); - } - - @Override - void close(SummariesStorage summariesStorage) { - ((MVStoreSummariesStorage) summariesStorage).close(); - } -} diff --git a/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesRepositoryTest.java b/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesRepositoryTest.java new file mode 100644 index 000000000000..b25a9be205b6 --- /dev/null +++ b/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesRepositoryTest.java @@ -0,0 +1,59 @@ +package org.jabref.logic.ai.summarization; + +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.util.Optional; + +import org.jabref.logic.ai.summarization.storages.SummariesRepository; +import org.jabref.model.ai.chatting.AiProvider; +import org.jabref.model.ai.summarization.Summary; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +abstract class SummariesRepositoryTest { + @TempDir Path tempDir; + + private SummariesRepository summariesRepository; + private Path bibPath; + + abstract SummariesRepository makeSummariesStorage(Path path); + + abstract void close(SummariesRepository summariesRepository); + + @BeforeEach + void setUp() { + bibPath = tempDir.resolve("test.bib"); + summariesRepository = makeSummariesStorage(tempDir.resolve("test.bib")); + } + + private void reopen() { + close(summariesRepository); + setUp(); + } + + @AfterEach + void tearDown() { + close(summariesRepository); + } + + @Test + void set() { + summariesRepository.set(bibPath, "citationKey", new Summary(LocalDateTime.now(), AiProvider.OPEN_AI, "model", "contents")); + reopen(); + assertEquals(Optional.of("contents"), summariesRepository.get(bibPath, "citationKey").map(Summary::content)); + } + + @Test + void clear() { + summariesRepository.set(bibPath, "citationKey", new Summary(LocalDateTime.now(), AiProvider.OPEN_AI, "model", "contents")); + reopen(); + summariesRepository.clear(bibPath, "citationKey"); + reopen(); + assertEquals(Optional.empty(), summariesRepository.get(bibPath, "citationKey")); + } +} diff --git a/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesStorageTest.java b/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesStorageTest.java deleted file mode 100644 index d030a7fe7087..000000000000 --- a/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesStorageTest.java +++ /dev/null @@ -1,59 +0,0 @@ -package org.jabref.logic.ai.summarization; - -import java.nio.file.Path; -import java.time.LocalDateTime; -import java.util.Optional; - -import org.jabref.logic.ai.summarization.storages.SummariesStorage; -import org.jabref.model.ai.chatting.AiProvider; -import org.jabref.model.ai.summarization.Summary; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -abstract class SummariesStorageTest { - @TempDir Path tempDir; - - private SummariesStorage summariesStorage; - private Path bibPath; - - abstract SummariesStorage makeSummariesStorage(Path path); - - abstract void close(SummariesStorage summariesStorage); - - @BeforeEach - void setUp() { - bibPath = tempDir.resolve("test.bib"); - summariesStorage = makeSummariesStorage(tempDir.resolve("test.bib")); - } - - private void reopen() { - close(summariesStorage); - setUp(); - } - - @AfterEach - void tearDown() { - close(summariesStorage); - } - - @Test - void set() { - summariesStorage.set(bibPath, "citationKey", new Summary(LocalDateTime.now(), AiProvider.OPEN_AI, "model", "contents")); - reopen(); - assertEquals(Optional.of("contents"), summariesStorage.get(bibPath, "citationKey").map(Summary::content)); - } - - @Test - void clear() { - summariesStorage.set(bibPath, "citationKey", new Summary(LocalDateTime.now(), AiProvider.OPEN_AI, "model", "contents")); - reopen(); - summariesStorage.clear(bibPath, "citationKey"); - reopen(); - assertEquals(Optional.empty(), summariesStorage.get(bibPath, "citationKey")); - } -} From b79c2233665a173d9c90c4afa5eac6a9cc7a5c31 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Mon, 24 Nov 2025 19:10:59 +0100 Subject: [PATCH 011/243] Fix wrong names --- .../logic/ai/summarization/SummariesService.java | 12 ++++++------ .../tasks/GenerateSummaryForSeveralTask.java | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java index 0207c74a7d00..be086f6535d4 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java @@ -47,7 +47,7 @@ public class SummariesService { private final List> listsUnderSummarization = new ArrayList<>(); private final AiPreferences aiPreferences; - private final SummariesStorage summariesStorage; + private final SummariesRepository summariesRepository; private final ChatModel chatLanguageModel; private final AiTemplatesService aiTemplatesService; private final BooleanProperty shutdownSignal; @@ -55,7 +55,7 @@ public class SummariesService { private final TaskExecutor taskExecutor; public SummariesService(AiPreferences aiPreferences, - SummariesStorage summariesStorage, + SummariesRepository summariesRepository, ChatModel chatLanguageModel, AiTemplatesService aiTemplatesService, BooleanProperty shutdownSignal, @@ -63,7 +63,7 @@ public SummariesService(AiPreferences aiPreferences, TaskExecutor taskExecutor ) { this.aiPreferences = aiPreferences; - this.summariesStorage = summariesStorage; + this.summariesRepository = summariesRepository; this.chatLanguageModel = chatLanguageModel; this.aiTemplatesService = aiTemplatesService; this.shutdownSignal = shutdownSignal; @@ -140,7 +140,7 @@ public void summarize(StringProperty groupName, List entries, BibDatab private void startSummarizationTask(BibEntry entry, BibDatabaseContext bibDatabaseContext, ProcessingInfo processingInfo) { processingInfo.setState(ProcessingState.PROCESSING); - new GenerateSummaryTask(entry, bibDatabaseContext, summariesStorage, chatLanguageModel, aiTemplatesService, shutdownSignal, aiPreferences, filePreferences) + new GenerateSummaryTask(entry, bibDatabaseContext, summariesRepository, chatLanguageModel, aiTemplatesService, shutdownSignal, aiPreferences, filePreferences) .onSuccess(processingInfo::setSuccess) .onFailure(processingInfo::setException) .executeWith(taskExecutor); @@ -149,7 +149,7 @@ private void startSummarizationTask(BibEntry entry, BibDatabaseContext bibDataba private void startSummarizationTask(StringProperty groupName, List> entries, BibDatabaseContext bibDatabaseContext) { entries.forEach(processingInfo -> processingInfo.setState(ProcessingState.PROCESSING)); - new GenerateSummaryForSeveralTask(groupName, entries, bibDatabaseContext, summariesStorage, chatLanguageModel, aiTemplatesService, shutdownSignal, aiPreferences, filePreferences, taskExecutor) + new GenerateSummaryForSeveralTask(groupName, entries, bibDatabaseContext, summariesRepository, chatLanguageModel, aiTemplatesService, shutdownSignal, aiPreferences, filePreferences, taskExecutor) .executeWith(taskExecutor); } @@ -165,7 +165,7 @@ public void regenerateSummary(BibEntry bibEntry, BibDatabaseContext bibDatabaseC } else if (bibEntry.getCitationKey().isEmpty() || CitationKeyCheck.citationKeyIsPresentAndUnique(bibDatabaseContext, bibEntry)) { LOGGER.info("No valid citation key is present. Could not clear stored summary for regeneration"); } else { - summariesStorage.clear(bibDatabaseContext.getDatabasePath().get(), bibEntry.getCitationKey().get()); + summariesRepository.clear(bibDatabaseContext.getDatabasePath().get(), bibEntry.getCitationKey().get()); } startSummarizationTask(bibEntry, bibDatabaseContext, processingInfo); diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java index 0e6764280e6e..5bcdec6f0684 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java @@ -38,7 +38,7 @@ public class GenerateSummaryForSeveralTask extends BackgroundTask { private final StringProperty groupName; private final List> entries; private final BibDatabaseContext bibDatabaseContext; - private final SummariesStorage summariesStorage; + private final SummariesRepository summariesRepository; private final ChatModel chatLanguageModel; private final AiTemplatesService aiTemplatesService; private final ReadOnlyBooleanProperty shutdownSignal; @@ -54,7 +54,7 @@ public GenerateSummaryForSeveralTask( StringProperty groupName, List> entries, BibDatabaseContext bibDatabaseContext, - SummariesStorage summariesStorage, + SummariesRepository summariesRepository, ChatModel chatLanguageModel, AiTemplatesService aiTemplatesService, ReadOnlyBooleanProperty shutdownSignal, @@ -65,7 +65,7 @@ public GenerateSummaryForSeveralTask( this.groupName = groupName; this.entries = entries; this.bibDatabaseContext = bibDatabaseContext; - this.summariesStorage = summariesStorage; + this.summariesRepository = summariesRepository; this.chatLanguageModel = chatLanguageModel; this.aiTemplatesService = aiTemplatesService; this.shutdownSignal = shutdownSignal; @@ -100,7 +100,7 @@ public Void call() throws ExecutionException, InterruptedException { new GenerateSummaryTask( processingInfo.getObject(), bibDatabaseContext, - summariesStorage, + summariesRepository, chatLanguageModel, aiTemplatesService, shutdownSignal, From 51a89651b12671288d7ef26453151a62e5f00d38 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Mon, 24 Nov 2025 19:33:01 +0100 Subject: [PATCH 012/243] Introduce `FileParser` --- .../storages/ChatHistoryRepository.java | 6 +- .../MVStoreEmbeddingStore.java | 0 .../logic/ai/rag/JabRefContentInjector.java | 72 ------------------- .../ai/rag/algorithms/parsing/FileParser.java | 10 +++ .../PdfFileParser.java} | 33 ++------- .../parsing/UniversalFileParser.java | 26 +++++++ .../ai/rag/tasks/GenerateEmbeddingsTask.java | 4 +- .../algorithms/BibEntrySummarizer.java | 4 +- 8 files changed, 46 insertions(+), 109 deletions(-) rename jablib/src/main/java/org/jabref/logic/ai/{rag/storages => customimplementations/embeddingstores}/MVStoreEmbeddingStore.java (100%) delete mode 100644 jablib/src/main/java/org/jabref/logic/ai/rag/JabRefContentInjector.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/FileParser.java rename jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/{FileToDocument.java => parsing/PdfFileParser.java} (53%) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/UniversalFileParser.java diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/storages/ChatHistoryRepository.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/storages/ChatHistoryRepository.java index 1166c037a279..1f199608f65f 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/storages/ChatHistoryRepository.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/storages/ChatHistoryRepository.java @@ -5,7 +5,7 @@ import dev.langchain4j.data.message.ChatMessage; -public interface ChatHistoryRepository { +public interface ChatHistoryRepository extends AutoCloseable { List loadMessagesForEntry(Path bibDatabasePath, String citationKey); void storeMessagesForEntry(Path bibDatabasePath, String citationKey, List messages); @@ -13,8 +13,4 @@ public interface ChatHistoryRepository { List loadMessagesForGroup(Path bibDatabasePath, String name); void storeMessagesForGroup(Path bibDatabasePath, String name, List messages); - - void commit(); - - void close(); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/storages/MVStoreEmbeddingStore.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingstores/MVStoreEmbeddingStore.java similarity index 100% rename from jablib/src/main/java/org/jabref/logic/ai/rag/storages/MVStoreEmbeddingStore.java rename to jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingstores/MVStoreEmbeddingStore.java diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/JabRefContentInjector.java b/jablib/src/main/java/org/jabref/logic/ai/rag/JabRefContentInjector.java deleted file mode 100644 index 5eaf0cd7a0c3..000000000000 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/JabRefContentInjector.java +++ /dev/null @@ -1,72 +0,0 @@ -package org.jabref.logic.ai.rag; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.entry.BibEntry; - -import dev.langchain4j.data.message.ChatMessage; -import dev.langchain4j.data.message.UserMessage; -import dev.langchain4j.model.input.PromptTemplate; -import dev.langchain4j.rag.content.Content; -import dev.langchain4j.rag.content.injector.ContentInjector; - -import static org.jabref.logic.ai.rag.storages.FileEmbeddingsManager.LINK_METADATA_KEY; - -public class JabRefContentInjector implements ContentInjector { - public static final PromptTemplate DEFAULT_PROMPT_TEMPLATE = PromptTemplate.from("{{userMessage}}\n\nAnswer using the following information:\n{{contents}}"); - - private final BibDatabaseContext bibDatabaseContext; - - public JabRefContentInjector(BibDatabaseContext bibDatabaseContext) { - this.bibDatabaseContext = bibDatabaseContext; - } - - @Override - public UserMessage inject(List contents, ChatMessage chatMessage) { - String contentText = contents.stream().map(this::contentToString).collect(Collectors.joining("\n\n")); - - if (chatMessage instanceof UserMessage userMessage) { - String res = applyPrompt(userMessage.singleText(), contentText); - return new UserMessage(res); - } - return new UserMessage("INVALID"); - } - - private String contentToString(Content content) { - String text = content.textSegment().text(); - - String link = content.textSegment().metadata().getString(LINK_METADATA_KEY); - if (link == null) { - return text; - } - - String keys = findEntriesByLink(link) - .filter(entry -> entry.getCitationKey().isPresent()) - .map(entry -> "@" + entry.getCitationKey().get()) - .collect(Collectors.joining(", ")); - - if (keys.isEmpty()) { - return text; - } else { - return keys + ":\n" + text; - } - } - - private Stream findEntriesByLink(String link) { - return bibDatabaseContext.getEntries().stream().filter(entry -> entry.getFiles().stream().anyMatch(file -> file.getLink().equals(link))); - } - - private String applyPrompt(String userMessage, String contents) { - Map variables = new HashMap<>(); - - variables.put("userMessage", userMessage); - variables.put("contents", contents); - - return DEFAULT_PROMPT_TEMPLATE.apply(variables).text(); - } -} diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/FileParser.java b/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/FileParser.java new file mode 100644 index 000000000000..eade1e5428cf --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/FileParser.java @@ -0,0 +1,10 @@ +package org.jabref.logic.ai.rag.algorithms; + +import java.nio.file.Path; +import java.util.Optional; + +import javafx.beans.property.ReadOnlyBooleanProperty; + +public interface FileParser { + Optional parse(Path path, ReadOnlyBooleanProperty shutdownSignal); +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/FileToDocument.java b/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/PdfFileParser.java similarity index 53% rename from jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/FileToDocument.java rename to jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/PdfFileParser.java index 8400183f5044..689990eed5fd 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/FileToDocument.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/PdfFileParser.java @@ -8,36 +8,17 @@ import javafx.beans.property.ReadOnlyBooleanProperty; import org.jabref.logic.pdf.InterruptablePDFTextStripper; -import org.jabref.logic.util.io.FileUtil; import org.jabref.logic.xmp.XmpUtilReader; -import dev.langchain4j.data.document.DefaultDocument; -import dev.langchain4j.data.document.Document; import org.apache.pdfbox.pdmodel.PDDocument; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class FileToDocument { - private static final Logger LOGGER = LoggerFactory.getLogger(FileToDocument.class); - - private final ReadOnlyBooleanProperty shutdownSignal; - - public FileToDocument(ReadOnlyBooleanProperty shutdownSignal) { - this.shutdownSignal = shutdownSignal; - } - - public Optional fromFile(Path path) { - if (FileUtil.isPDFFile(path)) { - return fromPdfFile(path); - } else { - LOGGER.info("Unsupported file type of file: {}. Currently, only PDF files are supported", path); - return Optional.empty(); - } - } - - private Optional fromPdfFile(Path path) { - // This method is private to ensure that the path is really pointing to PDF file (determined by extension). +public class PdfFileParser implements FileParser { + private static final Logger LOGGER = LoggerFactory.getLogger(PdfFileParser.class); + @Override + public Optional parse(Path path, ReadOnlyBooleanProperty shutdownSignal) { try (PDDocument document = new XmpUtilReader().loadWithAutomaticDecryption(path)) { int lastPage = document.getNumberOfPages(); StringWriter writer = new StringWriter(); @@ -51,14 +32,10 @@ private Optional fromPdfFile(Path path) { return Optional.empty(); } - return fromString(writer.toString()); + return Optional.of(writer.toString()); } catch (IOException e) { LOGGER.error("An error occurred while reading the PDF file: {}", path, e); return Optional.empty(); } } - - public Optional fromString(String content) { - return Optional.of(new DefaultDocument(content)); - } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/UniversalFileParser.java b/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/UniversalFileParser.java new file mode 100644 index 000000000000..b78ce5339fd4 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/UniversalFileParser.java @@ -0,0 +1,26 @@ +package org.jabref.logic.ai.rag.algorithms; + +import java.nio.file.Path; +import java.util.Optional; + +import javafx.beans.property.ReadOnlyBooleanProperty; + +import org.jabref.logic.util.io.FileUtil; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class UniversalFileParser implements FileParser { + private static final Logger LOGGER = LoggerFactory.getLogger(UniversalFileParser.class); + + private final PdfFileParser pdfFileParser = new PdfFileParser(); + + public Optional parse(Path path, ReadOnlyBooleanProperty shutdownSignal) { + if (FileUtil.isPDFFile(path)) { + return pdfFileParser.parse(path, shutdownSignal); + } else { + LOGGER.info("Unsupported file type of file: {}. Currently, only PDF files are supported", path); + return Optional.empty(); + } + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java index c2769b052a10..14dadf39da61 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java @@ -10,7 +10,7 @@ import javafx.beans.property.ReadOnlyBooleanProperty; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.rag.algorithms.FileToDocument; +import org.jabref.logic.ai.rag.algorithms.UniversalFileParser; import org.jabref.logic.ai.rag.storages.FileEmbeddingsManager; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; @@ -118,7 +118,7 @@ private void ingestLinkedFile(LinkedFile linkedFile) throws InterruptedException return; } - Optional document = new FileToDocument(shutdownSignal).fromFile(path.get()); + Optional document = new UniversalFileParser(shutdownSignal).fromFile(path.get()); if (document.isPresent()) { fileEmbeddingsManager.addDocument(linkedFile.getLink(), document.get(), modTime.orElse(0L), progressCounter.workDoneProperty(), progressCounter.workMaxProperty()); LOGGER.debug("Embeddings for file \"{}\" were generated successfully", linkedFile.getLink()); diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/BibEntrySummarizer.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/BibEntrySummarizer.java index 519bf6699af2..eff6af0c940b 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/BibEntrySummarizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/BibEntrySummarizer.java @@ -12,7 +12,7 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.rag.algorithms.FileToDocument; +import org.jabref.logic.ai.rag.algorithms.UniversalFileParser; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.ProgressCounter; @@ -98,7 +98,7 @@ private Optional generateSummary(LinkedFile linkedFile, BibDatabaseConte return Optional.empty(); } - Optional document = new FileToDocument(shutdownSignal).fromFile(path.get()); + Optional document = new UniversalFileParser(shutdownSignal).fromFile(path.get()); if (document.isEmpty()) { LOGGER.warn("Could not extract text from a linked file \"{}\" of entry {}. It will be skipped when generating a summary.", linkedFile.getLink(), citationKey); From bfeb1b0e83decd5556842d9a9bffa5c0516e16d8 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Mon, 24 Nov 2025 19:39:00 +0100 Subject: [PATCH 013/243] Fix run problems --- jablib/src/main/java/module-info.java | 3 --- .../org/jabref/logic/ai/chatting/ChatHistoryService.java | 3 +-- .../jabref/logic/ai/rag/storages/FileEmbeddingsManager.java | 4 +++- .../jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java | 4 ++-- .../ai/summarization/algorithms/BibEntrySummarizer.java | 6 +++--- 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index 21dc1bc734a0..b5076a7ad933 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -59,7 +59,6 @@ exports org.jabref.model.metadata.event; exports org.jabref.logic.ai.chatting; exports org.jabref.logic.ai.util; - exports org.jabref.model.ai; exports org.jabref.model.ai.processingstatus; exports org.jabref.logic.ai.summarization; exports org.jabref.logic.layout.format; @@ -120,7 +119,6 @@ exports org.jabref.model.sciteTallies; exports org.jabref.logic.ai.customimplementations.embeddingmodels; exports org.jabref.logic.ai.rag; - exports org.jabref.logic.ai.chatting.chathistory; exports org.jabref.logic.ai.summarization.tasks; exports org.jabref.logic.ai.summarization.storages; exports org.jabref.logic.ai.rag.tasks; @@ -281,6 +279,5 @@ requires org.jooq.jool; requires org.libreoffice.uno; requires transitive org.jspecify; - requires org.jabref.jablib; // endregion } diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java index 2096150d39bc..c40a0f68509f 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java @@ -196,14 +196,13 @@ private void tryToGenerateCitationKey(BibDatabaseContext bibDatabaseContext, Bib } @Override - public void close() { + public void close() throws Exception { // We need to clone `bibEntriesChatHistory.keySet()` because closeChatHistoryForEntry() modifies the `bibEntriesChatHistory` map. new HashSet<>(bibEntriesChatHistory.keySet()).forEach(this::closeChatHistoryForEntry); // Clone is for the same reason, as written above. new HashSet<>(groupsChatHistory.keySet()).forEach(this::closeChatHistoryForGroup); - implementation.commit(); implementation.close(); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FileEmbeddingsManager.java b/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FileEmbeddingsManager.java index e646624b15d6..8f69d6ce4eab 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FileEmbeddingsManager.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FileEmbeddingsManager.java @@ -10,6 +10,7 @@ import org.jabref.logic.ai.rag.algorithms.LowLevelIngestor; import org.jabref.model.entry.LinkedFile; +import dev.langchain4j.data.document.DefaultDocument; import dev.langchain4j.data.document.Document; import dev.langchain4j.data.segment.TextSegment; import dev.langchain4j.model.embedding.EmbeddingModel; @@ -57,7 +58,8 @@ private void setupListeningToPreferencesChanges() { aiPreferences.addListenerToEmbeddingsParametersChange(embeddingStore::removeAll); } - public void addDocument(String link, Document document, long modificationTimeInSeconds, IntegerProperty workDone, IntegerProperty workMax) throws InterruptedException { + public void addDocument(String link, String documentSource, long modificationTimeInSeconds, IntegerProperty workDone, IntegerProperty workMax) throws InterruptedException { + Document document = new DefaultDocument(documentSource); document.metadata().put(LINK_METADATA_KEY, link); lowLevelIngestor.ingestDocument(document, shutdownSignal, workDone, workMax); diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java index 14dadf39da61..9f5b4bc017b6 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java @@ -18,7 +18,6 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.LinkedFile; -import dev.langchain4j.data.document.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,6 +34,7 @@ public class GenerateEmbeddingsTask extends BackgroundTask { private final BibDatabaseContext bibDatabaseContext; private final FilePreferences filePreferences; private final ReadOnlyBooleanProperty shutdownSignal; + private final UniversalFileParser universalFileParser = new UniversalFileParser(); private final ProgressCounter progressCounter = new ProgressCounter(); @@ -118,7 +118,7 @@ private void ingestLinkedFile(LinkedFile linkedFile) throws InterruptedException return; } - Optional document = new UniversalFileParser(shutdownSignal).fromFile(path.get()); + Optional document = universalFileParser.parse(path.get(), shutdownSignal); if (document.isPresent()) { fileEmbeddingsManager.addDocument(linkedFile.getLink(), document.get(), modTime.orElse(0L), progressCounter.workDoneProperty(), progressCounter.workMaxProperty()); LOGGER.debug("Embeddings for file \"{}\" were generated successfully", linkedFile.getLink()); diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/BibEntrySummarizer.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/BibEntrySummarizer.java index eff6af0c940b..6a81c2f92977 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/BibEntrySummarizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/BibEntrySummarizer.java @@ -21,7 +21,6 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; -import dev.langchain4j.data.document.Document; import dev.langchain4j.model.chat.ChatModel; import org.slf4j.Logger; @@ -33,6 +32,7 @@ public class BibEntrySummarizer { private final FilePreferences filePreferences; private final ChunkedSummarizationAlgorithm chunkedSummarizationAlgorithm; + private final UniversalFileParser universalFileParser = new UniversalFileParser(); public BibEntrySummarizer( AiPreferences aiPreferences, @@ -98,7 +98,7 @@ private Optional generateSummary(LinkedFile linkedFile, BibDatabaseConte return Optional.empty(); } - Optional document = new UniversalFileParser(shutdownSignal).fromFile(path.get()); + Optional document = universalFileParser.parse(path.get(), shutdownSignal); if (document.isEmpty()) { LOGGER.warn("Could not extract text from a linked file \"{}\" of entry {}. It will be skipped when generating a summary.", linkedFile.getLink(), citationKey); @@ -106,7 +106,7 @@ private Optional generateSummary(LinkedFile linkedFile, BibDatabaseConte return Optional.empty(); } - String linkedFileSummary = chunkedSummarizationAlgorithm.summarize(document.get().text(), progressCounter, shutdownSignal); + String linkedFileSummary = chunkedSummarizationAlgorithm.summarize(document.get(), progressCounter, shutdownSignal); LOGGER.debug("Summary for file \"{}\" of entry {} was generated successfully", linkedFile.getLink(), citationKey); return Optional.of(linkedFileSummary); From 76dcd793c13353a180cf8b122301bd310e6a82c9 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Mon, 24 Nov 2025 19:39:54 +0100 Subject: [PATCH 014/243] Rename `storages` package to `repository` --- jablib/src/main/java/module-info.java | 6 +++--- jablib/src/main/java/org/jabref/logic/ai/AiService.java | 8 ++++---- .../org/jabref/logic/ai/chatting/ChatHistoryService.java | 2 +- .../ai/chatting/CurrentlySelectedChatLanguageModel.java | 2 +- .../jabref/logic/ai/chatting/algorithms/AiChatLogic.java | 2 +- .../{storages => repositories}/ChatHistoryRepository.java | 2 +- .../MVStoreChatHistoryRepository.java | 2 +- .../embeddingstores/MVStoreEmbeddingStore.java | 4 ++-- .../java/org/jabref/logic/ai/rag/IngestionService.java | 4 ++-- .../{storages => repositories}/FileEmbeddingsManager.java | 2 +- .../FullyIngestedDocumentsRepository.java | 2 +- .../MVStoreFullyIngestedDocumentsRepository.java | 2 +- .../ai/rag/tasks/GenerateEmbeddingsForSeveralTask.java | 2 +- .../jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java | 2 +- .../jabref/logic/ai/summarization/SummariesService.java | 2 +- .../algorithms/PersistentBibEntrySummarizer.java | 2 +- .../MVStoreSummariesRepository.java | 2 +- .../{storages => repositories}/SummariesRepository.java | 2 +- .../tasks/GenerateSummaryForSeveralTask.java | 2 +- .../logic/ai/summarization/tasks/GenerateSummaryTask.java | 2 +- .../chatting/chathistory/ChatHistoryRepositoryTest.java | 2 +- .../chathistory/MVStoreChatHistoryRepositoryTest.java | 4 ++-- .../ingestion/FullyIngestedDocumentsRepositoryTest.java | 2 +- .../MVStoreFullyIngestedDocumentsRepositoryTest.java | 4 ++-- .../ai/summarization/MVStoreSummariesRepositoryTest.java | 4 ++-- .../logic/ai/summarization/SummariesRepositoryTest.java | 2 +- 26 files changed, 36 insertions(+), 36 deletions(-) rename jablib/src/main/java/org/jabref/logic/ai/chatting/{storages => repositories}/ChatHistoryRepository.java (91%) rename jablib/src/main/java/org/jabref/logic/ai/chatting/{storages => repositories}/MVStoreChatHistoryRepository.java (99%) rename jablib/src/main/java/org/jabref/logic/ai/rag/{storages => repositories}/FileEmbeddingsManager.java (98%) rename jablib/src/main/java/org/jabref/logic/ai/rag/{storages => repositories}/FullyIngestedDocumentsRepository.java (92%) rename jablib/src/main/java/org/jabref/logic/ai/rag/{storages => repositories}/MVStoreFullyIngestedDocumentsRepository.java (98%) rename jablib/src/main/java/org/jabref/logic/ai/summarization/{storages => repositories}/MVStoreSummariesRepository.java (96%) rename jablib/src/main/java/org/jabref/logic/ai/summarization/{storages => repositories}/SummariesRepository.java (86%) diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index b5076a7ad933..45ab4730c665 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -120,9 +120,9 @@ exports org.jabref.logic.ai.customimplementations.embeddingmodels; exports org.jabref.logic.ai.rag; exports org.jabref.logic.ai.summarization.tasks; - exports org.jabref.logic.ai.summarization.storages; + exports org.jabref.logic.ai.summarization.repositories; exports org.jabref.logic.ai.rag.tasks; - exports org.jabref.logic.ai.rag.storages; + exports org.jabref.logic.ai.rag.repositories; exports org.jabref.logic.ai.rag.algorithms; exports org.jabref.logic.ai.chatting.algorithms; exports org.jabref.logic.ai.chatting.tasks; @@ -132,7 +132,7 @@ exports org.jabref.model.ai.rag; exports org.jabref.model.ai.embeddings; exports org.jabref.model.ai.summarization; - exports org.jabref.logic.ai.chatting.storages; + exports org.jabref.logic.ai.chatting.repositories; requires java.base; diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiService.java b/jablib/src/main/java/org/jabref/logic/ai/AiService.java index 98f303ea7dfb..7184d4a3bf30 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiService.java @@ -9,14 +9,14 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.chatting.ChatHistoryService; import org.jabref.logic.ai.chatting.CurrentlySelectedChatLanguageModel; -import org.jabref.logic.ai.chatting.storages.MVStoreChatHistoryRepository; +import org.jabref.logic.ai.chatting.repositories.MVStoreChatHistoryRepository; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.rag.CurrentlySelectedEmbeddingModel; import org.jabref.logic.ai.rag.IngestionService; -import org.jabref.logic.ai.rag.storages.MVStoreEmbeddingStore; -import org.jabref.logic.ai.rag.storages.MVStoreFullyIngestedDocumentsRepository; +import org.jabref.logic.ai.rag.repositories.MVStoreEmbeddingStore; +import org.jabref.logic.ai.rag.repositories.MVStoreFullyIngestedDocumentsRepository; import org.jabref.logic.ai.summarization.SummariesService; -import org.jabref.logic.ai.summarization.storages.MVStoreSummariesRepository; +import org.jabref.logic.ai.summarization.repositories.MVStoreSummariesRepository; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; import org.jabref.logic.util.Directories; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java index c40a0f68509f..ae708d0a29ac 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java @@ -9,7 +9,7 @@ import javafx.collections.FXCollections; import javafx.collections.ObservableList; -import org.jabref.logic.ai.chatting.storages.ChatHistoryRepository; +import org.jabref.logic.ai.chatting.repositories.ChatHistoryRepository; import org.jabref.logic.ai.util.CitationKeyCheck; import org.jabref.logic.citationkeypattern.CitationKeyGenerator; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java index 239a9e4cf6df..4e31c759d298 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java @@ -8,7 +8,7 @@ import java.util.concurrent.Executors; import org.jabref.logic.ai.chatting.algorithms.AiChatLogic; -import org.jabref.logic.ai.chatting.storages.ChatHistoryRepository; +import org.jabref.logic.ai.chatting.repositories.ChatHistoryRepository; import org.jabref.logic.ai.customimplementations.llms.Gpt4AllModel; import org.jabref.logic.ai.customimplementations.llms.JvmOpenAiChatLanguageModel; import org.jabref.logic.ai.preferences.AiPreferences; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/algorithms/AiChatLogic.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/algorithms/AiChatLogic.java index de6656f93129..ffba79de204a 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/algorithms/AiChatLogic.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/algorithms/AiChatLogic.java @@ -8,7 +8,7 @@ import javafx.collections.ObservableList; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.rag.storages.FileEmbeddingsManager; +import org.jabref.logic.ai.rag.repositories.FileEmbeddingsManager; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.model.ai.chatting.ErrorMessage; import org.jabref.model.ai.rag.PaperExcerpt; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/storages/ChatHistoryRepository.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/ChatHistoryRepository.java similarity index 91% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/storages/ChatHistoryRepository.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/ChatHistoryRepository.java index 1f199608f65f..db368bec61c6 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/storages/ChatHistoryRepository.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/ChatHistoryRepository.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.chatting.storages; +package org.jabref.logic.ai.chatting.repositories; import java.nio.file.Path; import java.util.List; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/storages/MVStoreChatHistoryRepository.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepository.java similarity index 99% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/storages/MVStoreChatHistoryRepository.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepository.java index eff1c6b39019..f35c255cc4b9 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/storages/MVStoreChatHistoryRepository.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepository.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.chatting.storages; +package org.jabref.logic.ai.chatting.repositories; import java.io.Serializable; import java.nio.file.Path; diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingstores/MVStoreEmbeddingStore.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingstores/MVStoreEmbeddingStore.java index 234cb13208b0..c4cb1835d468 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingstores/MVStoreEmbeddingStore.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingstores/MVStoreEmbeddingStore.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.rag.storages; +package org.jabref.logic.ai.rag.repositories; import java.io.Serializable; import java.nio.file.Path; @@ -34,7 +34,7 @@ import org.jspecify.annotations.Nullable; import static java.util.Comparator.comparingDouble; -import static org.jabref.logic.ai.rag.storages.FileEmbeddingsManager.LINK_METADATA_KEY; +import static org.jabref.logic.ai.rag.repositories.FileEmbeddingsManager.LINK_METADATA_KEY; /** * A custom implementation of langchain4j's {@link EmbeddingStore} that uses a {@link MVStore} as an embedded database. diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java b/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java index 1aab214cc2d7..957db16fb80c 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java @@ -10,8 +10,8 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.rag.storages.FileEmbeddingsManager; -import org.jabref.logic.ai.rag.storages.FullyIngestedDocumentsRepository; +import org.jabref.logic.ai.rag.repositories.FileEmbeddingsManager; +import org.jabref.logic.ai.rag.repositories.FullyIngestedDocumentsRepository; import org.jabref.logic.ai.rag.tasks.GenerateEmbeddingsForSeveralTask; import org.jabref.logic.ai.rag.tasks.GenerateEmbeddingsTask; import org.jabref.logic.util.TaskExecutor; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FileEmbeddingsManager.java b/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FileEmbeddingsManager.java similarity index 98% rename from jablib/src/main/java/org/jabref/logic/ai/rag/storages/FileEmbeddingsManager.java rename to jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FileEmbeddingsManager.java index 8f69d6ce4eab..201f02b6e70b 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FileEmbeddingsManager.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FileEmbeddingsManager.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.rag.storages; +package org.jabref.logic.ai.rag.repositories; import java.util.List; import java.util.Optional; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FullyIngestedDocumentsRepository.java b/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FullyIngestedDocumentsRepository.java similarity index 92% rename from jablib/src/main/java/org/jabref/logic/ai/rag/storages/FullyIngestedDocumentsRepository.java rename to jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FullyIngestedDocumentsRepository.java index 36e75eac6604..d9fa889f0cc9 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/storages/FullyIngestedDocumentsRepository.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FullyIngestedDocumentsRepository.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.rag.storages; +package org.jabref.logic.ai.rag.repositories; import java.util.Optional; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/storages/MVStoreFullyIngestedDocumentsRepository.java b/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/MVStoreFullyIngestedDocumentsRepository.java similarity index 98% rename from jablib/src/main/java/org/jabref/logic/ai/rag/storages/MVStoreFullyIngestedDocumentsRepository.java rename to jablib/src/main/java/org/jabref/logic/ai/rag/repositories/MVStoreFullyIngestedDocumentsRepository.java index ddabb60410f1..810bb3fd3cb3 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/storages/MVStoreFullyIngestedDocumentsRepository.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/MVStoreFullyIngestedDocumentsRepository.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.rag.storages; +package org.jabref.logic.ai.rag.repositories; import java.nio.file.Path; import java.util.Map; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsForSeveralTask.java b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsForSeveralTask.java index bd51b15e8f70..a767aedbd83f 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsForSeveralTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsForSeveralTask.java @@ -10,7 +10,7 @@ import javafx.util.Pair; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.rag.storages.FileEmbeddingsManager; +import org.jabref.logic.ai.rag.repositories.FileEmbeddingsManager; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.ProgressCounter; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java index 9f5b4bc017b6..f110dcc39788 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java @@ -11,7 +11,7 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.rag.algorithms.UniversalFileParser; -import org.jabref.logic.ai.rag.storages.FileEmbeddingsManager; +import org.jabref.logic.ai.rag.repositories.FileEmbeddingsManager; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.ProgressCounter; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java index be086f6535d4..2b3c345defbd 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java @@ -10,7 +10,7 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.summarization.storages.SummariesRepository; +import org.jabref.logic.ai.summarization.repositories.SummariesRepository; import org.jabref.logic.ai.summarization.tasks.GenerateSummaryForSeveralTask; import org.jabref.logic.ai.summarization.tasks.GenerateSummaryTask; import org.jabref.logic.ai.templates.AiTemplatesService; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/PersistentBibEntrySummarizer.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/PersistentBibEntrySummarizer.java index 43ce45613f62..a2899b65f71a 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/PersistentBibEntrySummarizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/PersistentBibEntrySummarizer.java @@ -6,7 +6,7 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.summarization.storages.SummariesRepository; +import org.jabref.logic.ai.summarization.repositories.SummariesRepository; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.ai.util.CitationKeyCheck; import org.jabref.logic.util.ProgressCounter; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/MVStoreSummariesRepository.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/repositories/MVStoreSummariesRepository.java similarity index 96% rename from jablib/src/main/java/org/jabref/logic/ai/summarization/storages/MVStoreSummariesRepository.java rename to jablib/src/main/java/org/jabref/logic/ai/summarization/repositories/MVStoreSummariesRepository.java index 280cce3a7775..b9b051aee569 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/MVStoreSummariesRepository.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/repositories/MVStoreSummariesRepository.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.summarization.storages; +package org.jabref.logic.ai.summarization.repositories; import java.nio.file.Path; import java.util.Map; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/SummariesRepository.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/repositories/SummariesRepository.java similarity index 86% rename from jablib/src/main/java/org/jabref/logic/ai/summarization/storages/SummariesRepository.java rename to jablib/src/main/java/org/jabref/logic/ai/summarization/repositories/SummariesRepository.java index cf332c139618..bfa913773f37 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/storages/SummariesRepository.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/repositories/SummariesRepository.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.summarization.storages; +package org.jabref.logic.ai.summarization.repositories; import java.nio.file.Path; import java.util.Optional; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java index 5bcdec6f0684..10a6b7f7f67f 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java @@ -11,7 +11,7 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.summarization.storages.SummariesRepository; +import org.jabref.logic.ai.summarization.repositories.SummariesRepository; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java index 5b93223ac9a7..6226f44b912d 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java @@ -6,7 +6,7 @@ import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.summarization.SummariesService; import org.jabref.logic.ai.summarization.algorithms.PersistentBibEntrySummarizer; -import org.jabref.logic.ai.summarization.storages.SummariesRepository; +import org.jabref.logic.ai.summarization.repositories.SummariesRepository; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; diff --git a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryRepositoryTest.java b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryRepositoryTest.java index 3c3206a8664f..6b42bc4a847a 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryRepositoryTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryRepositoryTest.java @@ -3,7 +3,7 @@ import java.nio.file.Path; import java.util.List; -import org.jabref.logic.ai.chatting.storages.ChatHistoryRepository; +import org.jabref.logic.ai.chatting.repositories.ChatHistoryRepository; import dev.langchain4j.data.message.AiMessage; import dev.langchain4j.data.message.ChatMessage; diff --git a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryTest.java b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryTest.java index 396b32a7bc1d..149e55c5d50b 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryTest.java @@ -2,8 +2,8 @@ import java.nio.file.Path; -import org.jabref.logic.ai.chatting.storages.ChatHistoryRepository; -import org.jabref.logic.ai.chatting.storages.MVStoreChatHistoryRepository; +import org.jabref.logic.ai.chatting.repositories.ChatHistoryRepository; +import org.jabref.logic.ai.chatting.repositories.MVStoreChatHistoryRepository; import org.jabref.logic.util.NotificationService; import static org.mockito.Mockito.mock; diff --git a/jablib/src/test/java/org/jabref/logic/ai/ingestion/FullyIngestedDocumentsRepositoryTest.java b/jablib/src/test/java/org/jabref/logic/ai/ingestion/FullyIngestedDocumentsRepositoryTest.java index 00ddca3d66e9..b0e661d98b53 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/ingestion/FullyIngestedDocumentsRepositoryTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/ingestion/FullyIngestedDocumentsRepositoryTest.java @@ -3,7 +3,7 @@ import java.nio.file.Path; import java.util.Optional; -import org.jabref.logic.ai.rag.storages.FullyIngestedDocumentsRepository; +import org.jabref.logic.ai.rag.repositories.FullyIngestedDocumentsRepository; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; diff --git a/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsRepositoryTest.java b/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsRepositoryTest.java index 459e5c837bd0..76a657b012e0 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsRepositoryTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsRepositoryTest.java @@ -2,8 +2,8 @@ import java.nio.file.Path; -import org.jabref.logic.ai.rag.storages.FullyIngestedDocumentsRepository; -import org.jabref.logic.ai.rag.storages.MVStoreFullyIngestedDocumentsRepository; +import org.jabref.logic.ai.rag.repositories.FullyIngestedDocumentsRepository; +import org.jabref.logic.ai.rag.repositories.MVStoreFullyIngestedDocumentsRepository; import org.jabref.logic.util.NotificationService; import static org.mockito.Mockito.mock; diff --git a/jablib/src/test/java/org/jabref/logic/ai/summarization/MVStoreSummariesRepositoryTest.java b/jablib/src/test/java/org/jabref/logic/ai/summarization/MVStoreSummariesRepositoryTest.java index 01b655560e47..0bf73edcacde 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/summarization/MVStoreSummariesRepositoryTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/summarization/MVStoreSummariesRepositoryTest.java @@ -2,8 +2,8 @@ import java.nio.file.Path; -import org.jabref.logic.ai.summarization.storages.MVStoreSummariesRepository; -import org.jabref.logic.ai.summarization.storages.SummariesRepository; +import org.jabref.logic.ai.summarization.repositories.MVStoreSummariesRepository; +import org.jabref.logic.ai.summarization.repositories.SummariesRepository; import org.jabref.logic.util.NotificationService; import static org.mockito.Mockito.mock; diff --git a/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesRepositoryTest.java b/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesRepositoryTest.java index b25a9be205b6..44da7f66bc94 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesRepositoryTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesRepositoryTest.java @@ -4,7 +4,7 @@ import java.time.LocalDateTime; import java.util.Optional; -import org.jabref.logic.ai.summarization.storages.SummariesRepository; +import org.jabref.logic.ai.summarization.repositories.SummariesRepository; import org.jabref.model.ai.chatting.AiProvider; import org.jabref.model.ai.summarization.Summary; From c0f51ac7f863d5a968cd0f4eeeb196a2b583f755 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Mon, 24 Nov 2025 19:43:10 +0100 Subject: [PATCH 015/243] Move CitationKeyCheck --- .../org/jabref/gui/ai/components/aichat/AiChatComponent.java | 2 +- .../jabref/gui/ai/components/summary/SummaryComponent.java | 4 ++-- .../src/main/java/org/jabref/gui/entryeditor/AiChatTab.java | 2 +- .../java/org/jabref/logic/ai/chatting/ChatHistoryService.java | 2 +- .../org/jabref/logic/ai/summarization/SummariesService.java | 2 +- .../algorithms/PersistentBibEntrySummarizer.java | 2 +- .../java/org/jabref/logic/{ai => }/util/CitationKeyCheck.java | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) rename jablib/src/main/java/org/jabref/logic/{ai => }/util/CitationKeyCheck.java (96%) diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java index c4f15779de2a..714896eaae9b 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java @@ -31,9 +31,9 @@ import org.jabref.logic.ai.chatting.algorithms.AiChatLogic; import org.jabref.logic.ai.chatting.tasks.GenerateAiResponseTask; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.util.CitationKeyCheck; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; +import org.jabref.logic.util.CitationKeyCheck; import org.jabref.logic.util.TaskExecutor; import org.jabref.logic.util.io.FileUtil; import org.jabref.model.ai.chatting.ErrorMessage; diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java index 6f5a78dfa812..9476c3959ed1 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java @@ -11,13 +11,13 @@ import org.jabref.gui.frame.ExternalApplicationsPreferences; import org.jabref.logic.ai.AiService; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.util.CitationKeyCheck; import org.jabref.logic.citationkeypattern.CitationKeyGenerator; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; import org.jabref.logic.l10n.Localization; +import org.jabref.logic.util.CitationKeyCheck; import org.jabref.logic.util.io.FileUtil; -import org.jabref.model.ai.summarization.Summary; import org.jabref.model.ai.processingstatus.ProcessingInfo; +import org.jabref.model.ai.summarization.Summary; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; diff --git a/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java b/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java index f64cdc533f76..2db36b1ae91f 100644 --- a/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java +++ b/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java @@ -19,10 +19,10 @@ import org.jabref.gui.preferences.GuiPreferences; import org.jabref.logic.ai.AiService; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.util.CitationKeyCheck; import org.jabref.logic.citationkeypattern.CitationKeyGenerator; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; import org.jabref.logic.l10n.Localization; +import org.jabref.logic.util.CitationKeyCheck; import org.jabref.logic.util.TaskExecutor; import org.jabref.logic.util.io.FileUtil; import org.jabref.model.database.BibDatabaseContext; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java index ae708d0a29ac..54764fed115d 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java @@ -10,9 +10,9 @@ import javafx.collections.ObservableList; import org.jabref.logic.ai.chatting.repositories.ChatHistoryRepository; -import org.jabref.logic.ai.util.CitationKeyCheck; import org.jabref.logic.citationkeypattern.CitationKeyGenerator; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; +import org.jabref.logic.util.CitationKeyCheck; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.event.FieldChangedEvent; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java index 2b3c345defbd..26b66873c224 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java @@ -14,7 +14,7 @@ import org.jabref.logic.ai.summarization.tasks.GenerateSummaryForSeveralTask; import org.jabref.logic.ai.summarization.tasks.GenerateSummaryTask; import org.jabref.logic.ai.templates.AiTemplatesService; -import org.jabref.logic.ai.util.CitationKeyCheck; +import org.jabref.logic.util.CitationKeyCheck; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.ai.processingstatus.ProcessingInfo; import org.jabref.model.ai.processingstatus.ProcessingState; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/PersistentBibEntrySummarizer.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/PersistentBibEntrySummarizer.java index a2899b65f71a..8ffe0d437832 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/PersistentBibEntrySummarizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/PersistentBibEntrySummarizer.java @@ -8,7 +8,7 @@ import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.summarization.repositories.SummariesRepository; import org.jabref.logic.ai.templates.AiTemplatesService; -import org.jabref.logic.ai.util.CitationKeyCheck; +import org.jabref.logic.util.CitationKeyCheck; import org.jabref.logic.util.ProgressCounter; import org.jabref.model.ai.summarization.Summary; import org.jabref.model.database.BibDatabaseContext; diff --git a/jablib/src/main/java/org/jabref/logic/ai/util/CitationKeyCheck.java b/jablib/src/main/java/org/jabref/logic/util/CitationKeyCheck.java similarity index 96% rename from jablib/src/main/java/org/jabref/logic/ai/util/CitationKeyCheck.java rename to jablib/src/main/java/org/jabref/logic/util/CitationKeyCheck.java index b841a528fcf1..4d8a705f9ed2 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/util/CitationKeyCheck.java +++ b/jablib/src/main/java/org/jabref/logic/util/CitationKeyCheck.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.util; +package org.jabref.logic.util; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; From 3fb2cd7ef37258abb997689029523b4f62eb8314 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Mon, 24 Nov 2025 20:36:42 +0100 Subject: [PATCH 016/243] Introduce AiIdentifiers --- jablib/src/main/java/module-info.java | 1 + .../logic/ai/chatting/ChatHistoryService.java | 34 +++++++++++++------ .../repositories/ChatHistoryRepository.java | 12 ++++--- .../MVStoreChatHistoryRepository.java | 26 +++++++------- .../ai/identifiers/BibEntryAiIdentifier.java | 6 ++++ .../ai/identifiers/GroupAiIdentifier.java | 6 ++++ .../ChatHistoryRepositoryTest.java | 19 ++++++++--- 7 files changed, 72 insertions(+), 32 deletions(-) create mode 100644 jablib/src/main/java/org/jabref/model/ai/identifiers/BibEntryAiIdentifier.java create mode 100644 jablib/src/main/java/org/jabref/model/ai/identifiers/GroupAiIdentifier.java diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index 45ab4730c665..4dde519d73c2 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -279,5 +279,6 @@ requires org.jooq.jool; requires org.libreoffice.uno; requires transitive org.jspecify; + requires org.jabref.jablib; // endregion } diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java index 54764fed115d..b76a2c4a9bdd 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java @@ -13,6 +13,8 @@ import org.jabref.logic.citationkeypattern.CitationKeyGenerator; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; import org.jabref.logic.util.CitationKeyCheck; +import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; +import org.jabref.model.ai.identifiers.GroupAiIdentifier; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.event.FieldChangedEvent; @@ -96,7 +98,8 @@ public ObservableList getChatHistoryForEntry(BibDatabaseContext bib if (entry.getCitationKey().isEmpty() || !correctCitationKey(bibDatabaseContext, entry) || bibDatabaseContext.getDatabasePath().isEmpty()) { chatHistory = FXCollections.observableArrayList(); } else { - List chatMessagesList = implementation.loadMessagesForEntry(bibDatabaseContext.getDatabasePath().get(), entry.getCitationKey().get()); + BibEntryAiIdentifier identifier = new BibEntryAiIdentifier(bibDatabaseContext.getDatabasePath().get(), entry.getCitationKey().get()); + List chatMessagesList = implementation.loadMessagesForEntry(identifier); chatHistory = FXCollections.observableArrayList(chatMessagesList); } @@ -124,9 +127,9 @@ public void closeChatHistoryForEntry(BibEntry entry) { if (bibDatabaseContext.isPresent() && entry.getCitationKey().isPresent() && correctCitationKey(bibDatabaseContext.get(), entry) && bibDatabaseContext.get().getDatabasePath().isPresent()) { // Method `correctCitationKey` will already check `entry.getCitationKey().isPresent()`, but it is still // there, to suppress warning from IntelliJ IDEA on `entry.getCitationKey().get()`. + BibEntryAiIdentifier identifier = new BibEntryAiIdentifier(bibDatabaseContext.get().getDatabasePath().get(), entry.getCitationKey().get()); implementation.storeMessagesForEntry( - bibDatabaseContext.get().getDatabasePath().get(), - entry.getCitationKey().get(), + identifier, chatHistoryManagementRecord.chatHistory() ); } @@ -142,9 +145,9 @@ public ObservableList getChatHistoryForGroup(BibDatabaseContext bib if (bibDatabaseContext.getDatabasePath().isEmpty()) { chatHistory = FXCollections.observableArrayList(); } else { + GroupAiIdentifier identifier = new GroupAiIdentifier(bibDatabaseContext.getDatabasePath().get(), group.getGroup().getName()); List chatMessagesList = implementation.loadMessagesForGroup( - bibDatabaseContext.getDatabasePath().get(), - group.getGroup().getName() + identifier ); chatHistory = FXCollections.observableArrayList(chatMessagesList); @@ -172,9 +175,9 @@ public void closeChatHistoryForGroup(GroupTreeNode group) { Optional bibDatabaseContext = chatHistoryManagementRecord.bibDatabaseContext(); if (bibDatabaseContext.isPresent() && bibDatabaseContext.get().getDatabasePath().isPresent()) { + GroupAiIdentifier identifier = new GroupAiIdentifier(bibDatabaseContext.get().getDatabasePath().get(), group.getGroup().getName()); implementation.storeMessagesForGroup( - bibDatabaseContext.get().getDatabasePath().get(), - group.getGroup().getName(), + identifier, chatHistoryManagementRecord.chatHistory() ); } @@ -214,8 +217,12 @@ private void transferGroupHistory(BibDatabaseContext bibDatabaseContext, GroupTr List chatMessages = groupsChatHistory.computeIfAbsent(groupTreeNode, e -> new ChatHistoryManagementRecord(Optional.of(bibDatabaseContext), FXCollections.observableArrayList())).chatHistory; - implementation.storeMessagesForGroup(bibDatabaseContext.getDatabasePath().get(), oldName, List.of()); - implementation.storeMessagesForGroup(bibDatabaseContext.getDatabasePath().get(), newName, chatMessages); + + GroupAiIdentifier oldIdentifier = new GroupAiIdentifier(bibDatabaseContext.getDatabasePath().get(), oldName); + GroupAiIdentifier newIdentifier = new GroupAiIdentifier(bibDatabaseContext.getDatabasePath().get(), newName); + + implementation.storeMessagesForGroup(oldIdentifier, List.of()); + implementation.storeMessagesForGroup(newIdentifier, chatMessages); } private void transferEntryHistory(BibDatabaseContext bibDatabaseContext, BibEntry entry, String oldCitationKey, String newCitationKey) { @@ -228,8 +235,13 @@ private void transferEntryHistory(BibDatabaseContext bibDatabaseContext, BibEntr List chatMessages = bibEntriesChatHistory.computeIfAbsent(entry, e -> new ChatHistoryManagementRecord(Optional.of(bibDatabaseContext), FXCollections.observableArrayList())).chatHistory; - implementation.storeMessagesForGroup(bibDatabaseContext.getDatabasePath().get(), oldCitationKey, List.of()); - implementation.storeMessagesForEntry(bibDatabaseContext.getDatabasePath().get(), newCitationKey, chatMessages); + + GroupAiIdentifier groupIdentifier = new GroupAiIdentifier(bibDatabaseContext.getDatabasePath().get(), oldCitationKey); + BibEntryAiIdentifier bibEntryIdentifier = new BibEntryAiIdentifier(bibDatabaseContext.getDatabasePath().get(), newCitationKey); + + // TODO: Why group here? + implementation.storeMessagesForGroup(groupIdentifier, List.of()); + implementation.storeMessagesForEntry(bibEntryIdentifier, chatMessages); } private class CitationKeyChangeListener { diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/ChatHistoryRepository.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/ChatHistoryRepository.java index db368bec61c6..66eb6a8021c7 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/ChatHistoryRepository.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/ChatHistoryRepository.java @@ -1,16 +1,18 @@ package org.jabref.logic.ai.chatting.repositories; -import java.nio.file.Path; import java.util.List; +import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; +import org.jabref.model.ai.identifiers.GroupAiIdentifier; + import dev.langchain4j.data.message.ChatMessage; public interface ChatHistoryRepository extends AutoCloseable { - List loadMessagesForEntry(Path bibDatabasePath, String citationKey); + List loadMessagesForEntry(BibEntryAiIdentifier identifier); - void storeMessagesForEntry(Path bibDatabasePath, String citationKey, List messages); + void storeMessagesForEntry(BibEntryAiIdentifier identifier, List messages); - List loadMessagesForGroup(Path bibDatabasePath, String name); + List loadMessagesForGroup(GroupAiIdentifier identifier); - void storeMessagesForGroup(Path bibDatabasePath, String name, List messages); + void storeMessagesForGroup(GroupAiIdentifier identifier, List messages); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepository.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepository.java index f35c255cc4b9..b16c478745bb 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepository.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepository.java @@ -10,6 +10,8 @@ import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.NotificationService; import org.jabref.model.ai.chatting.ErrorMessage; +import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; +import org.jabref.model.ai.identifiers.GroupAiIdentifier; import dev.langchain4j.data.message.AiMessage; import dev.langchain4j.data.message.ChatMessage; @@ -69,23 +71,23 @@ public MVStoreChatHistoryRepository(Path path, NotificationService dialogService } @Override - public List loadMessagesForEntry(Path bibDatabasePath, String citationKey) { - return loadMessagesFromMap(getMapForEntry(bibDatabasePath, citationKey)); + public List loadMessagesForEntry(BibEntryAiIdentifier identifier) { + return loadMessagesFromMap(getMapForEntry(identifier)); } @Override - public void storeMessagesForEntry(Path bibDatabasePath, String citationKey, List messages) { - storeMessagesForMap(getMapForEntry(bibDatabasePath, citationKey), messages); + public void storeMessagesForEntry(BibEntryAiIdentifier identifier, List messages) { + storeMessagesForMap(getMapForEntry(identifier), messages); } @Override - public List loadMessagesForGroup(Path bibDatabasePath, String name) { - return loadMessagesFromMap(getMapForGroup(bibDatabasePath, name)); + public List loadMessagesForGroup(GroupAiIdentifier identifier) { + return loadMessagesFromMap(getMapForGroup(identifier)); } @Override - public void storeMessagesForGroup(Path bibDatabasePath, String name, List messages) { - storeMessagesForMap(getMapForGroup(bibDatabasePath, name), messages); + public void storeMessagesForGroup(GroupAiIdentifier identifier, List messages) { + storeMessagesForMap(getMapForGroup(identifier), messages); } private List loadMessagesFromMap(Map map) { @@ -106,12 +108,12 @@ private void storeMessagesForMap(Map map, List getMapForEntry(Path bibDatabasePath, String citationKey) { - return getMap(bibDatabasePath, ENTRY_CHAT_HISTORY_PREFIX, citationKey); + private Map getMapForEntry(BibEntryAiIdentifier identifier) { + return getMap(identifier.databasePath(), ENTRY_CHAT_HISTORY_PREFIX, identifier.citationKey()); } - private Map getMapForGroup(Path bibDatabasePath, String name) { - return getMap(bibDatabasePath, GROUP_CHAT_HISTORY_PREFIX, name); + private Map getMapForGroup(GroupAiIdentifier identifier) { + return getMap(identifier.databasePath(), GROUP_CHAT_HISTORY_PREFIX, identifier.groupName()); } private Map getMap(Path bibDatabasePath, String type, String name) { diff --git a/jablib/src/main/java/org/jabref/model/ai/identifiers/BibEntryAiIdentifier.java b/jablib/src/main/java/org/jabref/model/ai/identifiers/BibEntryAiIdentifier.java new file mode 100644 index 000000000000..38edc938e9f7 --- /dev/null +++ b/jablib/src/main/java/org/jabref/model/ai/identifiers/BibEntryAiIdentifier.java @@ -0,0 +1,6 @@ +package org.jabref.model.ai.identifiers; + +import java.nio.file.Path; + +public record BibEntryAiIdentifier(Path databasePath, String citationKey) { +} diff --git a/jablib/src/main/java/org/jabref/model/ai/identifiers/GroupAiIdentifier.java b/jablib/src/main/java/org/jabref/model/ai/identifiers/GroupAiIdentifier.java new file mode 100644 index 000000000000..e7bd8eb49fb9 --- /dev/null +++ b/jablib/src/main/java/org/jabref/model/ai/identifiers/GroupAiIdentifier.java @@ -0,0 +1,6 @@ +package org.jabref.model.ai.identifiers; + +import java.nio.file.Path; + +public record GroupAiIdentifier(Path databasePath, String groupName) { +} diff --git a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryRepositoryTest.java b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryRepositoryTest.java index 6b42bc4a847a..eb9a87544d0c 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryRepositoryTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryRepositoryTest.java @@ -4,6 +4,8 @@ import java.util.List; import org.jabref.logic.ai.chatting.repositories.ChatHistoryRepository; +import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; +import org.jabref.model.ai.identifiers.GroupAiIdentifier; import dev.langchain4j.data.message.AiMessage; import dev.langchain4j.data.message.ChatMessage; @@ -46,9 +48,15 @@ void entryChatHistory() { new AiMessage("hello!") ); - storage.storeMessagesForEntry(tempDir.resolve("test.bib"), "citationKey", messages); + BibEntryAiIdentifier identifier = new BibEntryAiIdentifier( + tempDir.resolve("test.bib"), + "citationKey" + ); + storage.storeMessagesForEntry(identifier, messages); + reopen(); - assertEquals(messages, storage.loadMessagesForEntry(tempDir.resolve("test.bib"), "citationKey")); + + assertEquals(messages, storage.loadMessagesForEntry(identifier)); } @Test @@ -58,8 +66,11 @@ void groupChatHistory() { new AiMessage("hello!") ); - storage.storeMessagesForGroup(tempDir.resolve("test.bib"), "group", messages); + GroupAiIdentifier identifier = new GroupAiIdentifier(tempDir.resolve("test.bib"), "group"); + storage.storeMessagesForGroup(identifier, messages); + reopen(); - assertEquals(messages, storage.loadMessagesForGroup(tempDir.resolve("test.bib"), "group")); + + assertEquals(messages, storage.loadMessagesForGroup(identifier)); } } From 711847cd6e8310b0d9f052c66f36b7c65d82fb5d Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Mon, 24 Nov 2025 21:01:13 +0100 Subject: [PATCH 017/243] Remove AiDefaultPreferences.java --- .../gui/preferences/ai/AiTabViewModel.java | 26 ++-- jablib/src/main/java/module-info.java | 1 - .../ai/preferences/AiDefaultPreferences.java | 135 ------------------ .../ai/preferences/AiDefaultTemplates.java | 48 +++++++ .../ai/preferences/AiFallbackSettings.java | 5 + .../logic/ai/preferences/AiPreferences.java | 30 ++-- .../AiProviderDefaultChatModels.java | 19 +++ .../ai/preferences/PredefinedChatModel.java | 67 +++++++++ .../preferences/JabRefCliPreferences.java | 58 ++++---- 9 files changed, 200 insertions(+), 189 deletions(-) delete mode 100644 jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultPreferences.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultTemplates.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/preferences/AiFallbackSettings.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/preferences/AiProviderDefaultChatModels.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/preferences/PredefinedChatModel.java diff --git a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java index 612ad772b54f..40d2e72a64b5 100644 --- a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java +++ b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java @@ -20,16 +20,17 @@ import javafx.collections.FXCollections; import org.jabref.gui.preferences.PreferenceTabViewModel; -import org.jabref.logic.ai.preferences.AiDefaultPreferences; +import org.jabref.logic.ai.preferences.AiDefaultTemplates; import org.jabref.logic.ai.preferences.AiPreferences; +import org.jabref.logic.ai.preferences.PredefinedChatModel; import org.jabref.logic.l10n.Localization; import org.jabref.logic.preferences.CliPreferences; import org.jabref.logic.util.LocalizedNumbers; import org.jabref.logic.util.OptionalObjectProperty; import org.jabref.logic.util.strings.StringUtil; import org.jabref.model.ai.chatting.AiProvider; -import org.jabref.model.ai.templating.AiTemplate; import org.jabref.model.ai.embeddings.EmbeddingModel; +import org.jabref.model.ai.templating.AiTemplate; import de.saxsys.mvvmfx.utils.validation.FunctionBasedValidator; import de.saxsys.mvvmfx.utils.validation.ValidationMessage; @@ -136,7 +137,7 @@ public AiTabViewModel(CliPreferences preferences) { ); this.selectedAiProvider.addListener((_, oldValue, newValue) -> { - List models = AiDefaultPreferences.getAvailableModels(newValue); + List models = PredefinedChatModel.getAvailableModels(newValue); disableApiBaseUrl.set(newValue == AiProvider.HUGGING_FACE || newValue == AiProvider.GEMINI); @@ -223,7 +224,7 @@ public AiTabViewModel(CliPreferences preferences) { gpt4AllChatModel.set(newValue); } - contextWindowSize.set(AiDefaultPreferences.getContextWindowSize(selectedAiProvider.get(), newValue)); + contextWindowSize.set(PredefinedChatModel.getContextWindowSize(selectedAiProvider.get(), newValue)); }); this.currentApiKey.addListener((_, _, newValue) -> { @@ -407,23 +408,24 @@ public void resetExpertSettings() { String resetApiBaseUrl = selectedAiProvider.get().getApiUrl(); currentApiBaseUrl.set(resetApiBaseUrl); - contextWindowSize.set(AiDefaultPreferences.getContextWindowSize(selectedAiProvider.get(), currentChatModel.get())); + contextWindowSize.set(PredefinedChatModel.getContextWindowSize(selectedAiProvider.get(), currentChatModel.get())); - temperature.set(LocalizedNumbers.doubleToString(AiDefaultPreferences.TEMPERATURE)); - documentSplitterChunkSize.set(AiDefaultPreferences.DOCUMENT_SPLITTER_CHUNK_SIZE); - documentSplitterOverlapSize.set(AiDefaultPreferences.DOCUMENT_SPLITTER_OVERLAP); - ragMaxResultsCount.set(AiDefaultPreferences.RAG_MAX_RESULTS_COUNT); - ragMinScore.set(LocalizedNumbers.doubleToString(AiDefaultPreferences.RAG_MIN_SCORE)); + // TODO: Get default values. + temperature.set(LocalizedNumbers.doubleToString(0.7)); + documentSplitterChunkSize.set(300); + documentSplitterOverlapSize.set(100); + ragMaxResultsCount.set(10); + ragMinScore.set(LocalizedNumbers.doubleToString(0.3)); } public void resetTemplates() { Arrays.stream(AiTemplate.values()).forEach(template -> - templateSources.get(template).set(AiDefaultPreferences.TEMPLATES.get(template))); + templateSources.get(template).set(AiDefaultTemplates.getTemplate(template))); } public void resetCurrentTemplate() { selectedTemplateProperty().get().ifPresent(template -> { - String defaultTemplate = AiDefaultPreferences.TEMPLATES.get(template); + String defaultTemplate = AiDefaultTemplates.getTemplate(template); templateSources.get(template).set(defaultTemplate); }); } diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index 4dde519d73c2..45ab4730c665 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -279,6 +279,5 @@ requires org.jooq.jool; requires org.libreoffice.uno; requires transitive org.jspecify; - requires org.jabref.jablib; // endregion } diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultPreferences.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultPreferences.java deleted file mode 100644 index 5e5fcaee907b..000000000000 --- a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultPreferences.java +++ /dev/null @@ -1,135 +0,0 @@ -package org.jabref.logic.ai.preferences; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import org.jabref.model.ai.chatting.AiProvider; -import org.jabref.model.ai.templating.AiTemplate; -import org.jabref.model.ai.embeddings.EmbeddingModel; - -public class AiDefaultPreferences { - public enum PredefinedChatModel { - GPT_4O_MINI(AiProvider.OPEN_AI, "gpt-4o-mini", 128000), - GPT_4O(AiProvider.OPEN_AI, "gpt-4o", 128000), - GPT_4(AiProvider.OPEN_AI, "gpt-4", 8192), - GPT_4_TURBO(AiProvider.OPEN_AI, "gpt-4-turbo", 128000), - GPT_3_5_TURBO(AiProvider.OPEN_AI, "gpt-3.5-turbo", 16385), - OPEN_MISTRAL_NEMO(AiProvider.MISTRAL_AI, "open-mistral-nemo", 128000), - OPEN_MISTRAL_7B(AiProvider.MISTRAL_AI, "open-mistral-7b", 32000), - // "mixtral" is not a typo. - OPEN_MIXTRAL_8X7B(AiProvider.MISTRAL_AI, "open-mixtral-8x7b", 32000), - OPEN_MIXTRAL_8X22B(AiProvider.MISTRAL_AI, "open-mixtral-8x22b", 64000), - GEMINI_1_5_FLASH(AiProvider.GEMINI, "gemini-1.5-flash", 1048576), - GEMINI_1_5_PRO(AiProvider.GEMINI, "gemini-1.5-pro", 2097152), - GEMINI_1_0_PRO(AiProvider.GEMINI, "gemini-1.0-pro", 32000), - // Dummy variant for Hugging Face models. - // Blank entry used for cases where the model name is not specified. - BLANK_HUGGING_FACE(AiProvider.HUGGING_FACE, "", 0), - BLANK_GPT4ALL(AiProvider.GPT4ALL, "", 0); - - private final AiProvider aiProvider; - private final String name; - private final int contextWindowSize; - - PredefinedChatModel(AiProvider aiProvider, String name, int contextWindowSize) { - this.aiProvider = aiProvider; - this.name = name; - this.contextWindowSize = contextWindowSize; - } - - public AiProvider getAiProvider() { - return aiProvider; - } - - public String getName() { - return name; - } - - public int getContextWindowSize() { - return contextWindowSize; - } - - public String toString() { - return aiProvider.toString() + " " + name; - } - } - - public static final boolean ENABLE_CHAT = false; - public static final boolean AUTO_GENERATE_EMBEDDINGS = false; - public static final boolean AUTO_GENERATE_SUMMARIES = false; - - public static final AiProvider PROVIDER = AiProvider.OPEN_AI; - - public static final Map CHAT_MODELS = Map.of( - AiProvider.OPEN_AI, PredefinedChatModel.GPT_4O_MINI, - AiProvider.MISTRAL_AI, PredefinedChatModel.OPEN_MIXTRAL_8X22B, - AiProvider.GEMINI, PredefinedChatModel.GEMINI_1_5_FLASH, - AiProvider.HUGGING_FACE, PredefinedChatModel.BLANK_HUGGING_FACE, - AiProvider.GPT4ALL, PredefinedChatModel.BLANK_GPT4ALL - ); - - public static final boolean CUSTOMIZE_SETTINGS = false; - - public static final EmbeddingModel EMBEDDING_MODEL = EmbeddingModel.SENTENCE_TRANSFORMERS_ALL_MINILM_L12_V2; - public static final String SYSTEM_MESSAGE = "You are an AI assistant that analyses research papers. You answer questions about papers. You will be supplied with the necessary information. The supplied information will contain mentions of papers in form '@citationKey'. Whenever you refer to a paper, use its citation key in the same form with @ symbol. Whenever you find relevant information, always use the citation key. Here are the papers you are analyzing:\n"; - public static final double TEMPERATURE = 0.7; - public static final int DOCUMENT_SPLITTER_CHUNK_SIZE = 300; - public static final int DOCUMENT_SPLITTER_OVERLAP = 100; - public static final int RAG_MAX_RESULTS_COUNT = 10; - public static final double RAG_MIN_SCORE = 0.3; - - public static final int FALLBACK_CONTEXT_WINDOW_SIZE = 8196; - - public static final Map TEMPLATES = Map.of( - AiTemplate.CHATTING_SYSTEM_MESSAGE, """ - You are an AI assistant that analyses research papers. You answer questions about papers. - You will be supplied with the necessary information. The supplied information will contain mentions of papers in form '@citationKey'. - Whenever you refer to a paper, use its citation key in the same form with @ symbol. Whenever you find relevant information, always use the citation key. - - Here are the papers you are analyzing: - #foreach( $entry in $entries ) - ${CanonicalBibEntry.getCanonicalRepresentation($entry)} - #end""", - - AiTemplate.CHATTING_USER_MESSAGE, """ - $message - - Here is some relevant information for you: - #foreach( $excerpt in $excerpts ) - ${excerpt.citationKey()}: - ${excerpt.text()} - #end""", - - AiTemplate.SUMMARIZATION_CHUNK_SYSTEM_MESSAGE, """ - Please provide an overview of the following text. It is a part of a scientific paper. - The summary should include the main objectives, methodologies used, key findings, and conclusions. - Mention any significant experiments, data, or discussions presented in the paper.""", - - AiTemplate.SUMMARIZATION_CHUNK_USER_MESSAGE, "$text", - - AiTemplate.SUMMARIZATION_COMBINE_SYSTEM_MESSAGE, """ - You have written an overview of a scientific paper. You have been collecting notes from various parts - of the paper. Now your task is to combine all of the notes in one structured message.""", - - AiTemplate.SUMMARIZATION_COMBINE_USER_MESSAGE, "$chunks", - - AiTemplate.CITATION_PARSING_SYSTEM_MESSAGE, "You are a bot to convert a plain text citation to a BibTeX entry. The user you talk to understands only BibTeX code, so provide it plainly without any wrappings.", - AiTemplate.CITATION_PARSING_USER_MESSAGE, "Please convert this plain text citation to a BibTeX entry:\n$citation\nIn your output, please provide only BibTeX code as your message." - ); - - public static List getAvailableModels(AiProvider aiProvider) { - return Arrays.stream(AiDefaultPreferences.PredefinedChatModel.values()) - .filter(model -> model.getAiProvider() == aiProvider) - .map(AiDefaultPreferences.PredefinedChatModel::getName) - .toList(); - } - - public static int getContextWindowSize(AiProvider aiProvider, String modelName) { - return Arrays.stream(AiDefaultPreferences.PredefinedChatModel.values()) - .filter(model -> model.getAiProvider() == aiProvider && model.getName().equals(modelName)) - .map(AiDefaultPreferences.PredefinedChatModel::getContextWindowSize) - .findFirst() - .orElse(AiDefaultPreferences.FALLBACK_CONTEXT_WINDOW_SIZE); - } -} diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultTemplates.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultTemplates.java new file mode 100644 index 000000000000..8dd49f3b2bab --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultTemplates.java @@ -0,0 +1,48 @@ +package org.jabref.logic.ai.preferences; + +import java.util.Map; + +import org.jabref.model.ai.templating.AiTemplate; + +public class AiDefaultTemplates { + private static final Map TEMPLATES = Map.of( + AiTemplate.CHATTING_SYSTEM_MESSAGE, """ + You are an AI assistant that analyses research papers. You answer questions about papers. + You will be supplied with the necessary information. The supplied information will contain mentions of papers in form '@citationKey'. + Whenever you refer to a paper, use its citation key in the same form with @ symbol. Whenever you find relevant information, always use the citation key. + + Here are the papers you are analyzing: + #foreach( $entry in $entries ) + ${CanonicalBibEntry.getCanonicalRepresentation($entry)} + #end""", + + AiTemplate.CHATTING_USER_MESSAGE, """ + $message + + Here is some relevant information for you: + #foreach( $excerpt in $excerpts ) + ${excerpt.citationKey()}: + ${excerpt.text()} + #end""", + + AiTemplate.SUMMARIZATION_CHUNK_SYSTEM_MESSAGE, """ + Please provide an overview of the following text. It is a part of a scientific paper. + The summary should include the main objectives, methodologies used, key findings, and conclusions. + Mention any significant experiments, data, or discussions presented in the paper.""", + + AiTemplate.SUMMARIZATION_CHUNK_USER_MESSAGE, "$text", + + AiTemplate.SUMMARIZATION_COMBINE_SYSTEM_MESSAGE, """ + You have written an overview of a scientific paper. You have been collecting notes from various parts + of the paper. Now your task is to combine all of the notes in one structured message.""", + + AiTemplate.SUMMARIZATION_COMBINE_USER_MESSAGE, "$chunks", + + AiTemplate.CITATION_PARSING_SYSTEM_MESSAGE, "You are a bot to convert a plain text citation to a BibTeX entry. The user you talk to understands only BibTeX code, so provide it plainly without any wrappings.", + AiTemplate.CITATION_PARSING_USER_MESSAGE, "Please convert this plain text citation to a BibTeX entry:\n$citation\nIn your output, please provide only BibTeX code as your message." + ); + + public static String getTemplate(AiTemplate template) { + return TEMPLATES.get(template); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiFallbackSettings.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiFallbackSettings.java new file mode 100644 index 000000000000..dc42acd6c693 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiFallbackSettings.java @@ -0,0 +1,5 @@ +package org.jabref.logic.ai.preferences; + +public class AiFallbackSettings { + public static final int FALLBACK_CONTEXT_WINDOW_SIZE = 8196; +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java index 8e25cbab7601..b9c49a935d40 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java @@ -18,8 +18,8 @@ import org.jabref.logic.util.strings.StringUtil; import org.jabref.model.ai.chatting.AiProvider; -import org.jabref.model.ai.templating.AiTemplate; import org.jabref.model.ai.embeddings.EmbeddingModel; +import org.jabref.model.ai.templating.AiTemplate; import com.github.javakeyring.Keyring; import com.github.javakeyring.PasswordAccessException; @@ -287,7 +287,8 @@ public EmbeddingModel getEmbeddingModel() { if (getCustomizeExpertSettings()) { return embeddingModel.get(); } else { - return AiDefaultPreferences.EMBEDDING_MODEL; + // TODO: Think why this is? It was taken from AiDefaultSettings. + return EmbeddingModel.SENTENCE_TRANSFORMERS_ALL_MINILM_L12_V2; } } @@ -363,7 +364,8 @@ public double getTemperature() { if (getCustomizeExpertSettings()) { return temperature.get(); } else { - return AiDefaultPreferences.TEMPERATURE; + // TODO: default return values + return 0.7; } } @@ -381,15 +383,15 @@ public int getContextWindowSize() { } else { return switch (aiProvider.get()) { case OPEN_AI -> - AiDefaultPreferences.getContextWindowSize(AiProvider.OPEN_AI, openAiChatModel.get()); + PredefinedChatModel.getContextWindowSize(AiProvider.OPEN_AI, openAiChatModel.get()); case MISTRAL_AI -> - AiDefaultPreferences.getContextWindowSize(AiProvider.MISTRAL_AI, mistralAiChatModel.get()); + PredefinedChatModel.getContextWindowSize(AiProvider.MISTRAL_AI, mistralAiChatModel.get()); case HUGGING_FACE -> - AiDefaultPreferences.getContextWindowSize(AiProvider.HUGGING_FACE, huggingFaceChatModel.get()); + PredefinedChatModel.getContextWindowSize(AiProvider.HUGGING_FACE, huggingFaceChatModel.get()); case GEMINI -> - AiDefaultPreferences.getContextWindowSize(AiProvider.GEMINI, geminiChatModel.get()); + PredefinedChatModel.getContextWindowSize(AiProvider.GEMINI, geminiChatModel.get()); case GPT4ALL -> - AiDefaultPreferences.getContextWindowSize(AiProvider.GPT4ALL, gpt4AllChatModel.get()); + PredefinedChatModel.getContextWindowSize(AiProvider.GPT4ALL, gpt4AllChatModel.get()); }; } } @@ -406,7 +408,8 @@ public int getDocumentSplitterChunkSize() { if (getCustomizeExpertSettings()) { return documentSplitterChunkSize.get(); } else { - return AiDefaultPreferences.DOCUMENT_SPLITTER_CHUNK_SIZE; + // TODO: default value. + return 300; } } @@ -422,7 +425,8 @@ public int getDocumentSplitterOverlapSize() { if (getCustomizeExpertSettings()) { return documentSplitterOverlapSize.get(); } else { - return AiDefaultPreferences.DOCUMENT_SPLITTER_OVERLAP; + // TODO: default value + return 100; } } @@ -438,7 +442,8 @@ public int getRagMaxResultsCount() { if (getCustomizeExpertSettings()) { return ragMaxResultsCount.get(); } else { - return AiDefaultPreferences.RAG_MAX_RESULTS_COUNT; + // TODO: Default value + return 10; } } @@ -454,7 +459,8 @@ public double getRagMinScore() { if (getCustomizeExpertSettings()) { return ragMinScore.get(); } else { - return AiDefaultPreferences.RAG_MIN_SCORE; + // TODO: default value + return 0.3; } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiProviderDefaultChatModels.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiProviderDefaultChatModels.java new file mode 100644 index 000000000000..59542c659d7f --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiProviderDefaultChatModels.java @@ -0,0 +1,19 @@ +package org.jabref.logic.ai.preferences; + +import java.util.Map; + +import org.jabref.model.ai.chatting.AiProvider; + +public class AiProviderDefaultChatModels { + private static final Map CHAT_MODELS = Map.of( + AiProvider.OPEN_AI, PredefinedChatModel.GPT_4O_MINI, + AiProvider.MISTRAL_AI, PredefinedChatModel.OPEN_MIXTRAL_8X22B, + AiProvider.GEMINI, PredefinedChatModel.GEMINI_1_5_FLASH, + AiProvider.HUGGING_FACE, PredefinedChatModel.BLANK_HUGGING_FACE, + AiProvider.GPT4ALL, PredefinedChatModel.BLANK_GPT4ALL + ); + + public static PredefinedChatModel getDefaultChatModel(AiProvider aiProvider) { + return CHAT_MODELS.get(aiProvider); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/PredefinedChatModel.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/PredefinedChatModel.java new file mode 100644 index 000000000000..537e551b4f63 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/PredefinedChatModel.java @@ -0,0 +1,67 @@ +package org.jabref.logic.ai.preferences; + +import java.util.Arrays; +import java.util.List; + +import org.jabref.model.ai.chatting.AiProvider; + +public enum PredefinedChatModel { + GPT_4O_MINI(AiProvider.OPEN_AI, "gpt-4o-mini", 128000), + GPT_4O(AiProvider.OPEN_AI, "gpt-4o", 128000), + GPT_4(AiProvider.OPEN_AI, "gpt-4", 8192), + GPT_4_TURBO(AiProvider.OPEN_AI, "gpt-4-turbo", 128000), + GPT_3_5_TURBO(AiProvider.OPEN_AI, "gpt-3.5-turbo", 16385), + OPEN_MISTRAL_NEMO(AiProvider.MISTRAL_AI, "open-mistral-nemo", 128000), + OPEN_MISTRAL_7B(AiProvider.MISTRAL_AI, "open-mistral-7b", 32000), + // "mixtral" is not a typo. + OPEN_MIXTRAL_8X7B(AiProvider.MISTRAL_AI, "open-mixtral-8x7b", 32000), + OPEN_MIXTRAL_8X22B(AiProvider.MISTRAL_AI, "open-mixtral-8x22b", 64000), + GEMINI_1_5_FLASH(AiProvider.GEMINI, "gemini-1.5-flash", 1048576), + GEMINI_1_5_PRO(AiProvider.GEMINI, "gemini-1.5-pro", 2097152), + GEMINI_1_0_PRO(AiProvider.GEMINI, "gemini-1.0-pro", 32000), + // Dummy variant for Hugging Face models. + // Blank entry used for cases where the model name is not specified. + BLANK_HUGGING_FACE(AiProvider.HUGGING_FACE, "", 0), + BLANK_GPT4ALL(AiProvider.GPT4ALL, "", 0); + + private final AiProvider aiProvider; + private final String name; + private final int contextWindowSize; + + PredefinedChatModel(AiProvider aiProvider, String name, int contextWindowSize) { + this.aiProvider = aiProvider; + this.name = name; + this.contextWindowSize = contextWindowSize; + } + + public static List getAvailableModels(AiProvider aiProvider) { + return Arrays.stream(values()) + .filter(model -> model.getAiProvider() == aiProvider) + .map(PredefinedChatModel::getName) + .toList(); + } + + public static int getContextWindowSize(AiProvider aiProvider, String modelName) { + return Arrays.stream(values()) + .filter(model -> model.getAiProvider() == aiProvider && model.getName().equals(modelName)) + .map(PredefinedChatModel::getContextWindowSize) + .findFirst() + .orElse(AiFallbackSettings.FALLBACK_CONTEXT_WINDOW_SIZE); + } + + public AiProvider getAiProvider() { + return aiProvider; + } + + public String getName() { + return name; + } + + public int getContextWindowSize() { + return contextWindowSize; + } + + public String toString() { + return aiProvider.toString() + " " + name; + } +} diff --git a/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java b/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java index 6d4eceab24b6..f8038b139b90 100644 --- a/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java @@ -33,8 +33,10 @@ import org.jabref.logic.InternalPreferences; import org.jabref.logic.JabRefException; import org.jabref.logic.LibraryPreferences; -import org.jabref.logic.ai.preferences.AiDefaultPreferences; +import org.jabref.logic.ai.preferences.AiDefaultTemplates; import org.jabref.logic.ai.preferences.AiPreferences; +import org.jabref.logic.ai.preferences.AiProviderDefaultChatModels; +import org.jabref.logic.ai.preferences.PredefinedChatModel; import org.jabref.logic.bibtex.FieldPreferences; import org.jabref.logic.citationkeypattern.CitationKeyPattern; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; @@ -96,8 +98,8 @@ import org.jabref.logic.util.strings.StringUtil; import org.jabref.logic.xmp.XmpPreferences; import org.jabref.model.ai.chatting.AiProvider; -import org.jabref.model.ai.templating.AiTemplate; import org.jabref.model.ai.embeddings.EmbeddingModel; +import org.jabref.model.ai.templating.AiTemplate; import org.jabref.model.database.BibDatabaseMode; import org.jabref.model.entry.BibEntryPreferences; import org.jabref.model.entry.BibEntryType; @@ -392,7 +394,6 @@ public class JabRefCliPreferences implements CliPreferences { private static final String AI_GEMINI_API_BASE_URL = "aiGeminiApiBaseUrl"; private static final String AI_HUGGING_FACE_API_BASE_URL = "aiHuggingFaceApiBaseUrl"; private static final String AI_GPT_4_ALL_API_BASE_URL = "aiGpt4AllApiBaseUrl"; - private static final String AI_SYSTEM_MESSAGE = "aiSystemMessage"; private static final String AI_TEMPERATURE = "aiTemperature"; private static final String AI_CONTEXT_WINDOW_SIZE = "aiMessageWindowSize"; private static final String AI_DOCUMENT_SPLITTER_CHUNK_SIZE = "aiDocumentSplitterChunkSize"; @@ -725,39 +726,38 @@ public JabRefCliPreferences() { // endregion // region:AI - defaults.put(AI_ENABLED, AiDefaultPreferences.ENABLE_CHAT); - defaults.put(AI_AUTO_GENERATE_EMBEDDINGS, AiDefaultPreferences.AUTO_GENERATE_EMBEDDINGS); - defaults.put(AI_AUTO_GENERATE_SUMMARIES, AiDefaultPreferences.AUTO_GENERATE_SUMMARIES); - defaults.put(AI_PROVIDER, AiDefaultPreferences.PROVIDER.name()); - defaults.put(AI_OPEN_AI_CHAT_MODEL, AiDefaultPreferences.CHAT_MODELS.get(AiProvider.OPEN_AI).getName()); - defaults.put(AI_MISTRAL_AI_CHAT_MODEL, AiDefaultPreferences.CHAT_MODELS.get(AiProvider.MISTRAL_AI).getName()); - defaults.put(AI_GEMINI_CHAT_MODEL, AiDefaultPreferences.CHAT_MODELS.get(AiProvider.GEMINI).getName()); - defaults.put(AI_HUGGING_FACE_CHAT_MODEL, AiDefaultPreferences.CHAT_MODELS.get(AiProvider.HUGGING_FACE).getName()); - defaults.put(AI_GPT_4_ALL_MODEL, AiDefaultPreferences.CHAT_MODELS.get(AiProvider.GPT4ALL).getName()); - defaults.put(AI_CUSTOMIZE_SETTINGS, AiDefaultPreferences.CUSTOMIZE_SETTINGS); - defaults.put(AI_EMBEDDING_MODEL, AiDefaultPreferences.EMBEDDING_MODEL.name()); + defaults.put(AI_ENABLED, false); + defaults.put(AI_AUTO_GENERATE_EMBEDDINGS, false); + defaults.put(AI_AUTO_GENERATE_SUMMARIES, false); + defaults.put(AI_PROVIDER, AiProvider.OPEN_AI.name()); + defaults.put(AI_OPEN_AI_CHAT_MODEL, AiProviderDefaultChatModels.getDefaultChatModel(AiProvider.OPEN_AI).getName()); + defaults.put(AI_MISTRAL_AI_CHAT_MODEL, AiProviderDefaultChatModels.getDefaultChatModel(AiProvider.MISTRAL_AI).getName()); + defaults.put(AI_GEMINI_CHAT_MODEL, AiProviderDefaultChatModels.getDefaultChatModel(AiProvider.GEMINI).getName()); + defaults.put(AI_HUGGING_FACE_CHAT_MODEL, AiProviderDefaultChatModels.getDefaultChatModel(AiProvider.HUGGING_FACE).getName()); + defaults.put(AI_GPT_4_ALL_MODEL, AiProviderDefaultChatModels.getDefaultChatModel(AiProvider.GPT4ALL).getName()); + defaults.put(AI_CUSTOMIZE_SETTINGS, false); + defaults.put(AI_EMBEDDING_MODEL, EmbeddingModel.SENTENCE_TRANSFORMERS_ALL_MINILM_L12_V2.name()); defaults.put(AI_OPEN_AI_API_BASE_URL, AiProvider.OPEN_AI.getApiUrl()); defaults.put(AI_MISTRAL_AI_API_BASE_URL, AiProvider.MISTRAL_AI.getApiUrl()); defaults.put(AI_GEMINI_API_BASE_URL, AiProvider.GEMINI.getApiUrl()); defaults.put(AI_HUGGING_FACE_API_BASE_URL, AiProvider.HUGGING_FACE.getApiUrl()); defaults.put(AI_GPT_4_ALL_API_BASE_URL, AiProvider.GPT4ALL.getApiUrl()); - defaults.put(AI_SYSTEM_MESSAGE, AiDefaultPreferences.SYSTEM_MESSAGE); - defaults.put(AI_TEMPERATURE, AiDefaultPreferences.TEMPERATURE); - defaults.put(AI_CONTEXT_WINDOW_SIZE, AiDefaultPreferences.getContextWindowSize(AiDefaultPreferences.PROVIDER, AiDefaultPreferences.CHAT_MODELS.get(AiDefaultPreferences.PROVIDER).getName())); - defaults.put(AI_DOCUMENT_SPLITTER_CHUNK_SIZE, AiDefaultPreferences.DOCUMENT_SPLITTER_CHUNK_SIZE); - defaults.put(AI_DOCUMENT_SPLITTER_OVERLAP_SIZE, AiDefaultPreferences.DOCUMENT_SPLITTER_OVERLAP); - defaults.put(AI_RAG_MAX_RESULTS_COUNT, AiDefaultPreferences.RAG_MAX_RESULTS_COUNT); - defaults.put(AI_RAG_MIN_SCORE, AiDefaultPreferences.RAG_MIN_SCORE); + defaults.put(AI_TEMPERATURE, 0.7); + defaults.put(AI_CONTEXT_WINDOW_SIZE, PredefinedChatModel.getContextWindowSize((AiProvider) defaults.get(AI_PROVIDER), AiProviderDefaultChatModels.getDefaultChatModel((AiProvider) defaults.get(AI_PROVIDER)).getName())); + defaults.put(AI_DOCUMENT_SPLITTER_CHUNK_SIZE, 300); + defaults.put(AI_DOCUMENT_SPLITTER_OVERLAP_SIZE, 100); + defaults.put(AI_RAG_MAX_RESULTS_COUNT, 10); + defaults.put(AI_RAG_MIN_SCORE, 0.3); // region:AI templates - defaults.put(AI_CHATTING_SYSTEM_MESSAGE_TEMPLATE, AiDefaultPreferences.TEMPLATES.get(AiTemplate.CHATTING_SYSTEM_MESSAGE)); - defaults.put(AI_CHATTING_USER_MESSAGE_TEMPLATE, AiDefaultPreferences.TEMPLATES.get(AiTemplate.CHATTING_USER_MESSAGE)); - defaults.put(AI_SUMMARIZATION_CHUNK_SYSTEM_MESSAGE_TEMPLATE, AiDefaultPreferences.TEMPLATES.get(AiTemplate.SUMMARIZATION_CHUNK_SYSTEM_MESSAGE)); - defaults.put(AI_SUMMARIZATION_CHUNK_USER_MESSAGE_TEMPLATE, AiDefaultPreferences.TEMPLATES.get(AiTemplate.SUMMARIZATION_CHUNK_USER_MESSAGE)); - defaults.put(AI_SUMMARIZATION_COMBINE_SYSTEM_MESSAGE_TEMPLATE, AiDefaultPreferences.TEMPLATES.get(AiTemplate.SUMMARIZATION_COMBINE_SYSTEM_MESSAGE)); - defaults.put(AI_SUMMARIZATION_COMBINE_USER_MESSAGE_TEMPLATE, AiDefaultPreferences.TEMPLATES.get(AiTemplate.SUMMARIZATION_COMBINE_USER_MESSAGE)); - defaults.put(AI_CITATION_PARSING_SYSTEM_MESSAGE_TEMPLATE, AiDefaultPreferences.TEMPLATES.get(AiTemplate.CITATION_PARSING_SYSTEM_MESSAGE)); - defaults.put(AI_CITATION_PARSING_USER_MESSAGE_TEMPLATE, AiDefaultPreferences.TEMPLATES.get(AiTemplate.CITATION_PARSING_USER_MESSAGE)); + defaults.put(AI_CHATTING_SYSTEM_MESSAGE_TEMPLATE, AiDefaultTemplates.getTemplate(AiTemplate.CHATTING_SYSTEM_MESSAGE)); + defaults.put(AI_CHATTING_USER_MESSAGE_TEMPLATE, AiDefaultTemplates.getTemplate(AiTemplate.CHATTING_USER_MESSAGE)); + defaults.put(AI_SUMMARIZATION_CHUNK_SYSTEM_MESSAGE_TEMPLATE, AiDefaultTemplates.getTemplate(AiTemplate.SUMMARIZATION_CHUNK_SYSTEM_MESSAGE)); + defaults.put(AI_SUMMARIZATION_CHUNK_USER_MESSAGE_TEMPLATE, AiDefaultTemplates.getTemplate(AiTemplate.SUMMARIZATION_CHUNK_USER_MESSAGE)); + defaults.put(AI_SUMMARIZATION_COMBINE_SYSTEM_MESSAGE_TEMPLATE, AiDefaultTemplates.getTemplate(AiTemplate.SUMMARIZATION_COMBINE_SYSTEM_MESSAGE)); + defaults.put(AI_SUMMARIZATION_COMBINE_USER_MESSAGE_TEMPLATE, AiDefaultTemplates.getTemplate(AiTemplate.SUMMARIZATION_COMBINE_USER_MESSAGE)); + defaults.put(AI_CITATION_PARSING_SYSTEM_MESSAGE_TEMPLATE, AiDefaultTemplates.getTemplate(AiTemplate.CITATION_PARSING_SYSTEM_MESSAGE)); + defaults.put(AI_CITATION_PARSING_USER_MESSAGE_TEMPLATE, AiDefaultTemplates.getTemplate(AiTemplate.CITATION_PARSING_USER_MESSAGE)); // endregion // endregion From 0289b0afad5e728836f1582c2c0fc36fdeaf2828 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Mon, 24 Nov 2025 21:19:25 +0100 Subject: [PATCH 018/243] Refactor constructors --- .../ai/components/aichat/AiChatComponent.java | 10 +++---- jablib/src/main/java/module-info.java | 8 +++++- .../java/org/jabref/logic/ai/AiService.java | 23 +++++----------- .../logic/ai/chatting/ChatHistoryService.java | 5 +++- .../ai/chatting/algorithms/AiChatLogic.java | 19 +++++++------- .../MVStoreChatHistoryRepository.java | 2 +- .../tasks/GenerateAiResponseTask.java | 2 +- .../DeepJavaEmbeddingModel.java | 4 ++- .../MVStoreEmbeddingStore.java | 2 +- .../rag/CurrentlySelectedEmbeddingModel.java | 6 ++++- .../jabref/logic/ai/rag/IngestionService.java | 24 ++++++++--------- .../ai/rag/algorithms/LowLevelIngestor.java | 6 ++++- .../ai/rag/algorithms/parsing/FileParser.java | 2 +- .../rag/algorithms/parsing/PdfFileParser.java | 2 +- .../parsing/UniversalFileParser.java | 2 +- .../repositories/FileEmbeddingsManager.java | 13 +++++----- .../FullyIngestedDocumentsRepository.java | 6 +---- ...StoreFullyIngestedDocumentsRepository.java | 5 +++- .../GenerateEmbeddingsForSeveralTask.java | 13 ++++------ .../ai/rag/tasks/GenerateEmbeddingsTask.java | 13 +++++----- .../rag/tasks/UpdateEmbeddingModelTask.java | 5 +++- .../ai/summarization/SummariesService.java | 19 +++++++------- .../algorithms/BibEntrySummarizer.java | 2 +- .../algorithms/SummarizationAlgorithm.java | 6 ++++- .../MVStoreSummariesRepository.java | 2 +- .../tasks/GenerateSummaryForSeveralTask.java | 26 +++++++------------ .../tasks/GenerateSummaryTask.java | 17 ++++++------ .../MVStoreChatHistoryRepositoryTest.java | 2 +- ...eFullyIngestedDocumentsRepositoryTest.java | 2 +- .../MVStoreSummariesRepositoryTest.java | 2 +- 30 files changed, 128 insertions(+), 122 deletions(-) diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java index 714896eaae9b..9acd77576829 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java @@ -100,14 +100,10 @@ public AiChatComponent(AiService aiService, this.aiChatLogic = new AiChatLogic( aiPreferences, - aiService.getChatLanguageModel(), + aiService.getTemplatesService(), aiService.getChatLanguageModel(), aiService.getEmbeddingModel(), aiService.getEmbeddingStore(), - aiService.getTemplatesService(), - name, - chatHistory, - entries, - bibDatabaseContext + bibDatabaseContext, chatHistory, entries, name ); aiService.getIngestionService().ingest(name, ListUtil.getLinkedFiles(entries).toList(), bibDatabaseContext); @@ -293,7 +289,7 @@ private void onSendMessage(String userPrompt) { updatePromptHistory(); setLoading(true); - BackgroundTask task = new GenerateAiResponseTask(userMessage, aiChatLogic) + BackgroundTask task = new GenerateAiResponseTask(aiChatLogic, userMessage) .onSuccess(aiMessage -> { setLoading(false); chatPrompt.requestPromptFocus(); diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index 45ab4730c665..7fa4c78c2ec2 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -117,7 +117,11 @@ exports org.jabref.logic.git.merge.planning; exports org.jabref.logic.git.merge.execution; exports org.jabref.model.sciteTallies; + + // region: AI exports org.jabref.logic.ai.customimplementations.embeddingmodels; + exports org.jabref.logic.ai.customimplementations.embeddingstores; + exports org.jabref.logic.ai.customimplementations.llms; exports org.jabref.logic.ai.rag; exports org.jabref.logic.ai.summarization.tasks; exports org.jabref.logic.ai.summarization.repositories; @@ -127,12 +131,14 @@ exports org.jabref.logic.ai.chatting.algorithms; exports org.jabref.logic.ai.chatting.tasks; exports org.jabref.logic.ai.preferences; + exports org.jabref.logic.ai.chatting.repositories; exports org.jabref.model.ai.chatting; exports org.jabref.model.ai.templating; exports org.jabref.model.ai.rag; exports org.jabref.model.ai.embeddings; exports org.jabref.model.ai.summarization; - exports org.jabref.logic.ai.chatting.repositories; + exports org.jabref.model.ai.identifiers; + // endregion requires java.base; diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiService.java b/jablib/src/main/java/org/jabref/logic/ai/AiService.java index 7184d4a3bf30..db654d4811fb 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiService.java @@ -10,10 +10,10 @@ import org.jabref.logic.ai.chatting.ChatHistoryService; import org.jabref.logic.ai.chatting.CurrentlySelectedChatLanguageModel; import org.jabref.logic.ai.chatting.repositories.MVStoreChatHistoryRepository; +import org.jabref.logic.ai.customimplementations.embeddingstores.MVStoreEmbeddingStore; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.rag.CurrentlySelectedEmbeddingModel; import org.jabref.logic.ai.rag.IngestionService; -import org.jabref.logic.ai.rag.repositories.MVStoreEmbeddingStore; import org.jabref.logic.ai.rag.repositories.MVStoreFullyIngestedDocumentsRepository; import org.jabref.logic.ai.summarization.SummariesService; import org.jabref.logic.ai.summarization.repositories.MVStoreSummariesRepository; @@ -69,10 +69,10 @@ public AiService(AiPreferences aiPreferences, TaskExecutor taskExecutor ) { - this.mvStoreChatHistoryStorage = new MVStoreChatHistoryRepository(Directories.getAiFilesDirectory().resolve(CHAT_HISTORY_FILE_NAME), notificationService); + this.mvStoreChatHistoryStorage = new MVStoreChatHistoryRepository(notificationService, Directories.getAiFilesDirectory().resolve(CHAT_HISTORY_FILE_NAME)); this.mvStoreEmbeddingStore = new MVStoreEmbeddingStore(Directories.getAiFilesDirectory().resolve(EMBEDDINGS_FILE_NAME), notificationService); - this.mvStoreFullyIngestedDocumentsTracker = new MVStoreFullyIngestedDocumentsRepository(Directories.getAiFilesDirectory().resolve(FULLY_INGESTED_FILE_NAME), notificationService); - this.mvStoreSummariesStorage = new MVStoreSummariesRepository(Directories.getAiFilesDirectory().resolve(SUMMARIES_FILE_NAME), notificationService); + this.mvStoreFullyIngestedDocumentsTracker = new MVStoreFullyIngestedDocumentsRepository(notificationService, Directories.getAiFilesDirectory().resolve(FULLY_INGESTED_FILE_NAME)); + this.mvStoreSummariesStorage = new MVStoreSummariesRepository(notificationService, Directories.getAiFilesDirectory().resolve(SUMMARIES_FILE_NAME)); this.templatesService = new AiTemplatesService(aiPreferences); this.chatHistoryService = new ChatHistoryService(citationKeyPatternPreferences, mvStoreChatHistoryStorage); @@ -81,22 +81,13 @@ public AiService(AiPreferences aiPreferences, this.ingestionService = new IngestionService( aiPreferences, - shutdownSignal, - currentlySelectedEmbeddingModel, - mvStoreEmbeddingStore, - mvStoreFullyIngestedDocumentsTracker, - filePreferences, - taskExecutor + filePreferences, taskExecutor, currentlySelectedEmbeddingModel, mvStoreEmbeddingStore, mvStoreFullyIngestedDocumentsTracker, shutdownSignal ); this.summariesService = new SummariesService( aiPreferences, - mvStoreSummariesStorage, - currentlySelectedChatLanguageModel, - templatesService, - shutdownSignal, - filePreferences, - taskExecutor + filePreferences, templatesService, taskExecutor, currentlySelectedChatLanguageModel, mvStoreSummariesStorage, + shutdownSignal ); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java index b76a2c4a9bdd..cd826b0b1c15 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java @@ -67,7 +67,10 @@ private record ChatHistoryManagementRecord(Optional bibDatab // We use {@link TreeMap} for group chat history for the same reason as for {@link BibEntry}ies. private final TreeMap groupsChatHistory = new TreeMap<>(Comparator.comparing(GroupTreeNode::getName)); - public ChatHistoryService(CitationKeyPatternPreferences citationKeyPatternPreferences, ChatHistoryRepository implementation) { + public ChatHistoryService( + CitationKeyPatternPreferences citationKeyPatternPreferences, + ChatHistoryRepository implementation + ) { this.citationKeyPatternPreferences = citationKeyPatternPreferences; this.implementation = implementation; } diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/algorithms/AiChatLogic.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/algorithms/AiChatLogic.java index ffba79de204a..25844920a9e1 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/algorithms/AiChatLogic.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/algorithms/AiChatLogic.java @@ -56,15 +56,16 @@ public class AiChatLogic { private Optional filter = Optional.empty(); - public AiChatLogic(AiPreferences aiPreferences, - ChatModel chatLanguageModel, - EmbeddingModel embeddingModel, - EmbeddingStore embeddingStore, - AiTemplatesService aiTemplatesService, - StringProperty name, - ObservableList chatHistory, - ObservableList entries, - BibDatabaseContext bibDatabaseContext + public AiChatLogic( + AiPreferences aiPreferences, + AiTemplatesService aiTemplatesService, + ChatModel chatLanguageModel, + EmbeddingModel embeddingModel, + EmbeddingStore embeddingStore, + BibDatabaseContext bibDatabaseContext, + ObservableList chatHistory, + ObservableList entries, + StringProperty name ) { this.aiPreferences = aiPreferences; this.chatLanguageModel = chatLanguageModel; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepository.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepository.java index b16c478745bb..2230835d5904 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepository.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepository.java @@ -66,7 +66,7 @@ public ChatMessage toLangchainMessage() { } } - public MVStoreChatHistoryRepository(Path path, NotificationService dialogService) { + public MVStoreChatHistoryRepository(NotificationService dialogService, Path path) { super(path, dialogService); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/tasks/GenerateAiResponseTask.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/tasks/GenerateAiResponseTask.java index 8d4809ea7e50..d8591ef550e1 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/tasks/GenerateAiResponseTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/tasks/GenerateAiResponseTask.java @@ -11,7 +11,7 @@ public class GenerateAiResponseTask extends BackgroundTask { private final UserMessage userMessage; private final AiChatLogic aiChatLogic; - public GenerateAiResponseTask(UserMessage userMessage, AiChatLogic aiChatLogic) { + public GenerateAiResponseTask(AiChatLogic aiChatLogic, UserMessage userMessage) { this.userMessage = userMessage; this.aiChatLogic = aiChatLogic; diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingmodels/DeepJavaEmbeddingModel.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingmodels/DeepJavaEmbeddingModel.java index b8a4e457967b..e6556eb72279 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingmodels/DeepJavaEmbeddingModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingmodels/DeepJavaEmbeddingModel.java @@ -19,7 +19,9 @@ public class DeepJavaEmbeddingModel implements EmbeddingModel, AutoCloseable { private final ZooModel model; private final Predictor predictor; - public DeepJavaEmbeddingModel(Criteria criteria) throws ModelNotFoundException, MalformedModelException, IOException { + public DeepJavaEmbeddingModel( + Criteria criteria + ) throws ModelNotFoundException, MalformedModelException, IOException { this.model = criteria.loadModel(); this.predictor = model.newPredictor(); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingstores/MVStoreEmbeddingStore.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingstores/MVStoreEmbeddingStore.java index c4cb1835d468..0b1f919611e1 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingstores/MVStoreEmbeddingStore.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingstores/MVStoreEmbeddingStore.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.rag.repositories; +package org.jabref.logic.ai.customimplementations.embeddingstores; import java.io.Serializable; import java.nio.file.Path; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/CurrentlySelectedEmbeddingModel.java b/jablib/src/main/java/org/jabref/logic/ai/rag/CurrentlySelectedEmbeddingModel.java index 924a2767601b..cc126a6c34a6 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/CurrentlySelectedEmbeddingModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/CurrentlySelectedEmbeddingModel.java @@ -54,7 +54,11 @@ public static class EmbeddingModelBuildingErrorEvent { // Empty if there is no error. private String errorWhileBuildingModel = ""; - public CurrentlySelectedEmbeddingModel(AiPreferences aiPreferences, NotificationService notificationService, TaskExecutor taskExecutor) { + public CurrentlySelectedEmbeddingModel( + AiPreferences aiPreferences, + NotificationService notificationService, + TaskExecutor taskExecutor + ) { this.aiPreferences = aiPreferences; this.notificationService = notificationService; this.taskExecutor = taskExecutor; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java b/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java index 957db16fb80c..4e1b493646b7 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java @@ -46,13 +46,14 @@ public class IngestionService { private final ReadOnlyBooleanProperty shutdownSignal; - public IngestionService(AiPreferences aiPreferences, - ReadOnlyBooleanProperty shutdownSignal, - EmbeddingModel embeddingModel, - EmbeddingStore embeddingStore, - FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository, - FilePreferences filePreferences, - TaskExecutor taskExecutor + public IngestionService( + AiPreferences aiPreferences, + FilePreferences filePreferences, + TaskExecutor taskExecutor, + EmbeddingModel embeddingModel, + EmbeddingStore embeddingStore, + FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository, + ReadOnlyBooleanProperty shutdownSignal ) { this.aiPreferences = aiPreferences; this.filePreferences = filePreferences; @@ -60,10 +61,7 @@ public IngestionService(AiPreferences aiPreferences, this.fileEmbeddingsManager = new FileEmbeddingsManager( aiPreferences, - shutdownSignal, - embeddingModel, - embeddingStore, - fullyIngestedDocumentsRepository + embeddingModel, embeddingStore, fullyIngestedDocumentsRepository, shutdownSignal ); this.shutdownSignal = shutdownSignal; @@ -146,7 +144,7 @@ public List> ingest(StringProperty groupName, L private void startEmbeddingsGenerationTask(LinkedFile linkedFile, BibDatabaseContext bibDatabaseContext, ProcessingInfo processingInfo) { processingInfo.setState(ProcessingState.PROCESSING); - new GenerateEmbeddingsTask(linkedFile, fileEmbeddingsManager, bibDatabaseContext, filePreferences, shutdownSignal) + new GenerateEmbeddingsTask(filePreferences, fileEmbeddingsManager, bibDatabaseContext, linkedFile, shutdownSignal) .showToUser(true) .onSuccess(v -> processingInfo.setState(ProcessingState.SUCCESS)) .onFailure(processingInfo::setException) @@ -156,7 +154,7 @@ private void startEmbeddingsGenerationTask(LinkedFile linkedFile, BibDatabaseCon private void startEmbeddingsGenerationTask(StringProperty groupName, List> linkedFiles, BibDatabaseContext bibDatabaseContext) { linkedFiles.forEach(processingInfo -> processingInfo.setState(ProcessingState.PROCESSING)); - new GenerateEmbeddingsForSeveralTask(groupName, linkedFiles, fileEmbeddingsManager, bibDatabaseContext, filePreferences, taskExecutor, shutdownSignal) + new GenerateEmbeddingsForSeveralTask(filePreferences, taskExecutor, fileEmbeddingsManager, bibDatabaseContext, groupName, linkedFiles, shutdownSignal) .executeWith(taskExecutor); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/LowLevelIngestor.java b/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/LowLevelIngestor.java index 0d783a4c4e1c..5b28fd5a14b9 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/LowLevelIngestor.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/LowLevelIngestor.java @@ -25,7 +25,11 @@ public class LowLevelIngestor { private EmbeddingStoreIngestor ingestor; private DocumentSplitter documentSplitter; - public LowLevelIngestor(AiPreferences aiPreferences, EmbeddingStore embeddingStore, EmbeddingModel embeddingModel) { + public LowLevelIngestor( + AiPreferences aiPreferences, + EmbeddingModel embeddingModel, + EmbeddingStore embeddingStore + ) { this.aiPreferences = aiPreferences; this.embeddingStore = embeddingStore; this.embeddingModel = embeddingModel; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/FileParser.java b/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/FileParser.java index eade1e5428cf..d1748614ed4c 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/FileParser.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/FileParser.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.rag.algorithms; +package org.jabref.logic.ai.rag.algorithms.parsing; import java.nio.file.Path; import java.util.Optional; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/PdfFileParser.java b/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/PdfFileParser.java index 689990eed5fd..c8139c7e05e1 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/PdfFileParser.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/PdfFileParser.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.rag.algorithms; +package org.jabref.logic.ai.rag.algorithms.parsing; import java.io.IOException; import java.io.StringWriter; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/UniversalFileParser.java b/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/UniversalFileParser.java index b78ce5339fd4..2e847e8792b1 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/UniversalFileParser.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/UniversalFileParser.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.rag.algorithms; +package org.jabref.logic.ai.rag.algorithms.parsing; import java.nio.file.Path; import java.util.Optional; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FileEmbeddingsManager.java b/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FileEmbeddingsManager.java index 201f02b6e70b..023731c43738 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FileEmbeddingsManager.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FileEmbeddingsManager.java @@ -39,17 +39,18 @@ public class FileEmbeddingsManager { private final FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository; private final LowLevelIngestor lowLevelIngestor; - public FileEmbeddingsManager(AiPreferences aiPreferences, - ReadOnlyBooleanProperty shutdownSignal, - EmbeddingModel embeddingModel, - EmbeddingStore embeddingStore, - FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository + public FileEmbeddingsManager( + AiPreferences aiPreferences, + EmbeddingModel embeddingModel, + EmbeddingStore embeddingStore, + FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository, + ReadOnlyBooleanProperty shutdownSignal ) { this.aiPreferences = aiPreferences; this.shutdownSignal = shutdownSignal; this.embeddingStore = embeddingStore; this.fullyIngestedDocumentsRepository = fullyIngestedDocumentsRepository; - this.lowLevelIngestor = new LowLevelIngestor(aiPreferences, embeddingStore, embeddingModel); + this.lowLevelIngestor = new LowLevelIngestor(aiPreferences, embeddingModel, embeddingStore); setupListeningToPreferencesChanges(); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FullyIngestedDocumentsRepository.java b/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FullyIngestedDocumentsRepository.java index d9fa889f0cc9..bd0423a5ea5d 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FullyIngestedDocumentsRepository.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FullyIngestedDocumentsRepository.java @@ -7,14 +7,10 @@ *

* The class also records the document modification time. */ -public interface FullyIngestedDocumentsRepository { +public interface FullyIngestedDocumentsRepository extends AutoCloseable{ void markDocumentAsFullyIngested(String link, long modificationTimeInSeconds); Optional getIngestedDocumentModificationTimeInSeconds(String link); void unmarkDocumentAsFullyIngested(String link); - - void commit(); - - void close(); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/MVStoreFullyIngestedDocumentsRepository.java b/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/MVStoreFullyIngestedDocumentsRepository.java index 810bb3fd3cb3..ced1d3c9253e 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/MVStoreFullyIngestedDocumentsRepository.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/MVStoreFullyIngestedDocumentsRepository.java @@ -23,7 +23,10 @@ public class MVStoreFullyIngestedDocumentsRepository extends MVStoreBase impleme // it doesn't mean the document is fully ingested. private final Map ingestedMap; - public MVStoreFullyIngestedDocumentsRepository(Path path, NotificationService dialogService) { + public MVStoreFullyIngestedDocumentsRepository( + NotificationService dialogService, + Path path + ) { super(path, dialogService); this.ingestedMap = this.mvStore.openMap(INGESTED_MAP_NAME); diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsForSeveralTask.java b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsForSeveralTask.java index a767aedbd83f..41e15d79283f 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsForSeveralTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsForSeveralTask.java @@ -44,12 +44,12 @@ public class GenerateEmbeddingsForSeveralTask extends BackgroundTask { private String currentFile = ""; public GenerateEmbeddingsForSeveralTask( - StringProperty groupName, - List> linkedFiles, - FileEmbeddingsManager fileEmbeddingsManager, - BibDatabaseContext bibDatabaseContext, FilePreferences filePreferences, TaskExecutor taskExecutor, + FileEmbeddingsManager fileEmbeddingsManager, + BibDatabaseContext bibDatabaseContext, + StringProperty groupName, + List> linkedFiles, ReadOnlyBooleanProperty shutdownSignal ) { this.groupName = groupName; @@ -85,10 +85,7 @@ public Void call() throws ExecutionException, InterruptedException { processingInfo.setState(ProcessingState.PROCESSING); return new Pair<>( new GenerateEmbeddingsTask( - processingInfo.getObject(), - fileEmbeddingsManager, - bibDatabaseContext, - filePreferences, + filePreferences, fileEmbeddingsManager, bibDatabaseContext, processingInfo.getObject(), shutdownSignal ) .showToUser(false) diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java index f110dcc39788..e9e51c0b833c 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java @@ -10,7 +10,7 @@ import javafx.beans.property.ReadOnlyBooleanProperty; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.rag.algorithms.UniversalFileParser; +import org.jabref.logic.ai.rag.algorithms.parsing.UniversalFileParser; import org.jabref.logic.ai.rag.repositories.FileEmbeddingsManager; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; @@ -38,11 +38,12 @@ public class GenerateEmbeddingsTask extends BackgroundTask { private final ProgressCounter progressCounter = new ProgressCounter(); - public GenerateEmbeddingsTask(LinkedFile linkedFile, - FileEmbeddingsManager fileEmbeddingsManager, - BibDatabaseContext bibDatabaseContext, - FilePreferences filePreferences, - ReadOnlyBooleanProperty shutdownSignal + public GenerateEmbeddingsTask( + FilePreferences filePreferences, + FileEmbeddingsManager fileEmbeddingsManager, + BibDatabaseContext bibDatabaseContext, + LinkedFile linkedFile, + ReadOnlyBooleanProperty shutdownSignal ) { this.linkedFile = linkedFile; this.fileEmbeddingsManager = fileEmbeddingsManager; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/UpdateEmbeddingModelTask.java b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/UpdateEmbeddingModelTask.java index fcd6e3102d02..be3c7519ae44 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/UpdateEmbeddingModelTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/UpdateEmbeddingModelTask.java @@ -28,7 +28,10 @@ public class UpdateEmbeddingModelTask extends BackgroundTask { private final ProgressCounter progressCounter = new ProgressCounter(); - public UpdateEmbeddingModelTask(AiPreferences aiPreferences, ObjectProperty> predictorProperty) { + public UpdateEmbeddingModelTask( + AiPreferences aiPreferences, + ObjectProperty> predictorProperty + ) { this.aiPreferences = aiPreferences; this.predictorProperty = predictorProperty; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java index 26b66873c224..7e297932883a 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java @@ -54,13 +54,14 @@ public class SummariesService { private final FilePreferences filePreferences; private final TaskExecutor taskExecutor; - public SummariesService(AiPreferences aiPreferences, - SummariesRepository summariesRepository, - ChatModel chatLanguageModel, - AiTemplatesService aiTemplatesService, - BooleanProperty shutdownSignal, - FilePreferences filePreferences, - TaskExecutor taskExecutor + public SummariesService( + AiPreferences aiPreferences, + FilePreferences filePreferences, + AiTemplatesService aiTemplatesService, + TaskExecutor taskExecutor, + ChatModel chatLanguageModel, + SummariesRepository summariesRepository, + BooleanProperty shutdownSignal ) { this.aiPreferences = aiPreferences; this.summariesRepository = summariesRepository; @@ -140,7 +141,7 @@ public void summarize(StringProperty groupName, List entries, BibDatab private void startSummarizationTask(BibEntry entry, BibDatabaseContext bibDatabaseContext, ProcessingInfo processingInfo) { processingInfo.setState(ProcessingState.PROCESSING); - new GenerateSummaryTask(entry, bibDatabaseContext, summariesRepository, chatLanguageModel, aiTemplatesService, shutdownSignal, aiPreferences, filePreferences) + new GenerateSummaryTask(aiPreferences, filePreferences, aiTemplatesService, chatLanguageModel, summariesRepository, bibDatabaseContext, entry, shutdownSignal) .onSuccess(processingInfo::setSuccess) .onFailure(processingInfo::setException) .executeWith(taskExecutor); @@ -149,7 +150,7 @@ private void startSummarizationTask(BibEntry entry, BibDatabaseContext bibDataba private void startSummarizationTask(StringProperty groupName, List> entries, BibDatabaseContext bibDatabaseContext) { entries.forEach(processingInfo -> processingInfo.setState(ProcessingState.PROCESSING)); - new GenerateSummaryForSeveralTask(groupName, entries, bibDatabaseContext, summariesRepository, chatLanguageModel, aiTemplatesService, shutdownSignal, aiPreferences, filePreferences, taskExecutor) + new GenerateSummaryForSeveralTask(aiPreferences, filePreferences, aiTemplatesService, taskExecutor, chatLanguageModel, summariesRepository, bibDatabaseContext, groupName, entries, shutdownSignal) .executeWith(taskExecutor); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/BibEntrySummarizer.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/BibEntrySummarizer.java index 6a81c2f92977..2c7ebcaa37f3 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/BibEntrySummarizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/BibEntrySummarizer.java @@ -12,7 +12,7 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.rag.algorithms.UniversalFileParser; +import org.jabref.logic.ai.rag.algorithms.parsing.UniversalFileParser; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.ProgressCounter; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/SummarizationAlgorithm.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/SummarizationAlgorithm.java index cb34d2ea8ab6..95fed2aab2e6 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/SummarizationAlgorithm.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/SummarizationAlgorithm.java @@ -5,5 +5,9 @@ import org.jabref.logic.util.ProgressCounter; public interface SummarizationAlgorithm { - String summarize(String text, ProgressCounter progressCounter, ReadOnlyBooleanProperty shutdownSignal) throws InterruptedException; + String summarize( + String text, + ProgressCounter progressCounter, + ReadOnlyBooleanProperty shutdownSignal + ) throws InterruptedException; } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/repositories/MVStoreSummariesRepository.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/repositories/MVStoreSummariesRepository.java index b9b051aee569..dffef2fac417 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/repositories/MVStoreSummariesRepository.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/repositories/MVStoreSummariesRepository.java @@ -12,7 +12,7 @@ public class MVStoreSummariesRepository extends MVStoreBase implements SummariesRepository { private static final String SUMMARIES_MAP_PREFIX = "summaries"; - public MVStoreSummariesRepository(Path path, NotificationService dialogService) { + public MVStoreSummariesRepository(NotificationService dialogService, Path path) { super(path, dialogService); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java index 10a6b7f7f67f..ced909b3333b 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java @@ -51,16 +51,16 @@ public class GenerateSummaryForSeveralTask extends BackgroundTask { private String currentFile = ""; public GenerateSummaryForSeveralTask( - StringProperty groupName, - List> entries, - BibDatabaseContext bibDatabaseContext, - SummariesRepository summariesRepository, - ChatModel chatLanguageModel, - AiTemplatesService aiTemplatesService, - ReadOnlyBooleanProperty shutdownSignal, AiPreferences aiPreferences, FilePreferences filePreferences, - TaskExecutor taskExecutor + AiTemplatesService aiTemplatesService, + TaskExecutor taskExecutor, + ChatModel chatLanguageModel, + SummariesRepository summariesRepository, + BibDatabaseContext bibDatabaseContext, + StringProperty groupName, + List> entries, + ReadOnlyBooleanProperty shutdownSignal ) { this.groupName = groupName; this.entries = entries; @@ -98,14 +98,8 @@ public Void call() throws ExecutionException, InterruptedException { processingInfo.setState(ProcessingState.PROCESSING); return new Pair<>( new GenerateSummaryTask( - processingInfo.getObject(), - bibDatabaseContext, - summariesRepository, - chatLanguageModel, - aiTemplatesService, - shutdownSignal, - aiPreferences, - filePreferences + aiPreferences, filePreferences, aiTemplatesService, chatLanguageModel, summariesRepository, bibDatabaseContext, processingInfo.getObject(), + shutdownSignal ) .showToUser(false) .onSuccess(processingInfo::setSuccess) diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java index 6226f44b912d..0598dc0a406f 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java @@ -38,14 +38,15 @@ public class GenerateSummaryTask extends BackgroundTask

{ private final ProgressCounter progressCounter = new ProgressCounter(); - public GenerateSummaryTask(BibEntry entry, - BibDatabaseContext bibDatabaseContext, - SummariesRepository summariesRepository, - ChatModel chatLanguageModel, - AiTemplatesService aiTemplatesService, - ReadOnlyBooleanProperty shutdownSignal, - AiPreferences aiPreferences, - FilePreferences filePreferences + public GenerateSummaryTask( + AiPreferences aiPreferences, + FilePreferences filePreferences, + AiTemplatesService aiTemplatesService, + ChatModel chatLanguageModel, + SummariesRepository summariesRepository, + BibDatabaseContext bibDatabaseContext, + BibEntry entry, + ReadOnlyBooleanProperty shutdownSignal ) { this.bibDatabaseContext = bibDatabaseContext; this.entry = entry; diff --git a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryTest.java b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryTest.java index 149e55c5d50b..ad77ee96b1a6 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryTest.java @@ -11,7 +11,7 @@ class MVStoreChatHistoryRepositoryTest extends ChatHistoryRepositoryTest { @Override ChatHistoryRepository makeStorage(Path path) { - return new MVStoreChatHistoryRepository(path, mock(NotificationService.class)); + return new MVStoreChatHistoryRepository(mock(NotificationService.class), path); } @Override diff --git a/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsRepositoryTest.java b/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsRepositoryTest.java index 76a657b012e0..f0ccb775d4d7 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsRepositoryTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsRepositoryTest.java @@ -11,7 +11,7 @@ class MVStoreFullyIngestedDocumentsRepositoryTest extends FullyIngestedDocumentsRepositoryTest { @Override FullyIngestedDocumentsRepository makeTracker(Path path) { - return new MVStoreFullyIngestedDocumentsRepository(path, mock(NotificationService.class)); + return new MVStoreFullyIngestedDocumentsRepository(mock(NotificationService.class), path); } @Override diff --git a/jablib/src/test/java/org/jabref/logic/ai/summarization/MVStoreSummariesRepositoryTest.java b/jablib/src/test/java/org/jabref/logic/ai/summarization/MVStoreSummariesRepositoryTest.java index 0bf73edcacde..02c21b55a332 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/summarization/MVStoreSummariesRepositoryTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/summarization/MVStoreSummariesRepositoryTest.java @@ -11,7 +11,7 @@ class MVStoreSummariesRepositoryTest extends SummariesRepositoryTest { @Override SummariesRepository makeSummariesStorage(Path path) { - return new MVStoreSummariesRepository(path, mock(NotificationService.class)); + return new MVStoreSummariesRepository(mock(NotificationService.class), path); } @Override From 353b6cdc904ad4b294f0fa0ace7006df845f2abb Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Tue, 25 Nov 2025 22:36:31 +0100 Subject: [PATCH 019/243] Summarization overhaul --- .../components/summary/SummaryComponent.java | 16 +-- .../summary/SummaryShowingComponent.java | 14 +-- .../summary/SummaryShowingComponent.fxml | 2 +- jablib/src/main/java/module-info.java | 2 + .../java/org/jabref/logic/ai/AiService.java | 24 +++-- .../CurrentlySelectedChatLanguageModel.java | 11 ++ .../ai/preferences/AiDefaultTemplates.java | 2 +- .../logic/ai/preferences/AiPreferences.java | 63 +++++++---- ...rrentlySelectedSummarizationAlgorithm.java | 101 ++++++++++++++++++ .../ai/summarization/SummariesService.java | 101 +++++++++++++----- .../algorithms/BibEntrySummarizer.java | 86 +++++++++------ .../ChunkedSummarizationAlgorithm.java | 82 +++++++------- .../PersistentBibEntrySummarizer.java | 59 +++++----- .../algorithms/SummarizationAlgorithm.java | 14 +-- .../MVStoreSummariesRepository.java | 10 +- .../repositories/SummariesRepository.java | 6 +- .../tasks/GenerateSummaryForSeveralTask.java | 33 +++--- .../tasks/GenerateSummaryTask.java | 37 ++++--- ...mmarizationChunkSystemMessageTemplate.java | 22 ++++ ...SummarizationChunkUserMessageTemplate.java | 25 +++++ ...arizationCombineSystemMessageTemplate.java | 22 ++++ ...mmarizationCombineUserMessageTemplate.java | 27 +++++ .../ai/templates/AiTemplatesService.java | 30 +----- .../jabref/logic/ai/templates/Template.java | 42 ++++++++ .../jabref/logic/ai/util/LongTaskInfo.java | 8 ++ .../model/ai/chatting/ChatModelInfo.java | 6 ++ .../ai/summarization/BibEntrySummary.java | 15 +++ .../SummarizationAlgorithmName.java | 17 +++ .../model/ai/summarization/Summary.java | 9 -- .../SummariesRepositoryTest.java | 8 +- 30 files changed, 644 insertions(+), 250 deletions(-) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/CurrentlySelectedSummarizationAlgorithm.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationChunkSystemMessageTemplate.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationChunkUserMessageTemplate.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationCombineSystemMessageTemplate.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationCombineUserMessageTemplate.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/templates/Template.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/util/LongTaskInfo.java create mode 100644 jablib/src/main/java/org/jabref/model/ai/chatting/ChatModelInfo.java create mode 100644 jablib/src/main/java/org/jabref/model/ai/summarization/BibEntrySummary.java create mode 100644 jablib/src/main/java/org/jabref/model/ai/summarization/SummarizationAlgorithmName.java delete mode 100644 jablib/src/main/java/org/jabref/model/ai/summarization/Summary.java diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java index 9476c3959ed1..25b2465b84fe 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java @@ -17,7 +17,7 @@ import org.jabref.logic.util.CitationKeyCheck; import org.jabref.logic.util.io.FileUtil; import org.jabref.model.ai.processingstatus.ProcessingInfo; -import org.jabref.model.ai.summarization.Summary; +import org.jabref.model.ai.summarization.BibEntrySummary; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; @@ -104,7 +104,7 @@ private Node tryToGenerateCitationKeyThenBind(BibEntry entry) { } private Node tryToShowSummary() { - ProcessingInfo processingInfo = aiService.getSummariesService().summarize(entry, bibDatabaseContext); + ProcessingInfo processingInfo = aiService.getSummariesService().summarize(entry, bibDatabaseContext); return switch (processingInfo.getState()) { case SUCCESS -> { @@ -119,7 +119,7 @@ private Node tryToShowSummary() { }; } - private Node showErrorWhileSummarizing(ProcessingInfo processingInfo) { + private Node showErrorWhileSummarizing(ProcessingInfo processingInfo) { assert processingInfo.getException().isPresent(); // When the state is ERROR, the exception must be present. LOGGER.error("Got an error while generating a summary for entry {}", entry.getCitationKey().orElse(""), processingInfo.getException().get()); @@ -140,20 +140,20 @@ private Node showErrorNotSummarized() { ); } - private Node showSummary(Summary summary) { - return new SummaryShowingComponent(summary, () -> { + private Node showSummary(BibEntrySummary bibEntrySummary) { + return new SummaryShowingComponent(bibEntrySummary, () -> { if (bibDatabaseContext.getDatabasePath().isEmpty()) { - LOGGER.error("Bib database path is not set, but it was expected to be present. Unable to regenerate summary"); + LOGGER.error("Bib database path is not set, but it was expected to be present. Unable to regenerate bibEntrySummary"); return; } if (entry.getCitationKey().isEmpty()) { - LOGGER.error("Citation key is not set, but it was expected to be present. Unable to regenerate summary"); + LOGGER.error("Citation key is not set, but it was expected to be present. Unable to regenerate bibEntrySummary"); return; } aiService.getSummariesService().regenerateSummary(entry, bibDatabaseContext); - // No need to rebuildUi(), because this class listens to the state of ProcessingInfo of the summary. + // No need to rebuildUi(), because this class listens to the state of ProcessingInfo of the bibEntrySummary. }); } } diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryShowingComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryShowingComponent.java index ec08d85e4764..7d759d76baec 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryShowingComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryShowingComponent.java @@ -14,7 +14,7 @@ import org.jabref.gui.util.WebViewStore; import org.jabref.logic.layout.format.MarkdownFormatter; -import org.jabref.model.ai.summarization.Summary; +import org.jabref.model.ai.summarization.BibEntrySummary; import com.airhacks.afterburner.views.ViewLoader; @@ -24,11 +24,11 @@ public class SummaryShowingComponent extends VBox { @FXML private CheckBox markdownCheckbox; private WebView contentWebView; - private final Summary summary; + private final BibEntrySummary bibEntrySummary; private final Runnable regenerateCallback; - public SummaryShowingComponent(Summary summary, Runnable regenerateCallback) { - this.summary = summary; + public SummaryShowingComponent(BibEntrySummary bibEntrySummary, Runnable regenerateCallback) { + this.bibEntrySummary = bibEntrySummary; this.regenerateCallback = regenerateCallback; ViewLoader.view(this) @@ -51,7 +51,7 @@ private void initializeWebView() { } private void updateContent(boolean isMarkdown) { - String content = summary.content(); + String content = bibEntrySummary.content(); if (isMarkdown) { contentWebView.getEngine().loadContent(MARKDOWN_FORMATTER.format(content)); } else { @@ -67,8 +67,8 @@ private void updateContent(boolean isMarkdown) { private void updateInfoText() { String newInfo = summaryInfoText .getText() - .replaceAll("%0", formatTimestamp(summary.timestamp())) - .replaceAll("%1", summary.aiProvider().getLabel() + " " + summary.model()); + .replaceAll("%0", formatTimestamp(bibEntrySummary.timestamp())) + .replaceAll("%1", bibEntrySummary.aiProvider().getLabel() + " " + bibEntrySummary.model()); summaryInfoText.setText(newInfo); } diff --git a/jabgui/src/main/resources/org/jabref/gui/ai/components/summary/SummaryShowingComponent.fxml b/jabgui/src/main/resources/org/jabref/gui/ai/components/summary/SummaryShowingComponent.fxml index e0e0bfb6c66d..5a53e8011329 100644 --- a/jabgui/src/main/resources/org/jabref/gui/ai/components/summary/SummaryShowingComponent.fxml +++ b/jabgui/src/main/resources/org/jabref/gui/ai/components/summary/SummaryShowingComponent.fxml @@ -8,7 +8,7 @@ - + diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index 7fa4c78c2ec2..050a11fbe596 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -125,6 +125,8 @@ exports org.jabref.logic.ai.rag; exports org.jabref.logic.ai.summarization.tasks; exports org.jabref.logic.ai.summarization.repositories; + exports org.jabref.logic.ai.summarization.templates; + exports org.jabref.logic.ai.summarization.algorithms; exports org.jabref.logic.ai.rag.tasks; exports org.jabref.logic.ai.rag.repositories; exports org.jabref.logic.ai.rag.algorithms; diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiService.java b/jablib/src/main/java/org/jabref/logic/ai/AiService.java index db654d4811fb..bd4538ebcfa0 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiService.java @@ -62,11 +62,12 @@ public class AiService implements AutoCloseable { private final IngestionService ingestionService; private final SummariesService summariesService; - public AiService(AiPreferences aiPreferences, - FilePreferences filePreferences, - CitationKeyPatternPreferences citationKeyPatternPreferences, - NotificationService notificationService, - TaskExecutor taskExecutor + public AiService( + AiPreferences aiPreferences, + FilePreferences filePreferences, + CitationKeyPatternPreferences citationKeyPatternPreferences, + NotificationService notificationService, + TaskExecutor taskExecutor ) { this.mvStoreChatHistoryStorage = new MVStoreChatHistoryRepository(notificationService, Directories.getAiFilesDirectory().resolve(CHAT_HISTORY_FILE_NAME)); @@ -81,12 +82,21 @@ public AiService(AiPreferences aiPreferences, this.ingestionService = new IngestionService( aiPreferences, - filePreferences, taskExecutor, currentlySelectedEmbeddingModel, mvStoreEmbeddingStore, mvStoreFullyIngestedDocumentsTracker, shutdownSignal + filePreferences, + taskExecutor, + currentlySelectedEmbeddingModel, + mvStoreEmbeddingStore, + mvStoreFullyIngestedDocumentsTracker, + shutdownSignal ); this.summariesService = new SummariesService( aiPreferences, - filePreferences, templatesService, taskExecutor, currentlySelectedChatLanguageModel, mvStoreSummariesStorage, + filePreferences, + templatesService, + taskExecutor, + currentlySelectedChatLanguageModel.getChatModelInfo(), + mvStoreSummariesStorage, shutdownSignal ); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java index 4e31c759d298..274aa3d84dbd 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java @@ -14,6 +14,7 @@ import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.l10n.Localization; import org.jabref.model.ai.chatting.AiProvider; +import org.jabref.model.ai.chatting.ChatModelInfo; import com.google.common.util.concurrent.ThreadFactoryBuilder; import dev.langchain4j.data.message.ChatMessage; @@ -116,6 +117,16 @@ private void setupListeningToPreferencesChanges() { aiPreferences.setApiKeyChangeListener(() -> langchainChatModel = Optional.empty()); } + public ChatModelInfo getChatModelInfo() { + assert langchainChatModel.isPresent(); + return new ChatModelInfo( + langchainChatModel.get(), + aiPreferences.getAiProvider(), + aiPreferences.getSelectedChatModel(), + aiPreferences.getContextWindowSize() + ); + } + @Override public ChatResponse chat(List list) { // The rationale for RuntimeExceptions in this method: diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultTemplates.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultTemplates.java index 8dd49f3b2bab..a45949b699cd 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultTemplates.java +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultTemplates.java @@ -27,7 +27,7 @@ public class AiDefaultTemplates { AiTemplate.SUMMARIZATION_CHUNK_SYSTEM_MESSAGE, """ Please provide an overview of the following text. It is a part of a scientific paper. - The summary should include the main objectives, methodologies used, key findings, and conclusions. + The bibEntrySummary should include the main objectives, methodologies used, key findings, and conclusions. Mention any significant experiments, data, or discussions presented in the paper.""", AiTemplate.SUMMARIZATION_CHUNK_USER_MESSAGE, "$text", diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java index b9c49a935d40..e8684d16a458 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java @@ -19,6 +19,7 @@ import org.jabref.logic.util.strings.StringUtil; import org.jabref.model.ai.chatting.AiProvider; import org.jabref.model.ai.embeddings.EmbeddingModel; +import org.jabref.model.ai.summarization.SummarizationAlgorithmName; import org.jabref.model.ai.templating.AiTemplate; import com.github.javakeyring.Keyring; @@ -52,6 +53,7 @@ public class AiPreferences { private final StringProperty huggingFaceApiBaseUrl; private final StringProperty gpt4AllApiBaseUrl; + private final ObjectProperty defaultSummarizationAlgorithm; private final ObjectProperty embeddingModel; private final DoubleProperty temperature; private final IntegerProperty contextWindowSize; @@ -64,29 +66,31 @@ public class AiPreferences { private Runnable apiKeyChangeListener; - public AiPreferences(boolean enableAi, - boolean autoGenerateEmbeddings, - boolean autoGenerateSummaries, - AiProvider aiProvider, - String openAiChatModel, - String mistralAiChatModel, - String geminiChatModel, - String huggingFaceChatModel, - String gpt4AllModel, - boolean customizeExpertSettings, - String openAiApiBaseUrl, - String mistralAiApiBaseUrl, - String geminiApiBaseUrl, - String huggingFaceApiBaseUrl, - String gpt4AllApiBaseUrl, - EmbeddingModel embeddingModel, - double temperature, - int contextWindowSize, - int documentSplitterChunkSize, - int documentSplitterOverlapSize, - int ragMaxResultsCount, - double ragMinScore, - Map templates + public AiPreferences( + boolean enableAi, + boolean autoGenerateEmbeddings, + boolean autoGenerateSummaries, + AiProvider aiProvider, + String openAiChatModel, + String mistralAiChatModel, + String geminiChatModel, + String huggingFaceChatModel, + String gpt4AllModel, + boolean customizeExpertSettings, + String openAiApiBaseUrl, + String mistralAiApiBaseUrl, + String geminiApiBaseUrl, + String huggingFaceApiBaseUrl, + String gpt4AllApiBaseUrl, + SummarizationAlgorithmName defaultSummarizationAlgorithm, + EmbeddingModel embeddingModel, + double temperature, + int contextWindowSize, + int documentSplitterChunkSize, + int documentSplitterOverlapSize, + int ragMaxResultsCount, + double ragMinScore, + Map templates ) { this.enableAi = new SimpleBooleanProperty(enableAi); this.autoGenerateEmbeddings = new SimpleBooleanProperty(autoGenerateEmbeddings); @@ -108,6 +112,7 @@ public AiPreferences(boolean enableAi, this.huggingFaceApiBaseUrl = new SimpleStringProperty(huggingFaceApiBaseUrl); this.gpt4AllApiBaseUrl = new SimpleStringProperty(gpt4AllApiBaseUrl); + this.defaultSummarizationAlgorithm = new SimpleObjectProperty<>(defaultSummarizationAlgorithm); this.embeddingModel = new SimpleObjectProperty<>(embeddingModel); this.temperature = new SimpleDoubleProperty(temperature); this.contextWindowSize = new SimpleIntegerProperty(contextWindowSize); @@ -279,6 +284,18 @@ public void setCustomizeExpertSettings(boolean customizeExpertSettings) { this.customizeExpertSettings.set(customizeExpertSettings); } + public ObjectProperty defaultSummarizationAlgorithmProperty() { + return defaultSummarizationAlgorithm; + } + + public SummarizationAlgorithmName getDefaultSummarizationAlgorithm() { + return defaultSummarizationAlgorithm.get(); + } + + public void setDefaultSummarizationAlgorithm(SummarizationAlgorithmName defaultSummarizationAlgorithm) { + this.defaultSummarizationAlgorithm.set(defaultSummarizationAlgorithm); + } + public ObjectProperty embeddingModelProperty() { return embeddingModel; } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/CurrentlySelectedSummarizationAlgorithm.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/CurrentlySelectedSummarizationAlgorithm.java new file mode 100644 index 000000000000..8f88d56542a8 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/CurrentlySelectedSummarizationAlgorithm.java @@ -0,0 +1,101 @@ +package org.jabref.logic.ai.summarization; + +import java.util.Optional; + +import org.jabref.logic.ai.preferences.AiPreferences; +import org.jabref.logic.ai.summarization.algorithms.ChunkedSummarizationAlgorithm; +import org.jabref.logic.ai.summarization.algorithms.SummarizationAlgorithm; +import org.jabref.logic.ai.summarization.templates.SummarizationChunkSystemMessageTemplate; +import org.jabref.logic.ai.summarization.templates.SummarizationChunkUserMessageTemplate; +import org.jabref.logic.ai.summarization.templates.SummarizationCombineSystemMessageTemplate; +import org.jabref.logic.ai.summarization.templates.SummarizationCombineUserMessageTemplate; +import org.jabref.logic.ai.util.LongTaskInfo; +import org.jabref.model.ai.chatting.ChatModelInfo; +import org.jabref.model.ai.summarization.SummarizationAlgorithmName; +import org.jabref.model.ai.templating.AiTemplate; + +import org.jspecify.annotations.Nullable; + +public class CurrentlySelectedSummarizationAlgorithm implements SummarizationAlgorithm { + private final AiPreferences aiPreferences; + + @Nullable + private SummarizationAlgorithm summarizationAlgorithm = null; + + public CurrentlySelectedSummarizationAlgorithm( + AiPreferences aiPreferences + ) { + this.aiPreferences = aiPreferences; + + configure(); + } + + public Optional getSummarizationAlgorithm() { + return Optional.ofNullable(summarizationAlgorithm); + } + + private void configure() { + aiPreferences.defaultSummarizationAlgorithmProperty().addListener(_ -> { + updateAlgorithm(); + }); + + // Not efficient. + aiPreferences.templateProperty(AiTemplate.SUMMARIZATION_CHUNK_SYSTEM_MESSAGE).addListener(_ -> { + updateAlgorithm(); + }); + aiPreferences.templateProperty(AiTemplate.SUMMARIZATION_CHUNK_USER_MESSAGE).addListener(_ -> { + updateAlgorithm(); + }); + aiPreferences.templateProperty(AiTemplate.SUMMARIZATION_COMBINE_SYSTEM_MESSAGE).addListener(_ -> { + updateAlgorithm(); + }); + aiPreferences.templateProperty(AiTemplate.SUMMARIZATION_COMBINE_USER_MESSAGE).addListener(_ -> { + updateAlgorithm(); + }); + } + + private void updateAlgorithm() { + //noinspection SwitchStatementWithTooFewBranches + switch (aiPreferences.getDefaultSummarizationAlgorithm()) { + case SummarizationAlgorithmName.CHUNKED -> { + summarizationAlgorithm = createChunkedSummarizationAlgorithm(); + } + } + } + + private ChunkedSummarizationAlgorithm createChunkedSummarizationAlgorithm() { + return new ChunkedSummarizationAlgorithm( + new SummarizationChunkSystemMessageTemplate( + aiPreferences.getTemplate(AiTemplate.SUMMARIZATION_CHUNK_SYSTEM_MESSAGE) + ), + new SummarizationChunkUserMessageTemplate( + aiPreferences.getTemplate(AiTemplate.SUMMARIZATION_CHUNK_USER_MESSAGE) + ), + new SummarizationCombineSystemMessageTemplate( + aiPreferences.getTemplate(AiTemplate.SUMMARIZATION_COMBINE_SYSTEM_MESSAGE) + ), + new SummarizationCombineUserMessageTemplate( + aiPreferences.getTemplate(AiTemplate.SUMMARIZATION_COMBINE_USER_MESSAGE) + ) + ); + } + + @Override + public String summarize(ChatModelInfo chatModelInfo, LongTaskInfo longTaskInfo, String text) throws InterruptedException { + if (summarizationAlgorithm == null) { + return ""; + } + + return summarizationAlgorithm.summarize(chatModelInfo, longTaskInfo, text); + } + + @Override + public SummarizationAlgorithmName getName() { + if (summarizationAlgorithm == null) { + // Sadly. + return null; + } + + return summarizationAlgorithm.getName(); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java index 7e297932883a..4c2e4bae4166 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java @@ -10,15 +10,16 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.preferences.AiPreferences; +import org.jabref.logic.ai.summarization.algorithms.SummarizationAlgorithm; import org.jabref.logic.ai.summarization.repositories.SummariesRepository; import org.jabref.logic.ai.summarization.tasks.GenerateSummaryForSeveralTask; import org.jabref.logic.ai.summarization.tasks.GenerateSummaryTask; -import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.util.CitationKeyCheck; import org.jabref.logic.util.TaskExecutor; +import org.jabref.model.ai.chatting.ChatModelInfo; import org.jabref.model.ai.processingstatus.ProcessingInfo; import org.jabref.model.ai.processingstatus.ProcessingState; -import org.jabref.model.ai.summarization.Summary; +import org.jabref.model.ai.summarization.BibEntrySummary; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.event.EntriesAddedEvent; import org.jabref.model.entry.BibEntry; @@ -26,7 +27,6 @@ import org.jabref.model.entry.field.StandardField; import com.google.common.eventbus.Subscribe; -import dev.langchain4j.model.chat.ChatModel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,31 +42,32 @@ public class SummariesService { private static final Logger LOGGER = LoggerFactory.getLogger(SummariesService.class); - private final TreeMap> summariesStatusMap = new TreeMap<>(Comparator.comparing(BibEntry::getId)); + private final TreeMap> summariesStatusMap = new TreeMap<>(Comparator.comparing(BibEntry::getId)); private final List> listsUnderSummarization = new ArrayList<>(); private final AiPreferences aiPreferences; private final SummariesRepository summariesRepository; - private final ChatModel chatLanguageModel; - private final AiTemplatesService aiTemplatesService; + private final ChatModelInfo chatModelInfo; + private final SummarizationAlgorithm defaultSummarizationAlgorithm; private final BooleanProperty shutdownSignal; private final FilePreferences filePreferences; private final TaskExecutor taskExecutor; + // TODO: chat model should be argument. public SummariesService( AiPreferences aiPreferences, FilePreferences filePreferences, - AiTemplatesService aiTemplatesService, TaskExecutor taskExecutor, - ChatModel chatLanguageModel, + ChatModelInfo chatModelInfo, + SummarizationAlgorithm defaultSummarizationAlgorithm, SummariesRepository summariesRepository, BooleanProperty shutdownSignal ) { this.aiPreferences = aiPreferences; this.summariesRepository = summariesRepository; - this.chatLanguageModel = chatLanguageModel; - this.aiTemplatesService = aiTemplatesService; + this.chatModelInfo = chatModelInfo; + this.defaultSummarizationAlgorithm = defaultSummarizationAlgorithm; this.shutdownSignal = shutdownSignal; this.filePreferences = filePreferences; this.taskExecutor = taskExecutor; @@ -88,7 +89,7 @@ public EntriesChangedListener(BibDatabaseContext bibDatabaseContext) { public void listen(EntriesAddedEvent e) { e.getBibEntries().forEach(entry -> { if (aiPreferences.getAutoGenerateSummaries()) { - summarize(entry, bibDatabaseContext); + summarize(defaultSummarizationAlgorithm, entry, bibDatabaseContext); } }); } @@ -96,7 +97,7 @@ public void listen(EntriesAddedEvent e) { @Subscribe public void listen(FieldChangedEvent e) { if (e.getField() == StandardField.FILE && aiPreferences.getAutoGenerateSummaries()) { - summarize(e.getBibEntry(), bibDatabaseContext); + summarize(defaultSummarizationAlgorithm, e.getBibEntry(), bibDatabaseContext); } } } @@ -107,26 +108,31 @@ public void listen(FieldChangedEvent e) { * Returned {@link ProcessingInfo} is related to the passed {@link BibEntry}, so if you call this method twice * on the same {@link BibEntry}, the method will return the same {@link ProcessingInfo}. */ - public ProcessingInfo summarize(BibEntry bibEntry, BibDatabaseContext bibDatabaseContext) { - ProcessingInfo processingInfo = getProcessingInfo(bibEntry); + public ProcessingInfo summarize(SummarizationAlgorithm summarizationAlgorithm, BibEntry bibEntry, BibDatabaseContext bibDatabaseContext) { + ProcessingInfo processingInfo = getProcessingInfo(bibEntry); if (processingInfo.getState() == ProcessingState.STOPPED) { - startSummarizationTask(bibEntry, bibDatabaseContext, processingInfo); + startSummarizationTask(summarizationAlgorithm, bibEntry, bibDatabaseContext, processingInfo); } return processingInfo; } - public ProcessingInfo getProcessingInfo(BibEntry entry) { + public ProcessingInfo getProcessingInfo(BibEntry entry) { return summariesStatusMap.computeIfAbsent(entry, _ -> new ProcessingInfo<>(entry, ProcessingState.STOPPED)); } - public List> getProcessingInfo(List entries) { + public List> getProcessingInfo(List entries) { return entries.stream().map(this::getProcessingInfo).toList(); } - public void summarize(StringProperty groupName, List entries, BibDatabaseContext bibDatabaseContext) { - List> result = getProcessingInfo(entries); + public void summarize( + SummarizationAlgorithm summarizationAlgorithm, + StringProperty groupName, + List entries, + BibDatabaseContext bibDatabaseContext + ) { + List> result = getProcessingInfo(entries); if (listsUnderSummarization.contains(entries)) { return; @@ -134,31 +140,68 @@ public void summarize(StringProperty groupName, List entries, BibDatab listsUnderSummarization.add(entries); - List> needToProcess = result.stream().filter(processingInfo -> processingInfo.getState() == ProcessingState.STOPPED).toList(); - startSummarizationTask(groupName, needToProcess, bibDatabaseContext); + List> needToProcess = result.stream().filter(processingInfo -> processingInfo.getState() == ProcessingState.STOPPED).toList(); + startSummarizationTask( + summarizationAlgorithm, + groupName, + needToProcess, + bibDatabaseContext + ); } - private void startSummarizationTask(BibEntry entry, BibDatabaseContext bibDatabaseContext, ProcessingInfo processingInfo) { + private void startSummarizationTask( + SummarizationAlgorithm summarizationAlgorithm, + BibEntry entry, + BibDatabaseContext bibDatabaseContext, + ProcessingInfo processingInfo + ) { processingInfo.setState(ProcessingState.PROCESSING); - new GenerateSummaryTask(aiPreferences, filePreferences, aiTemplatesService, chatLanguageModel, summariesRepository, bibDatabaseContext, entry, shutdownSignal) + new GenerateSummaryTask( + filePreferences, + chatModelInfo, + summariesRepository, + summarizationAlgorithm, + bibDatabaseContext, + entry, + shutdownSignal + ) .onSuccess(processingInfo::setSuccess) .onFailure(processingInfo::setException) .executeWith(taskExecutor); } - private void startSummarizationTask(StringProperty groupName, List> entries, BibDatabaseContext bibDatabaseContext) { + private void startSummarizationTask( + SummarizationAlgorithm summarizationAlgorithm, + StringProperty groupName, + List> entries, + BibDatabaseContext bibDatabaseContext + ) { entries.forEach(processingInfo -> processingInfo.setState(ProcessingState.PROCESSING)); - new GenerateSummaryForSeveralTask(aiPreferences, filePreferences, aiTemplatesService, taskExecutor, chatLanguageModel, summariesRepository, bibDatabaseContext, groupName, entries, shutdownSignal) + new GenerateSummaryForSeveralTask( + filePreferences, + taskExecutor, + chatModelInfo, + summariesRepository, + summarizationAlgorithm, + bibDatabaseContext, + groupName, + entries, + shutdownSignal + ) .executeWith(taskExecutor); } /** - * Method, similar to {@link #summarize(BibEntry, BibDatabaseContext)}, but it allows you to regenerate summary. + * Method, similar to {@link #summarize(SummarizationAlgorithm, BibEntry, BibDatabaseContext)}, but it allows you to regenerate summary. */ - public void regenerateSummary(BibEntry bibEntry, BibDatabaseContext bibDatabaseContext) { - ProcessingInfo processingInfo = summarize(bibEntry, bibDatabaseContext); + public void regenerateSummary( + SummarizationAlgorithm summarizationAlgorithm, + BibEntry bibEntry, + BibDatabaseContext bibDatabaseContext + ) { + ProcessingInfo processingInfo = summarize(summarizationAlgorithm, bibEntry, bibDatabaseContext); processingInfo.setState(ProcessingState.PROCESSING); if (bibDatabaseContext.getDatabasePath().isEmpty()) { @@ -169,6 +212,6 @@ public void regenerateSummary(BibEntry bibEntry, BibDatabaseContext bibDatabaseC summariesRepository.clear(bibDatabaseContext.getDatabasePath().get(), bibEntry.getCitationKey().get()); } - startSummarizationTask(bibEntry, bibDatabaseContext, processingInfo); + startSummarizationTask(summarizationAlgorithm, bibEntry, bibDatabaseContext, processingInfo); } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/BibEntrySummarizer.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/BibEntrySummarizer.java index 2c7ebcaa37f3..269b75f88a4f 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/BibEntrySummarizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/BibEntrySummarizer.java @@ -8,45 +8,40 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import javafx.beans.property.ReadOnlyBooleanProperty; - import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.rag.algorithms.parsing.UniversalFileParser; -import org.jabref.logic.ai.templates.AiTemplatesService; +import org.jabref.logic.ai.util.LongTaskInfo; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.util.ProgressCounter; -import org.jabref.model.ai.summarization.Summary; +import org.jabref.model.ai.chatting.ChatModelInfo; +import org.jabref.model.ai.summarization.BibEntrySummary; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; -import dev.langchain4j.model.chat.ChatModel; import org.slf4j.Logger; public class BibEntrySummarizer { private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(BibEntrySummarizer.class); - // TODO: Same thing. - private final AiPreferences aiPreferences; private final FilePreferences filePreferences; - private final ChunkedSummarizationAlgorithm chunkedSummarizationAlgorithm; + private final SummarizationAlgorithm summarizationAlgorithm; private final UniversalFileParser universalFileParser = new UniversalFileParser(); public BibEntrySummarizer( - AiPreferences aiPreferences, FilePreferences filePreferences, - AiTemplatesService aiTemplatesService, - ChatModel chatModel + SummarizationAlgorithm summarizationAlgorithm ) { - this.aiPreferences = aiPreferences; this.filePreferences = filePreferences; - - this.chunkedSummarizationAlgorithm = new ChunkedSummarizationAlgorithm(aiPreferences, aiTemplatesService, chatModel); + this.summarizationAlgorithm = summarizationAlgorithm; } - public Summary summarize(BibEntry entry, BibDatabaseContext bibDatabaseContext, ProgressCounter progressCounter, ReadOnlyBooleanProperty shutdownSignal) throws InterruptedException { + public BibEntrySummary summarize( + ChatModelInfo chatModelInfo, + LongTaskInfo longTaskInfo, + BibDatabaseContext bibDatabaseContext, + BibEntry entry + ) throws InterruptedException { String citationKey = entry.getCitationKey().orElse(""); // Rationale for RuntimeException here: @@ -56,12 +51,18 @@ public Summary summarize(BibEntry entry, BibDatabaseContext bibDatabaseContext, // Stream API would look better here, but we need to catch InterruptedException. List linkedFilesSummary = new ArrayList<>(); for (LinkedFile linkedFile : entry.getFiles()) { - generateSummary(linkedFile, bibDatabaseContext, citationKey, progressCounter, shutdownSignal) + generateSummary( + chatModelInfo, + longTaskInfo, + bibDatabaseContext, + linkedFile, + citationKey + ) .ifPresent(linkedFilesSummary::add); } if (linkedFilesSummary.isEmpty()) { - progressCounter.increaseWorkDone(1); // Skipped generation of final summary. + longTaskInfo.progressCounter().increaseWorkDone(1); // Skipped generation of final summary. throw new RuntimeException(Localization.lang("No summary can be generated for entry '%0'. Could not find attached linked files.", citationKey)); } @@ -69,25 +70,36 @@ public Summary summarize(BibEntry entry, BibDatabaseContext bibDatabaseContext, String finalSummary; - progressCounter.increaseWorkMax(1); // For generating final summary. + longTaskInfo.progressCounter().increaseWorkMax(1); // For generating final summary. if (linkedFilesSummary.size() == 1) { finalSummary = linkedFilesSummary.getFirst(); } else { - finalSummary = summarizeSeveralDocuments(linkedFilesSummary.stream(), progressCounter, shutdownSignal); + finalSummary = summarizeSeveralDocuments( + chatModelInfo, + longTaskInfo, + linkedFilesSummary.stream() + ); } - progressCounter.increaseWorkDone(1); + longTaskInfo.progressCounter().increaseWorkDone(1); - return new Summary( + return new BibEntrySummary( LocalDateTime.now(), - aiPreferences.getAiProvider(), - aiPreferences.getSelectedChatModel(), + chatModelInfo.aiProvider(), + chatModelInfo.name(), + summarizationAlgorithm.getName(), finalSummary ); } - private Optional generateSummary(LinkedFile linkedFile, BibDatabaseContext bibDatabaseContext, String citationKey, ProgressCounter progressCounter, ReadOnlyBooleanProperty shutdownSignal) throws InterruptedException { + private Optional generateSummary( + ChatModelInfo chatModelInfo, + LongTaskInfo longTaskInfo, + BibDatabaseContext bibDatabaseContext, + LinkedFile linkedFile, + String citationKey + ) throws InterruptedException { LOGGER.debug("Generating summary for file \"{}\" of entry {}", linkedFile.getLink(), citationKey); Optional path = linkedFile.findIn(bibDatabaseContext, filePreferences); @@ -98,7 +110,7 @@ private Optional generateSummary(LinkedFile linkedFile, BibDatabaseConte return Optional.empty(); } - Optional document = universalFileParser.parse(path.get(), shutdownSignal); + Optional document = universalFileParser.parse(path.get(), longTaskInfo.shutdownSignal()); if (document.isEmpty()) { LOGGER.warn("Could not extract text from a linked file \"{}\" of entry {}. It will be skipped when generating a summary.", linkedFile.getLink(), citationKey); @@ -106,13 +118,25 @@ private Optional generateSummary(LinkedFile linkedFile, BibDatabaseConte return Optional.empty(); } - String linkedFileSummary = chunkedSummarizationAlgorithm.summarize(document.get(), progressCounter, shutdownSignal); + String linkedFileSummary = summarizationAlgorithm.summarize( + chatModelInfo, + longTaskInfo, + document.get() + ); - LOGGER.debug("Summary for file \"{}\" of entry {} was generated successfully", linkedFile.getLink(), citationKey); + LOGGER.debug("BibEntrySummary for file \"{}\" of entry {} was generated successfully", linkedFile.getLink(), citationKey); return Optional.of(linkedFileSummary); } - public String summarizeSeveralDocuments(Stream documents, ProgressCounter progressCounter, ReadOnlyBooleanProperty shutdownSignal) throws InterruptedException { - return chunkedSummarizationAlgorithm.summarize(documents.collect(Collectors.joining("\n\n")), progressCounter, shutdownSignal); + public String summarizeSeveralDocuments( + ChatModelInfo chatModelInfo, + LongTaskInfo longTaskInfo, + Stream documents + ) throws InterruptedException { + return summarizationAlgorithm.summarize( + chatModelInfo, + longTaskInfo, + documents.collect(Collectors.joining("\n\n")) + ); } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/ChunkedSummarizationAlgorithm.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/ChunkedSummarizationAlgorithm.java index cb07e382cfe0..17706f4e0107 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/ChunkedSummarizationAlgorithm.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/ChunkedSummarizationAlgorithm.java @@ -3,12 +3,13 @@ import java.util.ArrayList; import java.util.List; -import javafx.beans.property.ReadOnlyBooleanProperty; - -import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.templates.AiTemplatesService; -import org.jabref.logic.util.ProgressCounter; -import org.jabref.model.ai.templating.AiTemplate; +import org.jabref.logic.ai.summarization.templates.SummarizationChunkSystemMessageTemplate; +import org.jabref.logic.ai.summarization.templates.SummarizationChunkUserMessageTemplate; +import org.jabref.logic.ai.summarization.templates.SummarizationCombineSystemMessageTemplate; +import org.jabref.logic.ai.summarization.templates.SummarizationCombineUserMessageTemplate; +import org.jabref.logic.ai.util.LongTaskInfo; +import org.jabref.model.ai.chatting.ChatModelInfo; +import org.jabref.model.ai.summarization.SummarizationAlgorithmName; import dev.langchain4j.data.document.DefaultDocument; import dev.langchain4j.data.document.DocumentSplitter; @@ -16,7 +17,6 @@ import dev.langchain4j.data.message.SystemMessage; import dev.langchain4j.data.message.UserMessage; import dev.langchain4j.data.segment.TextSegment; -import dev.langchain4j.model.chat.ChatModel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,30 +26,35 @@ public class ChunkedSummarizationAlgorithm implements SummarizationAlgorithm { private static final int MAX_OVERLAP_SIZE_IN_CHARS = 100; private static final int CHAR_TOKEN_FACTOR = 4; // Means, every token is roughly 4 characters. - // TODO: AiPreferences are used here to determine the context window size of the chat model, but this is wrong, - // because AiPreferences determine the size based on the selected model, but this model can be supplied with other model. - private final AiPreferences aiPreferences; - private final AiTemplatesService aiTemplatesService; - private final ChatModel chatModel; + private final SummarizationChunkSystemMessageTemplate summarizationChunkSystemMessageTemplate; + private final SummarizationChunkUserMessageTemplate summarizationChunkUserMessageTemplate; + private final SummarizationCombineSystemMessageTemplate summarizationCombineSystemMessageTemplate; + private final SummarizationCombineUserMessageTemplate summarizationCombineUserMessageTemplate; public ChunkedSummarizationAlgorithm( - AiPreferences aiPreferences, - AiTemplatesService aiTemplatesService, - ChatModel chatModel + SummarizationChunkSystemMessageTemplate summarizationChunkSystemMessageTemplate, + SummarizationChunkUserMessageTemplate summarizationChunkUserMessageTemplate, + SummarizationCombineSystemMessageTemplate summarizationCombineSystemMessageTemplate, + SummarizationCombineUserMessageTemplate summarizationCombineUserMessageTemplate ) { - this.aiPreferences = aiPreferences; - this.aiTemplatesService = aiTemplatesService; - this.chatModel = chatModel; + this.summarizationChunkSystemMessageTemplate = summarizationChunkSystemMessageTemplate; + this.summarizationChunkUserMessageTemplate = summarizationChunkUserMessageTemplate; + this.summarizationCombineSystemMessageTemplate = summarizationCombineSystemMessageTemplate; + this.summarizationCombineUserMessageTemplate = summarizationCombineUserMessageTemplate; } @Override - public String summarize(String text, ProgressCounter progressCounter, ReadOnlyBooleanProperty shutdownSignal) throws InterruptedException { + public String summarize( + ChatModelInfo chatModelInfo, + LongTaskInfo longTaskInfo, + String text + ) throws InterruptedException { LOGGER.debug("Summarizing text ({} chars)", text.length()); - progressCounter.increaseWorkMax(1); // For the combination of summary chunks. + longTaskInfo.progressCounter().increaseWorkMax(1); // For the combination of summary chunks. DocumentSplitter documentSplitter = DocumentSplitters.recursive( - aiPreferences.getContextWindowSize() - MAX_OVERLAP_SIZE_IN_CHARS * 2 - estimateTokenCount(aiPreferences.getTemplate(AiTemplate.SUMMARIZATION_CHUNK_SYSTEM_MESSAGE)), + chatModelInfo.contextWindowSize() - MAX_OVERLAP_SIZE_IN_CHARS * 2 - estimateTokenCount(summarizationChunkSystemMessageTemplate.getSource()), MAX_OVERLAP_SIZE_IN_CHARS ); @@ -64,56 +69,61 @@ public String summarize(String text, ProgressCounter progressCounter, ReadOnlyBo passes++; LOGGER.debug("Summarizing {} chunk (of {}", passes, chunkSummaries.size()); - progressCounter.increaseWorkMax(chunkSummaries.size()); + longTaskInfo.progressCounter().increaseWorkMax(chunkSummaries.size()); List list = new ArrayList<>(); for (String chunkSummary : chunkSummaries) { - if (shutdownSignal.get()) { + if (longTaskInfo.shutdownSignal().get()) { throw new InterruptedException(); } - String systemMessage = aiTemplatesService.makeSummarizationChunkSystemMessage(); - String userMessage = aiTemplatesService.makeSummarizationChunkUserMessage(chunkSummary); + String systemMessage =summarizationChunkSystemMessageTemplate.render(); + String userMessage = summarizationChunkUserMessageTemplate.render(chunkSummary); LOGGER.debug("Sending request to AI provider to summarize a chunk"); - String chunk = chatModel.chat(List.of( + String chunk = chatModelInfo.chatModel().chat(List.of( new SystemMessage(systemMessage), new UserMessage(userMessage) )).aiMessage().text(); LOGGER.debug("Chunk {} summary was generated successfully", passes); list.add(chunk); - progressCounter.increaseWorkDone(1); + longTaskInfo.progressCounter().increaseWorkDone(1); } chunkSummaries = list; - } while (estimateTokenCount(chunkSummaries) > aiPreferences.getContextWindowSize() - estimateTokenCount(aiPreferences.getTemplate(AiTemplate.SUMMARIZATION_COMBINE_SYSTEM_MESSAGE))); + } while (estimateTokenCount(chunkSummaries) > chatModelInfo.contextWindowSize() - estimateTokenCount(summarizationCombineSystemMessageTemplate.getSource())); if (chunkSummaries.size() == 1) { - progressCounter.increaseWorkDone(1); // No need to call LLM for combination of summary chunks. - LOGGER.debug("Summary of the text was generated successfully"); + longTaskInfo.progressCounter().increaseWorkDone(1); // No need to call LLM for combination of summary chunks. + LOGGER.debug("BibEntrySummary of the text was generated successfully"); return chunkSummaries.getFirst(); } - String systemMessage = aiTemplatesService.makeSummarizationCombineSystemMessage(); - String userMessage = aiTemplatesService.makeSummarizationCombineUserMessage(chunkSummaries); + String systemMessage = summarizationCombineSystemMessageTemplate.render(); + String userMessage = summarizationCombineUserMessageTemplate.render(chunkSummaries); - if (shutdownSignal.get()) { + if (longTaskInfo.shutdownSignal().get()) { throw new InterruptedException(); } LOGGER.debug("Sending request to AI provider to combine summary chunks"); - String result = chatModel.chat(List.of( + String result = chatModelInfo.chatModel().chat(List.of( new SystemMessage(systemMessage), new UserMessage(userMessage) )).aiMessage().text(); - LOGGER.debug("Summary of the text was generated successfully"); + LOGGER.debug("BibEntrySummary of the text was generated successfully"); - progressCounter.increaseWorkDone(1); + longTaskInfo.progressCounter().increaseWorkDone(1); return result; } + @Override + public SummarizationAlgorithmName getName() { + return SummarizationAlgorithmName.CHUNKED; + } + private static int estimateTokenCount(List chunkSummaries) { return chunkSummaries.stream().mapToInt(ChunkedSummarizationAlgorithm::estimateTokenCount).sum(); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/PersistentBibEntrySummarizer.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/PersistentBibEntrySummarizer.java index 8ffe0d437832..d86b05db9332 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/PersistentBibEntrySummarizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/PersistentBibEntrySummarizer.java @@ -2,19 +2,15 @@ import java.util.Optional; -import javafx.beans.property.ReadOnlyBooleanProperty; - import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.summarization.repositories.SummariesRepository; -import org.jabref.logic.ai.templates.AiTemplatesService; +import org.jabref.logic.ai.util.LongTaskInfo; import org.jabref.logic.util.CitationKeyCheck; -import org.jabref.logic.util.ProgressCounter; -import org.jabref.model.ai.summarization.Summary; +import org.jabref.model.ai.chatting.ChatModelInfo; +import org.jabref.model.ai.summarization.BibEntrySummary; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; -import dev.langchain4j.model.chat.ChatModel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,40 +22,49 @@ public class PersistentBibEntrySummarizer { private final BibEntrySummarizer bibEntrySummarizer; public PersistentBibEntrySummarizer( - AiPreferences aiPreferences, FilePreferences filePreferences, SummariesRepository summariesRepository, - AiTemplatesService aiTemplatesService, - ChatModel chatModel + SummarizationAlgorithm summarizationAlgorithm ) { this.summariesRepository = summariesRepository; this.bibEntrySummarizer = new BibEntrySummarizer( - aiPreferences, filePreferences, - aiTemplatesService, - chatModel + summarizationAlgorithm ); } - public Summary summarize(BibEntry entry, BibDatabaseContext bibDatabaseContext, ProgressCounter progressCounter, ReadOnlyBooleanProperty shutdownSignal) { - Optional savedSummary = Optional.empty(); + public BibEntrySummary summarize( + ChatModelInfo chatModelInfo, + LongTaskInfo longTaskInfo, + BibDatabaseContext bibDatabaseContext, + BibEntry entry + ) { + Optional savedSummary = Optional.empty(); if (bibDatabaseContext.getDatabasePath().isEmpty()) { - LOGGER.info("No database path is present. Summary will not be stored in the next sessions"); + LOGGER.info("No database path is present. BibEntrySummary will not be stored in the next sessions"); } else if (entry.getCitationKey().isEmpty()) { - LOGGER.info("No citation key is present. Summary will not be stored in the next sessions"); + LOGGER.info("No citation key is present. BibEntrySummary will not be stored in the next sessions"); } else { - savedSummary = summariesRepository.get(bibDatabaseContext.getDatabasePath().get(), entry.getCitationKey().get()); + savedSummary = summariesRepository.get( + bibDatabaseContext.getDatabasePath().get(), + entry.getCitationKey().get() + ); } - Summary summary; + BibEntrySummary bibEntrySummary; if (savedSummary.isPresent()) { - summary = savedSummary.get(); + bibEntrySummary = savedSummary.get(); } else { try { - summary = bibEntrySummarizer.summarize(entry, bibDatabaseContext, progressCounter, shutdownSignal); + bibEntrySummary = bibEntrySummarizer.summarize( + chatModelInfo, + longTaskInfo, + bibDatabaseContext, + entry + ); } catch (InterruptedException e) { LOGGER.debug("There was a summarization task for {}. It will be canceled, because user quits JabRef.", entry.getCitationKey().orElse("")); return null; @@ -67,13 +72,17 @@ public Summary summarize(BibEntry entry, BibDatabaseContext bibDatabaseContext, } if (bibDatabaseContext.getDatabasePath().isEmpty()) { - LOGGER.info("No database path is present. Summary will not be stored in the next sessions"); + LOGGER.info("No database path is present. BibEntrySummary will not be stored in the next sessions"); } else if (CitationKeyCheck.citationKeyIsPresentAndUnique(bibDatabaseContext, entry)) { - LOGGER.info("No valid citation key is present. Summary will not be stored in the next sessions"); + LOGGER.info("No valid citation key is present. BibEntrySummary will not be stored in the next sessions"); } else { - summariesRepository.set(bibDatabaseContext.getDatabasePath().get(), entry.getCitationKey().get(), summary); + summariesRepository.set( + bibDatabaseContext.getDatabasePath().get(), + entry.getCitationKey().get(), + bibEntrySummary + ); } - return summary; + return bibEntrySummary; } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/SummarizationAlgorithm.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/SummarizationAlgorithm.java index 95fed2aab2e6..702a10a4ec58 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/SummarizationAlgorithm.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/SummarizationAlgorithm.java @@ -1,13 +1,15 @@ package org.jabref.logic.ai.summarization.algorithms; -import javafx.beans.property.ReadOnlyBooleanProperty; - -import org.jabref.logic.util.ProgressCounter; +import org.jabref.logic.ai.util.LongTaskInfo; +import org.jabref.model.ai.chatting.ChatModelInfo; +import org.jabref.model.ai.summarization.SummarizationAlgorithmName; public interface SummarizationAlgorithm { String summarize( - String text, - ProgressCounter progressCounter, - ReadOnlyBooleanProperty shutdownSignal + ChatModelInfo chatModelInfo, + LongTaskInfo longTaskInfo, + String text ) throws InterruptedException; + + SummarizationAlgorithmName getName(); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/repositories/MVStoreSummariesRepository.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/repositories/MVStoreSummariesRepository.java index dffef2fac417..df9beac79d1f 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/repositories/MVStoreSummariesRepository.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/repositories/MVStoreSummariesRepository.java @@ -7,7 +7,7 @@ import org.jabref.logic.ai.util.MVStoreBase; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.NotificationService; -import org.jabref.model.ai.summarization.Summary; +import org.jabref.model.ai.summarization.BibEntrySummary; public class MVStoreSummariesRepository extends MVStoreBase implements SummariesRepository { private static final String SUMMARIES_MAP_PREFIX = "summaries"; @@ -16,11 +16,11 @@ public MVStoreSummariesRepository(NotificationService dialogService, Path path) super(path, dialogService); } - public void set(Path bibDatabasePath, String citationKey, Summary summary) { - getMap(bibDatabasePath).put(citationKey, summary); + public void set(Path bibDatabasePath, String citationKey, BibEntrySummary bibEntrySummary) { + getMap(bibDatabasePath).put(citationKey, bibEntrySummary); } - public Optional get(Path bibDatabasePath, String citationKey) { + public Optional get(Path bibDatabasePath, String citationKey) { return Optional.ofNullable(getMap(bibDatabasePath).get(citationKey)); } @@ -28,7 +28,7 @@ public void clear(Path bibDatabasePath, String citationKey) { getMap(bibDatabasePath).remove(citationKey); } - private Map getMap(Path bibDatabasePath) { + private Map getMap(Path bibDatabasePath) { return mvStore.openMap(SUMMARIES_MAP_PREFIX + "-" + bibDatabasePath.toString()); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/repositories/SummariesRepository.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/repositories/SummariesRepository.java index bfa913773f37..bbc727272334 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/repositories/SummariesRepository.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/repositories/SummariesRepository.java @@ -3,12 +3,12 @@ import java.nio.file.Path; import java.util.Optional; -import org.jabref.model.ai.summarization.Summary; +import org.jabref.model.ai.summarization.BibEntrySummary; public interface SummariesRepository { - void set(Path bibDatabasePath, String citationKey, Summary summary); + void set(Path bibDatabasePath, String citationKey, BibEntrySummary bibEntrySummary); - Optional get(Path bibDatabasePath, String citationKey); + Optional get(Path bibDatabasePath, String citationKey); void clear(Path bibDatabasePath, String citationKey); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java index ced909b3333b..717b3c03c4d7 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java @@ -10,20 +10,19 @@ import javafx.util.Pair; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.preferences.AiPreferences; +import org.jabref.logic.ai.summarization.algorithms.SummarizationAlgorithm; import org.jabref.logic.ai.summarization.repositories.SummariesRepository; -import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.ProgressCounter; import org.jabref.logic.util.TaskExecutor; +import org.jabref.model.ai.chatting.ChatModelInfo; import org.jabref.model.ai.processingstatus.ProcessingInfo; import org.jabref.model.ai.processingstatus.ProcessingState; -import org.jabref.model.ai.summarization.Summary; +import org.jabref.model.ai.summarization.BibEntrySummary; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; -import dev.langchain4j.model.chat.ChatModel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,13 +35,12 @@ public class GenerateSummaryForSeveralTask extends BackgroundTask { private static final Logger LOGGER = LoggerFactory.getLogger(GenerateSummaryForSeveralTask.class); private final StringProperty groupName; - private final List> entries; + private final List> entries; private final BibDatabaseContext bibDatabaseContext; private final SummariesRepository summariesRepository; - private final ChatModel chatLanguageModel; - private final AiTemplatesService aiTemplatesService; + private final ChatModelInfo chatModelInfo; + private final SummarizationAlgorithm summarizationAlgorithm; private final ReadOnlyBooleanProperty shutdownSignal; - private final AiPreferences aiPreferences; private final FilePreferences filePreferences; private final TaskExecutor taskExecutor; @@ -51,25 +49,23 @@ public class GenerateSummaryForSeveralTask extends BackgroundTask { private String currentFile = ""; public GenerateSummaryForSeveralTask( - AiPreferences aiPreferences, FilePreferences filePreferences, - AiTemplatesService aiTemplatesService, TaskExecutor taskExecutor, - ChatModel chatLanguageModel, + ChatModelInfo chatModelInfo, SummariesRepository summariesRepository, + SummarizationAlgorithm summarizationAlgorithm, BibDatabaseContext bibDatabaseContext, StringProperty groupName, - List> entries, + List> entries, ReadOnlyBooleanProperty shutdownSignal ) { this.groupName = groupName; this.entries = entries; this.bibDatabaseContext = bibDatabaseContext; this.summariesRepository = summariesRepository; - this.chatLanguageModel = chatLanguageModel; - this.aiTemplatesService = aiTemplatesService; + this.chatModelInfo = chatModelInfo; + this.summarizationAlgorithm = summarizationAlgorithm; this.shutdownSignal = shutdownSignal; - this.aiPreferences = aiPreferences; this.filePreferences = filePreferences; this.taskExecutor = taskExecutor; @@ -98,7 +94,12 @@ public Void call() throws ExecutionException, InterruptedException { processingInfo.setState(ProcessingState.PROCESSING); return new Pair<>( new GenerateSummaryTask( - aiPreferences, filePreferences, aiTemplatesService, chatLanguageModel, summariesRepository, bibDatabaseContext, processingInfo.getObject(), + filePreferences, + chatModelInfo, + summariesRepository, + summarizationAlgorithm, + bibDatabaseContext, + processingInfo.getObject(), shutdownSignal ) .showToUser(false) diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java index 0598dc0a406f..275d7dd38d80 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java @@ -3,19 +3,19 @@ import javafx.beans.property.ReadOnlyBooleanProperty; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.summarization.SummariesService; import org.jabref.logic.ai.summarization.algorithms.PersistentBibEntrySummarizer; +import org.jabref.logic.ai.summarization.algorithms.SummarizationAlgorithm; import org.jabref.logic.ai.summarization.repositories.SummariesRepository; -import org.jabref.logic.ai.templates.AiTemplatesService; +import org.jabref.logic.ai.util.LongTaskInfo; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.ProgressCounter; -import org.jabref.model.ai.summarization.Summary; +import org.jabref.model.ai.chatting.ChatModelInfo; +import org.jabref.model.ai.summarization.BibEntrySummary; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; -import dev.langchain4j.model.chat.ChatModel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,9 +26,10 @@ *

* This task is created in the {@link SummariesService}, and stored then in a {@link SummariesRepository}. */ -public class GenerateSummaryTask extends BackgroundTask

{ +public class GenerateSummaryTask extends BackgroundTask { private static final Logger LOGGER = LoggerFactory.getLogger(GenerateSummaryTask.class); + private final ChatModelInfo chatModelInfo; private final BibDatabaseContext bibDatabaseContext; private final BibEntry entry; private final String citationKey; @@ -39,26 +40,24 @@ public class GenerateSummaryTask extends BackgroundTask { private final ProgressCounter progressCounter = new ProgressCounter(); public GenerateSummaryTask( - AiPreferences aiPreferences, FilePreferences filePreferences, - AiTemplatesService aiTemplatesService, - ChatModel chatLanguageModel, + ChatModelInfo chatModelInfo, SummariesRepository summariesRepository, + SummarizationAlgorithm summarizationAlgorithm, BibDatabaseContext bibDatabaseContext, BibEntry entry, ReadOnlyBooleanProperty shutdownSignal ) { + this.chatModelInfo = chatModelInfo; this.bibDatabaseContext = bibDatabaseContext; this.entry = entry; this.citationKey = entry.getCitationKey().orElse(""); this.shutdownSignal = shutdownSignal; this.persistentBibEntrySummarizer = new PersistentBibEntrySummarizer( - aiPreferences, filePreferences, summariesRepository, - aiTemplatesService, - chatLanguageModel + summarizationAlgorithm ); configure(); @@ -72,15 +71,25 @@ private void configure() { } @Override - public Summary call() { + public BibEntrySummary call() { LOGGER.debug("Starting summarization task for entry {}", citationKey); - Summary summary = persistentBibEntrySummarizer.summarize(entry, bibDatabaseContext, progressCounter, shutdownSignal); + LongTaskInfo longTaskInfo = new LongTaskInfo( + progressCounter, + shutdownSignal + ); + + BibEntrySummary bibEntrySummary = persistentBibEntrySummarizer.summarize( + chatModelInfo, + longTaskInfo, + bibDatabaseContext, + entry + ); LOGGER.debug("Finished summarization task for entry {}", citationKey); progressCounter.stop(); - return summary; + return bibEntrySummary; } private void updateProgress() { diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationChunkSystemMessageTemplate.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationChunkSystemMessageTemplate.java new file mode 100644 index 000000000000..ed225eeda4b2 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationChunkSystemMessageTemplate.java @@ -0,0 +1,22 @@ +package org.jabref.logic.ai.summarization.templates; + +import org.jabref.logic.ai.templates.Template; +import org.jabref.model.ai.templating.AiTemplate; + +import org.apache.velocity.VelocityContext; + +public class SummarizationChunkSystemMessageTemplate extends Template { + public SummarizationChunkSystemMessageTemplate(String source) { + super(source); + } + + public String render() { + VelocityContext context = makeContext(); + return render(context); + } + + @Override + public String getLogName() { + return AiTemplate.SUMMARIZATION_CHUNK_SYSTEM_MESSAGE.name(); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationChunkUserMessageTemplate.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationChunkUserMessageTemplate.java new file mode 100644 index 000000000000..916d7b263bab --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationChunkUserMessageTemplate.java @@ -0,0 +1,25 @@ +package org.jabref.logic.ai.summarization.templates; + +import org.jabref.logic.ai.templates.Template; +import org.jabref.model.ai.templating.AiTemplate; + +import org.apache.velocity.VelocityContext; + +public class SummarizationChunkUserMessageTemplate extends Template { + public SummarizationChunkUserMessageTemplate(String source) { + super(source); + } + + public String render(String text) { + VelocityContext context = makeContext(); + + context.put("text", text); + + return render(context); + } + + @Override + public String getLogName() { + return AiTemplate.SUMMARIZATION_CHUNK_USER_MESSAGE.name(); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationCombineSystemMessageTemplate.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationCombineSystemMessageTemplate.java new file mode 100644 index 000000000000..5615a7a56377 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationCombineSystemMessageTemplate.java @@ -0,0 +1,22 @@ +package org.jabref.logic.ai.summarization.templates; + +import org.jabref.logic.ai.templates.Template; +import org.jabref.model.ai.templating.AiTemplate; + +import org.apache.velocity.VelocityContext; + +public class SummarizationCombineSystemMessageTemplate extends Template { + public SummarizationCombineSystemMessageTemplate(String source) { + super(source); + } + + public String render() { + VelocityContext context = makeContext(); + return render(context); + } + + @Override + public String getLogName() { + return AiTemplate.SUMMARIZATION_COMBINE_SYSTEM_MESSAGE.name(); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationCombineUserMessageTemplate.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationCombineUserMessageTemplate.java new file mode 100644 index 000000000000..cf96b0c001b4 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationCombineUserMessageTemplate.java @@ -0,0 +1,27 @@ +package org.jabref.logic.ai.summarization.templates; + +import java.util.List; + +import org.jabref.logic.ai.templates.Template; +import org.jabref.model.ai.templating.AiTemplate; + +import org.apache.velocity.VelocityContext; + +public class SummarizationCombineUserMessageTemplate extends Template { + public SummarizationCombineUserMessageTemplate(String source) { + super(source); + } + + public String render(List chunks) { + VelocityContext context = makeContext(); + + context.put("chunks", chunks); + + return render(context); + } + + @Override + public String getLogName() { + return AiTemplate.SUMMARIZATION_COMBINE_USER_MESSAGE.name(); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java b/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java index ccb1552a2937..b09848835028 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java @@ -4,8 +4,8 @@ import java.util.List; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.model.ai.templating.AiTemplate; import org.jabref.model.ai.rag.PaperExcerpt; +import org.jabref.model.ai.templating.AiTemplate; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.CanonicalBibEntry; @@ -42,30 +42,6 @@ public String makeChattingUserMessage(List entries, String message, Li return makeTemplate(AiTemplate.CHATTING_USER_MESSAGE, context); } - public String makeSummarizationChunkSystemMessage() { - VelocityContext context = new VelocityContext(baseContext); - return makeTemplate(AiTemplate.SUMMARIZATION_CHUNK_SYSTEM_MESSAGE, context); - } - - public String makeSummarizationChunkUserMessage(String text) { - VelocityContext context = new VelocityContext(baseContext); - context.put("text", text); - - return makeTemplate(AiTemplate.SUMMARIZATION_CHUNK_USER_MESSAGE, context); - } - - public String makeSummarizationCombineSystemMessage() { - VelocityContext context = new VelocityContext(baseContext); - return makeTemplate(AiTemplate.SUMMARIZATION_COMBINE_SYSTEM_MESSAGE, context); - } - - public String makeSummarizationCombineUserMessage(List chunks) { - VelocityContext context = new VelocityContext(baseContext); - context.put("chunks", chunks); - - return makeTemplate(AiTemplate.SUMMARIZATION_COMBINE_USER_MESSAGE, context); - } - public String makeCitationParsingSystemMessage() { VelocityContext context = new VelocityContext(baseContext); return makeTemplate(AiTemplate.CITATION_PARSING_SYSTEM_MESSAGE, context); @@ -85,4 +61,8 @@ private String makeTemplate(AiTemplate template, VelocityContext context) { return writer.toString(); } + + public String getRawTemplate(AiTemplate template) { + return aiPreferences.getTemplate(template); + } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/templates/Template.java b/jablib/src/main/java/org/jabref/logic/ai/templates/Template.java new file mode 100644 index 000000000000..eb55ee643237 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/templates/Template.java @@ -0,0 +1,42 @@ +package org.jabref.logic.ai.templates; + +import java.io.StringWriter; + +import org.jabref.model.entry.CanonicalBibEntry; + +import org.apache.velocity.VelocityContext; +import org.apache.velocity.app.VelocityEngine; + +public abstract class Template { + private static final VelocityEngine VELOCITY_ENGINE = new VelocityEngine(); + private static final VelocityContext BASE_CONTEXT = new VelocityContext(); + + static { + VELOCITY_ENGINE.init(); + + BASE_CONTEXT.put("CanonicalBibEntry", CanonicalBibEntry.class); + } + + private final String source; + + public Template(String source) { + this.source = source; + } + + protected String render(VelocityContext context) { + StringWriter writer = new StringWriter(); + VELOCITY_ENGINE.evaluate(context, writer, getLogName(), source); + return writer.toString(); + } + + protected VelocityContext makeContext() { + return new VelocityContext(BASE_CONTEXT); + } + + public String getSource() { + return source; + } + + // Required by Velocity. + public abstract String getLogName(); +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/util/LongTaskInfo.java b/jablib/src/main/java/org/jabref/logic/ai/util/LongTaskInfo.java new file mode 100644 index 000000000000..e35f3a560fcd --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/util/LongTaskInfo.java @@ -0,0 +1,8 @@ +package org.jabref.logic.ai.util; + +import javafx.beans.property.ReadOnlyBooleanProperty; + +import org.jabref.logic.util.ProgressCounter; + +public record LongTaskInfo(ProgressCounter progressCounter, ReadOnlyBooleanProperty shutdownSignal) { +} diff --git a/jablib/src/main/java/org/jabref/model/ai/chatting/ChatModelInfo.java b/jablib/src/main/java/org/jabref/model/ai/chatting/ChatModelInfo.java new file mode 100644 index 000000000000..0eb847b48fc6 --- /dev/null +++ b/jablib/src/main/java/org/jabref/model/ai/chatting/ChatModelInfo.java @@ -0,0 +1,6 @@ +package org.jabref.model.ai.chatting; + +import dev.langchain4j.model.chat.ChatModel; + +public record ChatModelInfo(ChatModel chatModel, AiProvider aiProvider, String name, int contextWindowSize) { +} diff --git a/jablib/src/main/java/org/jabref/model/ai/summarization/BibEntrySummary.java b/jablib/src/main/java/org/jabref/model/ai/summarization/BibEntrySummary.java new file mode 100644 index 000000000000..595dcd1f9310 --- /dev/null +++ b/jablib/src/main/java/org/jabref/model/ai/summarization/BibEntrySummary.java @@ -0,0 +1,15 @@ +package org.jabref.model.ai.summarization; + +import java.io.Serializable; +import java.time.LocalDateTime; + +import org.jabref.model.ai.chatting.AiProvider; + +public record BibEntrySummary( + LocalDateTime timestamp, + AiProvider aiProvider, + String model, + SummarizationAlgorithmName summarizationAlgorithm, + String content +) implements Serializable { +} diff --git a/jablib/src/main/java/org/jabref/model/ai/summarization/SummarizationAlgorithmName.java b/jablib/src/main/java/org/jabref/model/ai/summarization/SummarizationAlgorithmName.java new file mode 100644 index 000000000000..8d064cb9a500 --- /dev/null +++ b/jablib/src/main/java/org/jabref/model/ai/summarization/SummarizationAlgorithmName.java @@ -0,0 +1,17 @@ +package org.jabref.model.ai.summarization; + +import org.jabref.logic.l10n.Localization; + +public enum SummarizationAlgorithmName { + CHUNKED(Localization.lang("Chunked Summarization")); + + private final String displayName; + + SummarizationAlgorithmName(String displayName) { + this.displayName = displayName; + } + + public String getDisplayName() { + return displayName; + } +} diff --git a/jablib/src/main/java/org/jabref/model/ai/summarization/Summary.java b/jablib/src/main/java/org/jabref/model/ai/summarization/Summary.java deleted file mode 100644 index fdba2fa7db8c..000000000000 --- a/jablib/src/main/java/org/jabref/model/ai/summarization/Summary.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.jabref.model.ai.summarization; - -import java.io.Serializable; -import java.time.LocalDateTime; - -import org.jabref.model.ai.chatting.AiProvider; - -public record Summary(LocalDateTime timestamp, AiProvider aiProvider, String model, String content) implements Serializable { -} diff --git a/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesRepositoryTest.java b/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesRepositoryTest.java index 44da7f66bc94..35ab11c4eace 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesRepositoryTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesRepositoryTest.java @@ -6,7 +6,7 @@ import org.jabref.logic.ai.summarization.repositories.SummariesRepository; import org.jabref.model.ai.chatting.AiProvider; -import org.jabref.model.ai.summarization.Summary; +import org.jabref.model.ai.summarization.BibEntrySummary; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -43,14 +43,14 @@ void tearDown() { @Test void set() { - summariesRepository.set(bibPath, "citationKey", new Summary(LocalDateTime.now(), AiProvider.OPEN_AI, "model", "contents")); + summariesRepository.set(bibPath, "citationKey", new BibEntrySummary(LocalDateTime.now(), AiProvider.OPEN_AI, "model", "contents")); reopen(); - assertEquals(Optional.of("contents"), summariesRepository.get(bibPath, "citationKey").map(Summary::content)); + assertEquals(Optional.of("contents"), summariesRepository.get(bibPath, "citationKey").map(BibEntrySummary::content)); } @Test void clear() { - summariesRepository.set(bibPath, "citationKey", new Summary(LocalDateTime.now(), AiProvider.OPEN_AI, "model", "contents")); + summariesRepository.set(bibPath, "citationKey", new BibEntrySummary(LocalDateTime.now(), AiProvider.OPEN_AI, "model", "contents")); reopen(); summariesRepository.clear(bibPath, "citationKey"); reopen(); From 17f5ba8508a2becfd9b5e2011ac6d3a32dae572b Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Wed, 26 Nov 2025 09:40:40 +0100 Subject: [PATCH 020/243] Rename `algorithms` to `logic` --- .../gui/ai/components/aichat/AiChatComponent.java | 2 +- .../gui/ai/components/aichat/AiChatComponentTest.java | 2 +- jablib/src/main/java/module-info.java | 7 ++++--- .../src/main/java/org/jabref/logic/ai/AiService.java | 2 +- .../chatting/CurrentlySelectedChatLanguageModel.java | 2 +- .../ai/chatting/{algorithms => logic}/AiChatLogic.java | 2 +- .../ai/chatting/tasks/GenerateAiResponseTask.java | 2 +- .../ai/rag/{algorithms => logic}/LowLevelIngestor.java | 2 +- .../rag/{algorithms => logic}/parsing/FileParser.java | 2 +- .../{algorithms => logic}/parsing/PdfFileParser.java | 2 +- .../parsing/UniversalFileParser.java | 2 +- .../ai/rag/repositories/FileEmbeddingsManager.java | 2 +- .../logic/ai/rag/tasks/GenerateEmbeddingsTask.java | 2 +- .../CurrentlySelectedSummarizationAlgorithm.java | 10 ++-------- .../logic/ai/summarization/SummariesService.java | 2 +- .../{algorithms => logic}/BibEntrySummarizer.java | 5 +++-- .../PersistentBibEntrySummarizer.java | 3 ++- .../ChunkedSummarizationAlgorithm.java | 2 +- .../SummarizationAlgorithm.java | 2 +- .../tasks/GenerateSummaryForSeveralTask.java | 2 +- .../ai/summarization/tasks/GenerateSummaryTask.java | 4 ++-- 21 files changed, 29 insertions(+), 32 deletions(-) rename jablib/src/main/java/org/jabref/logic/ai/chatting/{algorithms => logic}/AiChatLogic.java (99%) rename jablib/src/main/java/org/jabref/logic/ai/rag/{algorithms => logic}/LowLevelIngestor.java (98%) rename jablib/src/main/java/org/jabref/logic/ai/rag/{algorithms => logic}/parsing/FileParser.java (81%) rename jablib/src/main/java/org/jabref/logic/ai/rag/{algorithms => logic}/parsing/PdfFileParser.java (96%) rename jablib/src/main/java/org/jabref/logic/ai/rag/{algorithms => logic}/parsing/UniversalFileParser.java (93%) rename jablib/src/main/java/org/jabref/logic/ai/summarization/{algorithms => logic}/BibEntrySummarizer.java (96%) rename jablib/src/main/java/org/jabref/logic/ai/summarization/{algorithms => logic}/PersistentBibEntrySummarizer.java (95%) rename jablib/src/main/java/org/jabref/logic/ai/summarization/{algorithms => logic/summarizationalgorithms}/ChunkedSummarizationAlgorithm.java (98%) rename jablib/src/main/java/org/jabref/logic/ai/summarization/{algorithms => logic/summarizationalgorithms}/SummarizationAlgorithm.java (85%) diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java index 9acd77576829..e2f1b3d13512 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java @@ -28,7 +28,7 @@ import org.jabref.gui.icon.IconTheme; import org.jabref.gui.util.UiTaskExecutor; import org.jabref.logic.ai.AiService; -import org.jabref.logic.ai.chatting.algorithms.AiChatLogic; +import org.jabref.logic.ai.chatting.logic.AiChatLogic; import org.jabref.logic.ai.chatting.tasks.GenerateAiResponseTask; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.l10n.Localization; diff --git a/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java b/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java index 9226bffce365..aa12abbfd0f4 100644 --- a/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java +++ b/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java @@ -10,7 +10,7 @@ import org.jabref.gui.DialogService; import org.jabref.logic.ai.AiService; -import org.jabref.logic.ai.chatting.algorithms.AiChatLogic; +import org.jabref.logic.ai.chatting.logic.AiChatLogic; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.rag.IngestionService; import org.jabref.logic.l10n.Language; diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index 050a11fbe596..0e80610696ed 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -126,11 +126,11 @@ exports org.jabref.logic.ai.summarization.tasks; exports org.jabref.logic.ai.summarization.repositories; exports org.jabref.logic.ai.summarization.templates; - exports org.jabref.logic.ai.summarization.algorithms; + exports org.jabref.logic.ai.summarization.logic; exports org.jabref.logic.ai.rag.tasks; exports org.jabref.logic.ai.rag.repositories; - exports org.jabref.logic.ai.rag.algorithms; - exports org.jabref.logic.ai.chatting.algorithms; + exports org.jabref.logic.ai.rag.logic; + exports org.jabref.logic.ai.chatting.logic; exports org.jabref.logic.ai.chatting.tasks; exports org.jabref.logic.ai.preferences; exports org.jabref.logic.ai.chatting.repositories; @@ -140,6 +140,7 @@ exports org.jabref.model.ai.embeddings; exports org.jabref.model.ai.summarization; exports org.jabref.model.ai.identifiers; + exports org.jabref.logic.ai.summarization.logic.summarizationalgorithms; // endregion requires java.base; diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiService.java b/jablib/src/main/java/org/jabref/logic/ai/AiService.java index bd4538ebcfa0..fea3a2b13a42 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiService.java @@ -8,8 +8,8 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.chatting.ChatHistoryService; -import org.jabref.logic.ai.chatting.CurrentlySelectedChatLanguageModel; import org.jabref.logic.ai.chatting.repositories.MVStoreChatHistoryRepository; +import org.jabref.logic.ai.currentsettings.CurrentlySelectedChatLanguageModel; import org.jabref.logic.ai.customimplementations.embeddingstores.MVStoreEmbeddingStore; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.rag.CurrentlySelectedEmbeddingModel; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java index 274aa3d84dbd..b0fdb235fe85 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java @@ -7,7 +7,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import org.jabref.logic.ai.chatting.algorithms.AiChatLogic; +import org.jabref.logic.ai.chatting.logic.AiChatLogic; import org.jabref.logic.ai.chatting.repositories.ChatHistoryRepository; import org.jabref.logic.ai.customimplementations.llms.Gpt4AllModel; import org.jabref.logic.ai.customimplementations.llms.JvmOpenAiChatLanguageModel; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/algorithms/AiChatLogic.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java similarity index 99% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/algorithms/AiChatLogic.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java index 25844920a9e1..ee07893e68f4 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/algorithms/AiChatLogic.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.chatting.algorithms; +package org.jabref.logic.ai.chatting.logic; import java.util.List; import java.util.Optional; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/tasks/GenerateAiResponseTask.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/tasks/GenerateAiResponseTask.java index d8591ef550e1..53bbeede5c1e 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/tasks/GenerateAiResponseTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/tasks/GenerateAiResponseTask.java @@ -1,6 +1,6 @@ package org.jabref.logic.ai.chatting.tasks; -import org.jabref.logic.ai.chatting.algorithms.AiChatLogic; +import org.jabref.logic.ai.chatting.logic.AiChatLogic; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/LowLevelIngestor.java b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/LowLevelIngestor.java similarity index 98% rename from jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/LowLevelIngestor.java rename to jablib/src/main/java/org/jabref/logic/ai/rag/logic/LowLevelIngestor.java index 5b28fd5a14b9..2de142dd323d 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/LowLevelIngestor.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/LowLevelIngestor.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.rag.algorithms; +package org.jabref.logic.ai.rag.logic; import java.util.List; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/FileParser.java b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/FileParser.java similarity index 81% rename from jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/FileParser.java rename to jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/FileParser.java index d1748614ed4c..afa4a35d8116 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/FileParser.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/FileParser.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.rag.algorithms.parsing; +package org.jabref.logic.ai.rag.logic.parsing; import java.nio.file.Path; import java.util.Optional; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/PdfFileParser.java b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/PdfFileParser.java similarity index 96% rename from jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/PdfFileParser.java rename to jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/PdfFileParser.java index c8139c7e05e1..f5e8b9b73e5b 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/PdfFileParser.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/PdfFileParser.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.rag.algorithms.parsing; +package org.jabref.logic.ai.rag.logic.parsing; import java.io.IOException; import java.io.StringWriter; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/UniversalFileParser.java b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/UniversalFileParser.java similarity index 93% rename from jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/UniversalFileParser.java rename to jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/UniversalFileParser.java index 2e847e8792b1..750c1f84235d 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/algorithms/parsing/UniversalFileParser.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/UniversalFileParser.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.rag.algorithms.parsing; +package org.jabref.logic.ai.rag.logic.parsing; import java.nio.file.Path; import java.util.Optional; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FileEmbeddingsManager.java b/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FileEmbeddingsManager.java index 023731c43738..43767be6f82b 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FileEmbeddingsManager.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FileEmbeddingsManager.java @@ -7,7 +7,7 @@ import javafx.beans.property.ReadOnlyBooleanProperty; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.rag.algorithms.LowLevelIngestor; +import org.jabref.logic.ai.rag.logic.LowLevelIngestor; import org.jabref.model.entry.LinkedFile; import dev.langchain4j.data.document.DefaultDocument; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java index e9e51c0b833c..8d2b1b0abcf8 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java @@ -10,7 +10,7 @@ import javafx.beans.property.ReadOnlyBooleanProperty; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.rag.algorithms.parsing.UniversalFileParser; +import org.jabref.logic.ai.rag.logic.parsing.UniversalFileParser; import org.jabref.logic.ai.rag.repositories.FileEmbeddingsManager; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/CurrentlySelectedSummarizationAlgorithm.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/CurrentlySelectedSummarizationAlgorithm.java index 8f88d56542a8..c9b3702ab63e 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/CurrentlySelectedSummarizationAlgorithm.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/CurrentlySelectedSummarizationAlgorithm.java @@ -1,10 +1,8 @@ package org.jabref.logic.ai.summarization; -import java.util.Optional; - import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.summarization.algorithms.ChunkedSummarizationAlgorithm; -import org.jabref.logic.ai.summarization.algorithms.SummarizationAlgorithm; +import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.ChunkedSummarizationAlgorithm; +import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; import org.jabref.logic.ai.summarization.templates.SummarizationChunkSystemMessageTemplate; import org.jabref.logic.ai.summarization.templates.SummarizationChunkUserMessageTemplate; import org.jabref.logic.ai.summarization.templates.SummarizationCombineSystemMessageTemplate; @@ -30,10 +28,6 @@ public CurrentlySelectedSummarizationAlgorithm( configure(); } - public Optional getSummarizationAlgorithm() { - return Optional.ofNullable(summarizationAlgorithm); - } - private void configure() { aiPreferences.defaultSummarizationAlgorithmProperty().addListener(_ -> { updateAlgorithm(); diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java index 4c2e4bae4166..c75d86876e81 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java @@ -10,7 +10,7 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.summarization.algorithms.SummarizationAlgorithm; +import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; import org.jabref.logic.ai.summarization.repositories.SummariesRepository; import org.jabref.logic.ai.summarization.tasks.GenerateSummaryForSeveralTask; import org.jabref.logic.ai.summarization.tasks.GenerateSummaryTask; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/BibEntrySummarizer.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/BibEntrySummarizer.java similarity index 96% rename from jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/BibEntrySummarizer.java rename to jablib/src/main/java/org/jabref/logic/ai/summarization/logic/BibEntrySummarizer.java index 269b75f88a4f..15e3aff9830b 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/BibEntrySummarizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/BibEntrySummarizer.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.summarization.algorithms; +package org.jabref.logic.ai.summarization.logic; import java.nio.file.Path; import java.time.LocalDateTime; @@ -9,7 +9,8 @@ import java.util.stream.Stream; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.rag.algorithms.parsing.UniversalFileParser; +import org.jabref.logic.ai.rag.logic.parsing.UniversalFileParser; +import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; import org.jabref.logic.ai.util.LongTaskInfo; import org.jabref.logic.l10n.Localization; import org.jabref.model.ai.chatting.ChatModelInfo; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/PersistentBibEntrySummarizer.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/PersistentBibEntrySummarizer.java similarity index 95% rename from jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/PersistentBibEntrySummarizer.java rename to jablib/src/main/java/org/jabref/logic/ai/summarization/logic/PersistentBibEntrySummarizer.java index d86b05db9332..29b7deb2592f 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/PersistentBibEntrySummarizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/PersistentBibEntrySummarizer.java @@ -1,8 +1,9 @@ -package org.jabref.logic.ai.summarization.algorithms; +package org.jabref.logic.ai.summarization.logic; import java.util.Optional; import org.jabref.logic.FilePreferences; +import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; import org.jabref.logic.ai.summarization.repositories.SummariesRepository; import org.jabref.logic.ai.util.LongTaskInfo; import org.jabref.logic.util.CitationKeyCheck; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/ChunkedSummarizationAlgorithm.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/ChunkedSummarizationAlgorithm.java similarity index 98% rename from jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/ChunkedSummarizationAlgorithm.java rename to jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/ChunkedSummarizationAlgorithm.java index 17706f4e0107..f01d779a9900 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/ChunkedSummarizationAlgorithm.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/ChunkedSummarizationAlgorithm.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.summarization.algorithms; +package org.jabref.logic.ai.summarization.logic.summarizationalgorithms; import java.util.ArrayList; import java.util.List; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/SummarizationAlgorithm.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/SummarizationAlgorithm.java similarity index 85% rename from jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/SummarizationAlgorithm.java rename to jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/SummarizationAlgorithm.java index 702a10a4ec58..507da44c0cd9 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/algorithms/SummarizationAlgorithm.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/SummarizationAlgorithm.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.summarization.algorithms; +package org.jabref.logic.ai.summarization.logic.summarizationalgorithms; import org.jabref.logic.ai.util.LongTaskInfo; import org.jabref.model.ai.chatting.ChatModelInfo; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java index 717b3c03c4d7..b7f30261869d 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java @@ -10,7 +10,7 @@ import javafx.util.Pair; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.summarization.algorithms.SummarizationAlgorithm; +import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; import org.jabref.logic.ai.summarization.repositories.SummariesRepository; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java index 275d7dd38d80..e0c1ac821722 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java @@ -4,8 +4,8 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.summarization.SummariesService; -import org.jabref.logic.ai.summarization.algorithms.PersistentBibEntrySummarizer; -import org.jabref.logic.ai.summarization.algorithms.SummarizationAlgorithm; +import org.jabref.logic.ai.summarization.logic.PersistentBibEntrySummarizer; +import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; import org.jabref.logic.ai.summarization.repositories.SummariesRepository; import org.jabref.logic.ai.util.LongTaskInfo; import org.jabref.logic.l10n.Localization; From 28af1cf04ea0ca51e7234372b6fd3e81ec6c75cb Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Wed, 26 Nov 2025 14:34:59 +0100 Subject: [PATCH 021/243] Implement Tokenizers --- .../components/summary/SummaryComponent.java | 12 ++- .../jabref/gui/groups/GroupTreeViewModel.java | 1 + jablib/src/main/java/module-info.java | 3 +- .../java/org/jabref/logic/ai/AiService.java | 22 +++++- .../CurrentlySelectedChatLanguageModel.java | 12 ++- ...rentlySelectedTokenEstimationStrategy.java | 74 +++++++++++++++++++ .../algorithms/AverageTokenizer.java | 37 ++++++++++ .../algorithms/ByCharacterTokenizer.java | 39 ++++++++++ .../algorithms/ByWordsTokenizer.java | 39 ++++++++++ .../algorithms/MaximumTokenizer.java | 37 ++++++++++ .../algorithms/MinimumTokenizer.java | 37 ++++++++++ .../tokenization/algorithms/Tokenizer.java | 15 ++++ .../logic/ai/preferences/AiPreferences.java | 16 ++++ ...rrentlySelectedSummarizationAlgorithm.java | 4 +- .../ChunkedSummarizationAlgorithm.java | 18 +---- .../logic/ai/util/ChatMessagesUtil.java | 36 +++++++++ .../preferences/JabRefCliPreferences.java | 10 +++ .../model/ai/chatting/ChatModelInfo.java | 10 ++- .../tokenization/TokenEstimationStrategy.java | 19 +++++ 19 files changed, 414 insertions(+), 27 deletions(-) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/CurrentlySelectedTokenEstimationStrategy.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/AverageTokenizer.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByCharacterTokenizer.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByWordsTokenizer.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MaximumTokenizer.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MinimumTokenizer.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/Tokenizer.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/util/ChatMessagesUtil.java create mode 100644 jablib/src/main/java/org/jabref/model/ai/tokenization/TokenEstimationStrategy.java diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java index 25b2465b84fe..95f5d6fb4cf5 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java @@ -51,7 +51,11 @@ public SummaryComponent(BibDatabaseContext bibDatabaseContext, this.aiService = aiService; this.aiPreferences = aiPreferences; - aiService.getSummariesService().summarize(entry, bibDatabaseContext).stateProperty().addListener(o -> rebuildUi()); + aiService.getSummariesService().summarize( + aiService.getSummarizationAlgorithm(), + entry, + bibDatabaseContext + ).stateProperty().addListener(o -> rebuildUi()); rebuildUi(); } @@ -104,7 +108,7 @@ private Node tryToGenerateCitationKeyThenBind(BibEntry entry) { } private Node tryToShowSummary() { - ProcessingInfo processingInfo = aiService.getSummariesService().summarize(entry, bibDatabaseContext); + ProcessingInfo processingInfo = aiService.getSummariesService().summarize(aiService.getSummarizationAlgorithm(), entry, bibDatabaseContext); return switch (processingInfo.getState()) { case SUCCESS -> { @@ -129,7 +133,7 @@ private Node showErrorWhileSummarizing(ProcessingInfo Localization.lang("Got error while processing the file:"), processingInfo.getException().get().getLocalizedMessage(), Localization.lang("Regenerate"), - () -> aiService.getSummariesService().regenerateSummary(entry, bibDatabaseContext) + () -> aiService.getSummariesService().regenerateSummary(aiService.getSummarizationAlgorithm(), entry, bibDatabaseContext) ); } @@ -152,7 +156,7 @@ private Node showSummary(BibEntrySummary bibEntrySummary) { return; } - aiService.getSummariesService().regenerateSummary(entry, bibDatabaseContext); + aiService.getSummariesService().regenerateSummary(aiService.getSummarizationAlgorithm(), entry, bibDatabaseContext); // No need to rebuildUi(), because this class listens to the state of ProcessingInfo of the bibEntrySummary. }); } diff --git a/jabgui/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java b/jabgui/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java index 3048874495e9..b9f5dc07c650 100644 --- a/jabgui/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java +++ b/jabgui/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java @@ -534,6 +534,7 @@ public void generateSummaries(GroupNodeViewModel groupNode) { .toList(); aiService.getSummariesService().summarize( + aiService.getSummarizationAlgorithm(), group.nameProperty(), entries, currentDatabase.get() diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index 0e80610696ed..1c3403fe19bd 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -127,6 +127,7 @@ exports org.jabref.logic.ai.summarization.repositories; exports org.jabref.logic.ai.summarization.templates; exports org.jabref.logic.ai.summarization.logic; + exports org.jabref.logic.ai.summarization.logic.summarizationalgorithms; exports org.jabref.logic.ai.rag.tasks; exports org.jabref.logic.ai.rag.repositories; exports org.jabref.logic.ai.rag.logic; @@ -140,7 +141,7 @@ exports org.jabref.model.ai.embeddings; exports org.jabref.model.ai.summarization; exports org.jabref.model.ai.identifiers; - exports org.jabref.logic.ai.summarization.logic.summarizationalgorithms; + exports org.jabref.model.ai.tokenization; // endregion requires java.base; diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiService.java b/jablib/src/main/java/org/jabref/logic/ai/AiService.java index fea3a2b13a42..30541a865da1 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiService.java @@ -8,14 +8,17 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.chatting.ChatHistoryService; +import org.jabref.logic.ai.chatting.CurrentlySelectedChatLanguageModel; import org.jabref.logic.ai.chatting.repositories.MVStoreChatHistoryRepository; -import org.jabref.logic.ai.currentsettings.CurrentlySelectedChatLanguageModel; import org.jabref.logic.ai.customimplementations.embeddingstores.MVStoreEmbeddingStore; +import org.jabref.logic.ai.customimplementations.tokenization.CurrentlySelectedTokenEstimationStrategy; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.rag.CurrentlySelectedEmbeddingModel; import org.jabref.logic.ai.rag.IngestionService; import org.jabref.logic.ai.rag.repositories.MVStoreFullyIngestedDocumentsRepository; +import org.jabref.logic.ai.summarization.CurrentlySelectedSummarizationAlgorithm; import org.jabref.logic.ai.summarization.SummariesService; +import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; import org.jabref.logic.ai.summarization.repositories.MVStoreSummariesRepository; import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; @@ -57,8 +60,10 @@ public class AiService implements AutoCloseable { private final AiTemplatesService templatesService; private final ChatHistoryService chatHistoryService; + private final CurrentlySelectedTokenEstimationStrategy currentlySelectedTokenEstimationStrategy; private final CurrentlySelectedChatLanguageModel currentlySelectedChatLanguageModel; private final CurrentlySelectedEmbeddingModel currentlySelectedEmbeddingModel; + private final CurrentlySelectedSummarizationAlgorithm currentlySelectedSummarizationAlgorithm; private final IngestionService ingestionService; private final SummariesService summariesService; @@ -77,8 +82,10 @@ public AiService( this.templatesService = new AiTemplatesService(aiPreferences); this.chatHistoryService = new ChatHistoryService(citationKeyPatternPreferences, mvStoreChatHistoryStorage); - this.currentlySelectedChatLanguageModel = new CurrentlySelectedChatLanguageModel(aiPreferences); + this.currentlySelectedTokenEstimationStrategy = new CurrentlySelectedTokenEstimationStrategy(aiPreferences); + this.currentlySelectedChatLanguageModel = new CurrentlySelectedChatLanguageModel(aiPreferences, currentlySelectedTokenEstimationStrategy); this.currentlySelectedEmbeddingModel = new CurrentlySelectedEmbeddingModel(aiPreferences, notificationService, taskExecutor); + this.currentlySelectedSummarizationAlgorithm = new CurrentlySelectedSummarizationAlgorithm(aiPreferences); this.ingestionService = new IngestionService( aiPreferences, @@ -93,9 +100,9 @@ public AiService( this.summariesService = new SummariesService( aiPreferences, filePreferences, - templatesService, taskExecutor, currentlySelectedChatLanguageModel.getChatModelInfo(), + currentlySelectedSummarizationAlgorithm, mvStoreSummariesStorage, shutdownSignal ); @@ -129,6 +136,14 @@ public EmbeddingStore getEmbeddingStore() { return mvStoreEmbeddingStore; } + public SummarizationAlgorithm getSummarizationAlgorithm() { + return currentlySelectedSummarizationAlgorithm; + } + + public CurrentlySelectedTokenEstimationStrategy getTokenEstimationStrategy() { + return currentlySelectedTokenEstimationStrategy; + } + public void setupDatabase(BibDatabaseContext context) { chatHistoryService.setupDatabase(context); ingestionService.setupDatabase(context); @@ -143,6 +158,7 @@ public void close() { currentlySelectedChatLanguageModel.close(); currentlySelectedEmbeddingModel.close(); + mvStoreChatHistoryStorage.close(); mvStoreFullyIngestedDocumentsTracker.close(); mvStoreEmbeddingStore.close(); mvStoreSummariesStorage.close(); diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java index b0fdb235fe85..a646d5b6fa04 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java @@ -11,6 +11,7 @@ import org.jabref.logic.ai.chatting.repositories.ChatHistoryRepository; import org.jabref.logic.ai.customimplementations.llms.Gpt4AllModel; import org.jabref.logic.ai.customimplementations.llms.JvmOpenAiChatLanguageModel; +import org.jabref.logic.ai.customimplementations.tokenization.CurrentlySelectedTokenEstimationStrategy; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.l10n.Localization; import org.jabref.model.ai.chatting.AiProvider; @@ -36,6 +37,8 @@ public class CurrentlySelectedChatLanguageModel implements ChatModel, AutoClosea private final AiPreferences aiPreferences; + private final CurrentlySelectedTokenEstimationStrategy currentlySelectedTokenEstimationStrategy; + private final HttpClient httpClient; private final ExecutorService executorService = Executors.newSingleThreadExecutor( new ThreadFactoryBuilder().setNameFormat("ai-api-connection-pool-%d").build() @@ -43,8 +46,14 @@ public class CurrentlySelectedChatLanguageModel implements ChatModel, AutoClosea private Optional langchainChatModel = Optional.empty(); - public CurrentlySelectedChatLanguageModel(AiPreferences aiPreferences) { + public CurrentlySelectedChatLanguageModel( + AiPreferences aiPreferences, + CurrentlySelectedTokenEstimationStrategy currentlySelectedTokenEstimationStrategy + ) { this.aiPreferences = aiPreferences; + + this.currentlySelectedTokenEstimationStrategy = currentlySelectedTokenEstimationStrategy; + this.httpClient = HttpClient.newBuilder().connectTimeout(CONNECTION_TIMEOUT).executor(executorService).build(); setupListeningToPreferencesChanges(); @@ -121,6 +130,7 @@ public ChatModelInfo getChatModelInfo() { assert langchainChatModel.isPresent(); return new ChatModelInfo( langchainChatModel.get(), + currentlySelectedTokenEstimationStrategy, aiPreferences.getAiProvider(), aiPreferences.getSelectedChatModel(), aiPreferences.getContextWindowSize() diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/CurrentlySelectedTokenEstimationStrategy.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/CurrentlySelectedTokenEstimationStrategy.java new file mode 100644 index 000000000000..53e9b28d45e2 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/CurrentlySelectedTokenEstimationStrategy.java @@ -0,0 +1,74 @@ +package org.jabref.logic.ai.customimplementations.tokenization; + +import java.util.List; + +import org.jabref.logic.ai.customimplementations.tokenization.algorithms.AverageTokenizer; +import org.jabref.logic.ai.customimplementations.tokenization.algorithms.ByCharacterTokenizer; +import org.jabref.logic.ai.customimplementations.tokenization.algorithms.ByWordsTokenizer; +import org.jabref.logic.ai.customimplementations.tokenization.algorithms.MaximumTokenizer; +import org.jabref.logic.ai.customimplementations.tokenization.algorithms.MinimumTokenizer; +import org.jabref.logic.ai.customimplementations.tokenization.algorithms.Tokenizer; +import org.jabref.logic.ai.preferences.AiPreferences; +import org.jabref.model.ai.tokenization.TokenEstimationStrategy; + +import dev.langchain4j.data.message.ChatMessage; +import org.jspecify.annotations.Nullable; + +public class CurrentlySelectedTokenEstimationStrategy implements Tokenizer { + private final AiPreferences aiPreferences; + + @Nullable + private Tokenizer tokenizer; + + public CurrentlySelectedTokenEstimationStrategy( + AiPreferences aiPreferences + ) { + this.aiPreferences = aiPreferences; + + setupListeningToPreferences(); + } + + private void setupListeningToPreferences() { + aiPreferences.tokenEstimationStrategyProperty().addListener(_ -> { + createTokenizer(); + }); + } + + private void createTokenizer() { + switch (aiPreferences.getTokenEstimationStrategy()) { + case TokenEstimationStrategy.AVERAGE -> tokenizer = new AverageTokenizer(); + case TokenEstimationStrategy.MAX -> tokenizer = new MaximumTokenizer(); + case TokenEstimationStrategy.MIN -> tokenizer = new MinimumTokenizer(); + case TokenEstimationStrategy.CHARS -> tokenizer = new ByCharacterTokenizer(); + case TokenEstimationStrategy.WORDS -> tokenizer = new ByWordsTokenizer(); + } + } + + @Override + public int estimate(ChatMessage message) { + if (tokenizer == null) { + return 0; + } + + return tokenizer.estimate(message); + } + + @Override + public int estimate(List messages) { + if (tokenizer == null) { + return 0; + } + + return tokenizer.estimate(messages); + } + + @Override + public TokenEstimationStrategy getEstimationStrategy() { + // Sadly. + if (tokenizer == null) { + return null; + } + + return tokenizer.getEstimationStrategy(); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/AverageTokenizer.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/AverageTokenizer.java new file mode 100644 index 000000000000..68ba946f4e7f --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/AverageTokenizer.java @@ -0,0 +1,37 @@ +package org.jabref.logic.ai.customimplementations.tokenization.algorithms; + +import java.util.List; + +import org.jabref.model.ai.tokenization.TokenEstimationStrategy; + +import dev.langchain4j.data.message.ChatMessage; + +public class AverageTokenizer implements Tokenizer { + private final ByCharacterTokenizer byCharacterTokenizer = new ByCharacterTokenizer(); + private final ByWordsTokenizer byWordsTokenizer = new ByWordsTokenizer(); + + @Override + public int estimate(ChatMessage message) { + int byCharacter = byCharacterTokenizer.estimate(message); + int byWords = byWordsTokenizer.estimate(message); + + return calculate(byCharacter, byWords); + } + + @Override + public int estimate(List messages) { + int byCharacter = byCharacterTokenizer.estimate(messages); + int byWords = byWordsTokenizer.estimate(messages); + + return calculate(byCharacter, byWords); + } + + private int calculate(int byCharacter, int byWords) { + return Math.round((byWords + byCharacter) / 2.0f); + } + + @Override + public TokenEstimationStrategy getEstimationStrategy() { + return null; + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByCharacterTokenizer.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByCharacterTokenizer.java new file mode 100644 index 000000000000..10862d3f348f --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByCharacterTokenizer.java @@ -0,0 +1,39 @@ +package org.jabref.logic.ai.customimplementations.tokenization.algorithms; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.jabref.logic.ai.util.ChatMessagesUtil; +import org.jabref.model.ai.tokenization.TokenEstimationStrategy; + +import dev.langchain4j.data.message.ChatMessage; + +public class ByCharacterTokenizer implements Tokenizer { + private static final float CHAR_FACTOR = 4; + + @Override + public int estimate(ChatMessage message) { + return calculate(ChatMessagesUtil.getContent(message).orElse("")); + } + + @Override + public int estimate(List messages) { + String content = messages + .stream() + .map(ChatMessagesUtil::getContent) + .flatMap(Optional::stream) + .collect(Collectors.joining(" ")); + + return calculate(content); + } + + private int calculate(String content) { + return Math.round(content.length() / CHAR_FACTOR); + } + + @Override + public TokenEstimationStrategy getEstimationStrategy() { + return null; + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByWordsTokenizer.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByWordsTokenizer.java new file mode 100644 index 000000000000..704de3d10c8e --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByWordsTokenizer.java @@ -0,0 +1,39 @@ +package org.jabref.logic.ai.customimplementations.tokenization.algorithms; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.jabref.logic.ai.util.ChatMessagesUtil; +import org.jabref.model.ai.tokenization.TokenEstimationStrategy; + +import dev.langchain4j.data.message.ChatMessage; + +public class ByWordsTokenizer implements Tokenizer { + private static final float WORD_FACTOR = 0.75f; + + @Override + public int estimate(ChatMessage message) { + return calculate(ChatMessagesUtil.getContent(message).orElse("")); + } + + @Override + public int estimate(List messages) { + String content = messages + .stream() + .map(ChatMessagesUtil::getContent) + .flatMap(Optional::stream) + .collect(Collectors.joining(" ")); + + return calculate(content); + } + + private int calculate(String content) { + return Math.round(content.length() / WORD_FACTOR); + } + + @Override + public TokenEstimationStrategy getEstimationStrategy() { + return null; + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MaximumTokenizer.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MaximumTokenizer.java new file mode 100644 index 000000000000..c59d736161a4 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MaximumTokenizer.java @@ -0,0 +1,37 @@ +package org.jabref.logic.ai.customimplementations.tokenization.algorithms; + +import java.util.List; + +import org.jabref.model.ai.tokenization.TokenEstimationStrategy; + +import dev.langchain4j.data.message.ChatMessage; + +public class MaximumTokenizer implements Tokenizer { + private final ByCharacterTokenizer byCharacterTokenizer = new ByCharacterTokenizer(); + private final ByWordsTokenizer byWordsTokenizer = new ByWordsTokenizer(); + + @Override + public int estimate(ChatMessage message) { + int byWords = byWordsTokenizer.estimate(message); + int byCharacter = byCharacterTokenizer.estimate(message); + + return calculate(byWords, byCharacter); + } + + @Override + public int estimate(List messages) { + int byWords = byWordsTokenizer.estimate(messages); + int byCharacter = byCharacterTokenizer.estimate(messages); + + return calculate(byWords, byCharacter); + } + + private int calculate(int byWords, int byCharacters) { + return Math.max(byWords, byCharacters); + } + + @Override + public TokenEstimationStrategy getEstimationStrategy() { + return TokenEstimationStrategy.MAX; + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MinimumTokenizer.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MinimumTokenizer.java new file mode 100644 index 000000000000..5360292a9626 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MinimumTokenizer.java @@ -0,0 +1,37 @@ +package org.jabref.logic.ai.customimplementations.tokenization.algorithms; + +import java.util.List; + +import org.jabref.model.ai.tokenization.TokenEstimationStrategy; + +import dev.langchain4j.data.message.ChatMessage; + +public class MinimumTokenizer implements Tokenizer { + private final ByCharacterTokenizer byCharacterTokenizer = new ByCharacterTokenizer(); + private final ByWordsTokenizer byWordsTokenizer = new ByWordsTokenizer(); + + @Override + public int estimate(ChatMessage message) { + int byWords = byWordsTokenizer.estimate(message); + int byCharacter = byCharacterTokenizer.estimate(message); + + return calculate(byWords, byCharacter); + } + + @Override + public int estimate(List messages) { + int byWords = byWordsTokenizer.estimate(messages); + int byCharacter = byCharacterTokenizer.estimate(messages); + + return calculate(byWords, byCharacter); + } + + private int calculate(int byWords, int byCharacters) { + return Math.min(byWords, byCharacters); + } + + @Override + public TokenEstimationStrategy getEstimationStrategy() { + return TokenEstimationStrategy.MIN; + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/Tokenizer.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/Tokenizer.java new file mode 100644 index 000000000000..7f38e328df54 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/Tokenizer.java @@ -0,0 +1,15 @@ +package org.jabref.logic.ai.customimplementations.tokenization.algorithms; + +import java.util.List; + +import org.jabref.model.ai.tokenization.TokenEstimationStrategy; + +import dev.langchain4j.data.message.ChatMessage; + +public interface Tokenizer { + int estimate(ChatMessage message); + + int estimate(List messages); + + TokenEstimationStrategy getEstimationStrategy(); +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java index e8684d16a458..c2fe69116990 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java @@ -21,6 +21,7 @@ import org.jabref.model.ai.embeddings.EmbeddingModel; import org.jabref.model.ai.summarization.SummarizationAlgorithmName; import org.jabref.model.ai.templating.AiTemplate; +import org.jabref.model.ai.tokenization.TokenEstimationStrategy; import com.github.javakeyring.Keyring; import com.github.javakeyring.PasswordAccessException; @@ -54,6 +55,7 @@ public class AiPreferences { private final StringProperty gpt4AllApiBaseUrl; private final ObjectProperty defaultSummarizationAlgorithm; + private final ObjectProperty tokenEstimationStrategy; private final ObjectProperty embeddingModel; private final DoubleProperty temperature; private final IntegerProperty contextWindowSize; @@ -83,6 +85,7 @@ public AiPreferences( String huggingFaceApiBaseUrl, String gpt4AllApiBaseUrl, SummarizationAlgorithmName defaultSummarizationAlgorithm, + TokenEstimationStrategy tokenEstimationStrategy, EmbeddingModel embeddingModel, double temperature, int contextWindowSize, @@ -113,6 +116,7 @@ public AiPreferences( this.gpt4AllApiBaseUrl = new SimpleStringProperty(gpt4AllApiBaseUrl); this.defaultSummarizationAlgorithm = new SimpleObjectProperty<>(defaultSummarizationAlgorithm); + this.tokenEstimationStrategy = new SimpleObjectProperty<>(tokenEstimationStrategy); this.embeddingModel = new SimpleObjectProperty<>(embeddingModel); this.temperature = new SimpleDoubleProperty(temperature); this.contextWindowSize = new SimpleIntegerProperty(contextWindowSize); @@ -296,6 +300,18 @@ public void setDefaultSummarizationAlgorithm(SummarizationAlgorithmName defaultS this.defaultSummarizationAlgorithm.set(defaultSummarizationAlgorithm); } + public ObjectProperty tokenEstimationStrategyProperty() { + return tokenEstimationStrategy; + } + + public TokenEstimationStrategy getTokenEstimationStrategy() { + return tokenEstimationStrategy.get(); + } + + public void setTokenEstimationStrategy(TokenEstimationStrategy tokenEstimationStrategy) { + this.tokenEstimationStrategy.set(tokenEstimationStrategy); + } + public ObjectProperty embeddingModelProperty() { return embeddingModel; } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/CurrentlySelectedSummarizationAlgorithm.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/CurrentlySelectedSummarizationAlgorithm.java index c9b3702ab63e..655de326bf5b 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/CurrentlySelectedSummarizationAlgorithm.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/CurrentlySelectedSummarizationAlgorithm.java @@ -25,10 +25,10 @@ public CurrentlySelectedSummarizationAlgorithm( ) { this.aiPreferences = aiPreferences; - configure(); + setupListeningToPreferences(); } - private void configure() { + private void setupListeningToPreferences() { aiPreferences.defaultSummarizationAlgorithmProperty().addListener(_ -> { updateAlgorithm(); }); diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/ChunkedSummarizationAlgorithm.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/ChunkedSummarizationAlgorithm.java index f01d779a9900..2389706c6493 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/ChunkedSummarizationAlgorithm.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/ChunkedSummarizationAlgorithm.java @@ -23,8 +23,8 @@ public class ChunkedSummarizationAlgorithm implements SummarizationAlgorithm { private static final Logger LOGGER = LoggerFactory.getLogger(ChunkedSummarizationAlgorithm.class); + // TODO: Make a parameter? private static final int MAX_OVERLAP_SIZE_IN_CHARS = 100; - private static final int CHAR_TOKEN_FACTOR = 4; // Means, every token is roughly 4 characters. private final SummarizationChunkSystemMessageTemplate summarizationChunkSystemMessageTemplate; private final SummarizationChunkUserMessageTemplate summarizationChunkUserMessageTemplate; @@ -54,7 +54,7 @@ public String summarize( longTaskInfo.progressCounter().increaseWorkMax(1); // For the combination of summary chunks. DocumentSplitter documentSplitter = DocumentSplitters.recursive( - chatModelInfo.contextWindowSize() - MAX_OVERLAP_SIZE_IN_CHARS * 2 - estimateTokenCount(summarizationChunkSystemMessageTemplate.getSource()), + chatModelInfo.contextWindowSize() - MAX_OVERLAP_SIZE_IN_CHARS * 2 - chatModelInfo.tokenizer().estimate(new SystemMessage(summarizationChunkSystemMessageTemplate.getSource())), MAX_OVERLAP_SIZE_IN_CHARS ); @@ -93,7 +93,7 @@ public String summarize( } chunkSummaries = list; - } while (estimateTokenCount(chunkSummaries) > chatModelInfo.contextWindowSize() - estimateTokenCount(summarizationCombineSystemMessageTemplate.getSource())); + } while (chatModelInfo.tokenizer().estimate(chunkSummaries.stream().map(UserMessage::new).toList()) > chatModelInfo.contextWindowSize() - chatModelInfo.tokenizer().estimate(new SystemMessage(summarizationCombineSystemMessageTemplate.getSource()))); if (chunkSummaries.size() == 1) { longTaskInfo.progressCounter().increaseWorkDone(1); // No need to call LLM for combination of summary chunks. @@ -123,16 +123,4 @@ public String summarize( public SummarizationAlgorithmName getName() { return SummarizationAlgorithmName.CHUNKED; } - - private static int estimateTokenCount(List chunkSummaries) { - return chunkSummaries.stream().mapToInt(ChunkedSummarizationAlgorithm::estimateTokenCount).sum(); - } - - private static int estimateTokenCount(String string) { - return estimateTokenCount(string.length()); - } - - private static int estimateTokenCount(int numOfChars) { - return numOfChars / CHAR_TOKEN_FACTOR; - } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/util/ChatMessagesUtil.java b/jablib/src/main/java/org/jabref/logic/ai/util/ChatMessagesUtil.java new file mode 100644 index 000000000000..2f8b79a18bb2 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/util/ChatMessagesUtil.java @@ -0,0 +1,36 @@ +package org.jabref.logic.ai.util; + +import java.util.Optional; + +import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.data.message.ChatMessage; +import dev.langchain4j.data.message.SystemMessage; +import dev.langchain4j.data.message.UserMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ChatMessagesUtil { + private static final Logger LOGGER = LoggerFactory.getLogger(ChatMessagesUtil.class); + + private ChatMessagesUtil() { + throw new UnsupportedOperationException("unable to instantiate a utility class"); + } + + public static Optional getContent(ChatMessage chatMessage) { + switch (chatMessage) { + case SystemMessage systemMessage -> { + return Optional.of(systemMessage.text()); + } + case UserMessage userMessage -> { + return Optional.of(userMessage.singleText()); + } + case AiMessage aiMessage -> { + return Optional.of(aiMessage.text()); + } + default -> { + LOGGER.warn("Unable to get content from a message with type {}", chatMessage.type().name()); + return Optional.empty(); + } + } + } +} diff --git a/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java b/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java index f8038b139b90..c676909e2673 100644 --- a/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java @@ -99,7 +99,9 @@ import org.jabref.logic.xmp.XmpPreferences; import org.jabref.model.ai.chatting.AiProvider; import org.jabref.model.ai.embeddings.EmbeddingModel; +import org.jabref.model.ai.summarization.SummarizationAlgorithmName; import org.jabref.model.ai.templating.AiTemplate; +import org.jabref.model.ai.tokenization.TokenEstimationStrategy; import org.jabref.model.database.BibDatabaseMode; import org.jabref.model.entry.BibEntryPreferences; import org.jabref.model.entry.BibEntryType; @@ -394,6 +396,8 @@ public class JabRefCliPreferences implements CliPreferences { private static final String AI_GEMINI_API_BASE_URL = "aiGeminiApiBaseUrl"; private static final String AI_HUGGING_FACE_API_BASE_URL = "aiHuggingFaceApiBaseUrl"; private static final String AI_GPT_4_ALL_API_BASE_URL = "aiGpt4AllApiBaseUrl"; + private static final String AI_SUMMARIZATION_ALGORITHM = "aiSummarizationAlgorithm"; + private static final String AI_TOKEN_ESTIMATION_ALGORITHM = "aiTokenEstimationAlgorithm"; private static final String AI_TEMPERATURE = "aiTemperature"; private static final String AI_CONTEXT_WINDOW_SIZE = "aiMessageWindowSize"; private static final String AI_DOCUMENT_SPLITTER_CHUNK_SIZE = "aiDocumentSplitterChunkSize"; @@ -742,6 +746,8 @@ public JabRefCliPreferences() { defaults.put(AI_GEMINI_API_BASE_URL, AiProvider.GEMINI.getApiUrl()); defaults.put(AI_HUGGING_FACE_API_BASE_URL, AiProvider.HUGGING_FACE.getApiUrl()); defaults.put(AI_GPT_4_ALL_API_BASE_URL, AiProvider.GPT4ALL.getApiUrl()); + defaults.put(AI_SUMMARIZATION_ALGORITHM, SummarizationAlgorithmName.CHUNKED.name()); + defaults.put(AI_TOKEN_ESTIMATION_ALGORITHM, TokenEstimationStrategy.MAX.name()); defaults.put(AI_TEMPERATURE, 0.7); defaults.put(AI_CONTEXT_WINDOW_SIZE, PredefinedChatModel.getContextWindowSize((AiProvider) defaults.get(AI_PROVIDER), AiProviderDefaultChatModels.getDefaultChatModel((AiProvider) defaults.get(AI_PROVIDER)).getName())); defaults.put(AI_DOCUMENT_SPLITTER_CHUNK_SIZE, 300); @@ -2035,6 +2041,8 @@ public AiPreferences getAiPreferences() { get(AI_GEMINI_API_BASE_URL), get(AI_HUGGING_FACE_API_BASE_URL), get(AI_GPT_4_ALL_API_BASE_URL), + SummarizationAlgorithmName.valueOf(get(AI_SUMMARIZATION_ALGORITHM)), + TokenEstimationStrategy.valueOf(get(AI_TOKEN_ESTIMATION_ALGORITHM)), EmbeddingModel.valueOf(get(AI_EMBEDDING_MODEL)), getDouble(AI_TEMPERATURE), getInt(AI_CONTEXT_WINDOW_SIZE), @@ -2073,6 +2081,8 @@ AiTemplate.CITATION_PARSING_USER_MESSAGE, get(AI_CITATION_PARSING_USER_MESSAGE_T EasyBind.listen(aiPreferences.huggingFaceApiBaseUrlProperty(), (_, _, newValue) -> put(AI_HUGGING_FACE_API_BASE_URL, newValue)); EasyBind.listen(aiPreferences.gpt4AllApiBaseUrlProperty(), (_, _, newValue) -> put(AI_GPT_4_ALL_API_BASE_URL, newValue)); + EasyBind.listen(aiPreferences.defaultSummarizationAlgorithmProperty(), (_, _, newValue) -> put(AI_SUMMARIZATION_ALGORITHM, newValue.name())); + EasyBind.listen(aiPreferences.tokenEstimationStrategyProperty(), (_, _, newValue) -> put(AI_TOKEN_ESTIMATION_ALGORITHM, newValue.name())); EasyBind.listen(aiPreferences.embeddingModelProperty(), (_, _, newValue) -> put(AI_EMBEDDING_MODEL, newValue.name())); EasyBind.listen(aiPreferences.temperatureProperty(), (_, _, newValue) -> putDouble(AI_TEMPERATURE, newValue.doubleValue())); EasyBind.listen(aiPreferences.contextWindowSizeProperty(), (_, _, newValue) -> putInt(AI_CONTEXT_WINDOW_SIZE, newValue)); diff --git a/jablib/src/main/java/org/jabref/model/ai/chatting/ChatModelInfo.java b/jablib/src/main/java/org/jabref/model/ai/chatting/ChatModelInfo.java index 0eb847b48fc6..132f3d7a67fb 100644 --- a/jablib/src/main/java/org/jabref/model/ai/chatting/ChatModelInfo.java +++ b/jablib/src/main/java/org/jabref/model/ai/chatting/ChatModelInfo.java @@ -1,6 +1,14 @@ package org.jabref.model.ai.chatting; +import org.jabref.logic.ai.customimplementations.tokenization.algorithms.Tokenizer; + import dev.langchain4j.model.chat.ChatModel; -public record ChatModelInfo(ChatModel chatModel, AiProvider aiProvider, String name, int contextWindowSize) { +public record ChatModelInfo( + ChatModel chatModel, + Tokenizer tokenizer, + AiProvider aiProvider, + String name, + int contextWindowSize +) { } diff --git a/jablib/src/main/java/org/jabref/model/ai/tokenization/TokenEstimationStrategy.java b/jablib/src/main/java/org/jabref/model/ai/tokenization/TokenEstimationStrategy.java new file mode 100644 index 000000000000..90a2f7c300e1 --- /dev/null +++ b/jablib/src/main/java/org/jabref/model/ai/tokenization/TokenEstimationStrategy.java @@ -0,0 +1,19 @@ +package org.jabref.model.ai.tokenization; + +/// Idea took from: . +public enum TokenEstimationStrategy { + /// Average between (word count / 0.75) and (character count / 4). + AVERAGE, + + /// 0.75 words = 1 token. + WORDS, + + /// 4 characters = 1 token. + CHARS, + + /// Maximum between (word count / 0.75) and (character count / 4). + MAX, + + /// Minimum between (word count / 0.75) and (character count / 4). + MIN +} From 7e46164af6fbd9d12bdfb942076ecaa88ccb78d3 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Wed, 26 Nov 2025 14:56:39 +0100 Subject: [PATCH 022/243] Remove templates for chatting in AiTemplatesService --- .../ai/components/aichat/AiChatComponent.java | 26 +++++--- jablib/src/main/java/module-info.java | 2 + .../java/org/jabref/logic/ai/AiService.java | 9 ++- .../logic/ai/chatting/logic/AiChatLogic.java | 18 +++-- .../ChattingSystemMessageTemplate.java | 29 ++++++++ .../ChattingUserMessageTemplate.java | 32 +++++++++ ...rrentlySelectedSummarizationAlgorithm.java | 26 +++----- ...mmarizationChunkSystemMessageTemplate.java | 4 +- ...SummarizationChunkUserMessageTemplate.java | 4 +- ...arizationCombineSystemMessageTemplate.java | 4 +- ...mmarizationCombineUserMessageTemplate.java | 3 +- .../ai/templates/AiTemplatesService.java | 23 ------- .../ai/templates/CurrentAiTemplates.java | 66 +++++++++++++++++++ .../jabref/logic/ai/templates/Template.java | 11 ++-- 14 files changed, 190 insertions(+), 67 deletions(-) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/templates/ChattingSystemMessageTemplate.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/templates/ChattingUserMessageTemplate.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/templates/CurrentAiTemplates.java diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java index e2f1b3d13512..df4f2e197b5d 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java @@ -82,14 +82,15 @@ public class AiChatComponent extends VBox { private String noticeTemplate; - public AiChatComponent(AiService aiService, - StringProperty name, - ObservableList chatHistory, - ObservableList entries, - BibDatabaseContext bibDatabaseContext, - AiPreferences aiPreferences, - DialogService dialogService, - TaskExecutor taskExecutor + public AiChatComponent( + AiService aiService, + StringProperty name, + ObservableList chatHistory, + ObservableList entries, + BibDatabaseContext bibDatabaseContext, + AiPreferences aiPreferences, + DialogService dialogService, + TaskExecutor taskExecutor ) { this.aiService = aiService; this.entries = entries; @@ -100,10 +101,15 @@ public AiChatComponent(AiService aiService, this.aiChatLogic = new AiChatLogic( aiPreferences, - aiService.getTemplatesService(), aiService.getChatLanguageModel(), + aiService.getChatLanguageModel(), aiService.getEmbeddingModel(), aiService.getEmbeddingStore(), - bibDatabaseContext, chatHistory, entries, name + aiService.getCurrentAiTemplates().getChattingSystemMessageTemplate(), + aiService.getCurrentAiTemplates().getChattingUserMessageTemplate(), + bibDatabaseContext, + chatHistory, + entries, + name ); aiService.getIngestionService().ingest(name, ListUtil.getLinkedFiles(entries).toList(), bibDatabaseContext); diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index 1c3403fe19bd..a331753c7704 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -122,6 +122,7 @@ exports org.jabref.logic.ai.customimplementations.embeddingmodels; exports org.jabref.logic.ai.customimplementations.embeddingstores; exports org.jabref.logic.ai.customimplementations.llms; + exports org.jabref.logic.ai.customimplementations.tokenization; exports org.jabref.logic.ai.rag; exports org.jabref.logic.ai.summarization.tasks; exports org.jabref.logic.ai.summarization.repositories; @@ -133,6 +134,7 @@ exports org.jabref.logic.ai.rag.logic; exports org.jabref.logic.ai.chatting.logic; exports org.jabref.logic.ai.chatting.tasks; + exports org.jabref.logic.ai.chatting.templates; exports org.jabref.logic.ai.preferences; exports org.jabref.logic.ai.chatting.repositories; exports org.jabref.model.ai.chatting; diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiService.java b/jablib/src/main/java/org/jabref/logic/ai/AiService.java index 30541a865da1..a7af6b318acf 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiService.java @@ -21,6 +21,7 @@ import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; import org.jabref.logic.ai.summarization.repositories.MVStoreSummariesRepository; import org.jabref.logic.ai.templates.AiTemplatesService; +import org.jabref.logic.ai.templates.CurrentAiTemplates; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; import org.jabref.logic.util.Directories; import org.jabref.logic.util.NotificationService; @@ -59,6 +60,7 @@ public class AiService implements AutoCloseable { private final MVStoreSummariesRepository mvStoreSummariesStorage; private final AiTemplatesService templatesService; + private final CurrentAiTemplates currentAiTemplates; private final ChatHistoryService chatHistoryService; private final CurrentlySelectedTokenEstimationStrategy currentlySelectedTokenEstimationStrategy; private final CurrentlySelectedChatLanguageModel currentlySelectedChatLanguageModel; @@ -81,11 +83,12 @@ public AiService( this.mvStoreSummariesStorage = new MVStoreSummariesRepository(notificationService, Directories.getAiFilesDirectory().resolve(SUMMARIES_FILE_NAME)); this.templatesService = new AiTemplatesService(aiPreferences); + this.currentAiTemplates = new CurrentAiTemplates(aiPreferences); this.chatHistoryService = new ChatHistoryService(citationKeyPatternPreferences, mvStoreChatHistoryStorage); this.currentlySelectedTokenEstimationStrategy = new CurrentlySelectedTokenEstimationStrategy(aiPreferences); this.currentlySelectedChatLanguageModel = new CurrentlySelectedChatLanguageModel(aiPreferences, currentlySelectedTokenEstimationStrategy); this.currentlySelectedEmbeddingModel = new CurrentlySelectedEmbeddingModel(aiPreferences, notificationService, taskExecutor); - this.currentlySelectedSummarizationAlgorithm = new CurrentlySelectedSummarizationAlgorithm(aiPreferences); + this.currentlySelectedSummarizationAlgorithm = new CurrentlySelectedSummarizationAlgorithm(aiPreferences, currentAiTemplates); this.ingestionService = new IngestionService( aiPreferences, @@ -132,6 +135,10 @@ public AiTemplatesService getTemplatesService() { return templatesService; } + public CurrentAiTemplates getCurrentAiTemplates() { + return currentAiTemplates; + } + public EmbeddingStore getEmbeddingStore() { return mvStoreEmbeddingStore; } diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java index ee07893e68f4..2d4762a68c6d 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java @@ -7,9 +7,10 @@ import javafx.collections.ListChangeListener; import javafx.collections.ObservableList; +import org.jabref.logic.ai.chatting.templates.ChattingSystemMessageTemplate; +import org.jabref.logic.ai.chatting.templates.ChattingUserMessageTemplate; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.rag.repositories.FileEmbeddingsManager; -import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.model.ai.chatting.ErrorMessage; import org.jabref.model.ai.rag.PaperExcerpt; import org.jabref.model.ai.templating.AiTemplate; @@ -45,7 +46,8 @@ public class AiChatLogic { private final ChatModel chatLanguageModel; private final EmbeddingModel embeddingModel; private final EmbeddingStore embeddingStore; - private final AiTemplatesService aiTemplatesService; + private final ChattingSystemMessageTemplate chattingSystemMessageTemplate; + private final ChattingUserMessageTemplate chattingUserMessageTemplate; private final ObservableList chatHistory; private final ObservableList entries; @@ -58,10 +60,11 @@ public class AiChatLogic { public AiChatLogic( AiPreferences aiPreferences, - AiTemplatesService aiTemplatesService, ChatModel chatLanguageModel, EmbeddingModel embeddingModel, EmbeddingStore embeddingStore, + ChattingSystemMessageTemplate chattingSystemMessageTemplate, + ChattingUserMessageTemplate chattingUserMessageTemplate, BibDatabaseContext bibDatabaseContext, ObservableList chatHistory, ObservableList entries, @@ -71,7 +74,8 @@ public AiChatLogic( this.chatLanguageModel = chatLanguageModel; this.embeddingModel = embeddingModel; this.embeddingStore = embeddingStore; - this.aiTemplatesService = aiTemplatesService; + this.chattingSystemMessageTemplate = chattingSystemMessageTemplate; + this.chattingUserMessageTemplate = chattingUserMessageTemplate; this.chatHistory = chatHistory; this.entries = entries; this.name = name; @@ -87,7 +91,7 @@ private void setupListeningToPreferencesChanges() { aiPreferences .templateProperty(AiTemplate.CHATTING_SYSTEM_MESSAGE) .addListener(obs -> - setSystemMessage(aiTemplatesService.makeChattingSystemMessage(entries))); + setSystemMessage(chattingSystemMessageTemplate.render(entries))); aiPreferences.contextWindowSizeProperty().addListener(obs -> rebuildFull(chatMemory.messages())); } @@ -114,7 +118,7 @@ private void rebuildChatMemory(List chatMessages) { chatMessages.stream().filter(chatMessage -> !(chatMessage instanceof ErrorMessage)).forEach(chatMemory::add); - setSystemMessage(aiTemplatesService.makeChattingSystemMessage(entries)); + setSystemMessage(chattingSystemMessageTemplate.render(entries)); } private void rebuildFilter() { @@ -182,7 +186,7 @@ public AiMessage execute(UserMessage message) { chatMemory.messages().forEach(tempChatMemory::add); - tempChatMemory.add(new UserMessage(aiTemplatesService.makeChattingUserMessage(entries, message.singleText(), excerpts))); + tempChatMemory.add(new UserMessage(chattingUserMessageTemplate.render(entries, message.singleText(), excerpts))); chatMemory.add(message); AiMessage aiMessage = chatLanguageModel.chat(tempChatMemory.messages()).aiMessage(); diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/templates/ChattingSystemMessageTemplate.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/templates/ChattingSystemMessageTemplate.java new file mode 100644 index 000000000000..3dd60eecb897 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/templates/ChattingSystemMessageTemplate.java @@ -0,0 +1,29 @@ +package org.jabref.logic.ai.chatting.templates; + +import java.util.List; +import java.util.function.Supplier; + +import org.jabref.logic.ai.templates.Template; +import org.jabref.model.ai.templating.AiTemplate; +import org.jabref.model.entry.BibEntry; + +import org.apache.velocity.VelocityContext; + +public class ChattingSystemMessageTemplate extends Template { + public ChattingSystemMessageTemplate(Supplier source) { + super(source); + } + + public String render(List entries) { + VelocityContext context = makeContext(); + + context.put("entries", entries); + + return render(context); + } + + @Override + public String getLogName() { + return AiTemplate.CHATTING_SYSTEM_MESSAGE.name(); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/templates/ChattingUserMessageTemplate.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/templates/ChattingUserMessageTemplate.java new file mode 100644 index 000000000000..21141d6e350a --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/templates/ChattingUserMessageTemplate.java @@ -0,0 +1,32 @@ +package org.jabref.logic.ai.chatting.templates; + +import java.util.List; +import java.util.function.Supplier; + +import org.jabref.logic.ai.templates.Template; +import org.jabref.model.ai.rag.PaperExcerpt; +import org.jabref.model.ai.templating.AiTemplate; +import org.jabref.model.entry.BibEntry; + +import org.apache.velocity.VelocityContext; + +public class ChattingUserMessageTemplate extends Template { + public ChattingUserMessageTemplate(Supplier source) { + super(source); + } + + public String render(List entries, String message, List excerpts) { + VelocityContext context = makeContext(); + + context.put("entries", entries); + context.put("message", message); + context.put("excerpts", excerpts); + + return render(context); + } + + @Override + public String getLogName() { + return AiTemplate.CHATTING_USER_MESSAGE.name(); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/CurrentlySelectedSummarizationAlgorithm.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/CurrentlySelectedSummarizationAlgorithm.java index 655de326bf5b..3dd222351c89 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/CurrentlySelectedSummarizationAlgorithm.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/CurrentlySelectedSummarizationAlgorithm.java @@ -3,10 +3,7 @@ import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.ChunkedSummarizationAlgorithm; import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; -import org.jabref.logic.ai.summarization.templates.SummarizationChunkSystemMessageTemplate; -import org.jabref.logic.ai.summarization.templates.SummarizationChunkUserMessageTemplate; -import org.jabref.logic.ai.summarization.templates.SummarizationCombineSystemMessageTemplate; -import org.jabref.logic.ai.summarization.templates.SummarizationCombineUserMessageTemplate; +import org.jabref.logic.ai.templates.CurrentAiTemplates; import org.jabref.logic.ai.util.LongTaskInfo; import org.jabref.model.ai.chatting.ChatModelInfo; import org.jabref.model.ai.summarization.SummarizationAlgorithmName; @@ -16,14 +13,17 @@ public class CurrentlySelectedSummarizationAlgorithm implements SummarizationAlgorithm { private final AiPreferences aiPreferences; + private final CurrentAiTemplates currentAiTemplates; @Nullable private SummarizationAlgorithm summarizationAlgorithm = null; public CurrentlySelectedSummarizationAlgorithm( - AiPreferences aiPreferences + AiPreferences aiPreferences, + CurrentAiTemplates currentAiTemplates ) { this.aiPreferences = aiPreferences; + this.currentAiTemplates = currentAiTemplates; setupListeningToPreferences(); } @@ -59,18 +59,10 @@ private void updateAlgorithm() { private ChunkedSummarizationAlgorithm createChunkedSummarizationAlgorithm() { return new ChunkedSummarizationAlgorithm( - new SummarizationChunkSystemMessageTemplate( - aiPreferences.getTemplate(AiTemplate.SUMMARIZATION_CHUNK_SYSTEM_MESSAGE) - ), - new SummarizationChunkUserMessageTemplate( - aiPreferences.getTemplate(AiTemplate.SUMMARIZATION_CHUNK_USER_MESSAGE) - ), - new SummarizationCombineSystemMessageTemplate( - aiPreferences.getTemplate(AiTemplate.SUMMARIZATION_COMBINE_SYSTEM_MESSAGE) - ), - new SummarizationCombineUserMessageTemplate( - aiPreferences.getTemplate(AiTemplate.SUMMARIZATION_COMBINE_USER_MESSAGE) - ) + currentAiTemplates.getSummarizationChunkSystemMessageTemplate(), + currentAiTemplates.getSummarizationChunkUserMessageTemplate(), + currentAiTemplates.getSummarizationCombineSystemMessageTemplate(), + currentAiTemplates.getSummarizationCombineUserMessageTemplate() ); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationChunkSystemMessageTemplate.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationChunkSystemMessageTemplate.java index ed225eeda4b2..b54063842e19 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationChunkSystemMessageTemplate.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationChunkSystemMessageTemplate.java @@ -1,12 +1,14 @@ package org.jabref.logic.ai.summarization.templates; +import java.util.function.Supplier; + import org.jabref.logic.ai.templates.Template; import org.jabref.model.ai.templating.AiTemplate; import org.apache.velocity.VelocityContext; public class SummarizationChunkSystemMessageTemplate extends Template { - public SummarizationChunkSystemMessageTemplate(String source) { + public SummarizationChunkSystemMessageTemplate(Supplier source) { super(source); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationChunkUserMessageTemplate.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationChunkUserMessageTemplate.java index 916d7b263bab..8960ff8a3a29 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationChunkUserMessageTemplate.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationChunkUserMessageTemplate.java @@ -1,12 +1,14 @@ package org.jabref.logic.ai.summarization.templates; +import java.util.function.Supplier; + import org.jabref.logic.ai.templates.Template; import org.jabref.model.ai.templating.AiTemplate; import org.apache.velocity.VelocityContext; public class SummarizationChunkUserMessageTemplate extends Template { - public SummarizationChunkUserMessageTemplate(String source) { + public SummarizationChunkUserMessageTemplate(Supplier source) { super(source); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationCombineSystemMessageTemplate.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationCombineSystemMessageTemplate.java index 5615a7a56377..a3345e6aa7c8 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationCombineSystemMessageTemplate.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationCombineSystemMessageTemplate.java @@ -1,12 +1,14 @@ package org.jabref.logic.ai.summarization.templates; +import java.util.function.Supplier; + import org.jabref.logic.ai.templates.Template; import org.jabref.model.ai.templating.AiTemplate; import org.apache.velocity.VelocityContext; public class SummarizationCombineSystemMessageTemplate extends Template { - public SummarizationCombineSystemMessageTemplate(String source) { + public SummarizationCombineSystemMessageTemplate(Supplier source) { super(source); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationCombineUserMessageTemplate.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationCombineUserMessageTemplate.java index cf96b0c001b4..972b57a4892c 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationCombineUserMessageTemplate.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationCombineUserMessageTemplate.java @@ -1,6 +1,7 @@ package org.jabref.logic.ai.summarization.templates; import java.util.List; +import java.util.function.Supplier; import org.jabref.logic.ai.templates.Template; import org.jabref.model.ai.templating.AiTemplate; @@ -8,7 +9,7 @@ import org.apache.velocity.VelocityContext; public class SummarizationCombineUserMessageTemplate extends Template { - public SummarizationCombineUserMessageTemplate(String source) { + public SummarizationCombineUserMessageTemplate(Supplier source) { super(source); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java b/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java index b09848835028..0f92fe912db1 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java @@ -1,12 +1,9 @@ package org.jabref.logic.ai.templates; import java.io.StringWriter; -import java.util.List; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.model.ai.rag.PaperExcerpt; import org.jabref.model.ai.templating.AiTemplate; -import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.CanonicalBibEntry; import org.apache.velocity.VelocityContext; @@ -26,22 +23,6 @@ public AiTemplatesService(AiPreferences aiPreferences) { baseContext.put("CanonicalBibEntry", CanonicalBibEntry.class); } - public String makeChattingSystemMessage(List entries) { - VelocityContext context = new VelocityContext(baseContext); - context.put("entries", entries); - - return makeTemplate(AiTemplate.CHATTING_SYSTEM_MESSAGE, context); - } - - public String makeChattingUserMessage(List entries, String message, List excerpts) { - VelocityContext context = new VelocityContext(baseContext); - context.put("entries", entries); - context.put("message", message); - context.put("excerpts", excerpts); - - return makeTemplate(AiTemplate.CHATTING_USER_MESSAGE, context); - } - public String makeCitationParsingSystemMessage() { VelocityContext context = new VelocityContext(baseContext); return makeTemplate(AiTemplate.CITATION_PARSING_SYSTEM_MESSAGE, context); @@ -61,8 +42,4 @@ private String makeTemplate(AiTemplate template, VelocityContext context) { return writer.toString(); } - - public String getRawTemplate(AiTemplate template) { - return aiPreferences.getTemplate(template); - } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/templates/CurrentAiTemplates.java b/jablib/src/main/java/org/jabref/logic/ai/templates/CurrentAiTemplates.java new file mode 100644 index 000000000000..7a96a60407bf --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/templates/CurrentAiTemplates.java @@ -0,0 +1,66 @@ +package org.jabref.logic.ai.templates; + +import org.jabref.logic.ai.chatting.templates.ChattingSystemMessageTemplate; +import org.jabref.logic.ai.chatting.templates.ChattingUserMessageTemplate; +import org.jabref.logic.ai.preferences.AiPreferences; +import org.jabref.logic.ai.summarization.templates.SummarizationChunkSystemMessageTemplate; +import org.jabref.logic.ai.summarization.templates.SummarizationChunkUserMessageTemplate; +import org.jabref.logic.ai.summarization.templates.SummarizationCombineSystemMessageTemplate; +import org.jabref.logic.ai.summarization.templates.SummarizationCombineUserMessageTemplate; +import org.jabref.model.ai.templating.AiTemplate; + +public class CurrentAiTemplates { + private final SummarizationChunkSystemMessageTemplate summarizationChunkSystemMessageTemplate; + private final SummarizationChunkUserMessageTemplate summarizationChunkUserMessageTemplate; + private final SummarizationCombineSystemMessageTemplate summarizationCombineSystemMessageTemplate; + private final SummarizationCombineUserMessageTemplate summarizationCombineUserMessageTemplate; + + private final ChattingSystemMessageTemplate chattingSystemMessageTemplate; + private final ChattingUserMessageTemplate chattingUserMessageTemplate; + + public CurrentAiTemplates(AiPreferences aiPreferences) { + this.summarizationChunkSystemMessageTemplate = new SummarizationChunkSystemMessageTemplate( + () -> aiPreferences.getTemplate(AiTemplate.SUMMARIZATION_CHUNK_SYSTEM_MESSAGE) + ); + this.summarizationChunkUserMessageTemplate = new SummarizationChunkUserMessageTemplate( + () -> aiPreferences.getTemplate(AiTemplate.SUMMARIZATION_CHUNK_USER_MESSAGE) + ); + this.summarizationCombineSystemMessageTemplate = new SummarizationCombineSystemMessageTemplate( + () -> aiPreferences.getTemplate(AiTemplate.SUMMARIZATION_COMBINE_SYSTEM_MESSAGE) + ); + this.summarizationCombineUserMessageTemplate = new SummarizationCombineUserMessageTemplate( + () -> aiPreferences.getTemplate(AiTemplate.SUMMARIZATION_CHUNK_USER_MESSAGE) + ); + + this.chattingSystemMessageTemplate = new ChattingSystemMessageTemplate( + () -> aiPreferences.getTemplate(AiTemplate.CHATTING_SYSTEM_MESSAGE) + ); + this.chattingUserMessageTemplate = new ChattingUserMessageTemplate( + () -> aiPreferences.getTemplate(AiTemplate.CHATTING_USER_MESSAGE) + ); + } + + public SummarizationChunkSystemMessageTemplate getSummarizationChunkSystemMessageTemplate() { + return summarizationChunkSystemMessageTemplate; + } + + public SummarizationChunkUserMessageTemplate getSummarizationChunkUserMessageTemplate() { + return summarizationChunkUserMessageTemplate; + } + + public SummarizationCombineSystemMessageTemplate getSummarizationCombineSystemMessageTemplate() { + return summarizationCombineSystemMessageTemplate; + } + + public SummarizationCombineUserMessageTemplate getSummarizationCombineUserMessageTemplate() { + return summarizationCombineUserMessageTemplate; + } + + public ChattingSystemMessageTemplate getChattingSystemMessageTemplate() { + return chattingSystemMessageTemplate; + } + + public ChattingUserMessageTemplate getChattingUserMessageTemplate() { + return chattingUserMessageTemplate; + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/templates/Template.java b/jablib/src/main/java/org/jabref/logic/ai/templates/Template.java index eb55ee643237..b70661e38364 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/templates/Template.java +++ b/jablib/src/main/java/org/jabref/logic/ai/templates/Template.java @@ -1,6 +1,7 @@ package org.jabref.logic.ai.templates; import java.io.StringWriter; +import java.util.function.Supplier; import org.jabref.model.entry.CanonicalBibEntry; @@ -17,15 +18,15 @@ public abstract class Template { BASE_CONTEXT.put("CanonicalBibEntry", CanonicalBibEntry.class); } - private final String source; + private final Supplier sourceSupplier; - public Template(String source) { - this.source = source; + public Template(Supplier sourceSupplier) { + this.sourceSupplier = sourceSupplier; } protected String render(VelocityContext context) { StringWriter writer = new StringWriter(); - VELOCITY_ENGINE.evaluate(context, writer, getLogName(), source); + VELOCITY_ENGINE.evaluate(context, writer, getLogName(), sourceSupplier.get()); return writer.toString(); } @@ -34,7 +35,7 @@ protected VelocityContext makeContext() { } public String getSource() { - return source; + return sourceSupplier.get(); } // Required by Velocity. From 6feedc94a58547cc4cb12dde5653406298dbaad4 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Wed, 26 Nov 2025 15:48:38 +0100 Subject: [PATCH 023/243] Refactor LLM citation parsing + get rid of AiTemplatesService.java --- jablib/src/main/java/module-info.java | 5 +- .../java/org/jabref/logic/ai/AiService.java | 7 - .../logic/ParseCitationsWithLlm.java | 81 +++++++++++ .../CitationParsingSystemMessageTemplate.java | 24 ++++ .../CitationParsingUserMessageTemplate.java | 27 ++++ .../ai/templates/AiTemplatesService.java | 45 ------ .../ai/templates/CurrentAiTemplates.java | 20 +++ .../fileformat/pdf/CitationsFromPdf.java | 7 +- .../plaincitation/LlmPlainCitationParser.java | 69 ++++----- .../PlainCitationParserFactory.java | 2 +- .../java/org/jabref/logic/util/Result.java | 133 ++++++++++++++++++ 11 files changed, 323 insertions(+), 97 deletions(-) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/citationparsing/logic/ParseCitationsWithLlm.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/citationparsing/templates/CitationParsingSystemMessageTemplate.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/citationparsing/templates/CitationParsingUserMessageTemplate.java delete mode 100644 jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java create mode 100644 jablib/src/main/java/org/jabref/logic/util/Result.java diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index a331753c7704..fdde2da1c584 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -135,8 +135,10 @@ exports org.jabref.logic.ai.chatting.logic; exports org.jabref.logic.ai.chatting.tasks; exports org.jabref.logic.ai.chatting.templates; - exports org.jabref.logic.ai.preferences; exports org.jabref.logic.ai.chatting.repositories; + exports org.jabref.logic.ai.citationparsing.logic; + exports org.jabref.logic.ai.citationparsing.templates; + exports org.jabref.logic.ai.preferences; exports org.jabref.model.ai.chatting; exports org.jabref.model.ai.templating; exports org.jabref.model.ai.rag; @@ -291,5 +293,6 @@ requires org.jooq.jool; requires org.libreoffice.uno; requires transitive org.jspecify; + requires org.jabref.jablib; // endregion } diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiService.java b/jablib/src/main/java/org/jabref/logic/ai/AiService.java index a7af6b318acf..f3c6ad0cd4fd 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiService.java @@ -20,7 +20,6 @@ import org.jabref.logic.ai.summarization.SummariesService; import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; import org.jabref.logic.ai.summarization.repositories.MVStoreSummariesRepository; -import org.jabref.logic.ai.templates.AiTemplatesService; import org.jabref.logic.ai.templates.CurrentAiTemplates; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; import org.jabref.logic.util.Directories; @@ -59,7 +58,6 @@ public class AiService implements AutoCloseable { private final MVStoreFullyIngestedDocumentsRepository mvStoreFullyIngestedDocumentsTracker; private final MVStoreSummariesRepository mvStoreSummariesStorage; - private final AiTemplatesService templatesService; private final CurrentAiTemplates currentAiTemplates; private final ChatHistoryService chatHistoryService; private final CurrentlySelectedTokenEstimationStrategy currentlySelectedTokenEstimationStrategy; @@ -82,7 +80,6 @@ public AiService( this.mvStoreFullyIngestedDocumentsTracker = new MVStoreFullyIngestedDocumentsRepository(notificationService, Directories.getAiFilesDirectory().resolve(FULLY_INGESTED_FILE_NAME)); this.mvStoreSummariesStorage = new MVStoreSummariesRepository(notificationService, Directories.getAiFilesDirectory().resolve(SUMMARIES_FILE_NAME)); - this.templatesService = new AiTemplatesService(aiPreferences); this.currentAiTemplates = new CurrentAiTemplates(aiPreferences); this.chatHistoryService = new ChatHistoryService(citationKeyPatternPreferences, mvStoreChatHistoryStorage); this.currentlySelectedTokenEstimationStrategy = new CurrentlySelectedTokenEstimationStrategy(aiPreferences); @@ -131,10 +128,6 @@ public SummariesService getSummariesService() { return summariesService; } - public AiTemplatesService getTemplatesService() { - return templatesService; - } - public CurrentAiTemplates getCurrentAiTemplates() { return currentAiTemplates; } diff --git a/jablib/src/main/java/org/jabref/logic/ai/citationparsing/logic/ParseCitationsWithLlm.java b/jablib/src/main/java/org/jabref/logic/ai/citationparsing/logic/ParseCitationsWithLlm.java new file mode 100644 index 000000000000..3c517eb67380 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/citationparsing/logic/ParseCitationsWithLlm.java @@ -0,0 +1,81 @@ +package org.jabref.logic.ai.citationparsing.logic; + +import java.io.IOException; +import java.io.Reader; +import java.util.List; + +import org.jabref.logic.ai.citationparsing.templates.CitationParsingSystemMessageTemplate; +import org.jabref.logic.ai.citationparsing.templates.CitationParsingUserMessageTemplate; +import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.importer.ParserResult; +import org.jabref.logic.importer.fileformat.BibtexParser; +import org.jabref.logic.util.Result; +import org.jabref.model.ai.chatting.ChatModelInfo; +import org.jabref.model.entry.BibEntry; + +import dev.langchain4j.data.message.SystemMessage; +import dev.langchain4j.data.message.UserMessage; + +public class ParseCitationsWithLlm { + private final ImportFormatPreferences importFormatPreferences; + + private final CitationParsingSystemMessageTemplate citationParsingSystemMessageTemplate; + private final CitationParsingUserMessageTemplate citationParsingUserMessageTemplate; + + public ParseCitationsWithLlm( + ImportFormatPreferences importFormatPreferences, + CitationParsingSystemMessageTemplate citationParsingSystemMessageTemplate, + CitationParsingUserMessageTemplate citationParsingUserMessageTemplate + ) { + this.importFormatPreferences = importFormatPreferences; + + this.citationParsingSystemMessageTemplate = citationParsingSystemMessageTemplate; + this.citationParsingUserMessageTemplate = citationParsingUserMessageTemplate; + } + + public Result, IOException> parseMultiplePlainCitations( + ChatModelInfo chatModelInfo, + String text + ) { + String systemMessage = citationParsingSystemMessageTemplate.render(); + String userMessage = citationParsingUserMessageTemplate.render(text); + + // TODO: Clean possibly of backticks. + String llmResult = chatModelInfo.chatModel().chat( + List.of( + new SystemMessage(systemMessage), + new UserMessage(userMessage) + ) + ).aiMessage().text(); + + return parseBibEntryString(llmResult); + } + + private Result, IOException> parseBibEntryString(String text) { + Reader reader = Reader.of(text); + BibtexParser parser = new BibtexParser(importFormatPreferences); + ParserResult result; + try { + result = parser.parse(reader); + } catch (IOException e) { + return Result.err(e); + } + + return Result.ok(result.getDatabase().getEntries()); + } + + public String getBibtexStringFromLlm( + ChatModelInfo chatModelInfo, + String searchQuery + ) { + String systemMessage = citationParsingSystemMessageTemplate.render(); + String userMessage = citationParsingUserMessageTemplate.render(searchQuery); + + return chatModelInfo.chatModel().chat( + List.of( + new SystemMessage(systemMessage), + new UserMessage(userMessage) + ) + ).aiMessage().text(); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/citationparsing/templates/CitationParsingSystemMessageTemplate.java b/jablib/src/main/java/org/jabref/logic/ai/citationparsing/templates/CitationParsingSystemMessageTemplate.java new file mode 100644 index 000000000000..fabeafdd9e89 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/citationparsing/templates/CitationParsingSystemMessageTemplate.java @@ -0,0 +1,24 @@ +package org.jabref.logic.ai.citationparsing.templates; + +import java.util.function.Supplier; + +import org.jabref.logic.ai.templates.Template; +import org.jabref.model.ai.templating.AiTemplate; + +import org.apache.velocity.VelocityContext; + +public class CitationParsingSystemMessageTemplate extends Template { + public CitationParsingSystemMessageTemplate(Supplier sourceSupplier) { + super(sourceSupplier); + } + + public String render() { + VelocityContext context = makeContext(); + return render(context); + } + + @Override + public String getLogName() { + return AiTemplate.CITATION_PARSING_SYSTEM_MESSAGE.name(); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/citationparsing/templates/CitationParsingUserMessageTemplate.java b/jablib/src/main/java/org/jabref/logic/ai/citationparsing/templates/CitationParsingUserMessageTemplate.java new file mode 100644 index 000000000000..c0dc6b242a40 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/citationparsing/templates/CitationParsingUserMessageTemplate.java @@ -0,0 +1,27 @@ +package org.jabref.logic.ai.citationparsing.templates; + +import java.util.function.Supplier; + +import org.jabref.logic.ai.templates.Template; +import org.jabref.model.ai.templating.AiTemplate; + +import org.apache.velocity.VelocityContext; + +public class CitationParsingUserMessageTemplate extends Template { + public CitationParsingUserMessageTemplate(Supplier sourceSupplier) { + super(sourceSupplier); + } + + public String render(String citation) { + VelocityContext context = makeContext(); + + context.put("citation", citation); + + return render(context); + } + + @Override + public String getLogName() { + return AiTemplate.CITATION_PARSING_USER_MESSAGE.name(); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java b/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java deleted file mode 100644 index 0f92fe912db1..000000000000 --- a/jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesService.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.jabref.logic.ai.templates; - -import java.io.StringWriter; - -import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.model.ai.templating.AiTemplate; -import org.jabref.model.entry.CanonicalBibEntry; - -import org.apache.velocity.VelocityContext; -import org.apache.velocity.app.VelocityEngine; - -public class AiTemplatesService { - private final AiPreferences aiPreferences; - - private final VelocityEngine velocityEngine = new VelocityEngine(); - private final VelocityContext baseContext = new VelocityContext(); - - public AiTemplatesService(AiPreferences aiPreferences) { - this.aiPreferences = aiPreferences; - - velocityEngine.init(); - - baseContext.put("CanonicalBibEntry", CanonicalBibEntry.class); - } - - public String makeCitationParsingSystemMessage() { - VelocityContext context = new VelocityContext(baseContext); - return makeTemplate(AiTemplate.CITATION_PARSING_SYSTEM_MESSAGE, context); - } - - public String makeCitationParsingUserMessage(String citation) { - VelocityContext context = new VelocityContext(baseContext); - context.put("citation", citation); - - return makeTemplate(AiTemplate.CITATION_PARSING_USER_MESSAGE, context); - } - - private String makeTemplate(AiTemplate template, VelocityContext context) { - StringWriter writer = new StringWriter(); - - velocityEngine.evaluate(context, writer, template.name(), aiPreferences.getTemplate(template)); - - return writer.toString(); - } -} diff --git a/jablib/src/main/java/org/jabref/logic/ai/templates/CurrentAiTemplates.java b/jablib/src/main/java/org/jabref/logic/ai/templates/CurrentAiTemplates.java index 7a96a60407bf..0150076e4ac8 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/templates/CurrentAiTemplates.java +++ b/jablib/src/main/java/org/jabref/logic/ai/templates/CurrentAiTemplates.java @@ -2,6 +2,8 @@ import org.jabref.logic.ai.chatting.templates.ChattingSystemMessageTemplate; import org.jabref.logic.ai.chatting.templates.ChattingUserMessageTemplate; +import org.jabref.logic.ai.citationparsing.templates.CitationParsingSystemMessageTemplate; +import org.jabref.logic.ai.citationparsing.templates.CitationParsingUserMessageTemplate; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.summarization.templates.SummarizationChunkSystemMessageTemplate; import org.jabref.logic.ai.summarization.templates.SummarizationChunkUserMessageTemplate; @@ -18,6 +20,9 @@ public class CurrentAiTemplates { private final ChattingSystemMessageTemplate chattingSystemMessageTemplate; private final ChattingUserMessageTemplate chattingUserMessageTemplate; + private final CitationParsingSystemMessageTemplate citationParsingSystemMessageTemplate; + private final CitationParsingUserMessageTemplate citationParsingUserMessageTemplate; + public CurrentAiTemplates(AiPreferences aiPreferences) { this.summarizationChunkSystemMessageTemplate = new SummarizationChunkSystemMessageTemplate( () -> aiPreferences.getTemplate(AiTemplate.SUMMARIZATION_CHUNK_SYSTEM_MESSAGE) @@ -38,6 +43,13 @@ public CurrentAiTemplates(AiPreferences aiPreferences) { this.chattingUserMessageTemplate = new ChattingUserMessageTemplate( () -> aiPreferences.getTemplate(AiTemplate.CHATTING_USER_MESSAGE) ); + + this.citationParsingSystemMessageTemplate = new CitationParsingSystemMessageTemplate( + () -> aiPreferences.getTemplate(AiTemplate.CITATION_PARSING_SYSTEM_MESSAGE) + ); + this.citationParsingUserMessageTemplate = new CitationParsingUserMessageTemplate( + () -> aiPreferences.getTemplate(AiTemplate.CITATION_PARSING_USER_MESSAGE) + ); } public SummarizationChunkSystemMessageTemplate getSummarizationChunkSystemMessageTemplate() { @@ -63,4 +75,12 @@ public ChattingSystemMessageTemplate getChattingSystemMessageTemplate() { public ChattingUserMessageTemplate getChattingUserMessageTemplate() { return chattingUserMessageTemplate; } + + public CitationParsingSystemMessageTemplate getCitationParsingSystemMessageTemplate() { + return citationParsingSystemMessageTemplate; + } + + public CitationParsingUserMessageTemplate getCitationParsingUserMessageTemplate() { + return citationParsingUserMessageTemplate; + } } diff --git a/jablib/src/main/java/org/jabref/logic/importer/fileformat/pdf/CitationsFromPdf.java b/jablib/src/main/java/org/jabref/logic/importer/fileformat/pdf/CitationsFromPdf.java index f7f8bdd56aca..1d7f1fb48e49 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fileformat/pdf/CitationsFromPdf.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fileformat/pdf/CitationsFromPdf.java @@ -32,7 +32,12 @@ public static ParserResult extractCitationsUsingLLM(JabRefCliPreferences prefere preferences.getCitationKeyPatternPreferences(), notificationService, new CurrentThreadTaskExecutor())) { - LlmPlainCitationParser importer = new LlmPlainCitationParser(aiService.getTemplatesService(), preferences.getImportFormatPreferences(), aiService.getChatLanguageModel()); + LlmPlainCitationParser importer = new LlmPlainCitationParser( + aiService, + preferences.getImportFormatPreferences(), + aiService.getChatLanguageModel().getChatModelInfo() + ); + return importer.importDatabase(path); } } diff --git a/jablib/src/main/java/org/jabref/logic/importer/plaincitation/LlmPlainCitationParser.java b/jablib/src/main/java/org/jabref/logic/importer/plaincitation/LlmPlainCitationParser.java index 308a95b57cfa..950219392393 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/plaincitation/LlmPlainCitationParser.java +++ b/jablib/src/main/java/org/jabref/logic/importer/plaincitation/LlmPlainCitationParser.java @@ -1,33 +1,42 @@ package org.jabref.logic.importer.plaincitation; import java.io.IOException; -import java.io.Reader; import java.util.List; import java.util.Optional; -import org.jabref.logic.ai.templates.AiTemplatesService; +import org.jabref.logic.ai.AiService; +import org.jabref.logic.ai.citationparsing.logic.ParseCitationsWithLlm; import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ParseException; -import org.jabref.logic.importer.ParserResult; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.importer.fileformat.pdf.PdfImporterWithPlainCitationParser; import org.jabref.logic.l10n.Localization; +import org.jabref.logic.util.Result; +import org.jabref.model.ai.chatting.ChatModelInfo; import org.jabref.model.entry.BibEntry; -import dev.langchain4j.data.message.SystemMessage; -import dev.langchain4j.data.message.UserMessage; -import dev.langchain4j.model.chat.ChatModel; - public class LlmPlainCitationParser extends PdfImporterWithPlainCitationParser implements PlainCitationParser { - private final AiTemplatesService aiTemplatesService; private final ImportFormatPreferences importFormatPreferences; - private final ChatModel llm; - public LlmPlainCitationParser(AiTemplatesService aiTemplatesService, ImportFormatPreferences importFormatPreferences, ChatModel llm) { - this.aiTemplatesService = aiTemplatesService; + private final ChatModelInfo chatModelInfo; + + private final ParseCitationsWithLlm parseCitationsWithLlm; + + public LlmPlainCitationParser( + AiService aiService, + ImportFormatPreferences importFormatPreferences, + ChatModelInfo chatModelInfo + ) { this.importFormatPreferences = importFormatPreferences; - this.llm = llm; + + this.chatModelInfo = chatModelInfo; + + this.parseCitationsWithLlm = new ParseCitationsWithLlm( + importFormatPreferences, + aiService.getCurrentAiTemplates().getCitationParsingSystemMessageTemplate(), + aiService.getCurrentAiTemplates().getCitationParsingUserMessageTemplate() + ); } @Override @@ -48,7 +57,8 @@ public String getDescription() { @Override public Optional parsePlainCitation(String text) throws FetcherException { try { - return BibtexParser.singleFromString(getBibtexStringFromLlm(text), importFormatPreferences); + String string = parseCitationsWithLlm.getBibtexStringFromLlm(chatModelInfo, text); + return BibtexParser.singleFromString(string, importFormatPreferences); } catch (ParseException e) { throw new FetcherException("Could not parse BibTeX returned from LLM", e); } @@ -56,37 +66,12 @@ public Optional parsePlainCitation(String text) throws FetcherExceptio @Override public List parseMultiplePlainCitations(String text) throws FetcherException { - String systemMessage = aiTemplatesService.makeCitationParsingSystemMessage(); - String userMessage = aiTemplatesService.makeCitationParsingUserMessage(text); - - String llmResult = llm.chat( - List.of( - new SystemMessage(systemMessage), - new UserMessage(userMessage) - ) - ).aiMessage().text(); + Result, IOException> result = parseCitationsWithLlm.parseMultiplePlainCitations(chatModelInfo, text); - Reader reader = Reader.of(llmResult); - BibtexParser parser = new BibtexParser(importFormatPreferences); - ParserResult result; - try { - result = parser.parse(reader); - } catch (IOException e) { - throw new FetcherException("Could not parse BibTeX returned from LLM", e); + if (result.isErr()) { + throw new FetcherException("Could not parse BibTeX returned from LLM", result.getError()); } - return result.getDatabase().getEntries(); - } - - private String getBibtexStringFromLlm(String searchQuery) { - String systemMessage = aiTemplatesService.makeCitationParsingSystemMessage(); - String userMessage = aiTemplatesService.makeCitationParsingUserMessage(searchQuery); - - return llm.chat( - List.of( - new SystemMessage(systemMessage), - new UserMessage(userMessage) - ) - ).aiMessage().text(); + return result.getValue(); } } diff --git a/jablib/src/main/java/org/jabref/logic/importer/plaincitation/PlainCitationParserFactory.java b/jablib/src/main/java/org/jabref/logic/importer/plaincitation/PlainCitationParserFactory.java index 6286cfd85659..f915e96edf3f 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/plaincitation/PlainCitationParserFactory.java +++ b/jablib/src/main/java/org/jabref/logic/importer/plaincitation/PlainCitationParserFactory.java @@ -23,7 +23,7 @@ public static PlainCitationParser getPlainCitationParser(PlainCitationParserChoi case PlainCitationParserChoice.GROBID -> new GrobidPlainCitationParser(grobidPreferences, importFormatPreferences); case PlainCitationParserChoice.LLM -> - new LlmPlainCitationParser(aiService.getTemplatesService(), importFormatPreferences, aiService.getChatLanguageModel()); + new LlmPlainCitationParser(aiService, importFormatPreferences, aiService.getChatLanguageModel().getChatModelInfo()); }; } } diff --git a/jablib/src/main/java/org/jabref/logic/util/Result.java b/jablib/src/main/java/org/jabref/logic/util/Result.java new file mode 100644 index 000000000000..81437252536a --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/util/Result.java @@ -0,0 +1,133 @@ +package org.jabref.logic.util; + +/// A sealed interface representing a result of a computation that can either +/// succeed with a value of type T or fail with an error of type E. +/// +/// Similar to Either monad or [Result](https://doc.rust-lang.org/std/result/) in Rust +/// +/// @param Type of the success value +/// @param Type of the error +public sealed interface Result permits Result.Ok, Result.Err { + + /// Checks if this result is a success. + /// + /// @return true if this is an Ok result, false otherwise + boolean isOk(); + + /// Checks if this result is an error. + /// + /// @return true if this is an Err result, false otherwise + boolean isErr(); + + /// Returns the success value. + /// + /// @return the value of type T + /// @throws IllegalStateException if this result is an error + T getValue(); + + /// Returns the error value. + /// + /// @return the error of type E + /// @throws IllegalStateException if this result is a success + E getError(); + + /// Creates a success result containing the given value. + /// + /// @param value the success value + /// @param Type of the success value + /// @param Type of the error + /// @return an Ok result + static Result ok(T value) { + return new Ok<>(value); + } + + /// Creates an error result containing the given error. + /// + /// @param error the error value + /// @param Type of the success value + /// @param Type of the error + /// @return an Err result + static Result err(E error) { + return new Err<>(error); + } + + /// Represents a success result containing a value of type T. + /// + /// @param Type of the success value + /// @param Type of the error + final class Ok implements Result { + private final T value; + + /// Constructs a success result with the given value. + /// + /// @param value the success value + public Ok(T value) { + this.value = value; + } + + @Override + public boolean isOk() { + return true; + } + + @Override + public boolean isErr() { + return false; + } + + @Override + public T getValue() { + return value; + } + + @Override + public E getError() { + throw new IllegalStateException("No error in Ok"); + } + + @Override + public String toString() { + return "Ok(" + value + ")"; + } + } + + /// Represents an error result containing a value of type E. + /// + /// @param Type of the success value + /// @param Type of the error + final class Err implements Result { + private final E error; + + /// Constructs an error result with the given error value. + /// + /// @param error the error value + public Err(E error) { + this.error = error; + } + + @Override + public boolean isOk() { + return false; + } + + @Override + public boolean isErr() { + return true; + } + + @Override + public T getValue() { + throw new IllegalStateException("No value in Err"); + } + + @Override + public E getError() { + return error; + } + + @Override + public String toString() { + return "Err(" + error + ")"; + } + } +} From c25ccceb39054ea9af5c8d3cf9904b59dd5e707f Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Wed, 26 Nov 2025 15:51:41 +0100 Subject: [PATCH 024/243] Move "Currents" to `current` package --- .../util/EmbeddingModelGuardedComponent.java | 2 +- jablib/src/main/java/module-info.java | 1 + .../src/main/java/org/jabref/logic/ai/AiService.java | 10 +++++----- .../ai/{templates => current}/CurrentAiTemplates.java | 2 +- .../CurrentlySelectedChatLanguageModel.java | 3 +-- .../CurrentlySelectedEmbeddingModel.java | 2 +- .../CurrentlySelectedSummarizationAlgorithm.java | 3 +-- .../CurrentlySelectedTokenEstimationStrategy.java | 2 +- 8 files changed, 12 insertions(+), 13 deletions(-) rename jablib/src/main/java/org/jabref/logic/ai/{templates => current}/CurrentAiTemplates.java (99%) rename jablib/src/main/java/org/jabref/logic/ai/{chatting => current}/CurrentlySelectedChatLanguageModel.java (98%) rename jablib/src/main/java/org/jabref/logic/ai/{rag => current}/CurrentlySelectedEmbeddingModel.java (99%) rename jablib/src/main/java/org/jabref/logic/ai/{summarization => current}/CurrentlySelectedSummarizationAlgorithm.java (96%) rename jablib/src/main/java/org/jabref/logic/ai/{customimplementations/tokenization => current}/CurrentlySelectedTokenEstimationStrategy.java (97%) diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java index 077f1f8f8a6c..e2da5bbc4793 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java @@ -9,8 +9,8 @@ import org.jabref.gui.frame.ExternalApplicationsPreferences; import org.jabref.gui.util.UiTaskExecutor; import org.jabref.logic.ai.AiService; +import org.jabref.logic.ai.current.CurrentlySelectedEmbeddingModel; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.rag.CurrentlySelectedEmbeddingModel; import org.jabref.logic.l10n.Localization; import com.google.common.eventbus.Subscribe; diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index fdde2da1c584..4af3c849fcb4 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -146,6 +146,7 @@ exports org.jabref.model.ai.summarization; exports org.jabref.model.ai.identifiers; exports org.jabref.model.ai.tokenization; + exports org.jabref.logic.ai.current; // endregion requires java.base; diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiService.java b/jablib/src/main/java/org/jabref/logic/ai/AiService.java index f3c6ad0cd4fd..95de410c5050 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiService.java @@ -8,19 +8,19 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.chatting.ChatHistoryService; -import org.jabref.logic.ai.chatting.CurrentlySelectedChatLanguageModel; import org.jabref.logic.ai.chatting.repositories.MVStoreChatHistoryRepository; +import org.jabref.logic.ai.current.CurrentAiTemplates; +import org.jabref.logic.ai.current.CurrentlySelectedChatLanguageModel; +import org.jabref.logic.ai.current.CurrentlySelectedEmbeddingModel; +import org.jabref.logic.ai.current.CurrentlySelectedSummarizationAlgorithm; +import org.jabref.logic.ai.current.CurrentlySelectedTokenEstimationStrategy; import org.jabref.logic.ai.customimplementations.embeddingstores.MVStoreEmbeddingStore; -import org.jabref.logic.ai.customimplementations.tokenization.CurrentlySelectedTokenEstimationStrategy; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.rag.CurrentlySelectedEmbeddingModel; import org.jabref.logic.ai.rag.IngestionService; import org.jabref.logic.ai.rag.repositories.MVStoreFullyIngestedDocumentsRepository; -import org.jabref.logic.ai.summarization.CurrentlySelectedSummarizationAlgorithm; import org.jabref.logic.ai.summarization.SummariesService; import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; import org.jabref.logic.ai.summarization.repositories.MVStoreSummariesRepository; -import org.jabref.logic.ai.templates.CurrentAiTemplates; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; import org.jabref.logic.util.Directories; import org.jabref.logic.util.NotificationService; diff --git a/jablib/src/main/java/org/jabref/logic/ai/templates/CurrentAiTemplates.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentAiTemplates.java similarity index 99% rename from jablib/src/main/java/org/jabref/logic/ai/templates/CurrentAiTemplates.java rename to jablib/src/main/java/org/jabref/logic/ai/current/CurrentAiTemplates.java index 0150076e4ac8..cee57ee8a0df 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/templates/CurrentAiTemplates.java +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentAiTemplates.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.templates; +package org.jabref.logic.ai.current; import org.jabref.logic.ai.chatting.templates.ChattingSystemMessageTemplate; import org.jabref.logic.ai.chatting.templates.ChattingUserMessageTemplate; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedChatLanguageModel.java similarity index 98% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java rename to jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedChatLanguageModel.java index a646d5b6fa04..2030d9f2fe28 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/CurrentlySelectedChatLanguageModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedChatLanguageModel.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.chatting; +package org.jabref.logic.ai.current; import java.net.http.HttpClient; import java.time.Duration; @@ -11,7 +11,6 @@ import org.jabref.logic.ai.chatting.repositories.ChatHistoryRepository; import org.jabref.logic.ai.customimplementations.llms.Gpt4AllModel; import org.jabref.logic.ai.customimplementations.llms.JvmOpenAiChatLanguageModel; -import org.jabref.logic.ai.customimplementations.tokenization.CurrentlySelectedTokenEstimationStrategy; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.l10n.Localization; import org.jabref.model.ai.chatting.AiProvider; diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/CurrentlySelectedEmbeddingModel.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedEmbeddingModel.java similarity index 99% rename from jablib/src/main/java/org/jabref/logic/ai/rag/CurrentlySelectedEmbeddingModel.java rename to jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedEmbeddingModel.java index cc126a6c34a6..30a6b9728160 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/CurrentlySelectedEmbeddingModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedEmbeddingModel.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.rag; +package org.jabref.logic.ai.current; import java.util.List; import java.util.Optional; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/CurrentlySelectedSummarizationAlgorithm.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedSummarizationAlgorithm.java similarity index 96% rename from jablib/src/main/java/org/jabref/logic/ai/summarization/CurrentlySelectedSummarizationAlgorithm.java rename to jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedSummarizationAlgorithm.java index 3dd222351c89..ab405e31c290 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/CurrentlySelectedSummarizationAlgorithm.java +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedSummarizationAlgorithm.java @@ -1,9 +1,8 @@ -package org.jabref.logic.ai.summarization; +package org.jabref.logic.ai.current; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.ChunkedSummarizationAlgorithm; import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; -import org.jabref.logic.ai.templates.CurrentAiTemplates; import org.jabref.logic.ai.util.LongTaskInfo; import org.jabref.model.ai.chatting.ChatModelInfo; import org.jabref.model.ai.summarization.SummarizationAlgorithmName; diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/CurrentlySelectedTokenEstimationStrategy.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedTokenEstimationStrategy.java similarity index 97% rename from jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/CurrentlySelectedTokenEstimationStrategy.java rename to jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedTokenEstimationStrategy.java index 53e9b28d45e2..cc626c063a67 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/CurrentlySelectedTokenEstimationStrategy.java +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedTokenEstimationStrategy.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.customimplementations.tokenization; +package org.jabref.logic.ai.current; import java.util.List; From 80752c113eabd8a7c94286fac9be321a0b44deff Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Wed, 26 Nov 2025 16:02:17 +0100 Subject: [PATCH 025/243] Introduce AiDefaultExpertSettings --- jablib/src/main/java/module-info.java | 2 -- .../preferences/AiDefaultExpertSettings.java | 21 +++++++++++++++++++ .../ai/preferences/AiDefaultTemplates.java | 4 ++++ .../logic/ai/preferences/AiPreferences.java | 18 ++++++---------- .../preferences/JabRefCliPreferences.java | 17 ++++++++------- 5 files changed, 40 insertions(+), 22 deletions(-) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultExpertSettings.java diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index 4af3c849fcb4..16a3a180958e 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -122,7 +122,6 @@ exports org.jabref.logic.ai.customimplementations.embeddingmodels; exports org.jabref.logic.ai.customimplementations.embeddingstores; exports org.jabref.logic.ai.customimplementations.llms; - exports org.jabref.logic.ai.customimplementations.tokenization; exports org.jabref.logic.ai.rag; exports org.jabref.logic.ai.summarization.tasks; exports org.jabref.logic.ai.summarization.repositories; @@ -294,6 +293,5 @@ requires org.jooq.jool; requires org.libreoffice.uno; requires transitive org.jspecify; - requires org.jabref.jablib; // endregion } diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultExpertSettings.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultExpertSettings.java new file mode 100644 index 000000000000..57ea42930b03 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultExpertSettings.java @@ -0,0 +1,21 @@ +package org.jabref.logic.ai.preferences; + +import org.jabref.model.ai.embeddings.EmbeddingModel; +import org.jabref.model.ai.summarization.SummarizationAlgorithmName; +import org.jabref.model.ai.tokenization.TokenEstimationStrategy; + +/// A collection of values for the default settings of AI in the expert section. +/// +/// This collection was made because "Expert settings" in the AI settings is resettable. +/// There are facilities in JabRef codebase to reset either all settings or 1 section, but not a part of a section. +public class AiDefaultExpertSettings { + public static final EmbeddingModel EMBEDDING_MODEL = EmbeddingModel.SENTENCE_TRANSFORMERS_ALL_MINILM_L12_V2; + public static final SummarizationAlgorithmName SUMMARIZATION_ALGORITHM_NAME = SummarizationAlgorithmName.CHUNKED; + public static final TokenEstimationStrategy TOKEN_ESTIMATION_STRATEGY = TokenEstimationStrategy.MAX; + public static final float TEMPERATURE = 0.7f; + public static final int CONTEXT_WINDOW_SIZE = 8192; + public static final int DOCUMENT_SPLITTER_CHUNK_SIZE = 300; + public static final int DOCUMENT_SPLITTER_OVERLAP_SIZE = 100; + public static final int RAG_MAX_RESULTS_COUNT = 10; + public static final float RAG_MIN_SCORE = 0.3f; +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultTemplates.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultTemplates.java index a45949b699cd..743d4cb206e9 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultTemplates.java +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultTemplates.java @@ -4,6 +4,10 @@ import org.jabref.model.ai.templating.AiTemplate; +/// A collection of default AI templates. +/// +/// This collection is made into a separate class (instead of putting into defaults at [org.jabref.logic.preferences.JabRefCliPreferences]), +/// because they are too big. public class AiDefaultTemplates { private static final Map TEMPLATES = Map.of( AiTemplate.CHATTING_SYSTEM_MESSAGE, """ diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java index c2fe69116990..40b6429520ef 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java @@ -320,8 +320,7 @@ public EmbeddingModel getEmbeddingModel() { if (getCustomizeExpertSettings()) { return embeddingModel.get(); } else { - // TODO: Think why this is? It was taken from AiDefaultSettings. - return EmbeddingModel.SENTENCE_TRANSFORMERS_ALL_MINILM_L12_V2; + return AiDefaultExpertSettings.EMBEDDING_MODEL; } } @@ -397,8 +396,7 @@ public double getTemperature() { if (getCustomizeExpertSettings()) { return temperature.get(); } else { - // TODO: default return values - return 0.7; + return AiDefaultExpertSettings.TEMPERATURE; } } @@ -441,8 +439,7 @@ public int getDocumentSplitterChunkSize() { if (getCustomizeExpertSettings()) { return documentSplitterChunkSize.get(); } else { - // TODO: default value. - return 300; + return AiDefaultExpertSettings.DOCUMENT_SPLITTER_CHUNK_SIZE; } } @@ -458,8 +455,7 @@ public int getDocumentSplitterOverlapSize() { if (getCustomizeExpertSettings()) { return documentSplitterOverlapSize.get(); } else { - // TODO: default value - return 100; + return AiDefaultExpertSettings.DOCUMENT_SPLITTER_OVERLAP_SIZE; } } @@ -475,8 +471,7 @@ public int getRagMaxResultsCount() { if (getCustomizeExpertSettings()) { return ragMaxResultsCount.get(); } else { - // TODO: Default value - return 10; + return AiDefaultExpertSettings.RAG_MAX_RESULTS_COUNT; } } @@ -492,8 +487,7 @@ public double getRagMinScore() { if (getCustomizeExpertSettings()) { return ragMinScore.get(); } else { - // TODO: default value - return 0.3; + return AiDefaultExpertSettings.RAG_MIN_SCORE; } } diff --git a/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java b/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java index c676909e2673..b0c4cecbc21e 100644 --- a/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java @@ -33,6 +33,7 @@ import org.jabref.logic.InternalPreferences; import org.jabref.logic.JabRefException; import org.jabref.logic.LibraryPreferences; +import org.jabref.logic.ai.preferences.AiDefaultExpertSettings; import org.jabref.logic.ai.preferences.AiDefaultTemplates; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.preferences.AiProviderDefaultChatModels; @@ -740,20 +741,20 @@ public JabRefCliPreferences() { defaults.put(AI_HUGGING_FACE_CHAT_MODEL, AiProviderDefaultChatModels.getDefaultChatModel(AiProvider.HUGGING_FACE).getName()); defaults.put(AI_GPT_4_ALL_MODEL, AiProviderDefaultChatModels.getDefaultChatModel(AiProvider.GPT4ALL).getName()); defaults.put(AI_CUSTOMIZE_SETTINGS, false); - defaults.put(AI_EMBEDDING_MODEL, EmbeddingModel.SENTENCE_TRANSFORMERS_ALL_MINILM_L12_V2.name()); + defaults.put(AI_EMBEDDING_MODEL, AiDefaultExpertSettings.EMBEDDING_MODEL.name()); defaults.put(AI_OPEN_AI_API_BASE_URL, AiProvider.OPEN_AI.getApiUrl()); defaults.put(AI_MISTRAL_AI_API_BASE_URL, AiProvider.MISTRAL_AI.getApiUrl()); defaults.put(AI_GEMINI_API_BASE_URL, AiProvider.GEMINI.getApiUrl()); defaults.put(AI_HUGGING_FACE_API_BASE_URL, AiProvider.HUGGING_FACE.getApiUrl()); defaults.put(AI_GPT_4_ALL_API_BASE_URL, AiProvider.GPT4ALL.getApiUrl()); - defaults.put(AI_SUMMARIZATION_ALGORITHM, SummarizationAlgorithmName.CHUNKED.name()); - defaults.put(AI_TOKEN_ESTIMATION_ALGORITHM, TokenEstimationStrategy.MAX.name()); - defaults.put(AI_TEMPERATURE, 0.7); + defaults.put(AI_SUMMARIZATION_ALGORITHM, AiDefaultExpertSettings.SUMMARIZATION_ALGORITHM_NAME.name()); + defaults.put(AI_TOKEN_ESTIMATION_ALGORITHM, AiDefaultExpertSettings.TOKEN_ESTIMATION_STRATEGY.name()); + defaults.put(AI_TEMPERATURE, AiDefaultExpertSettings.TEMPERATURE); defaults.put(AI_CONTEXT_WINDOW_SIZE, PredefinedChatModel.getContextWindowSize((AiProvider) defaults.get(AI_PROVIDER), AiProviderDefaultChatModels.getDefaultChatModel((AiProvider) defaults.get(AI_PROVIDER)).getName())); - defaults.put(AI_DOCUMENT_SPLITTER_CHUNK_SIZE, 300); - defaults.put(AI_DOCUMENT_SPLITTER_OVERLAP_SIZE, 100); - defaults.put(AI_RAG_MAX_RESULTS_COUNT, 10); - defaults.put(AI_RAG_MIN_SCORE, 0.3); + defaults.put(AI_DOCUMENT_SPLITTER_CHUNK_SIZE, AiDefaultExpertSettings.DOCUMENT_SPLITTER_CHUNK_SIZE); + defaults.put(AI_DOCUMENT_SPLITTER_OVERLAP_SIZE, AiDefaultExpertSettings.DOCUMENT_SPLITTER_OVERLAP_SIZE); + defaults.put(AI_RAG_MAX_RESULTS_COUNT, AiDefaultExpertSettings.RAG_MAX_RESULTS_COUNT); + defaults.put(AI_RAG_MIN_SCORE, AiDefaultExpertSettings.RAG_MIN_SCORE); // region:AI templates defaults.put(AI_CHATTING_SYSTEM_MESSAGE_TEMPLATE, AiDefaultTemplates.getTemplate(AiTemplate.CHATTING_SYSTEM_MESSAGE)); From e1812238873f5e4949b779b6a065d9db8f8cd4df Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Wed, 26 Nov 2025 16:04:41 +0100 Subject: [PATCH 026/243] Finish previous --- .../org/jabref/logic/ai/preferences/AiFallbackSettings.java | 5 ----- .../org/jabref/logic/ai/preferences/PredefinedChatModel.java | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) delete mode 100644 jablib/src/main/java/org/jabref/logic/ai/preferences/AiFallbackSettings.java diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiFallbackSettings.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiFallbackSettings.java deleted file mode 100644 index dc42acd6c693..000000000000 --- a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiFallbackSettings.java +++ /dev/null @@ -1,5 +0,0 @@ -package org.jabref.logic.ai.preferences; - -public class AiFallbackSettings { - public static final int FALLBACK_CONTEXT_WINDOW_SIZE = 8196; -} diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/PredefinedChatModel.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/PredefinedChatModel.java index 537e551b4f63..3c097bfc294d 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/preferences/PredefinedChatModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/PredefinedChatModel.java @@ -46,7 +46,7 @@ public static int getContextWindowSize(AiProvider aiProvider, String modelName) .filter(model -> model.getAiProvider() == aiProvider && model.getName().equals(modelName)) .map(PredefinedChatModel::getContextWindowSize) .findFirst() - .orElse(AiFallbackSettings.FALLBACK_CONTEXT_WINDOW_SIZE); + .orElse(AiDefaultExpertSettings.CONTEXT_WINDOW_SIZE); } public AiProvider getAiProvider() { From ff866d732d63f08d2c6b666f012a719b89247739 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Wed, 26 Nov 2025 19:51:10 +0100 Subject: [PATCH 027/243] Make summary work again --- docs/code-howtos/ai.md | 8 ++ .../summary/SummaryShowingComponent.fxml | 42 +++++-- jabgui/src/main/resources/tinylog.properties | 2 +- jablib/src/main/java/module-info.java | 2 + .../java/org/jabref/logic/ai/AiService.java | 11 +- .../logic/ai/chatting/logic/AiChatLogic.java | 6 +- .../logic/ParseCitationsWithLlm.java | 10 +- .../CurrentlySelectedChatLanguageModel.java | 84 +++++++------- .../CurrentlySelectedDocumentSplitter.java | 64 +++++++++++ ...rrentlySelectedSummarizationAlgorithm.java | 10 +- ...rentlySelectedTokenEstimationStrategy.java | 1 + .../MVStoreEmbeddingStore.java | 2 +- .../customimplementations/llms/ChatModel.java | 11 ++ .../preferences/AiDefaultExpertSettings.java | 2 + .../logic/ai/preferences/AiPreferences.java | 20 ++++ .../jabref/logic/ai/rag/IngestionService.java | 57 ++++++++-- .../logic/ai/rag/logic/EmbeddingsCleaner.java | 45 ++++++++ .../logic/ai/rag/logic/LowLevelIngestor.java | 81 ------------- .../DocumentSplitterAlgorithm.java | 12 ++ .../SlidingWindowDocumentSplitter.java | 36 ++++++ .../ai/rag/logic/ingestion/FileIngestor.java | 56 +++++++++ .../logic/ingestion/LinkedFileIngestor.java | 40 +++++++ .../PersistentLinkedFileIngestor.java | 101 +++++++++++++++++ .../ai/rag/logic/ingestion/TextIngestor.java | 51 +++++++++ .../ai/rag/logic/parsing/FileParser.java | 4 +- .../ai/rag/logic/parsing/PdfFileParser.java | 10 +- .../logic/parsing/UniversalFileParser.java | 7 +- .../repositories/FileEmbeddingsManager.java | 88 -------------- .../GenerateEmbeddingsForSeveralTask.java | 49 +++++--- .../ai/rag/tasks/GenerateEmbeddingsTask.java | 107 ++++++------------ .../ai/summarization/SummariesService.java | 12 +- .../logic/BibEntrySummarizer.java | 23 ++-- .../logic/PersistentBibEntrySummarizer.java | 10 +- .../ChunkedSummarizationAlgorithm.java | 13 ++- .../SummarizationAlgorithm.java | 4 +- .../tasks/GenerateSummaryForSeveralTask.java | 10 +- .../tasks/GenerateSummaryTask.java | 10 +- .../fileformat/pdf/CitationsFromPdf.java | 2 +- .../plaincitation/LlmPlainCitationParser.java | 12 +- .../PlainCitationParserFactory.java | 2 +- .../preferences/JabRefCliPreferences.java | 12 +- .../model/ai/chatting/ChatModelInfo.java | 14 --- .../ai/rag/DocumentSplittingStrategy.java | 5 + 43 files changed, 745 insertions(+), 403 deletions(-) create mode 100644 docs/code-howtos/ai.md create mode 100644 jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedDocumentSplitter.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/ChatModel.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/rag/logic/EmbeddingsCleaner.java delete mode 100644 jablib/src/main/java/org/jabref/logic/ai/rag/logic/LowLevelIngestor.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/rag/logic/documentsplitting/DocumentSplitterAlgorithm.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/rag/logic/documentsplitting/SlidingWindowDocumentSplitter.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/FileIngestor.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/LinkedFileIngestor.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/PersistentLinkedFileIngestor.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/TextIngestor.java delete mode 100644 jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FileEmbeddingsManager.java delete mode 100644 jablib/src/main/java/org/jabref/model/ai/chatting/ChatModelInfo.java create mode 100644 jablib/src/main/java/org/jabref/model/ai/rag/DocumentSplittingStrategy.java diff --git a/docs/code-howtos/ai.md b/docs/code-howtos/ai.md new file mode 100644 index 000000000000..7729bee2ae7b --- /dev/null +++ b/docs/code-howtos/ai.md @@ -0,0 +1,8 @@ +- ai package organization +- gui / logic / model +- how to make ui -> view view-model pattern +- currents +- task organization +- generic classess +- long task and shutdown signal +- ensure to use ChatModel from jabref diff --git a/jabgui/src/main/resources/org/jabref/gui/ai/components/summary/SummaryShowingComponent.fxml b/jabgui/src/main/resources/org/jabref/gui/ai/components/summary/SummaryShowingComponent.fxml index 5a53e8011329..90222cf46575 100644 --- a/jabgui/src/main/resources/org/jabref/gui/ai/components/summary/SummaryShowingComponent.fxml +++ b/jabgui/src/main/resources/org/jabref/gui/ai/components/summary/SummaryShowingComponent.fxml @@ -1,26 +1,48 @@ - - + - + - - + - + - +
-
- + - + diff --git a/jabgui/src/main/resources/tinylog.properties b/jabgui/src/main/resources/tinylog.properties index b3106ac889de..b019da09a145 100644 --- a/jabgui/src/main/resources/tinylog.properties +++ b/jabgui/src/main/resources/tinylog.properties @@ -24,6 +24,6 @@ level@org.freedesktop.dbus.connections.transports.TransportBuilder = warn #level@org.jabref.logic.ai.AiService = trace #level@org.jabref.logic.ai.chathistory.BibDatabaseChatHistory = trace #level@org.jabref.logic.ai.chathistory.AiChatHistoryManager = trace -#level@org.jabref.logic.ai.FileEmbeddingsManager = trace +#level@org.jabref.logic.ai.EmbeddingsCleaner = trace #level@org.jabref.logic.ai.impl.embeddings.LowLevelIngestor = trace #level@org.jabref.logic.ai.chat.AiChatLogic = trace diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index 16a3a180958e..da9fc13faf15 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -123,6 +123,7 @@ exports org.jabref.logic.ai.customimplementations.embeddingstores; exports org.jabref.logic.ai.customimplementations.llms; exports org.jabref.logic.ai.rag; + exports org.jabref.logic.ai.rag.logic.documentsplitting; exports org.jabref.logic.ai.summarization.tasks; exports org.jabref.logic.ai.summarization.repositories; exports org.jabref.logic.ai.summarization.templates; @@ -132,6 +133,7 @@ exports org.jabref.logic.ai.rag.repositories; exports org.jabref.logic.ai.rag.logic; exports org.jabref.logic.ai.chatting.logic; + exports org.jabref.logic.ai.customimplementations.tokenization.algorithms; exports org.jabref.logic.ai.chatting.tasks; exports org.jabref.logic.ai.chatting.templates; exports org.jabref.logic.ai.chatting.repositories; diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiService.java b/jablib/src/main/java/org/jabref/logic/ai/AiService.java index 95de410c5050..17177de192a4 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiService.java @@ -11,6 +11,7 @@ import org.jabref.logic.ai.chatting.repositories.MVStoreChatHistoryRepository; import org.jabref.logic.ai.current.CurrentAiTemplates; import org.jabref.logic.ai.current.CurrentlySelectedChatLanguageModel; +import org.jabref.logic.ai.current.CurrentlySelectedDocumentSplitter; import org.jabref.logic.ai.current.CurrentlySelectedEmbeddingModel; import org.jabref.logic.ai.current.CurrentlySelectedSummarizationAlgorithm; import org.jabref.logic.ai.current.CurrentlySelectedTokenEstimationStrategy; @@ -60,6 +61,7 @@ public class AiService implements AutoCloseable { private final CurrentAiTemplates currentAiTemplates; private final ChatHistoryService chatHistoryService; + private final CurrentlySelectedDocumentSplitter currentlySelectedDocumentSplitter; private final CurrentlySelectedTokenEstimationStrategy currentlySelectedTokenEstimationStrategy; private final CurrentlySelectedChatLanguageModel currentlySelectedChatLanguageModel; private final CurrentlySelectedEmbeddingModel currentlySelectedEmbeddingModel; @@ -82,6 +84,8 @@ public AiService( this.currentAiTemplates = new CurrentAiTemplates(aiPreferences); this.chatHistoryService = new ChatHistoryService(citationKeyPatternPreferences, mvStoreChatHistoryStorage); + + this.currentlySelectedDocumentSplitter = new CurrentlySelectedDocumentSplitter(aiPreferences); this.currentlySelectedTokenEstimationStrategy = new CurrentlySelectedTokenEstimationStrategy(aiPreferences); this.currentlySelectedChatLanguageModel = new CurrentlySelectedChatLanguageModel(aiPreferences, currentlySelectedTokenEstimationStrategy); this.currentlySelectedEmbeddingModel = new CurrentlySelectedEmbeddingModel(aiPreferences, notificationService, taskExecutor); @@ -93,6 +97,7 @@ public AiService( taskExecutor, currentlySelectedEmbeddingModel, mvStoreEmbeddingStore, + currentlySelectedDocumentSplitter, mvStoreFullyIngestedDocumentsTracker, shutdownSignal ); @@ -101,13 +106,17 @@ public AiService( aiPreferences, filePreferences, taskExecutor, - currentlySelectedChatLanguageModel.getChatModelInfo(), + currentlySelectedChatLanguageModel, currentlySelectedSummarizationAlgorithm, mvStoreSummariesStorage, shutdownSignal ); } + public CurrentlySelectedDocumentSplitter getDocumentSplitter() { + return currentlySelectedDocumentSplitter; + } + public CurrentlySelectedChatLanguageModel getChatLanguageModel() { return currentlySelectedChatLanguageModel; } diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java index 2d4762a68c6d..26f96a8d4989 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java @@ -10,7 +10,7 @@ import org.jabref.logic.ai.chatting.templates.ChattingSystemMessageTemplate; import org.jabref.logic.ai.chatting.templates.ChattingUserMessageTemplate; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.rag.repositories.FileEmbeddingsManager; +import org.jabref.logic.ai.rag.logic.EmbeddingsCleaner; import org.jabref.model.ai.chatting.ErrorMessage; import org.jabref.model.ai.rag.PaperExcerpt; import org.jabref.model.ai.templating.AiTemplate; @@ -128,7 +128,7 @@ private void rebuildFilter() { filter = Optional.empty(); } else { filter = Optional.of(MetadataFilterBuilder - .metadataKey(FileEmbeddingsManager.LINK_METADATA_KEY) + .metadataKey(EmbeddingsCleaner.LINK_METADATA_KEY) .isIn(linkedFiles .stream() .map(LinkedFile::getLink) @@ -166,7 +166,7 @@ public AiMessage execute(UserMessage message) { .stream() .map(EmbeddingMatch::embedded) .map(textSegment -> { - String link = textSegment.metadata().getString(FileEmbeddingsManager.LINK_METADATA_KEY); + String link = textSegment.metadata().getString(EmbeddingsCleaner.LINK_METADATA_KEY); if (link == null) { return new PaperExcerpt("", textSegment.text()); diff --git a/jablib/src/main/java/org/jabref/logic/ai/citationparsing/logic/ParseCitationsWithLlm.java b/jablib/src/main/java/org/jabref/logic/ai/citationparsing/logic/ParseCitationsWithLlm.java index 3c517eb67380..2c407bf4baf2 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/citationparsing/logic/ParseCitationsWithLlm.java +++ b/jablib/src/main/java/org/jabref/logic/ai/citationparsing/logic/ParseCitationsWithLlm.java @@ -6,11 +6,11 @@ import org.jabref.logic.ai.citationparsing.templates.CitationParsingSystemMessageTemplate; import org.jabref.logic.ai.citationparsing.templates.CitationParsingUserMessageTemplate; +import org.jabref.logic.ai.customimplementations.llms.ChatModel; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ParserResult; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.util.Result; -import org.jabref.model.ai.chatting.ChatModelInfo; import org.jabref.model.entry.BibEntry; import dev.langchain4j.data.message.SystemMessage; @@ -34,14 +34,14 @@ public ParseCitationsWithLlm( } public Result, IOException> parseMultiplePlainCitations( - ChatModelInfo chatModelInfo, + ChatModel chatModel, String text ) { String systemMessage = citationParsingSystemMessageTemplate.render(); String userMessage = citationParsingUserMessageTemplate.render(text); // TODO: Clean possibly of backticks. - String llmResult = chatModelInfo.chatModel().chat( + String llmResult = chatModel.chat( List.of( new SystemMessage(systemMessage), new UserMessage(userMessage) @@ -65,13 +65,13 @@ private Result, IOException> parseBibEntryString(String text) { } public String getBibtexStringFromLlm( - ChatModelInfo chatModelInfo, + ChatModel chatModel, String searchQuery ) { String systemMessage = citationParsingSystemMessageTemplate.render(); String userMessage = citationParsingUserMessageTemplate.render(searchQuery); - return chatModelInfo.chatModel().chat( + return chatModel.chat( List.of( new SystemMessage(systemMessage), new UserMessage(userMessage) diff --git a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedChatLanguageModel.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedChatLanguageModel.java index 2030d9f2fe28..3a4813f5561f 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedChatLanguageModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedChatLanguageModel.java @@ -3,27 +3,27 @@ import java.net.http.HttpClient; import java.time.Duration; import java.util.List; -import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import org.jabref.logic.ai.chatting.logic.AiChatLogic; import org.jabref.logic.ai.chatting.repositories.ChatHistoryRepository; +import org.jabref.logic.ai.customimplementations.llms.ChatModel; import org.jabref.logic.ai.customimplementations.llms.Gpt4AllModel; import org.jabref.logic.ai.customimplementations.llms.JvmOpenAiChatLanguageModel; +import org.jabref.logic.ai.customimplementations.tokenization.algorithms.Tokenizer; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.l10n.Localization; import org.jabref.model.ai.chatting.AiProvider; -import org.jabref.model.ai.chatting.ChatModelInfo; import com.google.common.util.concurrent.ThreadFactoryBuilder; import dev.langchain4j.data.message.ChatMessage; import dev.langchain4j.memory.ChatMemory; -import dev.langchain4j.model.chat.ChatModel; import dev.langchain4j.model.chat.response.ChatResponse; import dev.langchain4j.model.googleai.GoogleAiGeminiChatModel; import dev.langchain4j.model.huggingface.HuggingFaceChatModel; import dev.langchain4j.model.mistralai.MistralAiChatModel; +import jakarta.annotation.Nullable; /** * Wrapper around langchain4j chat language model. @@ -43,7 +43,8 @@ public class CurrentlySelectedChatLanguageModel implements ChatModel, AutoClosea new ThreadFactoryBuilder().setNameFormat("ai-api-connection-pool-%d").build() ); - private Optional langchainChatModel = Optional.empty(); + @Nullable + private dev.langchain4j.model.chat.ChatModel langchainChatModel = null; public CurrentlySelectedChatLanguageModel( AiPreferences aiPreferences, @@ -67,19 +68,19 @@ public CurrentlySelectedChatLanguageModel( private void rebuild() { String apiKey = aiPreferences.getApiKeyForAiProvider(aiPreferences.getAiProvider()); if (!aiPreferences.getEnableAi() || (apiKey.isEmpty() && aiPreferences.getAiProvider() != AiProvider.GPT4ALL)) { - langchainChatModel = Optional.empty(); + langchainChatModel = null; return; } switch (aiPreferences.getAiProvider()) { case OPEN_AI -> - langchainChatModel = Optional.of(new JvmOpenAiChatLanguageModel(aiPreferences, httpClient)); + langchainChatModel = new JvmOpenAiChatLanguageModel(aiPreferences, httpClient); case GPT4ALL -> - langchainChatModel = Optional.of(new Gpt4AllModel(aiPreferences, httpClient)); + langchainChatModel = new Gpt4AllModel(aiPreferences, httpClient); case MISTRAL_AI -> - langchainChatModel = Optional.of(MistralAiChatModel + langchainChatModel = MistralAiChatModel .builder() .apiKey(apiKey) .modelName(aiPreferences.getSelectedChatModel()) @@ -87,53 +88,40 @@ private void rebuild() { .baseUrl(aiPreferences.getSelectedApiBaseUrl()) .logRequests(true) .logResponses(true) - .build() - ); + .build(); case GEMINI -> // NOTE: {@link GoogleAiGeminiChatModel} doesn't support API base url. - langchainChatModel = Optional.of(GoogleAiGeminiChatModel + langchainChatModel = GoogleAiGeminiChatModel .builder() .apiKey(apiKey) .modelName(aiPreferences.getSelectedChatModel()) .temperature(aiPreferences.getTemperature()) .logRequestsAndResponses(true) - .build() - ); + .build(); case HUGGING_FACE -> // NOTE: {@link HuggingFaceChatModel} doesn't support API base url. - langchainChatModel = Optional.of(HuggingFaceChatModel + langchainChatModel = HuggingFaceChatModel .builder() .accessToken(apiKey) .modelId(aiPreferences.getSelectedChatModel()) .temperature(aiPreferences.getTemperature()) .timeout(Duration.ofMinutes(2)) - .build() - ); + .build(); } } private void setupListeningToPreferencesChanges() { - // Setting "langchainChatModel" to "Optional.empty()" will trigger a rebuild on the next usage + // TODO: written below is weird, but works. + // Setting "langchainChatModel" to "null" will trigger a rebuild on the next usage - aiPreferences.enableAiProperty().addListener(_ -> langchainChatModel = Optional.empty()); - aiPreferences.aiProviderProperty().addListener(_ -> langchainChatModel = Optional.empty()); - aiPreferences.customizeExpertSettingsProperty().addListener(_ -> langchainChatModel = Optional.empty()); - aiPreferences.temperatureProperty().addListener(_ -> langchainChatModel = Optional.empty()); + aiPreferences.enableAiProperty().addListener(_ -> langchainChatModel = null); + aiPreferences.aiProviderProperty().addListener(_ -> langchainChatModel = null); + aiPreferences.customizeExpertSettingsProperty().addListener(_ -> langchainChatModel = null); + aiPreferences.temperatureProperty().addListener(_ -> langchainChatModel = null); - aiPreferences.addListenerToChatModels(() -> langchainChatModel = Optional.empty()); - aiPreferences.addListenerToApiBaseUrls(() -> langchainChatModel = Optional.empty()); - aiPreferences.setApiKeyChangeListener(() -> langchainChatModel = Optional.empty()); - } - - public ChatModelInfo getChatModelInfo() { - assert langchainChatModel.isPresent(); - return new ChatModelInfo( - langchainChatModel.get(), - currentlySelectedTokenEstimationStrategy, - aiPreferences.getAiProvider(), - aiPreferences.getSelectedChatModel(), - aiPreferences.getContextWindowSize() - ); + aiPreferences.addListenerToChatModels(() -> langchainChatModel = null); + aiPreferences.addListenerToApiBaseUrls(() -> langchainChatModel = null); + aiPreferences.setApiKeyChangeListener(() -> langchainChatModel = null); } @Override @@ -146,20 +134,20 @@ public ChatResponse chat(List list) { // in the result type, nor "throws" in method signature. Actually, // it's possible, but langchain4j doesn't do it. - if (langchainChatModel.isEmpty()) { + if (langchainChatModel == null) { if (!aiPreferences.getEnableAi()) { throw new RuntimeException(Localization.lang("In order to use AI chat, you need to enable chatting with attached PDF files in JabRef preferences (AI tab).")); } else if (aiPreferences.getApiKeyForAiProvider(aiPreferences.getAiProvider()).isEmpty() && aiPreferences.getAiProvider() != AiProvider.GPT4ALL) { throw new RuntimeException(Localization.lang("In order to use AI chat, set an API key inside JabRef preferences (AI tab).")); } else { rebuild(); - if (langchainChatModel.isEmpty()) { + if (langchainChatModel == null) { throw new RuntimeException(Localization.lang("Unable to chat with AI.")); } } } - return langchainChatModel.get().chat(list); + return langchainChatModel.chat(list); } @Override @@ -167,4 +155,24 @@ public void close() { httpClient.shutdownNow(); executorService.shutdownNow(); } + + @Override + public Tokenizer getTokenizer() { + return currentlySelectedTokenEstimationStrategy; + } + + @Override + public AiProvider getAiProvider() { + return aiPreferences.getAiProvider(); + } + + @Override + public String getName() { + return aiPreferences.getSelectedChatModel(); + } + + @Override + public int getContextWindowSize() { + return aiPreferences.getContextWindowSize(); + } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedDocumentSplitter.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedDocumentSplitter.java new file mode 100644 index 000000000000..eebcdc69a077 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedDocumentSplitter.java @@ -0,0 +1,64 @@ +package org.jabref.logic.ai.current; + +import java.util.stream.Stream; + +import org.jabref.logic.ai.preferences.AiPreferences; +import org.jabref.logic.ai.rag.logic.documentsplitting.DocumentSplitterAlgorithm; +import org.jabref.logic.ai.rag.logic.documentsplitting.SlidingWindowDocumentSplitter; +import org.jabref.logic.ai.util.LongTaskInfo; +import org.jabref.model.ai.rag.DocumentSplittingStrategy; + +import org.jspecify.annotations.Nullable; + +public class CurrentlySelectedDocumentSplitter implements DocumentSplitterAlgorithm { + private final AiPreferences aiPreferences; + + @Nullable + private DocumentSplitterAlgorithm documentSplitterAlgorithm = null; + + public CurrentlySelectedDocumentSplitter(AiPreferences aiPreferences) { + this.aiPreferences = aiPreferences; + + update(); + configure(); + } + + private void configure() { + aiPreferences.customizeExpertSettingsProperty().addListener(_ -> update()); + aiPreferences.documentSplittingStrategyProperty().addListener(_ -> update()); + aiPreferences.documentSplitterChunkSizeProperty().addListener(_ -> update()); + aiPreferences.documentSplitterOverlapSizeProperty().addListener(_ -> update()); + } + + private void update() { + // Because in the future there will be more strategies. + //noinspection SwitchStatementWithTooFewBranches + switch (aiPreferences.getDocumentSplittingStrategy()) { + case DocumentSplittingStrategy.SLIDING_WINDOW -> { + documentSplitterAlgorithm = new SlidingWindowDocumentSplitter( + aiPreferences.getDocumentSplitterChunkSize(), + aiPreferences.getDocumentSplitterOverlapSize() + ); + } + } + } + + @Override + public Stream split(LongTaskInfo longTaskInfo, String text) throws InterruptedException { + if (documentSplitterAlgorithm == null) { + return Stream.of(text); + } + + return documentSplitterAlgorithm.split(longTaskInfo, text); + } + + @Override + public DocumentSplittingStrategy getStrategy() { + if (documentSplitterAlgorithm == null) { + // Unfortunately. + return null; + } + + return documentSplitterAlgorithm.getStrategy(); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedSummarizationAlgorithm.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedSummarizationAlgorithm.java index ab405e31c290..bb9a41b60a3d 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedSummarizationAlgorithm.java +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedSummarizationAlgorithm.java @@ -1,10 +1,10 @@ package org.jabref.logic.ai.current; +import org.jabref.logic.ai.customimplementations.llms.ChatModel; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.ChunkedSummarizationAlgorithm; import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; import org.jabref.logic.ai.util.LongTaskInfo; -import org.jabref.model.ai.chatting.ChatModelInfo; import org.jabref.model.ai.summarization.SummarizationAlgorithmName; import org.jabref.model.ai.templating.AiTemplate; @@ -24,6 +24,7 @@ public CurrentlySelectedSummarizationAlgorithm( this.aiPreferences = aiPreferences; this.currentAiTemplates = currentAiTemplates; + updateAlgorithm(); setupListeningToPreferences(); } @@ -48,6 +49,7 @@ private void setupListeningToPreferences() { } private void updateAlgorithm() { + // Because in the future there will be more strategies. //noinspection SwitchStatementWithTooFewBranches switch (aiPreferences.getDefaultSummarizationAlgorithm()) { case SummarizationAlgorithmName.CHUNKED -> { @@ -66,12 +68,12 @@ private ChunkedSummarizationAlgorithm createChunkedSummarizationAlgorithm() { } @Override - public String summarize(ChatModelInfo chatModelInfo, LongTaskInfo longTaskInfo, String text) throws InterruptedException { + public String summarize(ChatModel chatModel, LongTaskInfo longTaskInfo, String text) throws InterruptedException { if (summarizationAlgorithm == null) { - return ""; + throw new RuntimeException("No summarization algorithm selected."); } - return summarizationAlgorithm.summarize(chatModelInfo, longTaskInfo, text); + return summarizationAlgorithm.summarize(chatModel, longTaskInfo, text); } @Override diff --git a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedTokenEstimationStrategy.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedTokenEstimationStrategy.java index cc626c063a67..00564e14ad13 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedTokenEstimationStrategy.java +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedTokenEstimationStrategy.java @@ -25,6 +25,7 @@ public CurrentlySelectedTokenEstimationStrategy( ) { this.aiPreferences = aiPreferences; + createTokenizer(); setupListeningToPreferences(); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingstores/MVStoreEmbeddingStore.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingstores/MVStoreEmbeddingStore.java index 0b1f919611e1..518c9d5eecbc 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingstores/MVStoreEmbeddingStore.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingstores/MVStoreEmbeddingStore.java @@ -34,7 +34,7 @@ import org.jspecify.annotations.Nullable; import static java.util.Comparator.comparingDouble; -import static org.jabref.logic.ai.rag.repositories.FileEmbeddingsManager.LINK_METADATA_KEY; +import static org.jabref.logic.ai.rag.logic.EmbeddingsCleaner.LINK_METADATA_KEY; /** * A custom implementation of langchain4j's {@link EmbeddingStore} that uses a {@link MVStore} as an embedded database. diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/ChatModel.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/ChatModel.java new file mode 100644 index 000000000000..bd39593da802 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/ChatModel.java @@ -0,0 +1,11 @@ +package org.jabref.logic.ai.customimplementations.llms; + +import org.jabref.logic.ai.customimplementations.tokenization.algorithms.Tokenizer; +import org.jabref.model.ai.chatting.AiProvider; + +public interface ChatModel extends dev.langchain4j.model.chat.ChatModel { + Tokenizer getTokenizer(); + AiProvider getAiProvider(); + String getName(); + int getContextWindowSize(); +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultExpertSettings.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultExpertSettings.java index 57ea42930b03..c2db723fa305 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultExpertSettings.java +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultExpertSettings.java @@ -1,6 +1,7 @@ package org.jabref.logic.ai.preferences; import org.jabref.model.ai.embeddings.EmbeddingModel; +import org.jabref.model.ai.rag.DocumentSplittingStrategy; import org.jabref.model.ai.summarization.SummarizationAlgorithmName; import org.jabref.model.ai.tokenization.TokenEstimationStrategy; @@ -14,6 +15,7 @@ public class AiDefaultExpertSettings { public static final TokenEstimationStrategy TOKEN_ESTIMATION_STRATEGY = TokenEstimationStrategy.MAX; public static final float TEMPERATURE = 0.7f; public static final int CONTEXT_WINDOW_SIZE = 8192; + public static final DocumentSplittingStrategy DOCUMENT_SPLITTING_STRATEGY = DocumentSplittingStrategy.SLIDING_WINDOW; public static final int DOCUMENT_SPLITTER_CHUNK_SIZE = 300; public static final int DOCUMENT_SPLITTER_OVERLAP_SIZE = 100; public static final int RAG_MAX_RESULTS_COUNT = 10; diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java index 40b6429520ef..3dad367ec079 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java @@ -19,6 +19,7 @@ import org.jabref.logic.util.strings.StringUtil; import org.jabref.model.ai.chatting.AiProvider; import org.jabref.model.ai.embeddings.EmbeddingModel; +import org.jabref.model.ai.rag.DocumentSplittingStrategy; import org.jabref.model.ai.summarization.SummarizationAlgorithmName; import org.jabref.model.ai.templating.AiTemplate; import org.jabref.model.ai.tokenization.TokenEstimationStrategy; @@ -59,8 +60,11 @@ public class AiPreferences { private final ObjectProperty embeddingModel; private final DoubleProperty temperature; private final IntegerProperty contextWindowSize; + + private final ObjectProperty documentSplittingStrategy; private final IntegerProperty documentSplitterChunkSize; private final IntegerProperty documentSplitterOverlapSize; + private final IntegerProperty ragMaxResultsCount; private final DoubleProperty ragMinScore; @@ -89,6 +93,7 @@ public AiPreferences( EmbeddingModel embeddingModel, double temperature, int contextWindowSize, + DocumentSplittingStrategy documentSplittingStrategy, int documentSplitterChunkSize, int documentSplitterOverlapSize, int ragMaxResultsCount, @@ -120,8 +125,11 @@ public AiPreferences( this.embeddingModel = new SimpleObjectProperty<>(embeddingModel); this.temperature = new SimpleDoubleProperty(temperature); this.contextWindowSize = new SimpleIntegerProperty(contextWindowSize); + + this.documentSplittingStrategy = new SimpleObjectProperty<>(documentSplittingStrategy); this.documentSplitterChunkSize = new SimpleIntegerProperty(documentSplitterChunkSize); this.documentSplitterOverlapSize = new SimpleIntegerProperty(documentSplitterOverlapSize); + this.ragMaxResultsCount = new SimpleIntegerProperty(ragMaxResultsCount); this.ragMinScore = new SimpleDoubleProperty(ragMinScore); @@ -431,6 +439,18 @@ public void setContextWindowSize(int contextWindowSize) { this.contextWindowSize.set(contextWindowSize); } + public ObjectProperty documentSplittingStrategyProperty() { + return documentSplittingStrategy; + } + + public DocumentSplittingStrategy getDocumentSplittingStrategy() { + return documentSplittingStrategy.get(); + } + + public void setDocumentSplittingStrategy(DocumentSplittingStrategy documentSplittingStrategy) { + this.documentSplittingStrategy.set(documentSplittingStrategy); + } + public IntegerProperty documentSplitterChunkSizeProperty() { return documentSplitterChunkSize; } diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java b/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java index 4e1b493646b7..73159ba1f762 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java @@ -10,7 +10,8 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.rag.repositories.FileEmbeddingsManager; +import org.jabref.logic.ai.rag.logic.EmbeddingsCleaner; +import org.jabref.logic.ai.rag.logic.documentsplitting.DocumentSplitterAlgorithm; import org.jabref.logic.ai.rag.repositories.FullyIngestedDocumentsRepository; import org.jabref.logic.ai.rag.tasks.GenerateEmbeddingsForSeveralTask; import org.jabref.logic.ai.rag.tasks.GenerateEmbeddingsTask; @@ -42,7 +43,11 @@ public class IngestionService { private final FilePreferences filePreferences; private final TaskExecutor taskExecutor; - private final FileEmbeddingsManager fileEmbeddingsManager; + private final EmbeddingModel embeddingModel; + private final EmbeddingStore embeddingStore; + private final DocumentSplitterAlgorithm documentSplitterAlgorithm; + private final FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository; + private final EmbeddingsCleaner embeddingsCleaner; private final ReadOnlyBooleanProperty shutdownSignal; @@ -52,6 +57,7 @@ public IngestionService( TaskExecutor taskExecutor, EmbeddingModel embeddingModel, EmbeddingStore embeddingStore, + DocumentSplitterAlgorithm documentSplitterAlgorithm, FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository, ReadOnlyBooleanProperty shutdownSignal ) { @@ -59,9 +65,14 @@ public IngestionService( this.filePreferences = filePreferences; this.taskExecutor = taskExecutor; - this.fileEmbeddingsManager = new FileEmbeddingsManager( + this.embeddingModel = embeddingModel; + this.embeddingStore = embeddingStore; + this.documentSplitterAlgorithm = documentSplitterAlgorithm; + this.fullyIngestedDocumentsRepository = fullyIngestedDocumentsRepository; + this.embeddingsCleaner = new EmbeddingsCleaner( aiPreferences, - embeddingModel, embeddingStore, fullyIngestedDocumentsRepository, shutdownSignal + embeddingStore, + fullyIngestedDocumentsRepository ); this.shutdownSignal = shutdownSignal; @@ -126,7 +137,11 @@ public List> getProcessingInfo(List return linkedFiles.stream().map(this::getProcessingInfo).toList(); } - public List> ingest(StringProperty groupName, List linkedFiles, BibDatabaseContext bibDatabaseContext) { + public List> ingest( + StringProperty groupName, + List linkedFiles, + BibDatabaseContext bibDatabaseContext + ) { List> result = getProcessingInfo(linkedFiles); if (listsUnderIngestion.contains(linkedFiles)) { @@ -141,10 +156,23 @@ public List> ingest(StringProperty groupName, L return result; } - private void startEmbeddingsGenerationTask(LinkedFile linkedFile, BibDatabaseContext bibDatabaseContext, ProcessingInfo processingInfo) { + private void startEmbeddingsGenerationTask( + LinkedFile linkedFile, + BibDatabaseContext bibDatabaseContext, + ProcessingInfo processingInfo + ) { processingInfo.setState(ProcessingState.PROCESSING); - new GenerateEmbeddingsTask(filePreferences, fileEmbeddingsManager, bibDatabaseContext, linkedFile, shutdownSignal) + new GenerateEmbeddingsTask( + filePreferences, + fullyIngestedDocumentsRepository, + embeddingStore, + embeddingModel, + documentSplitterAlgorithm, + bibDatabaseContext, + linkedFile, + shutdownSignal + ) .showToUser(true) .onSuccess(v -> processingInfo.setState(ProcessingState.SUCCESS)) .onFailure(processingInfo::setException) @@ -154,12 +182,23 @@ private void startEmbeddingsGenerationTask(LinkedFile linkedFile, BibDatabaseCon private void startEmbeddingsGenerationTask(StringProperty groupName, List> linkedFiles, BibDatabaseContext bibDatabaseContext) { linkedFiles.forEach(processingInfo -> processingInfo.setState(ProcessingState.PROCESSING)); - new GenerateEmbeddingsForSeveralTask(filePreferences, taskExecutor, fileEmbeddingsManager, bibDatabaseContext, groupName, linkedFiles, shutdownSignal) + new GenerateEmbeddingsForSeveralTask( + filePreferences, + fullyIngestedDocumentsRepository, + embeddingStore, + embeddingModel, + documentSplitterAlgorithm, + bibDatabaseContext, + groupName, + linkedFiles, + shutdownSignal, + taskExecutor + ) .executeWith(taskExecutor); } public void clearEmbeddingsFor(List linkedFiles) { - fileEmbeddingsManager.clearEmbeddingsFor(linkedFiles); + embeddingsCleaner.clearEmbeddingsFor(linkedFiles); ingestionStatusMap.values().forEach(processingInfo -> processingInfo.setState(ProcessingState.STOPPED)); } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/EmbeddingsCleaner.java b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/EmbeddingsCleaner.java new file mode 100644 index 000000000000..147cb335e933 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/EmbeddingsCleaner.java @@ -0,0 +1,45 @@ +package org.jabref.logic.ai.rag.logic; + +import java.util.List; + +import org.jabref.logic.ai.preferences.AiPreferences; +import org.jabref.logic.ai.rag.repositories.FullyIngestedDocumentsRepository; +import org.jabref.model.entry.LinkedFile; + +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.store.embedding.EmbeddingStore; +import dev.langchain4j.store.embedding.filter.MetadataFilterBuilder; + +public class EmbeddingsCleaner { + public static final String LINK_METADATA_KEY = "link"; + + private final AiPreferences aiPreferences; + + private final EmbeddingStore embeddingStore; + private final FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository; + + public EmbeddingsCleaner( + AiPreferences aiPreferences, + EmbeddingStore embeddingStore, + FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository + ) { + this.aiPreferences = aiPreferences; + this.embeddingStore = embeddingStore; + this.fullyIngestedDocumentsRepository = fullyIngestedDocumentsRepository; + + setupListeningToPreferencesChanges(); + } + + private void setupListeningToPreferencesChanges() { + aiPreferences.addListenerToEmbeddingsParametersChange(embeddingStore::removeAll); + } + + public void removeDocument(String link) { + embeddingStore.removeAll(MetadataFilterBuilder.metadataKey(LINK_METADATA_KEY).isEqualTo(link)); + fullyIngestedDocumentsRepository.unmarkDocumentAsFullyIngested(link); + } + + public void clearEmbeddingsFor(List linkedFiles) { + linkedFiles.stream().map(LinkedFile::getLink).forEach(this::removeDocument); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/LowLevelIngestor.java b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/LowLevelIngestor.java deleted file mode 100644 index 2de142dd323d..000000000000 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/LowLevelIngestor.java +++ /dev/null @@ -1,81 +0,0 @@ -package org.jabref.logic.ai.rag.logic; - -import java.util.List; - -import javafx.beans.property.IntegerProperty; -import javafx.beans.property.ReadOnlyBooleanProperty; - -import org.jabref.logic.ai.preferences.AiPreferences; - -import dev.langchain4j.data.document.DefaultDocument; -import dev.langchain4j.data.document.Document; -import dev.langchain4j.data.document.DocumentSplitter; -import dev.langchain4j.data.document.splitter.DocumentSplitters; -import dev.langchain4j.data.segment.TextSegment; -import dev.langchain4j.model.embedding.EmbeddingModel; -import dev.langchain4j.store.embedding.EmbeddingStore; -import dev.langchain4j.store.embedding.EmbeddingStoreIngestor; - -public class LowLevelIngestor { - private final AiPreferences aiPreferences; - - private final EmbeddingStore embeddingStore; - private final EmbeddingModel embeddingModel; - - private EmbeddingStoreIngestor ingestor; - private DocumentSplitter documentSplitter; - - public LowLevelIngestor( - AiPreferences aiPreferences, - EmbeddingModel embeddingModel, - EmbeddingStore embeddingStore - ) { - this.aiPreferences = aiPreferences; - this.embeddingStore = embeddingStore; - this.embeddingModel = embeddingModel; - - rebuild(); - - setupListeningToPreferencesChanges(); - } - - private void rebuild() { - this.documentSplitter = DocumentSplitters - .recursive(aiPreferences.getDocumentSplitterChunkSize(), - aiPreferences.getDocumentSplitterOverlapSize()); - - this.ingestor = EmbeddingStoreIngestor - .builder() - .embeddingStore(embeddingStore) - .embeddingModel(embeddingModel) - .documentSplitter(documentSplitter) - .build(); - } - - private void setupListeningToPreferencesChanges() { - aiPreferences.customizeExpertSettingsProperty().addListener(_ -> rebuild()); - aiPreferences.addListenerToEmbeddingsParametersChange(this::rebuild); - } - - /** - * Add document to embedding store. - * This method does not check if file was already ingested. - * - * @param document - document to add. - * @param stopProperty - in case you want to stop the ingestion process, set this property to true. - */ - public void ingestDocument(Document document, ReadOnlyBooleanProperty stopProperty, IntegerProperty workDone, IntegerProperty workMax) throws InterruptedException { - List textSegments = documentSplitter.split(document); - workMax.set(textSegments.size()); - - for (TextSegment documentPart : textSegments) { - if (stopProperty.get()) { - throw new InterruptedException(); - } - - ingestor.ingest(new DefaultDocument(documentPart.text(), document.metadata())); - - workDone.set(workDone.get() + 1); - } - } -} diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/documentsplitting/DocumentSplitterAlgorithm.java b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/documentsplitting/DocumentSplitterAlgorithm.java new file mode 100644 index 000000000000..93d23ce48899 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/documentsplitting/DocumentSplitterAlgorithm.java @@ -0,0 +1,12 @@ +package org.jabref.logic.ai.rag.logic.documentsplitting; + +import java.util.stream.Stream; + +import org.jabref.logic.ai.util.LongTaskInfo; +import org.jabref.model.ai.rag.DocumentSplittingStrategy; + +public interface DocumentSplitterAlgorithm { + Stream split(LongTaskInfo longTaskInfo, String text) throws InterruptedException; + + DocumentSplittingStrategy getStrategy(); +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/documentsplitting/SlidingWindowDocumentSplitter.java b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/documentsplitting/SlidingWindowDocumentSplitter.java new file mode 100644 index 000000000000..60fcfecc953b --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/documentsplitting/SlidingWindowDocumentSplitter.java @@ -0,0 +1,36 @@ +package org.jabref.logic.ai.rag.logic.documentsplitting; + +import java.util.stream.Stream; + +import org.jabref.logic.ai.util.LongTaskInfo; +import org.jabref.model.ai.rag.DocumentSplittingStrategy; + +import dev.langchain4j.data.document.DefaultDocument; +import dev.langchain4j.data.document.DocumentSplitter; +import dev.langchain4j.data.document.splitter.DocumentSplitters; +import dev.langchain4j.data.segment.TextSegment; + +public class SlidingWindowDocumentSplitter implements DocumentSplitterAlgorithm { + private final DocumentSplitter langchainDocumentSplitter; + + public SlidingWindowDocumentSplitter(int chunkSize, int chunkOverlap) { + this.langchainDocumentSplitter = DocumentSplitters.recursive( + chunkSize, + chunkOverlap + ); + } + + @Override + public Stream split(LongTaskInfo longTaskInfo, String text) throws InterruptedException { + // NOTE: Unfortunately, there is no way to handle the shutdown signal. It returns a `List` and not a stream or an iterator. + return langchainDocumentSplitter + .split(new DefaultDocument(text)) + .stream() + .map(TextSegment::text); + } + + @Override + public DocumentSplittingStrategy getStrategy() { + return DocumentSplittingStrategy.SLIDING_WINDOW; + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/FileIngestor.java b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/FileIngestor.java new file mode 100644 index 000000000000..9d8552355101 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/FileIngestor.java @@ -0,0 +1,56 @@ +package org.jabref.logic.ai.rag.logic.ingestion; + +import java.nio.file.Path; +import java.util.Optional; + +import org.jabref.logic.ai.rag.logic.documentsplitting.DocumentSplitterAlgorithm; +import org.jabref.logic.ai.rag.logic.parsing.UniversalFileParser; +import org.jabref.logic.ai.util.LongTaskInfo; +import org.jabref.logic.l10n.Localization; + +import dev.langchain4j.data.document.Metadata; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.store.embedding.EmbeddingStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FileIngestor { + private static final Logger LOGGER = LoggerFactory.getLogger(FileIngestor.class); + + private final UniversalFileParser universalFileParser; + private final TextIngestor textIngestor; + + public FileIngestor( + EmbeddingStore embeddingStore, + EmbeddingModel embeddingModel, + DocumentSplitterAlgorithm documentSplitterAlgorithm + ) { + this.universalFileParser = new UniversalFileParser(); + this.textIngestor = new TextIngestor( + embeddingStore, + embeddingModel, + documentSplitterAlgorithm + ); + } + + public void ingest( + LongTaskInfo longTaskInfo, + Metadata metadata, + Path path + ) throws InterruptedException { + Optional document = universalFileParser.parse(longTaskInfo, path); + + if (document.isPresent()) { + textIngestor.ingest( + longTaskInfo, + metadata, + document.get() + ); + LOGGER.debug("Embeddings for file \"{}\" were generated successfully", path.toString()); + } else { + LOGGER.error("Unable to generate embeddings for file \"{}\", because JabRef was unable to extract text from the file", path.toString()); + throw new RuntimeException(Localization.lang("Unable to generate embeddings for file '%0', because JabRef was unable to extract text from the file", path.toString())); + } + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/LinkedFileIngestor.java b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/LinkedFileIngestor.java new file mode 100644 index 000000000000..1c97fdbd4d84 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/LinkedFileIngestor.java @@ -0,0 +1,40 @@ +package org.jabref.logic.ai.rag.logic.ingestion; + +import java.nio.file.Path; + +import org.jabref.logic.ai.rag.logic.EmbeddingsCleaner; +import org.jabref.logic.ai.rag.logic.documentsplitting.DocumentSplitterAlgorithm; +import org.jabref.logic.ai.util.LongTaskInfo; +import org.jabref.model.entry.LinkedFile; + +import dev.langchain4j.data.document.Metadata; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.store.embedding.EmbeddingStore; + +public class LinkedFileIngestor { + private final FileIngestor fileIngestor; + + public LinkedFileIngestor( + EmbeddingStore embeddingStore, + EmbeddingModel embeddingModel, + DocumentSplitterAlgorithm documentSplitterAlgorithm + ) { + this.fileIngestor = new FileIngestor( + embeddingStore, + embeddingModel, + documentSplitterAlgorithm + ); + } + + public void ingest( + LongTaskInfo longTaskInfo, + LinkedFile linkedFile, + Path path + ) throws InterruptedException { + Metadata metadata = new Metadata(); + metadata.put(EmbeddingsCleaner.LINK_METADATA_KEY, linkedFile.getLink()); + + fileIngestor.ingest(longTaskInfo, metadata, path); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/PersistentLinkedFileIngestor.java b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/PersistentLinkedFileIngestor.java new file mode 100644 index 000000000000..fe8d92d07b21 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/PersistentLinkedFileIngestor.java @@ -0,0 +1,101 @@ +package org.jabref.logic.ai.rag.logic.ingestion; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +import org.jabref.logic.FilePreferences; +import org.jabref.logic.ai.rag.logic.documentsplitting.DocumentSplitterAlgorithm; +import org.jabref.logic.ai.rag.repositories.FullyIngestedDocumentsRepository; +import org.jabref.logic.ai.util.LongTaskInfo; +import org.jabref.logic.l10n.Localization; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.LinkedFile; + +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.store.embedding.EmbeddingStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PersistentLinkedFileIngestor { + private static final Logger LOGGER = LoggerFactory.getLogger(PersistentLinkedFileIngestor.class); + + private final FilePreferences filePreferences; + private final FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository; + private final LinkedFileIngestor linkedFileIngestor; + + public PersistentLinkedFileIngestor( + FilePreferences filePreferences, + FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository, + EmbeddingStore embeddingStore, + EmbeddingModel embeddingModel, + DocumentSplitterAlgorithm documentSplitterAlgorithm + ) { + this.filePreferences = filePreferences; + this.fullyIngestedDocumentsRepository = fullyIngestedDocumentsRepository; + this.linkedFileIngestor = new LinkedFileIngestor( + embeddingStore, + embeddingModel, + documentSplitterAlgorithm + ); + } + + public void ingest( + BibDatabaseContext bibDatabaseContext, + LongTaskInfo longTaskInfo, + LinkedFile linkedFile + ) throws InterruptedException { + // TODO: Simplify this method. + // Rationale for RuntimeException here: + // See org.jabref.logic.ai.summarization.tasks.GenerateSummaryTask.summarizeAll + + LOGGER.debug("Generating embeddings for file \"{}\"", linkedFile.getLink()); + + Optional path = linkedFile.findIn(bibDatabaseContext, filePreferences); + + if (path.isEmpty()) { + LOGGER.error("Could not find path for a linked file \"{}\", while generating embeddings", linkedFile.getLink()); + LOGGER.debug("Unable to generate embeddings for file \"{}\", because it was not found while generating embeddings", linkedFile.getLink()); + throw new RuntimeException(Localization.lang("Could not find path for a linked file '%0' while generating embeddings.", linkedFile.getLink())); + } + + Optional modTime = Optional.empty(); + boolean shouldIngest = true; + + try { + BasicFileAttributes attributes = Files.readAttributes(path.get(), BasicFileAttributes.class); + + long currentModificationTimeInSeconds = attributes.lastModifiedTime().to(TimeUnit.SECONDS); + + Optional ingestedModificationTimeInSeconds = fullyIngestedDocumentsRepository.getIngestedDocumentModificationTimeInSeconds(linkedFile.getLink()); + + if (ingestedModificationTimeInSeconds.isEmpty()) { + modTime = Optional.of(currentModificationTimeInSeconds); + } else { + if (currentModificationTimeInSeconds > ingestedModificationTimeInSeconds.get()) { + modTime = Optional.of(currentModificationTimeInSeconds); + } else { + LOGGER.debug("No need to generate embeddings for file \"{}\", because it was already generated", linkedFile.getLink()); + shouldIngest = false; + } + } + } catch (IOException e) { + LOGGER.error("Could not retrieve attributes of a linked file \"{}\"", linkedFile.getLink(), e); + LOGGER.warn("Possibly regenerating embeddings for linked file \"{}\"", linkedFile.getLink()); + } + + if (!shouldIngest) { + return; + } + + linkedFileIngestor.ingest(longTaskInfo, linkedFile, path.get()); + + if (!longTaskInfo.shutdownSignal().get()) { + fullyIngestedDocumentsRepository.markDocumentAsFullyIngested(linkedFile.getLink(), modTime.orElse(0L)); + } + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/TextIngestor.java b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/TextIngestor.java new file mode 100644 index 000000000000..d1bb072ff6e3 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/TextIngestor.java @@ -0,0 +1,51 @@ +package org.jabref.logic.ai.rag.logic.ingestion; + +import java.util.List; + +import org.jabref.logic.ai.rag.logic.documentsplitting.DocumentSplitterAlgorithm; +import org.jabref.logic.ai.util.LongTaskInfo; + +import dev.langchain4j.data.document.DefaultDocument; +import dev.langchain4j.data.document.Metadata; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.store.embedding.EmbeddingStore; +import dev.langchain4j.store.embedding.EmbeddingStoreIngestor; + +public class TextIngestor { + private final EmbeddingStoreIngestor ingestor; + private final DocumentSplitterAlgorithm documentSplitterAlgorithm; + + public TextIngestor( + EmbeddingStore embeddingStore, + EmbeddingModel embeddingModel, + DocumentSplitterAlgorithm documentSplitterAlgorithm + ) { + this.ingestor = EmbeddingStoreIngestor + .builder() + .embeddingStore(embeddingStore) + .embeddingModel(embeddingModel) + .build(); + + this.documentSplitterAlgorithm = documentSplitterAlgorithm; + } + + public void ingest( + LongTaskInfo longTaskInfo, + Metadata metadata, + String text + ) throws InterruptedException { + List chunks = documentSplitterAlgorithm.split(longTaskInfo, text).toList(); + longTaskInfo.progressCounter().increaseWorkMax(chunks.size()); + + for (String documentPart : chunks) { + if (longTaskInfo.shutdownSignal().get()) { + throw new InterruptedException(); + } + + ingestor.ingest(new DefaultDocument(documentPart, metadata)); + + longTaskInfo.progressCounter().increaseWorkDone(1); + } + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/FileParser.java b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/FileParser.java index afa4a35d8116..05367d2710d4 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/FileParser.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/FileParser.java @@ -3,8 +3,8 @@ import java.nio.file.Path; import java.util.Optional; -import javafx.beans.property.ReadOnlyBooleanProperty; +import org.jabref.logic.ai.util.LongTaskInfo; public interface FileParser { - Optional parse(Path path, ReadOnlyBooleanProperty shutdownSignal); + Optional parse(LongTaskInfo longTaskInfo, Path path); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/PdfFileParser.java b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/PdfFileParser.java index f5e8b9b73e5b..55d3c84fdcf9 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/PdfFileParser.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/PdfFileParser.java @@ -5,8 +5,7 @@ import java.nio.file.Path; import java.util.Optional; -import javafx.beans.property.ReadOnlyBooleanProperty; - +import org.jabref.logic.ai.util.LongTaskInfo; import org.jabref.logic.pdf.InterruptablePDFTextStripper; import org.jabref.logic.xmp.XmpUtilReader; @@ -18,17 +17,18 @@ public class PdfFileParser implements FileParser { private static final Logger LOGGER = LoggerFactory.getLogger(PdfFileParser.class); @Override - public Optional parse(Path path, ReadOnlyBooleanProperty shutdownSignal) { + public Optional parse(LongTaskInfo longTaskInfo, Path path) { try (PDDocument document = new XmpUtilReader().loadWithAutomaticDecryption(path)) { int lastPage = document.getNumberOfPages(); StringWriter writer = new StringWriter(); - InterruptablePDFTextStripper stripper = new InterruptablePDFTextStripper(shutdownSignal); + InterruptablePDFTextStripper stripper = new InterruptablePDFTextStripper(longTaskInfo.shutdownSignal()); stripper.setStartPage(1); stripper.setEndPage(lastPage); stripper.writeText(document, writer); - if (shutdownSignal.get()) { + if (longTaskInfo.shutdownSignal().get()) { + // TODO: Why not throw interrupted exception? return Optional.empty(); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/UniversalFileParser.java b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/UniversalFileParser.java index 750c1f84235d..9b30b1b94251 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/UniversalFileParser.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/UniversalFileParser.java @@ -3,8 +3,7 @@ import java.nio.file.Path; import java.util.Optional; -import javafx.beans.property.ReadOnlyBooleanProperty; - +import org.jabref.logic.ai.util.LongTaskInfo; import org.jabref.logic.util.io.FileUtil; import org.slf4j.Logger; @@ -15,9 +14,9 @@ public class UniversalFileParser implements FileParser { private final PdfFileParser pdfFileParser = new PdfFileParser(); - public Optional parse(Path path, ReadOnlyBooleanProperty shutdownSignal) { + public Optional parse(LongTaskInfo longTaskInfo, Path path) { if (FileUtil.isPDFFile(path)) { - return pdfFileParser.parse(path, shutdownSignal); + return pdfFileParser.parse(longTaskInfo, path); } else { LOGGER.info("Unsupported file type of file: {}. Currently, only PDF files are supported", path); return Optional.empty(); diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FileEmbeddingsManager.java b/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FileEmbeddingsManager.java deleted file mode 100644 index 43767be6f82b..000000000000 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FileEmbeddingsManager.java +++ /dev/null @@ -1,88 +0,0 @@ -package org.jabref.logic.ai.rag.repositories; - -import java.util.List; -import java.util.Optional; - -import javafx.beans.property.IntegerProperty; -import javafx.beans.property.ReadOnlyBooleanProperty; - -import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.rag.logic.LowLevelIngestor; -import org.jabref.model.entry.LinkedFile; - -import dev.langchain4j.data.document.DefaultDocument; -import dev.langchain4j.data.document.Document; -import dev.langchain4j.data.segment.TextSegment; -import dev.langchain4j.model.embedding.EmbeddingModel; -import dev.langchain4j.store.embedding.EmbeddingStore; -import dev.langchain4j.store.embedding.filter.MetadataFilterBuilder; - -/** - * This class is responsible for managing the embeddings cache. The cache is saved in a local user directory. - *

- * MVStore is used as an embedded database. It stores the embeddings and what files have been fully ingested. - * {@link org.jabref.model.entry.LinkedFile} and embeddings are connected with LinkedFile.getLink(). - *

- * In case an error occurs while opening an MVStore, the class will notify the user of this error and continue - * with in-memory store (meaning all embeddings will be thrown away on exit). - *

- * This class also listens for changes of embeddings parameters (in AI "Expert settings" section). In case any of them - * changes, the embeddings should be invalidated (cleared). - */ -public class FileEmbeddingsManager { - public static final String LINK_METADATA_KEY = "link"; - - private final AiPreferences aiPreferences; - private final ReadOnlyBooleanProperty shutdownSignal; - - private final EmbeddingStore embeddingStore; - private final FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository; - private final LowLevelIngestor lowLevelIngestor; - - public FileEmbeddingsManager( - AiPreferences aiPreferences, - EmbeddingModel embeddingModel, - EmbeddingStore embeddingStore, - FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository, - ReadOnlyBooleanProperty shutdownSignal - ) { - this.aiPreferences = aiPreferences; - this.shutdownSignal = shutdownSignal; - this.embeddingStore = embeddingStore; - this.fullyIngestedDocumentsRepository = fullyIngestedDocumentsRepository; - this.lowLevelIngestor = new LowLevelIngestor(aiPreferences, embeddingModel, embeddingStore); - - setupListeningToPreferencesChanges(); - } - - private void setupListeningToPreferencesChanges() { - aiPreferences.addListenerToEmbeddingsParametersChange(embeddingStore::removeAll); - } - - public void addDocument(String link, String documentSource, long modificationTimeInSeconds, IntegerProperty workDone, IntegerProperty workMax) throws InterruptedException { - Document document = new DefaultDocument(documentSource); - document.metadata().put(LINK_METADATA_KEY, link); - lowLevelIngestor.ingestDocument(document, shutdownSignal, workDone, workMax); - - if (!shutdownSignal.get()) { - fullyIngestedDocumentsRepository.markDocumentAsFullyIngested(link, modificationTimeInSeconds); - } - } - - public void removeDocument(String link) { - embeddingStore.removeAll(MetadataFilterBuilder.metadataKey(LINK_METADATA_KEY).isEqualTo(link)); - fullyIngestedDocumentsRepository.unmarkDocumentAsFullyIngested(link); - } - - public EmbeddingStore getEmbeddingsStore() { - return embeddingStore; - } - - public Optional getIngestedDocumentModificationTimeInSeconds(String link) { - return fullyIngestedDocumentsRepository.getIngestedDocumentModificationTimeInSeconds(link); - } - - public void clearEmbeddingsFor(List linkedFiles) { - linkedFiles.stream().map(LinkedFile::getLink).forEach(this::removeDocument); - } -} diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsForSeveralTask.java b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsForSeveralTask.java index 41e15d79283f..9b18cac80497 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsForSeveralTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsForSeveralTask.java @@ -10,7 +10,8 @@ import javafx.util.Pair; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.rag.repositories.FileEmbeddingsManager; +import org.jabref.logic.ai.rag.logic.documentsplitting.DocumentSplitterAlgorithm; +import org.jabref.logic.ai.rag.repositories.FullyIngestedDocumentsRepository; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.ProgressCounter; @@ -20,6 +21,9 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.LinkedFile; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.store.embedding.EmbeddingStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,34 +35,43 @@ public class GenerateEmbeddingsForSeveralTask extends BackgroundTask { private static final Logger LOGGER = LoggerFactory.getLogger(GenerateEmbeddingsForSeveralTask.class); + private final FilePreferences filePreferences; + private final FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository; + private final EmbeddingStore embeddingStore; + private final EmbeddingModel embeddingModel; + private final DocumentSplitterAlgorithm documentSplitterAlgorithm; + private final BibDatabaseContext bibDatabaseContext; private final StringProperty groupName; private final List> linkedFiles; - private final FileEmbeddingsManager fileEmbeddingsManager; - private final BibDatabaseContext bibDatabaseContext; - private final FilePreferences filePreferences; - private final TaskExecutor taskExecutor; + private final ProgressCounter progressCounter; private final ReadOnlyBooleanProperty shutdownSignal; - - private final ProgressCounter progressCounter = new ProgressCounter(); + private final TaskExecutor taskExecutor; private String currentFile = ""; public GenerateEmbeddingsForSeveralTask( FilePreferences filePreferences, - TaskExecutor taskExecutor, - FileEmbeddingsManager fileEmbeddingsManager, + FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository, + EmbeddingStore embeddingStore, + EmbeddingModel embeddingModel, + DocumentSplitterAlgorithm documentSplitterAlgorithm, BibDatabaseContext bibDatabaseContext, StringProperty groupName, List> linkedFiles, - ReadOnlyBooleanProperty shutdownSignal + ReadOnlyBooleanProperty shutdownSignal, + TaskExecutor taskExecutor ) { + this.filePreferences = filePreferences; + this.fullyIngestedDocumentsRepository = fullyIngestedDocumentsRepository; + this.embeddingStore = embeddingStore; + this.embeddingModel = embeddingModel; + this.documentSplitterAlgorithm = documentSplitterAlgorithm; + this.bibDatabaseContext = bibDatabaseContext; this.groupName = groupName; this.linkedFiles = linkedFiles; - this.fileEmbeddingsManager = fileEmbeddingsManager; - this.bibDatabaseContext = bibDatabaseContext; - this.filePreferences = filePreferences; - this.taskExecutor = taskExecutor; + this.progressCounter = new ProgressCounter(); this.shutdownSignal = shutdownSignal; + this.taskExecutor = taskExecutor; configure(groupName); } @@ -85,7 +98,13 @@ public Void call() throws ExecutionException, InterruptedException { processingInfo.setState(ProcessingState.PROCESSING); return new Pair<>( new GenerateEmbeddingsTask( - filePreferences, fileEmbeddingsManager, bibDatabaseContext, processingInfo.getObject(), + filePreferences, + fullyIngestedDocumentsRepository, + embeddingStore, + embeddingModel, + documentSplitterAlgorithm, + bibDatabaseContext, + processingInfo.getObject(), shutdownSignal ) .showToUser(false) diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java index 8d2b1b0abcf8..9ad4b84f0b0a 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java @@ -1,23 +1,21 @@ package org.jabref.logic.ai.rag.tasks; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.Optional; -import java.util.concurrent.TimeUnit; - import javafx.beans.property.ReadOnlyBooleanProperty; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.rag.logic.parsing.UniversalFileParser; -import org.jabref.logic.ai.rag.repositories.FileEmbeddingsManager; +import org.jabref.logic.ai.rag.logic.documentsplitting.DocumentSplitterAlgorithm; +import org.jabref.logic.ai.rag.logic.ingestion.PersistentLinkedFileIngestor; +import org.jabref.logic.ai.rag.repositories.FullyIngestedDocumentsRepository; +import org.jabref.logic.ai.util.LongTaskInfo; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.ProgressCounter; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.LinkedFile; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.store.embedding.EmbeddingStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,27 +27,34 @@ public class GenerateEmbeddingsTask extends BackgroundTask { private static final Logger LOGGER = LoggerFactory.getLogger(GenerateEmbeddingsTask.class); - private final LinkedFile linkedFile; - private final FileEmbeddingsManager fileEmbeddingsManager; private final BibDatabaseContext bibDatabaseContext; - private final FilePreferences filePreferences; + private final LinkedFile linkedFile; + private final ProgressCounter progressCounter; + private final PersistentLinkedFileIngestor persistentLinkedFileIngestor; private final ReadOnlyBooleanProperty shutdownSignal; - private final UniversalFileParser universalFileParser = new UniversalFileParser(); - - private final ProgressCounter progressCounter = new ProgressCounter(); public GenerateEmbeddingsTask( FilePreferences filePreferences, - FileEmbeddingsManager fileEmbeddingsManager, + FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository, + EmbeddingStore embeddingStore, + EmbeddingModel embeddingModel, + DocumentSplitterAlgorithm documentSplitterAlgorithm, BibDatabaseContext bibDatabaseContext, LinkedFile linkedFile, ReadOnlyBooleanProperty shutdownSignal ) { this.linkedFile = linkedFile; - this.fileEmbeddingsManager = fileEmbeddingsManager; - this.bibDatabaseContext = bibDatabaseContext; - this.filePreferences = filePreferences; this.shutdownSignal = shutdownSignal; + this.bibDatabaseContext = bibDatabaseContext; + this.progressCounter = new ProgressCounter(); + + this.persistentLinkedFileIngestor = new PersistentLinkedFileIngestor( + filePreferences, + fullyIngestedDocumentsRepository, + embeddingStore, + embeddingModel, + documentSplitterAlgorithm + ); configure(); } @@ -66,7 +71,16 @@ public Void call() { LOGGER.debug("Starting embeddings generation task for file \"{}\"", linkedFile.getLink()); try { - ingestLinkedFile(linkedFile); + LongTaskInfo longTaskInfo = new LongTaskInfo( + progressCounter, + shutdownSignal + ); + + persistentLinkedFileIngestor.ingest( + bibDatabaseContext, + longTaskInfo, + linkedFile + ); } catch (InterruptedException e) { LOGGER.debug("There is a embeddings generation task for file \"{}\". It will be cancelled, because user quits JabRef.", linkedFile.getLink()); } @@ -76,59 +90,6 @@ public Void call() { return null; } - private void ingestLinkedFile(LinkedFile linkedFile) throws InterruptedException { - // Rationale for RuntimeException here: - // See org.jabref.logic.ai.summarization.tasks.GenerateSummaryTask.summarizeAll - - LOGGER.debug("Generating embeddings for file \"{}\"", linkedFile.getLink()); - - Optional path = linkedFile.findIn(bibDatabaseContext, filePreferences); - - if (path.isEmpty()) { - LOGGER.error("Could not find path for a linked file \"{}\", while generating embeddings", linkedFile.getLink()); - LOGGER.debug("Unable to generate embeddings for file \"{}\", because it was not found while generating embeddings", linkedFile.getLink()); - throw new RuntimeException(Localization.lang("Could not find path for a linked file '%0' while generating embeddings.", linkedFile.getLink())); - } - - Optional modTime = Optional.empty(); - boolean shouldIngest = true; - - try { - BasicFileAttributes attributes = Files.readAttributes(path.get(), BasicFileAttributes.class); - - long currentModificationTimeInSeconds = attributes.lastModifiedTime().to(TimeUnit.SECONDS); - - Optional ingestedModificationTimeInSeconds = fileEmbeddingsManager.getIngestedDocumentModificationTimeInSeconds(linkedFile.getLink()); - - if (ingestedModificationTimeInSeconds.isEmpty()) { - modTime = Optional.of(currentModificationTimeInSeconds); - } else { - if (currentModificationTimeInSeconds > ingestedModificationTimeInSeconds.get()) { - modTime = Optional.of(currentModificationTimeInSeconds); - } else { - LOGGER.debug("No need to generate embeddings for file \"{}\", because it was already generated", linkedFile.getLink()); - shouldIngest = false; - } - } - } catch (IOException e) { - LOGGER.error("Could not retrieve attributes of a linked file \"{}\"", linkedFile.getLink(), e); - LOGGER.warn("Possibly regenerating embeddings for linked file \"{}\"", linkedFile.getLink()); - } - - if (!shouldIngest) { - return; - } - - Optional document = universalFileParser.parse(path.get(), shutdownSignal); - if (document.isPresent()) { - fileEmbeddingsManager.addDocument(linkedFile.getLink(), document.get(), modTime.orElse(0L), progressCounter.workDoneProperty(), progressCounter.workMaxProperty()); - LOGGER.debug("Embeddings for file \"{}\" were generated successfully", linkedFile.getLink()); - } else { - LOGGER.error("Unable to generate embeddings for file \"{}\", because JabRef was unable to extract text from the file", linkedFile.getLink()); - throw new RuntimeException(Localization.lang("Unable to generate embeddings for file '%0', because JabRef was unable to extract text from the file", linkedFile.getLink())); - } - } - private void updateProgress() { updateProgress(progressCounter.getWorkDone(), progressCounter.getWorkMax()); updateMessage(progressCounter.getMessage()); diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java index c75d86876e81..ec420ac943f5 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java @@ -9,6 +9,7 @@ import javafx.beans.property.StringProperty; import org.jabref.logic.FilePreferences; +import org.jabref.logic.ai.customimplementations.llms.ChatModel; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; import org.jabref.logic.ai.summarization.repositories.SummariesRepository; @@ -16,7 +17,6 @@ import org.jabref.logic.ai.summarization.tasks.GenerateSummaryTask; import org.jabref.logic.util.CitationKeyCheck; import org.jabref.logic.util.TaskExecutor; -import org.jabref.model.ai.chatting.ChatModelInfo; import org.jabref.model.ai.processingstatus.ProcessingInfo; import org.jabref.model.ai.processingstatus.ProcessingState; import org.jabref.model.ai.summarization.BibEntrySummary; @@ -48,7 +48,7 @@ public class SummariesService { private final AiPreferences aiPreferences; private final SummariesRepository summariesRepository; - private final ChatModelInfo chatModelInfo; + private final ChatModel chatModel; private final SummarizationAlgorithm defaultSummarizationAlgorithm; private final BooleanProperty shutdownSignal; private final FilePreferences filePreferences; @@ -59,14 +59,14 @@ public SummariesService( AiPreferences aiPreferences, FilePreferences filePreferences, TaskExecutor taskExecutor, - ChatModelInfo chatModelInfo, + ChatModel chatModel, SummarizationAlgorithm defaultSummarizationAlgorithm, SummariesRepository summariesRepository, BooleanProperty shutdownSignal ) { this.aiPreferences = aiPreferences; this.summariesRepository = summariesRepository; - this.chatModelInfo = chatModelInfo; + this.chatModel = chatModel; this.defaultSummarizationAlgorithm = defaultSummarizationAlgorithm; this.shutdownSignal = shutdownSignal; this.filePreferences = filePreferences; @@ -159,7 +159,7 @@ private void startSummarizationTask( new GenerateSummaryTask( filePreferences, - chatModelInfo, + chatModel, summariesRepository, summarizationAlgorithm, bibDatabaseContext, @@ -182,7 +182,7 @@ private void startSummarizationTask( new GenerateSummaryForSeveralTask( filePreferences, taskExecutor, - chatModelInfo, + chatModel, summariesRepository, summarizationAlgorithm, bibDatabaseContext, diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/BibEntrySummarizer.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/BibEntrySummarizer.java index 15e3aff9830b..a1b4deb6b8ce 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/BibEntrySummarizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/BibEntrySummarizer.java @@ -9,11 +9,11 @@ import java.util.stream.Stream; import org.jabref.logic.FilePreferences; +import org.jabref.logic.ai.customimplementations.llms.ChatModel; import org.jabref.logic.ai.rag.logic.parsing.UniversalFileParser; import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; import org.jabref.logic.ai.util.LongTaskInfo; import org.jabref.logic.l10n.Localization; -import org.jabref.model.ai.chatting.ChatModelInfo; import org.jabref.model.ai.summarization.BibEntrySummary; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -22,6 +22,7 @@ import org.slf4j.Logger; public class BibEntrySummarizer { + // TODO: Simplify this class. private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(BibEntrySummarizer.class); private final FilePreferences filePreferences; @@ -38,7 +39,7 @@ public BibEntrySummarizer( } public BibEntrySummary summarize( - ChatModelInfo chatModelInfo, + ChatModel chatModel, LongTaskInfo longTaskInfo, BibDatabaseContext bibDatabaseContext, BibEntry entry @@ -53,7 +54,7 @@ public BibEntrySummary summarize( List linkedFilesSummary = new ArrayList<>(); for (LinkedFile linkedFile : entry.getFiles()) { generateSummary( - chatModelInfo, + chatModel, longTaskInfo, bibDatabaseContext, linkedFile, @@ -77,7 +78,7 @@ public BibEntrySummary summarize( finalSummary = linkedFilesSummary.getFirst(); } else { finalSummary = summarizeSeveralDocuments( - chatModelInfo, + chatModel, longTaskInfo, linkedFilesSummary.stream() ); @@ -87,15 +88,15 @@ public BibEntrySummary summarize( return new BibEntrySummary( LocalDateTime.now(), - chatModelInfo.aiProvider(), - chatModelInfo.name(), + chatModel.getAiProvider(), + chatModel.getName(), summarizationAlgorithm.getName(), finalSummary ); } private Optional generateSummary( - ChatModelInfo chatModelInfo, + ChatModel chatModel, LongTaskInfo longTaskInfo, BibDatabaseContext bibDatabaseContext, LinkedFile linkedFile, @@ -111,7 +112,7 @@ private Optional generateSummary( return Optional.empty(); } - Optional document = universalFileParser.parse(path.get(), longTaskInfo.shutdownSignal()); + Optional document = universalFileParser.parse(longTaskInfo, path.get()); if (document.isEmpty()) { LOGGER.warn("Could not extract text from a linked file \"{}\" of entry {}. It will be skipped when generating a summary.", linkedFile.getLink(), citationKey); @@ -120,7 +121,7 @@ private Optional generateSummary( } String linkedFileSummary = summarizationAlgorithm.summarize( - chatModelInfo, + chatModel, longTaskInfo, document.get() ); @@ -130,12 +131,12 @@ private Optional generateSummary( } public String summarizeSeveralDocuments( - ChatModelInfo chatModelInfo, + ChatModel chatModel, LongTaskInfo longTaskInfo, Stream documents ) throws InterruptedException { return summarizationAlgorithm.summarize( - chatModelInfo, + chatModel, longTaskInfo, documents.collect(Collectors.joining("\n\n")) ); diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/PersistentBibEntrySummarizer.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/PersistentBibEntrySummarizer.java index 29b7deb2592f..207fef033e48 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/PersistentBibEntrySummarizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/PersistentBibEntrySummarizer.java @@ -3,11 +3,11 @@ import java.util.Optional; import org.jabref.logic.FilePreferences; +import org.jabref.logic.ai.customimplementations.llms.ChatModel; import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; import org.jabref.logic.ai.summarization.repositories.SummariesRepository; import org.jabref.logic.ai.util.LongTaskInfo; import org.jabref.logic.util.CitationKeyCheck; -import org.jabref.model.ai.chatting.ChatModelInfo; import org.jabref.model.ai.summarization.BibEntrySummary; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -36,7 +36,7 @@ public PersistentBibEntrySummarizer( } public BibEntrySummary summarize( - ChatModelInfo chatModelInfo, + ChatModel chatModel, LongTaskInfo longTaskInfo, BibDatabaseContext bibDatabaseContext, BibEntry entry @@ -61,7 +61,7 @@ public BibEntrySummary summarize( } else { try { bibEntrySummary = bibEntrySummarizer.summarize( - chatModelInfo, + chatModel, longTaskInfo, bibDatabaseContext, entry @@ -74,8 +74,8 @@ public BibEntrySummary summarize( if (bibDatabaseContext.getDatabasePath().isEmpty()) { LOGGER.info("No database path is present. BibEntrySummary will not be stored in the next sessions"); - } else if (CitationKeyCheck.citationKeyIsPresentAndUnique(bibDatabaseContext, entry)) { - LOGGER.info("No valid citation key is present. BibEntrySummary will not be stored in the next sessions"); + } else if (!CitationKeyCheck.citationKeyIsPresentAndUnique(bibDatabaseContext, entry)) { + LOGGER.info("No valid citation key is present. Summary will not be stored in the next sessions"); } else { summariesRepository.set( bibDatabaseContext.getDatabasePath().get(), diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/ChunkedSummarizationAlgorithm.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/ChunkedSummarizationAlgorithm.java index 2389706c6493..8ead55def29b 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/ChunkedSummarizationAlgorithm.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/ChunkedSummarizationAlgorithm.java @@ -3,12 +3,12 @@ import java.util.ArrayList; import java.util.List; +import org.jabref.logic.ai.customimplementations.llms.ChatModel; import org.jabref.logic.ai.summarization.templates.SummarizationChunkSystemMessageTemplate; import org.jabref.logic.ai.summarization.templates.SummarizationChunkUserMessageTemplate; import org.jabref.logic.ai.summarization.templates.SummarizationCombineSystemMessageTemplate; import org.jabref.logic.ai.summarization.templates.SummarizationCombineUserMessageTemplate; import org.jabref.logic.ai.util.LongTaskInfo; -import org.jabref.model.ai.chatting.ChatModelInfo; import org.jabref.model.ai.summarization.SummarizationAlgorithmName; import dev.langchain4j.data.document.DefaultDocument; @@ -45,16 +45,17 @@ public ChunkedSummarizationAlgorithm( @Override public String summarize( - ChatModelInfo chatModelInfo, + ChatModel chatModel, LongTaskInfo longTaskInfo, String text ) throws InterruptedException { + // TODO: Simplify. LOGGER.debug("Summarizing text ({} chars)", text.length()); longTaskInfo.progressCounter().increaseWorkMax(1); // For the combination of summary chunks. DocumentSplitter documentSplitter = DocumentSplitters.recursive( - chatModelInfo.contextWindowSize() - MAX_OVERLAP_SIZE_IN_CHARS * 2 - chatModelInfo.tokenizer().estimate(new SystemMessage(summarizationChunkSystemMessageTemplate.getSource())), + chatModel.getContextWindowSize() - MAX_OVERLAP_SIZE_IN_CHARS * 2 - chatModel.getTokenizer().estimate(new SystemMessage(summarizationChunkSystemMessageTemplate.getSource())), MAX_OVERLAP_SIZE_IN_CHARS ); @@ -82,7 +83,7 @@ public String summarize( String userMessage = summarizationChunkUserMessageTemplate.render(chunkSummary); LOGGER.debug("Sending request to AI provider to summarize a chunk"); - String chunk = chatModelInfo.chatModel().chat(List.of( + String chunk = chatModel.chat(List.of( new SystemMessage(systemMessage), new UserMessage(userMessage) )).aiMessage().text(); @@ -93,7 +94,7 @@ public String summarize( } chunkSummaries = list; - } while (chatModelInfo.tokenizer().estimate(chunkSummaries.stream().map(UserMessage::new).toList()) > chatModelInfo.contextWindowSize() - chatModelInfo.tokenizer().estimate(new SystemMessage(summarizationCombineSystemMessageTemplate.getSource()))); + } while (chatModel.getTokenizer().estimate(chunkSummaries.stream().map(UserMessage::new).toList()) > chatModel.getContextWindowSize() - chatModel.getTokenizer().estimate(new SystemMessage(summarizationCombineSystemMessageTemplate.getSource()))); if (chunkSummaries.size() == 1) { longTaskInfo.progressCounter().increaseWorkDone(1); // No need to call LLM for combination of summary chunks. @@ -109,7 +110,7 @@ public String summarize( } LOGGER.debug("Sending request to AI provider to combine summary chunks"); - String result = chatModelInfo.chatModel().chat(List.of( + String result = chatModel.chat(List.of( new SystemMessage(systemMessage), new UserMessage(userMessage) )).aiMessage().text(); diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/SummarizationAlgorithm.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/SummarizationAlgorithm.java index 507da44c0cd9..58e444902cc2 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/SummarizationAlgorithm.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/SummarizationAlgorithm.java @@ -1,12 +1,12 @@ package org.jabref.logic.ai.summarization.logic.summarizationalgorithms; +import org.jabref.logic.ai.customimplementations.llms.ChatModel; import org.jabref.logic.ai.util.LongTaskInfo; -import org.jabref.model.ai.chatting.ChatModelInfo; import org.jabref.model.ai.summarization.SummarizationAlgorithmName; public interface SummarizationAlgorithm { String summarize( - ChatModelInfo chatModelInfo, + ChatModel chatModel, LongTaskInfo longTaskInfo, String text ) throws InterruptedException; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java index b7f30261869d..149764903de7 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java @@ -10,13 +10,13 @@ import javafx.util.Pair; import org.jabref.logic.FilePreferences; +import org.jabref.logic.ai.customimplementations.llms.ChatModel; import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; import org.jabref.logic.ai.summarization.repositories.SummariesRepository; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.ProgressCounter; import org.jabref.logic.util.TaskExecutor; -import org.jabref.model.ai.chatting.ChatModelInfo; import org.jabref.model.ai.processingstatus.ProcessingInfo; import org.jabref.model.ai.processingstatus.ProcessingState; import org.jabref.model.ai.summarization.BibEntrySummary; @@ -38,7 +38,7 @@ public class GenerateSummaryForSeveralTask extends BackgroundTask { private final List> entries; private final BibDatabaseContext bibDatabaseContext; private final SummariesRepository summariesRepository; - private final ChatModelInfo chatModelInfo; + private final ChatModel chatModel; private final SummarizationAlgorithm summarizationAlgorithm; private final ReadOnlyBooleanProperty shutdownSignal; private final FilePreferences filePreferences; @@ -51,7 +51,7 @@ public class GenerateSummaryForSeveralTask extends BackgroundTask { public GenerateSummaryForSeveralTask( FilePreferences filePreferences, TaskExecutor taskExecutor, - ChatModelInfo chatModelInfo, + ChatModel chatModel, SummariesRepository summariesRepository, SummarizationAlgorithm summarizationAlgorithm, BibDatabaseContext bibDatabaseContext, @@ -63,7 +63,7 @@ public GenerateSummaryForSeveralTask( this.entries = entries; this.bibDatabaseContext = bibDatabaseContext; this.summariesRepository = summariesRepository; - this.chatModelInfo = chatModelInfo; + this.chatModel = chatModel; this.summarizationAlgorithm = summarizationAlgorithm; this.shutdownSignal = shutdownSignal; this.filePreferences = filePreferences; @@ -95,7 +95,7 @@ public Void call() throws ExecutionException, InterruptedException { return new Pair<>( new GenerateSummaryTask( filePreferences, - chatModelInfo, + chatModel, summariesRepository, summarizationAlgorithm, bibDatabaseContext, diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java index e0c1ac821722..57f381b11ffe 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java @@ -3,6 +3,7 @@ import javafx.beans.property.ReadOnlyBooleanProperty; import org.jabref.logic.FilePreferences; +import org.jabref.logic.ai.customimplementations.llms.ChatModel; import org.jabref.logic.ai.summarization.SummariesService; import org.jabref.logic.ai.summarization.logic.PersistentBibEntrySummarizer; import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; @@ -11,7 +12,6 @@ import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.ProgressCounter; -import org.jabref.model.ai.chatting.ChatModelInfo; import org.jabref.model.ai.summarization.BibEntrySummary; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -29,7 +29,7 @@ public class GenerateSummaryTask extends BackgroundTask { private static final Logger LOGGER = LoggerFactory.getLogger(GenerateSummaryTask.class); - private final ChatModelInfo chatModelInfo; + private final ChatModel chatModel; private final BibDatabaseContext bibDatabaseContext; private final BibEntry entry; private final String citationKey; @@ -41,14 +41,14 @@ public class GenerateSummaryTask extends BackgroundTask { public GenerateSummaryTask( FilePreferences filePreferences, - ChatModelInfo chatModelInfo, + ChatModel chatModel, SummariesRepository summariesRepository, SummarizationAlgorithm summarizationAlgorithm, BibDatabaseContext bibDatabaseContext, BibEntry entry, ReadOnlyBooleanProperty shutdownSignal ) { - this.chatModelInfo = chatModelInfo; + this.chatModel = chatModel; this.bibDatabaseContext = bibDatabaseContext; this.entry = entry; this.citationKey = entry.getCitationKey().orElse(""); @@ -80,7 +80,7 @@ public BibEntrySummary call() { ); BibEntrySummary bibEntrySummary = persistentBibEntrySummarizer.summarize( - chatModelInfo, + chatModel, longTaskInfo, bibDatabaseContext, entry diff --git a/jablib/src/main/java/org/jabref/logic/importer/fileformat/pdf/CitationsFromPdf.java b/jablib/src/main/java/org/jabref/logic/importer/fileformat/pdf/CitationsFromPdf.java index 1d7f1fb48e49..d31565b17f54 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fileformat/pdf/CitationsFromPdf.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fileformat/pdf/CitationsFromPdf.java @@ -35,7 +35,7 @@ public static ParserResult extractCitationsUsingLLM(JabRefCliPreferences prefere LlmPlainCitationParser importer = new LlmPlainCitationParser( aiService, preferences.getImportFormatPreferences(), - aiService.getChatLanguageModel().getChatModelInfo() + aiService.getChatLanguageModel() ); return importer.importDatabase(path); diff --git a/jablib/src/main/java/org/jabref/logic/importer/plaincitation/LlmPlainCitationParser.java b/jablib/src/main/java/org/jabref/logic/importer/plaincitation/LlmPlainCitationParser.java index 950219392393..923cdefb7fc9 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/plaincitation/LlmPlainCitationParser.java +++ b/jablib/src/main/java/org/jabref/logic/importer/plaincitation/LlmPlainCitationParser.java @@ -6,6 +6,7 @@ import org.jabref.logic.ai.AiService; import org.jabref.logic.ai.citationparsing.logic.ParseCitationsWithLlm; +import org.jabref.logic.ai.customimplementations.llms.ChatModel; import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ParseException; @@ -13,24 +14,23 @@ import org.jabref.logic.importer.fileformat.pdf.PdfImporterWithPlainCitationParser; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.Result; -import org.jabref.model.ai.chatting.ChatModelInfo; import org.jabref.model.entry.BibEntry; public class LlmPlainCitationParser extends PdfImporterWithPlainCitationParser implements PlainCitationParser { private final ImportFormatPreferences importFormatPreferences; - private final ChatModelInfo chatModelInfo; + private final ChatModel chatModel; private final ParseCitationsWithLlm parseCitationsWithLlm; public LlmPlainCitationParser( AiService aiService, ImportFormatPreferences importFormatPreferences, - ChatModelInfo chatModelInfo + ChatModel chatModel ) { this.importFormatPreferences = importFormatPreferences; - this.chatModelInfo = chatModelInfo; + this.chatModel = chatModel; this.parseCitationsWithLlm = new ParseCitationsWithLlm( importFormatPreferences, @@ -57,7 +57,7 @@ public String getDescription() { @Override public Optional parsePlainCitation(String text) throws FetcherException { try { - String string = parseCitationsWithLlm.getBibtexStringFromLlm(chatModelInfo, text); + String string = parseCitationsWithLlm.getBibtexStringFromLlm(chatModel, text); return BibtexParser.singleFromString(string, importFormatPreferences); } catch (ParseException e) { throw new FetcherException("Could not parse BibTeX returned from LLM", e); @@ -66,7 +66,7 @@ public Optional parsePlainCitation(String text) throws FetcherExceptio @Override public List parseMultiplePlainCitations(String text) throws FetcherException { - Result, IOException> result = parseCitationsWithLlm.parseMultiplePlainCitations(chatModelInfo, text); + Result, IOException> result = parseCitationsWithLlm.parseMultiplePlainCitations(chatModel, text); if (result.isErr()) { throw new FetcherException("Could not parse BibTeX returned from LLM", result.getError()); diff --git a/jablib/src/main/java/org/jabref/logic/importer/plaincitation/PlainCitationParserFactory.java b/jablib/src/main/java/org/jabref/logic/importer/plaincitation/PlainCitationParserFactory.java index f915e96edf3f..1d49ad3d68a3 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/plaincitation/PlainCitationParserFactory.java +++ b/jablib/src/main/java/org/jabref/logic/importer/plaincitation/PlainCitationParserFactory.java @@ -23,7 +23,7 @@ public static PlainCitationParser getPlainCitationParser(PlainCitationParserChoi case PlainCitationParserChoice.GROBID -> new GrobidPlainCitationParser(grobidPreferences, importFormatPreferences); case PlainCitationParserChoice.LLM -> - new LlmPlainCitationParser(aiService, importFormatPreferences, aiService.getChatLanguageModel().getChatModelInfo()); + new LlmPlainCitationParser(aiService, importFormatPreferences, aiService.getChatLanguageModel()); }; } } diff --git a/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java b/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java index b0c4cecbc21e..da30afcf649e 100644 --- a/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java @@ -100,6 +100,7 @@ import org.jabref.logic.xmp.XmpPreferences; import org.jabref.model.ai.chatting.AiProvider; import org.jabref.model.ai.embeddings.EmbeddingModel; +import org.jabref.model.ai.rag.DocumentSplittingStrategy; import org.jabref.model.ai.summarization.SummarizationAlgorithmName; import org.jabref.model.ai.templating.AiTemplate; import org.jabref.model.ai.tokenization.TokenEstimationStrategy; @@ -401,6 +402,7 @@ public class JabRefCliPreferences implements CliPreferences { private static final String AI_TOKEN_ESTIMATION_ALGORITHM = "aiTokenEstimationAlgorithm"; private static final String AI_TEMPERATURE = "aiTemperature"; private static final String AI_CONTEXT_WINDOW_SIZE = "aiMessageWindowSize"; + private static final String AI_DOCUMENT_SPLITTING_STRATEGY = "aiDocumentSplittingStrategy"; private static final String AI_DOCUMENT_SPLITTER_CHUNK_SIZE = "aiDocumentSplitterChunkSize"; private static final String AI_DOCUMENT_SPLITTER_OVERLAP_SIZE = "aiDocumentSplitterOverlapSize"; private static final String AI_RAG_MAX_RESULTS_COUNT = "aiRagMaxResultsCount"; @@ -750,7 +752,13 @@ public JabRefCliPreferences() { defaults.put(AI_SUMMARIZATION_ALGORITHM, AiDefaultExpertSettings.SUMMARIZATION_ALGORITHM_NAME.name()); defaults.put(AI_TOKEN_ESTIMATION_ALGORITHM, AiDefaultExpertSettings.TOKEN_ESTIMATION_STRATEGY.name()); defaults.put(AI_TEMPERATURE, AiDefaultExpertSettings.TEMPERATURE); - defaults.put(AI_CONTEXT_WINDOW_SIZE, PredefinedChatModel.getContextWindowSize((AiProvider) defaults.get(AI_PROVIDER), AiProviderDefaultChatModels.getDefaultChatModel((AiProvider) defaults.get(AI_PROVIDER)).getName())); + defaults.put(AI_CONTEXT_WINDOW_SIZE, + PredefinedChatModel.getContextWindowSize( + AiProvider.valueOf((String) defaults.get(AI_PROVIDER)), + AiProviderDefaultChatModels.getDefaultChatModel(AiProvider.valueOf((String) defaults.get(AI_PROVIDER))).getName() + ) + ); + defaults.put(AI_DOCUMENT_SPLITTING_STRATEGY, AiDefaultExpertSettings.DOCUMENT_SPLITTING_STRATEGY.name()); defaults.put(AI_DOCUMENT_SPLITTER_CHUNK_SIZE, AiDefaultExpertSettings.DOCUMENT_SPLITTER_CHUNK_SIZE); defaults.put(AI_DOCUMENT_SPLITTER_OVERLAP_SIZE, AiDefaultExpertSettings.DOCUMENT_SPLITTER_OVERLAP_SIZE); defaults.put(AI_RAG_MAX_RESULTS_COUNT, AiDefaultExpertSettings.RAG_MAX_RESULTS_COUNT); @@ -2047,6 +2055,7 @@ public AiPreferences getAiPreferences() { EmbeddingModel.valueOf(get(AI_EMBEDDING_MODEL)), getDouble(AI_TEMPERATURE), getInt(AI_CONTEXT_WINDOW_SIZE), + DocumentSplittingStrategy.valueOf(get(AI_DOCUMENT_SPLITTING_STRATEGY)), getInt(AI_DOCUMENT_SPLITTER_CHUNK_SIZE), getInt(AI_DOCUMENT_SPLITTER_OVERLAP_SIZE), getInt(AI_RAG_MAX_RESULTS_COUNT), @@ -2087,6 +2096,7 @@ AiTemplate.CITATION_PARSING_USER_MESSAGE, get(AI_CITATION_PARSING_USER_MESSAGE_T EasyBind.listen(aiPreferences.embeddingModelProperty(), (_, _, newValue) -> put(AI_EMBEDDING_MODEL, newValue.name())); EasyBind.listen(aiPreferences.temperatureProperty(), (_, _, newValue) -> putDouble(AI_TEMPERATURE, newValue.doubleValue())); EasyBind.listen(aiPreferences.contextWindowSizeProperty(), (_, _, newValue) -> putInt(AI_CONTEXT_WINDOW_SIZE, newValue)); + EasyBind.listen(aiPreferences.documentSplittingStrategyProperty(), (_, _, newValue) -> put(AI_DOCUMENT_SPLITTING_STRATEGY, newValue.name())); EasyBind.listen(aiPreferences.documentSplitterChunkSizeProperty(), (_, _, newValue) -> putInt(AI_DOCUMENT_SPLITTER_CHUNK_SIZE, newValue)); EasyBind.listen(aiPreferences.documentSplitterOverlapSizeProperty(), (_, _, newValue) -> putInt(AI_DOCUMENT_SPLITTER_OVERLAP_SIZE, newValue)); EasyBind.listen(aiPreferences.ragMaxResultsCountProperty(), (_, _, newValue) -> putInt(AI_RAG_MAX_RESULTS_COUNT, newValue)); diff --git a/jablib/src/main/java/org/jabref/model/ai/chatting/ChatModelInfo.java b/jablib/src/main/java/org/jabref/model/ai/chatting/ChatModelInfo.java deleted file mode 100644 index 132f3d7a67fb..000000000000 --- a/jablib/src/main/java/org/jabref/model/ai/chatting/ChatModelInfo.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.jabref.model.ai.chatting; - -import org.jabref.logic.ai.customimplementations.tokenization.algorithms.Tokenizer; - -import dev.langchain4j.model.chat.ChatModel; - -public record ChatModelInfo( - ChatModel chatModel, - Tokenizer tokenizer, - AiProvider aiProvider, - String name, - int contextWindowSize -) { -} diff --git a/jablib/src/main/java/org/jabref/model/ai/rag/DocumentSplittingStrategy.java b/jablib/src/main/java/org/jabref/model/ai/rag/DocumentSplittingStrategy.java new file mode 100644 index 000000000000..dd46aeff10f8 --- /dev/null +++ b/jablib/src/main/java/org/jabref/model/ai/rag/DocumentSplittingStrategy.java @@ -0,0 +1,5 @@ +package org.jabref.model.ai.rag; + +public enum DocumentSplittingStrategy { + SLIDING_WINDOW; +} From 1dac75a23e821c2a9991e32741b89a2631cf4c2c Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Wed, 26 Nov 2025 21:30:30 +0100 Subject: [PATCH 028/243] Fix error --- .../org/jabref/logic/ai/rag/logic/ingestion/TextIngestor.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/TextIngestor.java b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/TextIngestor.java index d1bb072ff6e3..c219eff8edd7 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/TextIngestor.java +++ b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/TextIngestor.java @@ -25,6 +25,8 @@ public TextIngestor( .builder() .embeddingStore(embeddingStore) .embeddingModel(embeddingModel) + // TODO: remove this stub. + .documentSplitter(document -> List.of(new TextSegment(document.text(), document.metadata()))) .build(); this.documentSplitterAlgorithm = documentSplitterAlgorithm; From 39df01b0b01f06c08c22451d8ee52496fa292b01 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Fri, 28 Nov 2025 20:46:59 +0100 Subject: [PATCH 029/243] Big renaming --- .../org/jabref/gui/preferences/ai/AiTab.java | 10 ++-- .../gui/preferences/ai/AiTabViewModel.java | 12 ++-- .../aichat/AiChatComponentTest.java | 2 +- jablib/src/main/java/module-info.java | 10 ++-- .../java/org/jabref/logic/ai/AiService.java | 18 +++--- .../logic/ai/chatting/logic/AiChatLogic.java | 2 +- .../CurrentlySelectedDocumentSplitter.java | 24 ++++---- .../CurrentlySelectedEmbeddingModel.java | 2 +- ...ava => CurrentlySelectedSummarizator.java} | 30 +++++----- ...rentlySelectedTokenEstimationStrategy.java | 16 +++--- .../MVStoreEmbeddingStore.java | 2 +- .../algorithms/AverageTokenizer.java | 4 +- .../algorithms/ByCharacterTokenizer.java | 4 +- .../algorithms/ByWordsTokenizer.java | 4 +- .../algorithms/MaximumTokenizer.java | 6 +- .../algorithms/MinimumTokenizer.java | 6 +- .../tokenization/algorithms/Tokenizer.java | 4 +- .../{rag => pipeline}/IngestionService.java | 34 +++++------ .../logic/EmbeddingsCleaner.java | 12 ++-- .../documentsplitting/DocumentSplitter.java | 12 ++++ .../SlidingWindowDocumentSplitter.java | 13 ++--- .../logic/ingestion/FileIngestor.java | 14 ++--- .../logic/ingestion/LinkedFileIngestor.java | 10 ++-- .../PersistentLinkedFileIngestor.java | 20 +++---- .../logic/ingestion/TextIngestor.java | 12 ++-- .../logic/parsing/FileContentParser.java} | 4 +- .../logic/parsing/PdfContentParser.java} | 6 +- .../parsing/UniversalContentParser.java} | 8 +-- .../IngestedDocumentsRepository.java} | 4 +- .../MVStoreIngestedDocumentsRepository.java} | 6 +- .../GenerateEmbeddingsForSeveralTask.java | 22 ++++---- .../tasks/GenerateEmbeddingsTask.java | 16 +++--- .../tasks/UpdateEmbeddingModelTask.java | 2 +- .../preferences/AiDefaultExpertSettings.java | 16 +++--- .../logic/ai/preferences/AiPreferences.java | 56 +++++++++---------- .../DocumentSplitterAlgorithm.java | 12 ---- .../ai/summarization/SummariesService.java | 36 ++++++------ ...marizer.java => BibEntrySummarizator.java} | 24 ++++---- ...va => PersistentBibEntrySummarizator.java} | 18 +++--- ...lgorithm.java => ChunkedSummarizator.java} | 12 ++-- ...zationAlgorithm.java => Summarizator.java} | 6 +- .../tasks/GenerateSummaryForSeveralTask.java | 10 ++-- .../tasks/GenerateSummaryTask.java | 14 ++--- .../preferences/JabRefCliPreferences.java | 16 +++--- ...el.java => EmbeddingModelEnumeration.java} | 4 +- .../ai/processingstatus/ProcessingInfo.java | 1 + .../ai/processingstatus/ProcessingState.java | 1 + ...trategy.java => DocumentSplitterKind.java} | 2 +- .../ai/summarization/BibEntrySummary.java | 2 +- ...gorithmName.java => SummarizatorKind.java} | 4 +- ...nStrategy.java => TokenEstimatorKind.java} | 4 +- ...a => IngestedDocumentsRepositoryTest.java} | 10 ++-- ...eFullyIngestedDocumentsRepositoryTest.java | 21 ------- ...VStoreIngestedDocumentsRepositoryTest.java | 21 +++++++ 54 files changed, 321 insertions(+), 320 deletions(-) rename jablib/src/main/java/org/jabref/logic/ai/current/{CurrentlySelectedSummarizationAlgorithm.java => CurrentlySelectedSummarizator.java} (74%) rename jablib/src/main/java/org/jabref/logic/ai/{rag => pipeline}/IngestionService.java (87%) rename jablib/src/main/java/org/jabref/logic/ai/{rag => pipeline}/logic/EmbeddingsCleaner.java (74%) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/documentsplitting/DocumentSplitter.java rename jablib/src/main/java/org/jabref/logic/ai/{rag => pipeline}/logic/documentsplitting/SlidingWindowDocumentSplitter.java (72%) rename jablib/src/main/java/org/jabref/logic/ai/{rag => pipeline}/logic/ingestion/FileIngestor.java (80%) rename jablib/src/main/java/org/jabref/logic/ai/{rag => pipeline}/logic/ingestion/LinkedFileIngestor.java (77%) rename jablib/src/main/java/org/jabref/logic/ai/{rag => pipeline}/logic/ingestion/PersistentLinkedFileIngestor.java (81%) rename jablib/src/main/java/org/jabref/logic/ai/{rag => pipeline}/logic/ingestion/TextIngestor.java (78%) rename jablib/src/main/java/org/jabref/logic/ai/{rag/logic/parsing/FileParser.java => pipeline/logic/parsing/FileContentParser.java} (65%) rename jablib/src/main/java/org/jabref/logic/ai/{rag/logic/parsing/PdfFileParser.java => pipeline/logic/parsing/PdfContentParser.java} (90%) rename jablib/src/main/java/org/jabref/logic/ai/{rag/logic/parsing/UniversalFileParser.java => pipeline/logic/parsing/UniversalContentParser.java} (73%) rename jablib/src/main/java/org/jabref/logic/ai/{rag/repositories/FullyIngestedDocumentsRepository.java => pipeline/repositories/IngestedDocumentsRepository.java} (78%) rename jablib/src/main/java/org/jabref/logic/ai/{rag/repositories/MVStoreFullyIngestedDocumentsRepository.java => pipeline/repositories/MVStoreIngestedDocumentsRepository.java} (90%) rename jablib/src/main/java/org/jabref/logic/ai/{rag => pipeline}/tasks/GenerateEmbeddingsForSeveralTask.java (86%) rename jablib/src/main/java/org/jabref/logic/ai/{rag => pipeline}/tasks/GenerateEmbeddingsTask.java (86%) rename jablib/src/main/java/org/jabref/logic/ai/{rag => pipeline}/tasks/UpdateEmbeddingModelTask.java (98%) delete mode 100644 jablib/src/main/java/org/jabref/logic/ai/rag/logic/documentsplitting/DocumentSplitterAlgorithm.java rename jablib/src/main/java/org/jabref/logic/ai/summarization/logic/{BibEntrySummarizer.java => BibEntrySummarizator.java} (88%) rename jablib/src/main/java/org/jabref/logic/ai/summarization/logic/{PersistentBibEntrySummarizer.java => PersistentBibEntrySummarizator.java} (87%) rename jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/{ChunkedSummarizationAlgorithm.java => ChunkedSummarizator.java} (94%) rename jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/{SummarizationAlgorithm.java => Summarizator.java} (69%) rename jablib/src/main/java/org/jabref/model/ai/embeddings/{EmbeddingModel.java => EmbeddingModelEnumeration.java} (99%) rename jablib/src/main/java/org/jabref/model/ai/rag/{DocumentSplittingStrategy.java => DocumentSplitterKind.java} (58%) rename jablib/src/main/java/org/jabref/model/ai/summarization/{SummarizationAlgorithmName.java => SummarizatorKind.java} (76%) rename jablib/src/main/java/org/jabref/model/ai/tokenization/{TokenEstimationStrategy.java => TokenEstimatorKind.java} (71%) rename jablib/src/test/java/org/jabref/logic/ai/ingestion/{FullyIngestedDocumentsRepositoryTest.java => IngestedDocumentsRepositoryTest.java} (78%) delete mode 100644 jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsRepositoryTest.java create mode 100644 jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreIngestedDocumentsRepositoryTest.java diff --git a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java index 6f388ea854b7..1cfa87a54fb4 100644 --- a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java +++ b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java @@ -23,9 +23,9 @@ import org.jabref.gui.util.ViewModelListCellFactory; import org.jabref.logic.help.HelpFile; import org.jabref.logic.l10n.Localization; -import org.jabref.model.ai.chatting.AiProvider; +import org.jabref.model.ai.chatting.AiProviderType; +import org.jabref.model.ai.embeddings.EmbeddingModelEnumeration; import org.jabref.model.ai.templating.AiTemplate; -import org.jabref.model.ai.embeddings.EmbeddingModel; import com.airhacks.afterburner.views.ViewLoader; import com.dlsc.unitfx.IntegerInputField; @@ -49,7 +49,7 @@ public class AiTab extends AbstractPreferenceTabView implements @FXML private VBox expertSettingsPane; @FXML private TextField apiBaseUrlTextField; - @FXML private SearchableComboBox embeddingModelComboBox; + @FXML private SearchableComboBox embeddingModelComboBox; @FXML private TextField temperatureTextField; @FXML private IntegerInputField contextWindowSizeTextField; @FXML private IntegerInputField documentSplitterChunkSizeTextField; @@ -145,8 +145,8 @@ private void initializeExpertSettings() { expertSettingsPane.visibleProperty().bind(customizeExpertSettingsCheckbox.selectedProperty()); expertSettingsPane.managedProperty().bind(customizeExpertSettingsCheckbox.selectedProperty()); - new ViewModelListCellFactory() - .withText(EmbeddingModel::fullInfo) + new ViewModelListCellFactory() + .withText(EmbeddingModelEnumeration::fullInfo) .install(embeddingModelComboBox); embeddingModelComboBox.setItems(viewModel.embeddingModelsProperty()); embeddingModelComboBox.valueProperty().bindBidirectional(viewModel.selectedEmbeddingModelProperty()); diff --git a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java index 40d2e72a64b5..35be8dd8932e 100644 --- a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java +++ b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java @@ -29,7 +29,7 @@ import org.jabref.logic.util.OptionalObjectProperty; import org.jabref.logic.util.strings.StringUtil; import org.jabref.model.ai.chatting.AiProvider; -import org.jabref.model.ai.embeddings.EmbeddingModel; +import org.jabref.model.ai.embeddings.EmbeddingModelEnumeration; import org.jabref.model.ai.templating.AiTemplate; import de.saxsys.mvvmfx.utils.validation.FunctionBasedValidator; @@ -71,9 +71,9 @@ public class AiTabViewModel implements PreferenceTabViewModel { private final BooleanProperty customizeExpertSettings = new SimpleBooleanProperty(); - private final ListProperty embeddingModelsList = - new SimpleListProperty<>(FXCollections.observableArrayList(EmbeddingModel.values())); - private final ObjectProperty selectedEmbeddingModel = new SimpleObjectProperty<>(); + private final ListProperty embeddingModelsList = + new SimpleListProperty<>(FXCollections.observableArrayList(EmbeddingModelEnumeration.values())); + private final ObjectProperty selectedEmbeddingModel = new SimpleObjectProperty<>(); private final StringProperty currentApiBaseUrl = new SimpleStringProperty(); private final BooleanProperty disableApiBaseUrl = new SimpleBooleanProperty(true); // {@link HuggingFaceChatModel} and {@link GoogleAiGeminiChatModel} doesn't support setting API base URL @@ -513,11 +513,11 @@ public BooleanProperty customizeExpertSettingsProperty() { return customizeExpertSettings; } - public ReadOnlyListProperty embeddingModelsProperty() { + public ReadOnlyListProperty embeddingModelsProperty() { return embeddingModelsList; } - public ObjectProperty selectedEmbeddingModelProperty() { + public ObjectProperty selectedEmbeddingModelProperty() { return selectedEmbeddingModel; } diff --git a/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java b/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java index aa12abbfd0f4..30469f7dade3 100644 --- a/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java +++ b/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java @@ -11,8 +11,8 @@ import org.jabref.gui.DialogService; import org.jabref.logic.ai.AiService; import org.jabref.logic.ai.chatting.logic.AiChatLogic; +import org.jabref.logic.ai.pipeline.IngestionService; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.rag.IngestionService; import org.jabref.logic.l10n.Language; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.TaskExecutor; diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index da9fc13faf15..be5da2c8110e 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -122,16 +122,16 @@ exports org.jabref.logic.ai.customimplementations.embeddingmodels; exports org.jabref.logic.ai.customimplementations.embeddingstores; exports org.jabref.logic.ai.customimplementations.llms; - exports org.jabref.logic.ai.rag; - exports org.jabref.logic.ai.rag.logic.documentsplitting; + exports org.jabref.logic.ai.pipeline; + exports org.jabref.logic.ai.pipeline.logic.documentsplitting; exports org.jabref.logic.ai.summarization.tasks; exports org.jabref.logic.ai.summarization.repositories; exports org.jabref.logic.ai.summarization.templates; exports org.jabref.logic.ai.summarization.logic; exports org.jabref.logic.ai.summarization.logic.summarizationalgorithms; - exports org.jabref.logic.ai.rag.tasks; - exports org.jabref.logic.ai.rag.repositories; - exports org.jabref.logic.ai.rag.logic; + exports org.jabref.logic.ai.pipeline.tasks; + exports org.jabref.logic.ai.pipeline.repositories; + exports org.jabref.logic.ai.pipeline.logic; exports org.jabref.logic.ai.chatting.logic; exports org.jabref.logic.ai.customimplementations.tokenization.algorithms; exports org.jabref.logic.ai.chatting.tasks; diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiService.java b/jablib/src/main/java/org/jabref/logic/ai/AiService.java index 17177de192a4..1a46cf87fefb 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiService.java @@ -13,14 +13,14 @@ import org.jabref.logic.ai.current.CurrentlySelectedChatLanguageModel; import org.jabref.logic.ai.current.CurrentlySelectedDocumentSplitter; import org.jabref.logic.ai.current.CurrentlySelectedEmbeddingModel; -import org.jabref.logic.ai.current.CurrentlySelectedSummarizationAlgorithm; +import org.jabref.logic.ai.current.CurrentlySelectedSummarizator; import org.jabref.logic.ai.current.CurrentlySelectedTokenEstimationStrategy; import org.jabref.logic.ai.customimplementations.embeddingstores.MVStoreEmbeddingStore; +import org.jabref.logic.ai.pipeline.IngestionService; +import org.jabref.logic.ai.pipeline.repositories.MVStoreIngestedDocumentsRepository; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.rag.IngestionService; -import org.jabref.logic.ai.rag.repositories.MVStoreFullyIngestedDocumentsRepository; import org.jabref.logic.ai.summarization.SummariesService; -import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; +import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.Summarizator; import org.jabref.logic.ai.summarization.repositories.MVStoreSummariesRepository; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; import org.jabref.logic.util.Directories; @@ -56,7 +56,7 @@ public class AiService implements AutoCloseable { private final MVStoreChatHistoryRepository mvStoreChatHistoryStorage; private final MVStoreEmbeddingStore mvStoreEmbeddingStore; - private final MVStoreFullyIngestedDocumentsRepository mvStoreFullyIngestedDocumentsTracker; + private final MVStoreIngestedDocumentsRepository mvStoreFullyIngestedDocumentsTracker; private final MVStoreSummariesRepository mvStoreSummariesStorage; private final CurrentAiTemplates currentAiTemplates; @@ -65,7 +65,7 @@ public class AiService implements AutoCloseable { private final CurrentlySelectedTokenEstimationStrategy currentlySelectedTokenEstimationStrategy; private final CurrentlySelectedChatLanguageModel currentlySelectedChatLanguageModel; private final CurrentlySelectedEmbeddingModel currentlySelectedEmbeddingModel; - private final CurrentlySelectedSummarizationAlgorithm currentlySelectedSummarizationAlgorithm; + private final CurrentlySelectedSummarizator currentlySelectedSummarizationAlgorithm; private final IngestionService ingestionService; private final SummariesService summariesService; @@ -79,7 +79,7 @@ public AiService( this.mvStoreChatHistoryStorage = new MVStoreChatHistoryRepository(notificationService, Directories.getAiFilesDirectory().resolve(CHAT_HISTORY_FILE_NAME)); this.mvStoreEmbeddingStore = new MVStoreEmbeddingStore(Directories.getAiFilesDirectory().resolve(EMBEDDINGS_FILE_NAME), notificationService); - this.mvStoreFullyIngestedDocumentsTracker = new MVStoreFullyIngestedDocumentsRepository(notificationService, Directories.getAiFilesDirectory().resolve(FULLY_INGESTED_FILE_NAME)); + this.mvStoreFullyIngestedDocumentsTracker = new MVStoreIngestedDocumentsRepository(notificationService, Directories.getAiFilesDirectory().resolve(FULLY_INGESTED_FILE_NAME)); this.mvStoreSummariesStorage = new MVStoreSummariesRepository(notificationService, Directories.getAiFilesDirectory().resolve(SUMMARIES_FILE_NAME)); this.currentAiTemplates = new CurrentAiTemplates(aiPreferences); @@ -89,7 +89,7 @@ public AiService( this.currentlySelectedTokenEstimationStrategy = new CurrentlySelectedTokenEstimationStrategy(aiPreferences); this.currentlySelectedChatLanguageModel = new CurrentlySelectedChatLanguageModel(aiPreferences, currentlySelectedTokenEstimationStrategy); this.currentlySelectedEmbeddingModel = new CurrentlySelectedEmbeddingModel(aiPreferences, notificationService, taskExecutor); - this.currentlySelectedSummarizationAlgorithm = new CurrentlySelectedSummarizationAlgorithm(aiPreferences, currentAiTemplates); + this.currentlySelectedSummarizationAlgorithm = new CurrentlySelectedSummarizator(aiPreferences, currentAiTemplates); this.ingestionService = new IngestionService( aiPreferences, @@ -145,7 +145,7 @@ public EmbeddingStore getEmbeddingStore() { return mvStoreEmbeddingStore; } - public SummarizationAlgorithm getSummarizationAlgorithm() { + public Summarizator getSummarizationAlgorithm() { return currentlySelectedSummarizationAlgorithm; } diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java index 26f96a8d4989..9f42a4f37459 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java @@ -9,8 +9,8 @@ import org.jabref.logic.ai.chatting.templates.ChattingSystemMessageTemplate; import org.jabref.logic.ai.chatting.templates.ChattingUserMessageTemplate; +import org.jabref.logic.ai.pipeline.logic.EmbeddingsCleaner; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.rag.logic.EmbeddingsCleaner; import org.jabref.model.ai.chatting.ErrorMessage; import org.jabref.model.ai.rag.PaperExcerpt; import org.jabref.model.ai.templating.AiTemplate; diff --git a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedDocumentSplitter.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedDocumentSplitter.java index eebcdc69a077..051f3730eb57 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedDocumentSplitter.java +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedDocumentSplitter.java @@ -2,19 +2,19 @@ import java.util.stream.Stream; +import org.jabref.logic.ai.pipeline.logic.documentsplitting.DocumentSplitter; +import org.jabref.logic.ai.pipeline.logic.documentsplitting.SlidingWindowDocumentSplitter; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.rag.logic.documentsplitting.DocumentSplitterAlgorithm; -import org.jabref.logic.ai.rag.logic.documentsplitting.SlidingWindowDocumentSplitter; import org.jabref.logic.ai.util.LongTaskInfo; -import org.jabref.model.ai.rag.DocumentSplittingStrategy; +import org.jabref.model.ai.rag.DocumentSplitterKind; import org.jspecify.annotations.Nullable; -public class CurrentlySelectedDocumentSplitter implements DocumentSplitterAlgorithm { +public class CurrentlySelectedDocumentSplitter implements DocumentSplitter { private final AiPreferences aiPreferences; @Nullable - private DocumentSplitterAlgorithm documentSplitterAlgorithm = null; + private DocumentSplitter documentSplitter = null; public CurrentlySelectedDocumentSplitter(AiPreferences aiPreferences) { this.aiPreferences = aiPreferences; @@ -34,8 +34,8 @@ private void update() { // Because in the future there will be more strategies. //noinspection SwitchStatementWithTooFewBranches switch (aiPreferences.getDocumentSplittingStrategy()) { - case DocumentSplittingStrategy.SLIDING_WINDOW -> { - documentSplitterAlgorithm = new SlidingWindowDocumentSplitter( + case DocumentSplitterKind.SLIDING_WINDOW -> { + documentSplitter = new SlidingWindowDocumentSplitter( aiPreferences.getDocumentSplitterChunkSize(), aiPreferences.getDocumentSplitterOverlapSize() ); @@ -45,20 +45,20 @@ private void update() { @Override public Stream split(LongTaskInfo longTaskInfo, String text) throws InterruptedException { - if (documentSplitterAlgorithm == null) { + if (documentSplitter == null) { return Stream.of(text); } - return documentSplitterAlgorithm.split(longTaskInfo, text); + return documentSplitter.split(longTaskInfo, text); } @Override - public DocumentSplittingStrategy getStrategy() { - if (documentSplitterAlgorithm == null) { + public DocumentSplitterKind getKind() { + if (documentSplitter == null) { // Unfortunately. return null; } - return documentSplitterAlgorithm.getStrategy(); + return documentSplitter.getKind(); } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedEmbeddingModel.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedEmbeddingModel.java index 30a6b9728160..180cebb23324 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedEmbeddingModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedEmbeddingModel.java @@ -9,8 +9,8 @@ import javafx.beans.property.SimpleObjectProperty; import org.jabref.logic.ai.customimplementations.embeddingmodels.DeepJavaEmbeddingModel; +import org.jabref.logic.ai.pipeline.tasks.UpdateEmbeddingModelTask; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.rag.tasks.UpdateEmbeddingModelTask; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.NotificationService; import org.jabref.logic.util.TaskExecutor; diff --git a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedSummarizationAlgorithm.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedSummarizator.java similarity index 74% rename from jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedSummarizationAlgorithm.java rename to jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedSummarizator.java index bb9a41b60a3d..727772080dc9 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedSummarizationAlgorithm.java +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedSummarizator.java @@ -2,22 +2,22 @@ import org.jabref.logic.ai.customimplementations.llms.ChatModel; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.ChunkedSummarizationAlgorithm; -import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; +import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.ChunkedSummarizator; +import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.Summarizator; import org.jabref.logic.ai.util.LongTaskInfo; -import org.jabref.model.ai.summarization.SummarizationAlgorithmName; +import org.jabref.model.ai.summarization.SummarizatorKind; import org.jabref.model.ai.templating.AiTemplate; import org.jspecify.annotations.Nullable; -public class CurrentlySelectedSummarizationAlgorithm implements SummarizationAlgorithm { +public class CurrentlySelectedSummarizator implements Summarizator { private final AiPreferences aiPreferences; private final CurrentAiTemplates currentAiTemplates; @Nullable - private SummarizationAlgorithm summarizationAlgorithm = null; + private Summarizator summarizator = null; - public CurrentlySelectedSummarizationAlgorithm( + public CurrentlySelectedSummarizator( AiPreferences aiPreferences, CurrentAiTemplates currentAiTemplates ) { @@ -52,14 +52,14 @@ private void updateAlgorithm() { // Because in the future there will be more strategies. //noinspection SwitchStatementWithTooFewBranches switch (aiPreferences.getDefaultSummarizationAlgorithm()) { - case SummarizationAlgorithmName.CHUNKED -> { - summarizationAlgorithm = createChunkedSummarizationAlgorithm(); + case SummarizatorKind.CHUNKED -> { + summarizator = createChunkedSummarizationAlgorithm(); } } } - private ChunkedSummarizationAlgorithm createChunkedSummarizationAlgorithm() { - return new ChunkedSummarizationAlgorithm( + private ChunkedSummarizator createChunkedSummarizationAlgorithm() { + return new ChunkedSummarizator( currentAiTemplates.getSummarizationChunkSystemMessageTemplate(), currentAiTemplates.getSummarizationChunkUserMessageTemplate(), currentAiTemplates.getSummarizationCombineSystemMessageTemplate(), @@ -69,20 +69,20 @@ private ChunkedSummarizationAlgorithm createChunkedSummarizationAlgorithm() { @Override public String summarize(ChatModel chatModel, LongTaskInfo longTaskInfo, String text) throws InterruptedException { - if (summarizationAlgorithm == null) { + if (summarizator == null) { throw new RuntimeException("No summarization algorithm selected."); } - return summarizationAlgorithm.summarize(chatModel, longTaskInfo, text); + return summarizator.summarize(chatModel, longTaskInfo, text); } @Override - public SummarizationAlgorithmName getName() { - if (summarizationAlgorithm == null) { + public SummarizatorKind getName() { + if (summarizator == null) { // Sadly. return null; } - return summarizationAlgorithm.getName(); + return summarizator.getName(); } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedTokenEstimationStrategy.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedTokenEstimationStrategy.java index 00564e14ad13..ca57019dd77b 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedTokenEstimationStrategy.java +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedTokenEstimationStrategy.java @@ -9,7 +9,7 @@ import org.jabref.logic.ai.customimplementations.tokenization.algorithms.MinimumTokenizer; import org.jabref.logic.ai.customimplementations.tokenization.algorithms.Tokenizer; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.model.ai.tokenization.TokenEstimationStrategy; +import org.jabref.model.ai.tokenization.TokenEstimatorKind; import dev.langchain4j.data.message.ChatMessage; import org.jspecify.annotations.Nullable; @@ -37,11 +37,11 @@ private void setupListeningToPreferences() { private void createTokenizer() { switch (aiPreferences.getTokenEstimationStrategy()) { - case TokenEstimationStrategy.AVERAGE -> tokenizer = new AverageTokenizer(); - case TokenEstimationStrategy.MAX -> tokenizer = new MaximumTokenizer(); - case TokenEstimationStrategy.MIN -> tokenizer = new MinimumTokenizer(); - case TokenEstimationStrategy.CHARS -> tokenizer = new ByCharacterTokenizer(); - case TokenEstimationStrategy.WORDS -> tokenizer = new ByWordsTokenizer(); + case TokenEstimatorKind.AVERAGE -> tokenizer = new AverageTokenizer(); + case TokenEstimatorKind.MAX -> tokenizer = new MaximumTokenizer(); + case TokenEstimatorKind.MIN -> tokenizer = new MinimumTokenizer(); + case TokenEstimatorKind.CHARS -> tokenizer = new ByCharacterTokenizer(); + case TokenEstimatorKind.WORDS -> tokenizer = new ByWordsTokenizer(); } } @@ -64,12 +64,12 @@ public int estimate(List messages) { } @Override - public TokenEstimationStrategy getEstimationStrategy() { + public TokenEstimatorKind getKind() { // Sadly. if (tokenizer == null) { return null; } - return tokenizer.getEstimationStrategy(); + return tokenizer.getKind(); } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingstores/MVStoreEmbeddingStore.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingstores/MVStoreEmbeddingStore.java index 518c9d5eecbc..309012f18d61 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingstores/MVStoreEmbeddingStore.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/embeddingstores/MVStoreEmbeddingStore.java @@ -34,7 +34,7 @@ import org.jspecify.annotations.Nullable; import static java.util.Comparator.comparingDouble; -import static org.jabref.logic.ai.rag.logic.EmbeddingsCleaner.LINK_METADATA_KEY; +import static org.jabref.logic.ai.pipeline.logic.EmbeddingsCleaner.LINK_METADATA_KEY; /** * A custom implementation of langchain4j's {@link EmbeddingStore} that uses a {@link MVStore} as an embedded database. diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/AverageTokenizer.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/AverageTokenizer.java index 68ba946f4e7f..87eacf80a679 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/AverageTokenizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/AverageTokenizer.java @@ -2,7 +2,7 @@ import java.util.List; -import org.jabref.model.ai.tokenization.TokenEstimationStrategy; +import org.jabref.model.ai.tokenization.TokenEstimatorKind; import dev.langchain4j.data.message.ChatMessage; @@ -31,7 +31,7 @@ private int calculate(int byCharacter, int byWords) { } @Override - public TokenEstimationStrategy getEstimationStrategy() { + public TokenEstimatorKind getKind() { return null; } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByCharacterTokenizer.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByCharacterTokenizer.java index 10862d3f348f..46dd849f5e6c 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByCharacterTokenizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByCharacterTokenizer.java @@ -5,7 +5,7 @@ import java.util.stream.Collectors; import org.jabref.logic.ai.util.ChatMessagesUtil; -import org.jabref.model.ai.tokenization.TokenEstimationStrategy; +import org.jabref.model.ai.tokenization.TokenEstimatorKind; import dev.langchain4j.data.message.ChatMessage; @@ -33,7 +33,7 @@ private int calculate(String content) { } @Override - public TokenEstimationStrategy getEstimationStrategy() { + public TokenEstimatorKind getKind() { return null; } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByWordsTokenizer.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByWordsTokenizer.java index 704de3d10c8e..9a535357c183 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByWordsTokenizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByWordsTokenizer.java @@ -5,7 +5,7 @@ import java.util.stream.Collectors; import org.jabref.logic.ai.util.ChatMessagesUtil; -import org.jabref.model.ai.tokenization.TokenEstimationStrategy; +import org.jabref.model.ai.tokenization.TokenEstimatorKind; import dev.langchain4j.data.message.ChatMessage; @@ -33,7 +33,7 @@ private int calculate(String content) { } @Override - public TokenEstimationStrategy getEstimationStrategy() { + public TokenEstimatorKind getKind() { return null; } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MaximumTokenizer.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MaximumTokenizer.java index c59d736161a4..4e7bef244464 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MaximumTokenizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MaximumTokenizer.java @@ -2,7 +2,7 @@ import java.util.List; -import org.jabref.model.ai.tokenization.TokenEstimationStrategy; +import org.jabref.model.ai.tokenization.TokenEstimatorKind; import dev.langchain4j.data.message.ChatMessage; @@ -31,7 +31,7 @@ private int calculate(int byWords, int byCharacters) { } @Override - public TokenEstimationStrategy getEstimationStrategy() { - return TokenEstimationStrategy.MAX; + public TokenEstimatorKind getKind() { + return TokenEstimatorKind.MAX; } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MinimumTokenizer.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MinimumTokenizer.java index 5360292a9626..71a84a55934f 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MinimumTokenizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MinimumTokenizer.java @@ -2,7 +2,7 @@ import java.util.List; -import org.jabref.model.ai.tokenization.TokenEstimationStrategy; +import org.jabref.model.ai.tokenization.TokenEstimatorKind; import dev.langchain4j.data.message.ChatMessage; @@ -31,7 +31,7 @@ private int calculate(int byWords, int byCharacters) { } @Override - public TokenEstimationStrategy getEstimationStrategy() { - return TokenEstimationStrategy.MIN; + public TokenEstimatorKind getKind() { + return TokenEstimatorKind.MIN; } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/Tokenizer.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/Tokenizer.java index 7f38e328df54..194047208dfb 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/Tokenizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/Tokenizer.java @@ -2,7 +2,7 @@ import java.util.List; -import org.jabref.model.ai.tokenization.TokenEstimationStrategy; +import org.jabref.model.ai.tokenization.TokenEstimatorKind; import dev.langchain4j.data.message.ChatMessage; @@ -11,5 +11,5 @@ public interface Tokenizer { int estimate(List messages); - TokenEstimationStrategy getEstimationStrategy(); + TokenEstimatorKind getKind(); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/IngestionService.java similarity index 87% rename from jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java rename to jablib/src/main/java/org/jabref/logic/ai/pipeline/IngestionService.java index 73159ba1f762..bf149764e4d9 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/IngestionService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/IngestionService.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.rag; +package org.jabref.logic.ai.pipeline; import java.util.ArrayList; import java.util.Comparator; @@ -9,12 +9,12 @@ import javafx.beans.property.StringProperty; import org.jabref.logic.FilePreferences; +import org.jabref.logic.ai.pipeline.logic.EmbeddingsCleaner; +import org.jabref.logic.ai.pipeline.logic.documentsplitting.DocumentSplitter; +import org.jabref.logic.ai.pipeline.repositories.IngestedDocumentsRepository; +import org.jabref.logic.ai.pipeline.tasks.GenerateEmbeddingsForSeveralTask; +import org.jabref.logic.ai.pipeline.tasks.GenerateEmbeddingsTask; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.rag.logic.EmbeddingsCleaner; -import org.jabref.logic.ai.rag.logic.documentsplitting.DocumentSplitterAlgorithm; -import org.jabref.logic.ai.rag.repositories.FullyIngestedDocumentsRepository; -import org.jabref.logic.ai.rag.tasks.GenerateEmbeddingsForSeveralTask; -import org.jabref.logic.ai.rag.tasks.GenerateEmbeddingsTask; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.ai.processingstatus.ProcessingInfo; import org.jabref.model.ai.processingstatus.ProcessingState; @@ -45,8 +45,8 @@ public class IngestionService { private final EmbeddingModel embeddingModel; private final EmbeddingStore embeddingStore; - private final DocumentSplitterAlgorithm documentSplitterAlgorithm; - private final FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository; + private final DocumentSplitter documentSplitter; + private final IngestedDocumentsRepository ingestedDocumentsRepository; private final EmbeddingsCleaner embeddingsCleaner; private final ReadOnlyBooleanProperty shutdownSignal; @@ -57,8 +57,8 @@ public IngestionService( TaskExecutor taskExecutor, EmbeddingModel embeddingModel, EmbeddingStore embeddingStore, - DocumentSplitterAlgorithm documentSplitterAlgorithm, - FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository, + DocumentSplitter documentSplitter, + IngestedDocumentsRepository ingestedDocumentsRepository, ReadOnlyBooleanProperty shutdownSignal ) { this.aiPreferences = aiPreferences; @@ -67,12 +67,12 @@ public IngestionService( this.embeddingModel = embeddingModel; this.embeddingStore = embeddingStore; - this.documentSplitterAlgorithm = documentSplitterAlgorithm; - this.fullyIngestedDocumentsRepository = fullyIngestedDocumentsRepository; + this.documentSplitter = documentSplitter; + this.ingestedDocumentsRepository = ingestedDocumentsRepository; this.embeddingsCleaner = new EmbeddingsCleaner( aiPreferences, embeddingStore, - fullyIngestedDocumentsRepository + ingestedDocumentsRepository ); this.shutdownSignal = shutdownSignal; @@ -165,10 +165,10 @@ private void startEmbeddingsGenerationTask( new GenerateEmbeddingsTask( filePreferences, - fullyIngestedDocumentsRepository, + ingestedDocumentsRepository, embeddingStore, embeddingModel, - documentSplitterAlgorithm, + documentSplitter, bibDatabaseContext, linkedFile, shutdownSignal @@ -184,10 +184,10 @@ private void startEmbeddingsGenerationTask(StringProperty groupName, List embeddingStore; - private final FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository; + private final IngestedDocumentsRepository ingestedDocumentsRepository; public EmbeddingsCleaner( AiPreferences aiPreferences, EmbeddingStore embeddingStore, - FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository + IngestedDocumentsRepository ingestedDocumentsRepository ) { this.aiPreferences = aiPreferences; this.embeddingStore = embeddingStore; - this.fullyIngestedDocumentsRepository = fullyIngestedDocumentsRepository; + this.ingestedDocumentsRepository = ingestedDocumentsRepository; setupListeningToPreferencesChanges(); } @@ -36,7 +36,7 @@ private void setupListeningToPreferencesChanges() { public void removeDocument(String link) { embeddingStore.removeAll(MetadataFilterBuilder.metadataKey(LINK_METADATA_KEY).isEqualTo(link)); - fullyIngestedDocumentsRepository.unmarkDocumentAsFullyIngested(link); + ingestedDocumentsRepository.unmarkDocumentAsFullyIngested(link); } public void clearEmbeddingsFor(List linkedFiles) { diff --git a/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/documentsplitting/DocumentSplitter.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/documentsplitting/DocumentSplitter.java new file mode 100644 index 000000000000..4d1d787b57cf --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/documentsplitting/DocumentSplitter.java @@ -0,0 +1,12 @@ +package org.jabref.logic.ai.pipeline.logic.documentsplitting; + +import java.util.stream.Stream; + +import org.jabref.logic.ai.util.LongTaskInfo; +import org.jabref.model.ai.rag.DocumentSplitterKind; + +public interface DocumentSplitter { + Stream split(LongTaskInfo longTaskInfo, String text) throws InterruptedException; + + DocumentSplitterKind getKind(); +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/documentsplitting/SlidingWindowDocumentSplitter.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/documentsplitting/SlidingWindowDocumentSplitter.java similarity index 72% rename from jablib/src/main/java/org/jabref/logic/ai/rag/logic/documentsplitting/SlidingWindowDocumentSplitter.java rename to jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/documentsplitting/SlidingWindowDocumentSplitter.java index 60fcfecc953b..8635e4fadbe6 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/documentsplitting/SlidingWindowDocumentSplitter.java +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/documentsplitting/SlidingWindowDocumentSplitter.java @@ -1,17 +1,16 @@ -package org.jabref.logic.ai.rag.logic.documentsplitting; +package org.jabref.logic.ai.pipeline.logic.documentsplitting; import java.util.stream.Stream; import org.jabref.logic.ai.util.LongTaskInfo; -import org.jabref.model.ai.rag.DocumentSplittingStrategy; +import org.jabref.model.ai.rag.DocumentSplitterKind; import dev.langchain4j.data.document.DefaultDocument; -import dev.langchain4j.data.document.DocumentSplitter; import dev.langchain4j.data.document.splitter.DocumentSplitters; import dev.langchain4j.data.segment.TextSegment; -public class SlidingWindowDocumentSplitter implements DocumentSplitterAlgorithm { - private final DocumentSplitter langchainDocumentSplitter; +public class SlidingWindowDocumentSplitter implements DocumentSplitter { + private final dev.langchain4j.data.document.DocumentSplitter langchainDocumentSplitter; public SlidingWindowDocumentSplitter(int chunkSize, int chunkOverlap) { this.langchainDocumentSplitter = DocumentSplitters.recursive( @@ -30,7 +29,7 @@ public Stream split(LongTaskInfo longTaskInfo, String text) throws Inter } @Override - public DocumentSplittingStrategy getStrategy() { - return DocumentSplittingStrategy.SLIDING_WINDOW; + public DocumentSplitterKind getKind() { + return DocumentSplitterKind.SLIDING_WINDOW; } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/FileIngestor.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/ingestion/FileIngestor.java similarity index 80% rename from jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/FileIngestor.java rename to jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/ingestion/FileIngestor.java index 9d8552355101..b0af662d7bff 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/FileIngestor.java +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/ingestion/FileIngestor.java @@ -1,10 +1,10 @@ -package org.jabref.logic.ai.rag.logic.ingestion; +package org.jabref.logic.ai.pipeline.logic.ingestion; import java.nio.file.Path; import java.util.Optional; -import org.jabref.logic.ai.rag.logic.documentsplitting.DocumentSplitterAlgorithm; -import org.jabref.logic.ai.rag.logic.parsing.UniversalFileParser; +import org.jabref.logic.ai.pipeline.logic.documentsplitting.DocumentSplitter; +import org.jabref.logic.ai.pipeline.logic.parsing.UniversalContentParser; import org.jabref.logic.ai.util.LongTaskInfo; import org.jabref.logic.l10n.Localization; @@ -18,19 +18,19 @@ public class FileIngestor { private static final Logger LOGGER = LoggerFactory.getLogger(FileIngestor.class); - private final UniversalFileParser universalFileParser; + private final UniversalContentParser universalFileParser; private final TextIngestor textIngestor; public FileIngestor( EmbeddingStore embeddingStore, EmbeddingModel embeddingModel, - DocumentSplitterAlgorithm documentSplitterAlgorithm + DocumentSplitter documentSplitter ) { - this.universalFileParser = new UniversalFileParser(); + this.universalFileParser = new UniversalContentParser(); this.textIngestor = new TextIngestor( embeddingStore, embeddingModel, - documentSplitterAlgorithm + documentSplitter ); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/LinkedFileIngestor.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/ingestion/LinkedFileIngestor.java similarity index 77% rename from jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/LinkedFileIngestor.java rename to jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/ingestion/LinkedFileIngestor.java index 1c97fdbd4d84..341ef92574e0 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/LinkedFileIngestor.java +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/ingestion/LinkedFileIngestor.java @@ -1,9 +1,9 @@ -package org.jabref.logic.ai.rag.logic.ingestion; +package org.jabref.logic.ai.pipeline.logic.ingestion; import java.nio.file.Path; -import org.jabref.logic.ai.rag.logic.EmbeddingsCleaner; -import org.jabref.logic.ai.rag.logic.documentsplitting.DocumentSplitterAlgorithm; +import org.jabref.logic.ai.pipeline.logic.EmbeddingsCleaner; +import org.jabref.logic.ai.pipeline.logic.documentsplitting.DocumentSplitter; import org.jabref.logic.ai.util.LongTaskInfo; import org.jabref.model.entry.LinkedFile; @@ -18,12 +18,12 @@ public class LinkedFileIngestor { public LinkedFileIngestor( EmbeddingStore embeddingStore, EmbeddingModel embeddingModel, - DocumentSplitterAlgorithm documentSplitterAlgorithm + DocumentSplitter documentSplitter ) { this.fileIngestor = new FileIngestor( embeddingStore, embeddingModel, - documentSplitterAlgorithm + documentSplitter ); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/PersistentLinkedFileIngestor.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/ingestion/PersistentLinkedFileIngestor.java similarity index 81% rename from jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/PersistentLinkedFileIngestor.java rename to jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/ingestion/PersistentLinkedFileIngestor.java index fe8d92d07b21..e94ed887f70d 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/PersistentLinkedFileIngestor.java +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/ingestion/PersistentLinkedFileIngestor.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.rag.logic.ingestion; +package org.jabref.logic.ai.pipeline.logic.ingestion; import java.io.IOException; import java.nio.file.Files; @@ -8,8 +8,8 @@ import java.util.concurrent.TimeUnit; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.rag.logic.documentsplitting.DocumentSplitterAlgorithm; -import org.jabref.logic.ai.rag.repositories.FullyIngestedDocumentsRepository; +import org.jabref.logic.ai.pipeline.logic.documentsplitting.DocumentSplitter; +import org.jabref.logic.ai.pipeline.repositories.IngestedDocumentsRepository; import org.jabref.logic.ai.util.LongTaskInfo; import org.jabref.logic.l10n.Localization; import org.jabref.model.database.BibDatabaseContext; @@ -25,22 +25,22 @@ public class PersistentLinkedFileIngestor { private static final Logger LOGGER = LoggerFactory.getLogger(PersistentLinkedFileIngestor.class); private final FilePreferences filePreferences; - private final FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository; + private final IngestedDocumentsRepository ingestedDocumentsRepository; private final LinkedFileIngestor linkedFileIngestor; public PersistentLinkedFileIngestor( FilePreferences filePreferences, - FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository, + IngestedDocumentsRepository ingestedDocumentsRepository, EmbeddingStore embeddingStore, EmbeddingModel embeddingModel, - DocumentSplitterAlgorithm documentSplitterAlgorithm + DocumentSplitter documentSplitter ) { this.filePreferences = filePreferences; - this.fullyIngestedDocumentsRepository = fullyIngestedDocumentsRepository; + this.ingestedDocumentsRepository = ingestedDocumentsRepository; this.linkedFileIngestor = new LinkedFileIngestor( embeddingStore, embeddingModel, - documentSplitterAlgorithm + documentSplitter ); } @@ -71,7 +71,7 @@ public void ingest( long currentModificationTimeInSeconds = attributes.lastModifiedTime().to(TimeUnit.SECONDS); - Optional ingestedModificationTimeInSeconds = fullyIngestedDocumentsRepository.getIngestedDocumentModificationTimeInSeconds(linkedFile.getLink()); + Optional ingestedModificationTimeInSeconds = ingestedDocumentsRepository.getIngestedDocumentModificationTimeInSeconds(linkedFile.getLink()); if (ingestedModificationTimeInSeconds.isEmpty()) { modTime = Optional.of(currentModificationTimeInSeconds); @@ -95,7 +95,7 @@ public void ingest( linkedFileIngestor.ingest(longTaskInfo, linkedFile, path.get()); if (!longTaskInfo.shutdownSignal().get()) { - fullyIngestedDocumentsRepository.markDocumentAsFullyIngested(linkedFile.getLink(), modTime.orElse(0L)); + ingestedDocumentsRepository.markDocumentAsFullyIngested(linkedFile.getLink(), modTime.orElse(0L)); } } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/TextIngestor.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/ingestion/TextIngestor.java similarity index 78% rename from jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/TextIngestor.java rename to jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/ingestion/TextIngestor.java index c219eff8edd7..402b6e0ee231 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/ingestion/TextIngestor.java +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/ingestion/TextIngestor.java @@ -1,8 +1,8 @@ -package org.jabref.logic.ai.rag.logic.ingestion; +package org.jabref.logic.ai.pipeline.logic.ingestion; import java.util.List; -import org.jabref.logic.ai.rag.logic.documentsplitting.DocumentSplitterAlgorithm; +import org.jabref.logic.ai.pipeline.logic.documentsplitting.DocumentSplitter; import org.jabref.logic.ai.util.LongTaskInfo; import dev.langchain4j.data.document.DefaultDocument; @@ -14,12 +14,12 @@ public class TextIngestor { private final EmbeddingStoreIngestor ingestor; - private final DocumentSplitterAlgorithm documentSplitterAlgorithm; + private final DocumentSplitter documentSplitter; public TextIngestor( EmbeddingStore embeddingStore, EmbeddingModel embeddingModel, - DocumentSplitterAlgorithm documentSplitterAlgorithm + DocumentSplitter documentSplitter ) { this.ingestor = EmbeddingStoreIngestor .builder() @@ -29,7 +29,7 @@ public TextIngestor( .documentSplitter(document -> List.of(new TextSegment(document.text(), document.metadata()))) .build(); - this.documentSplitterAlgorithm = documentSplitterAlgorithm; + this.documentSplitter = documentSplitter; } public void ingest( @@ -37,7 +37,7 @@ public void ingest( Metadata metadata, String text ) throws InterruptedException { - List chunks = documentSplitterAlgorithm.split(longTaskInfo, text).toList(); + List chunks = documentSplitter.split(longTaskInfo, text).toList(); longTaskInfo.progressCounter().increaseWorkMax(chunks.size()); for (String documentPart : chunks) { diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/FileParser.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/parsing/FileContentParser.java similarity index 65% rename from jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/FileParser.java rename to jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/parsing/FileContentParser.java index 05367d2710d4..f472260227ac 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/FileParser.java +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/parsing/FileContentParser.java @@ -1,10 +1,10 @@ -package org.jabref.logic.ai.rag.logic.parsing; +package org.jabref.logic.ai.pipeline.logic.parsing; import java.nio.file.Path; import java.util.Optional; import org.jabref.logic.ai.util.LongTaskInfo; -public interface FileParser { +public interface FileContentParser { Optional parse(LongTaskInfo longTaskInfo, Path path); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/PdfFileParser.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/parsing/PdfContentParser.java similarity index 90% rename from jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/PdfFileParser.java rename to jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/parsing/PdfContentParser.java index 55d3c84fdcf9..7a9855503e09 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/PdfFileParser.java +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/parsing/PdfContentParser.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.rag.logic.parsing; +package org.jabref.logic.ai.pipeline.logic.parsing; import java.io.IOException; import java.io.StringWriter; @@ -13,8 +13,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class PdfFileParser implements FileParser { - private static final Logger LOGGER = LoggerFactory.getLogger(PdfFileParser.class); +public class PdfContentParser implements FileContentParser { + private static final Logger LOGGER = LoggerFactory.getLogger(PdfContentParser.class); @Override public Optional parse(LongTaskInfo longTaskInfo, Path path) { diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/UniversalFileParser.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/parsing/UniversalContentParser.java similarity index 73% rename from jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/UniversalFileParser.java rename to jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/parsing/UniversalContentParser.java index 9b30b1b94251..63ff0e15b3d2 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/parsing/UniversalFileParser.java +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/parsing/UniversalContentParser.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.rag.logic.parsing; +package org.jabref.logic.ai.pipeline.logic.parsing; import java.nio.file.Path; import java.util.Optional; @@ -9,10 +9,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class UniversalFileParser implements FileParser { - private static final Logger LOGGER = LoggerFactory.getLogger(UniversalFileParser.class); +public class UniversalContentParser implements FileContentParser { + private static final Logger LOGGER = LoggerFactory.getLogger(UniversalContentParser.class); - private final PdfFileParser pdfFileParser = new PdfFileParser(); + private final PdfContentParser pdfFileParser = new PdfContentParser(); public Optional parse(LongTaskInfo longTaskInfo, Path path) { if (FileUtil.isPDFFile(path)) { diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FullyIngestedDocumentsRepository.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/repositories/IngestedDocumentsRepository.java similarity index 78% rename from jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FullyIngestedDocumentsRepository.java rename to jablib/src/main/java/org/jabref/logic/ai/pipeline/repositories/IngestedDocumentsRepository.java index bd0423a5ea5d..6ebc66907194 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/FullyIngestedDocumentsRepository.java +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/repositories/IngestedDocumentsRepository.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.rag.repositories; +package org.jabref.logic.ai.pipeline.repositories; import java.util.Optional; @@ -7,7 +7,7 @@ *

* The class also records the document modification time. */ -public interface FullyIngestedDocumentsRepository extends AutoCloseable{ +public interface IngestedDocumentsRepository extends AutoCloseable{ void markDocumentAsFullyIngested(String link, long modificationTimeInSeconds); Optional getIngestedDocumentModificationTimeInSeconds(String link); diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/MVStoreFullyIngestedDocumentsRepository.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/repositories/MVStoreIngestedDocumentsRepository.java similarity index 90% rename from jablib/src/main/java/org/jabref/logic/ai/rag/repositories/MVStoreFullyIngestedDocumentsRepository.java rename to jablib/src/main/java/org/jabref/logic/ai/pipeline/repositories/MVStoreIngestedDocumentsRepository.java index ced1d3c9253e..2c016f58bd4d 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/repositories/MVStoreFullyIngestedDocumentsRepository.java +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/repositories/MVStoreIngestedDocumentsRepository.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.rag.repositories; +package org.jabref.logic.ai.pipeline.repositories; import java.nio.file.Path; import java.util.Map; @@ -13,7 +13,7 @@ *

* The class also records the document modification time. */ -public class MVStoreFullyIngestedDocumentsRepository extends MVStoreBase implements FullyIngestedDocumentsRepository { +public class MVStoreIngestedDocumentsRepository extends MVStoreBase implements IngestedDocumentsRepository { private static final String INGESTED_MAP_NAME = "ingested"; // This map stores the ingested documents. The key is LinkedDocument.getLink(), and the value is the modification time in seconds. @@ -23,7 +23,7 @@ public class MVStoreFullyIngestedDocumentsRepository extends MVStoreBase impleme // it doesn't mean the document is fully ingested. private final Map ingestedMap; - public MVStoreFullyIngestedDocumentsRepository( + public MVStoreIngestedDocumentsRepository( NotificationService dialogService, Path path ) { diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsForSeveralTask.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/tasks/GenerateEmbeddingsForSeveralTask.java similarity index 86% rename from jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsForSeveralTask.java rename to jablib/src/main/java/org/jabref/logic/ai/pipeline/tasks/GenerateEmbeddingsForSeveralTask.java index 9b18cac80497..0afa721d22b1 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsForSeveralTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/tasks/GenerateEmbeddingsForSeveralTask.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.rag.tasks; +package org.jabref.logic.ai.pipeline.tasks; import java.util.ArrayList; import java.util.List; @@ -10,8 +10,8 @@ import javafx.util.Pair; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.rag.logic.documentsplitting.DocumentSplitterAlgorithm; -import org.jabref.logic.ai.rag.repositories.FullyIngestedDocumentsRepository; +import org.jabref.logic.ai.pipeline.logic.documentsplitting.DocumentSplitter; +import org.jabref.logic.ai.pipeline.repositories.IngestedDocumentsRepository; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.ProgressCounter; @@ -36,10 +36,10 @@ public class GenerateEmbeddingsForSeveralTask extends BackgroundTask { private static final Logger LOGGER = LoggerFactory.getLogger(GenerateEmbeddingsForSeveralTask.class); private final FilePreferences filePreferences; - private final FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository; + private final IngestedDocumentsRepository ingestedDocumentsRepository; private final EmbeddingStore embeddingStore; private final EmbeddingModel embeddingModel; - private final DocumentSplitterAlgorithm documentSplitterAlgorithm; + private final DocumentSplitter documentSplitter; private final BibDatabaseContext bibDatabaseContext; private final StringProperty groupName; private final List> linkedFiles; @@ -51,10 +51,10 @@ public class GenerateEmbeddingsForSeveralTask extends BackgroundTask { public GenerateEmbeddingsForSeveralTask( FilePreferences filePreferences, - FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository, + IngestedDocumentsRepository ingestedDocumentsRepository, EmbeddingStore embeddingStore, EmbeddingModel embeddingModel, - DocumentSplitterAlgorithm documentSplitterAlgorithm, + DocumentSplitter documentSplitter, BibDatabaseContext bibDatabaseContext, StringProperty groupName, List> linkedFiles, @@ -62,10 +62,10 @@ public GenerateEmbeddingsForSeveralTask( TaskExecutor taskExecutor ) { this.filePreferences = filePreferences; - this.fullyIngestedDocumentsRepository = fullyIngestedDocumentsRepository; + this.ingestedDocumentsRepository = ingestedDocumentsRepository; this.embeddingStore = embeddingStore; this.embeddingModel = embeddingModel; - this.documentSplitterAlgorithm = documentSplitterAlgorithm; + this.documentSplitter = documentSplitter; this.bibDatabaseContext = bibDatabaseContext; this.groupName = groupName; this.linkedFiles = linkedFiles; @@ -99,10 +99,10 @@ public Void call() throws ExecutionException, InterruptedException { return new Pair<>( new GenerateEmbeddingsTask( filePreferences, - fullyIngestedDocumentsRepository, + ingestedDocumentsRepository, embeddingStore, embeddingModel, - documentSplitterAlgorithm, + documentSplitter, bibDatabaseContext, processingInfo.getObject(), shutdownSignal diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/tasks/GenerateEmbeddingsTask.java similarity index 86% rename from jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java rename to jablib/src/main/java/org/jabref/logic/ai/pipeline/tasks/GenerateEmbeddingsTask.java index 9ad4b84f0b0a..3117b9806c8a 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/GenerateEmbeddingsTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/tasks/GenerateEmbeddingsTask.java @@ -1,11 +1,11 @@ -package org.jabref.logic.ai.rag.tasks; +package org.jabref.logic.ai.pipeline.tasks; import javafx.beans.property.ReadOnlyBooleanProperty; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.rag.logic.documentsplitting.DocumentSplitterAlgorithm; -import org.jabref.logic.ai.rag.logic.ingestion.PersistentLinkedFileIngestor; -import org.jabref.logic.ai.rag.repositories.FullyIngestedDocumentsRepository; +import org.jabref.logic.ai.pipeline.logic.documentsplitting.DocumentSplitter; +import org.jabref.logic.ai.pipeline.logic.ingestion.PersistentLinkedFileIngestor; +import org.jabref.logic.ai.pipeline.repositories.IngestedDocumentsRepository; import org.jabref.logic.ai.util.LongTaskInfo; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; @@ -35,10 +35,10 @@ public class GenerateEmbeddingsTask extends BackgroundTask { public GenerateEmbeddingsTask( FilePreferences filePreferences, - FullyIngestedDocumentsRepository fullyIngestedDocumentsRepository, + IngestedDocumentsRepository ingestedDocumentsRepository, EmbeddingStore embeddingStore, EmbeddingModel embeddingModel, - DocumentSplitterAlgorithm documentSplitterAlgorithm, + DocumentSplitter documentSplitter, BibDatabaseContext bibDatabaseContext, LinkedFile linkedFile, ReadOnlyBooleanProperty shutdownSignal @@ -50,10 +50,10 @@ public GenerateEmbeddingsTask( this.persistentLinkedFileIngestor = new PersistentLinkedFileIngestor( filePreferences, - fullyIngestedDocumentsRepository, + ingestedDocumentsRepository, embeddingStore, embeddingModel, - documentSplitterAlgorithm + documentSplitter ); configure(); diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/UpdateEmbeddingModelTask.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/tasks/UpdateEmbeddingModelTask.java similarity index 98% rename from jablib/src/main/java/org/jabref/logic/ai/rag/tasks/UpdateEmbeddingModelTask.java rename to jablib/src/main/java/org/jabref/logic/ai/pipeline/tasks/UpdateEmbeddingModelTask.java index be3c7519ae44..fc78a3454de0 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/tasks/UpdateEmbeddingModelTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/tasks/UpdateEmbeddingModelTask.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.rag.tasks; +package org.jabref.logic.ai.pipeline.tasks; import java.io.IOException; import java.util.Optional; diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultExpertSettings.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultExpertSettings.java index c2db723fa305..b2726cc6b696 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultExpertSettings.java +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultExpertSettings.java @@ -1,21 +1,21 @@ package org.jabref.logic.ai.preferences; -import org.jabref.model.ai.embeddings.EmbeddingModel; -import org.jabref.model.ai.rag.DocumentSplittingStrategy; -import org.jabref.model.ai.summarization.SummarizationAlgorithmName; -import org.jabref.model.ai.tokenization.TokenEstimationStrategy; +import org.jabref.model.ai.embeddings.EmbeddingModelEnumeration; +import org.jabref.model.ai.rag.DocumentSplitterKind; +import org.jabref.model.ai.summarization.SummarizatorKind; +import org.jabref.model.ai.tokenization.TokenEstimatorKind; /// A collection of values for the default settings of AI in the expert section. /// /// This collection was made because "Expert settings" in the AI settings is resettable. /// There are facilities in JabRef codebase to reset either all settings or 1 section, but not a part of a section. public class AiDefaultExpertSettings { - public static final EmbeddingModel EMBEDDING_MODEL = EmbeddingModel.SENTENCE_TRANSFORMERS_ALL_MINILM_L12_V2; - public static final SummarizationAlgorithmName SUMMARIZATION_ALGORITHM_NAME = SummarizationAlgorithmName.CHUNKED; - public static final TokenEstimationStrategy TOKEN_ESTIMATION_STRATEGY = TokenEstimationStrategy.MAX; + public static final EmbeddingModelEnumeration EMBEDDING_MODEL = EmbeddingModelEnumeration.SENTENCE_TRANSFORMERS_ALL_MINILM_L12_V2; + public static final SummarizatorKind SUMMARIZATION_ALGORITHM_NAME = SummarizatorKind.CHUNKED; + public static final TokenEstimatorKind TOKEN_ESTIMATION_STRATEGY = TokenEstimatorKind.MAX; public static final float TEMPERATURE = 0.7f; public static final int CONTEXT_WINDOW_SIZE = 8192; - public static final DocumentSplittingStrategy DOCUMENT_SPLITTING_STRATEGY = DocumentSplittingStrategy.SLIDING_WINDOW; + public static final DocumentSplitterKind DOCUMENT_SPLITTING_STRATEGY = DocumentSplitterKind.SLIDING_WINDOW; public static final int DOCUMENT_SPLITTER_CHUNK_SIZE = 300; public static final int DOCUMENT_SPLITTER_OVERLAP_SIZE = 100; public static final int RAG_MAX_RESULTS_COUNT = 10; diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java index 3dad367ec079..67c8f0be1dc8 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java @@ -18,11 +18,11 @@ import org.jabref.logic.util.strings.StringUtil; import org.jabref.model.ai.chatting.AiProvider; -import org.jabref.model.ai.embeddings.EmbeddingModel; -import org.jabref.model.ai.rag.DocumentSplittingStrategy; -import org.jabref.model.ai.summarization.SummarizationAlgorithmName; +import org.jabref.model.ai.embeddings.EmbeddingModelEnumeration; +import org.jabref.model.ai.rag.DocumentSplitterKind; +import org.jabref.model.ai.summarization.SummarizatorKind; import org.jabref.model.ai.templating.AiTemplate; -import org.jabref.model.ai.tokenization.TokenEstimationStrategy; +import org.jabref.model.ai.tokenization.TokenEstimatorKind; import com.github.javakeyring.Keyring; import com.github.javakeyring.PasswordAccessException; @@ -55,13 +55,13 @@ public class AiPreferences { private final StringProperty huggingFaceApiBaseUrl; private final StringProperty gpt4AllApiBaseUrl; - private final ObjectProperty defaultSummarizationAlgorithm; - private final ObjectProperty tokenEstimationStrategy; - private final ObjectProperty embeddingModel; + private final ObjectProperty defaultSummarizationAlgorithm; + private final ObjectProperty tokenEstimationStrategy; + private final ObjectProperty embeddingModel; private final DoubleProperty temperature; private final IntegerProperty contextWindowSize; - private final ObjectProperty documentSplittingStrategy; + private final ObjectProperty documentSplittingStrategy; private final IntegerProperty documentSplitterChunkSize; private final IntegerProperty documentSplitterOverlapSize; @@ -88,12 +88,12 @@ public AiPreferences( String geminiApiBaseUrl, String huggingFaceApiBaseUrl, String gpt4AllApiBaseUrl, - SummarizationAlgorithmName defaultSummarizationAlgorithm, - TokenEstimationStrategy tokenEstimationStrategy, - EmbeddingModel embeddingModel, + SummarizatorKind defaultSummarizationAlgorithm, + TokenEstimatorKind tokenEstimatorKind, + EmbeddingModelEnumeration embeddingModel, double temperature, int contextWindowSize, - DocumentSplittingStrategy documentSplittingStrategy, + DocumentSplitterKind documentSplitterKind, int documentSplitterChunkSize, int documentSplitterOverlapSize, int ragMaxResultsCount, @@ -121,12 +121,12 @@ public AiPreferences( this.gpt4AllApiBaseUrl = new SimpleStringProperty(gpt4AllApiBaseUrl); this.defaultSummarizationAlgorithm = new SimpleObjectProperty<>(defaultSummarizationAlgorithm); - this.tokenEstimationStrategy = new SimpleObjectProperty<>(tokenEstimationStrategy); + this.tokenEstimationStrategy = new SimpleObjectProperty<>(tokenEstimatorKind); this.embeddingModel = new SimpleObjectProperty<>(embeddingModel); this.temperature = new SimpleDoubleProperty(temperature); this.contextWindowSize = new SimpleIntegerProperty(contextWindowSize); - this.documentSplittingStrategy = new SimpleObjectProperty<>(documentSplittingStrategy); + this.documentSplittingStrategy = new SimpleObjectProperty<>(documentSplitterKind); this.documentSplitterChunkSize = new SimpleIntegerProperty(documentSplitterChunkSize); this.documentSplitterOverlapSize = new SimpleIntegerProperty(documentSplitterOverlapSize); @@ -296,35 +296,35 @@ public void setCustomizeExpertSettings(boolean customizeExpertSettings) { this.customizeExpertSettings.set(customizeExpertSettings); } - public ObjectProperty defaultSummarizationAlgorithmProperty() { + public ObjectProperty defaultSummarizationAlgorithmProperty() { return defaultSummarizationAlgorithm; } - public SummarizationAlgorithmName getDefaultSummarizationAlgorithm() { + public SummarizatorKind getDefaultSummarizationAlgorithm() { return defaultSummarizationAlgorithm.get(); } - public void setDefaultSummarizationAlgorithm(SummarizationAlgorithmName defaultSummarizationAlgorithm) { + public void setDefaultSummarizationAlgorithm(SummarizatorKind defaultSummarizationAlgorithm) { this.defaultSummarizationAlgorithm.set(defaultSummarizationAlgorithm); } - public ObjectProperty tokenEstimationStrategyProperty() { + public ObjectProperty tokenEstimationStrategyProperty() { return tokenEstimationStrategy; } - public TokenEstimationStrategy getTokenEstimationStrategy() { + public TokenEstimatorKind getTokenEstimationStrategy() { return tokenEstimationStrategy.get(); } - public void setTokenEstimationStrategy(TokenEstimationStrategy tokenEstimationStrategy) { - this.tokenEstimationStrategy.set(tokenEstimationStrategy); + public void setTokenEstimationStrategy(TokenEstimatorKind tokenEstimatorKind) { + this.tokenEstimationStrategy.set(tokenEstimatorKind); } - public ObjectProperty embeddingModelProperty() { + public ObjectProperty embeddingModelProperty() { return embeddingModel; } - public EmbeddingModel getEmbeddingModel() { + public EmbeddingModelEnumeration getEmbeddingModel() { if (getCustomizeExpertSettings()) { return embeddingModel.get(); } else { @@ -332,7 +332,7 @@ public EmbeddingModel getEmbeddingModel() { } } - public void setEmbeddingModel(EmbeddingModel embeddingModel) { + public void setEmbeddingModel(EmbeddingModelEnumeration embeddingModel) { this.embeddingModel.set(embeddingModel); } @@ -439,16 +439,16 @@ public void setContextWindowSize(int contextWindowSize) { this.contextWindowSize.set(contextWindowSize); } - public ObjectProperty documentSplittingStrategyProperty() { + public ObjectProperty documentSplittingStrategyProperty() { return documentSplittingStrategy; } - public DocumentSplittingStrategy getDocumentSplittingStrategy() { + public DocumentSplitterKind getDocumentSplittingStrategy() { return documentSplittingStrategy.get(); } - public void setDocumentSplittingStrategy(DocumentSplittingStrategy documentSplittingStrategy) { - this.documentSplittingStrategy.set(documentSplittingStrategy); + public void setDocumentSplittingStrategy(DocumentSplitterKind documentSplitterKind) { + this.documentSplittingStrategy.set(documentSplitterKind); } public IntegerProperty documentSplitterChunkSizeProperty() { diff --git a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/documentsplitting/DocumentSplitterAlgorithm.java b/jablib/src/main/java/org/jabref/logic/ai/rag/logic/documentsplitting/DocumentSplitterAlgorithm.java deleted file mode 100644 index 93d23ce48899..000000000000 --- a/jablib/src/main/java/org/jabref/logic/ai/rag/logic/documentsplitting/DocumentSplitterAlgorithm.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.jabref.logic.ai.rag.logic.documentsplitting; - -import java.util.stream.Stream; - -import org.jabref.logic.ai.util.LongTaskInfo; -import org.jabref.model.ai.rag.DocumentSplittingStrategy; - -public interface DocumentSplitterAlgorithm { - Stream split(LongTaskInfo longTaskInfo, String text) throws InterruptedException; - - DocumentSplittingStrategy getStrategy(); -} diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java index ec420ac943f5..749115ed1751 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java @@ -11,7 +11,7 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.customimplementations.llms.ChatModel; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; +import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.Summarizator; import org.jabref.logic.ai.summarization.repositories.SummariesRepository; import org.jabref.logic.ai.summarization.tasks.GenerateSummaryForSeveralTask; import org.jabref.logic.ai.summarization.tasks.GenerateSummaryTask; @@ -49,7 +49,7 @@ public class SummariesService { private final AiPreferences aiPreferences; private final SummariesRepository summariesRepository; private final ChatModel chatModel; - private final SummarizationAlgorithm defaultSummarizationAlgorithm; + private final Summarizator defaultSummarizator; private final BooleanProperty shutdownSignal; private final FilePreferences filePreferences; private final TaskExecutor taskExecutor; @@ -60,14 +60,14 @@ public SummariesService( FilePreferences filePreferences, TaskExecutor taskExecutor, ChatModel chatModel, - SummarizationAlgorithm defaultSummarizationAlgorithm, + Summarizator defaultSummarizator, SummariesRepository summariesRepository, BooleanProperty shutdownSignal ) { this.aiPreferences = aiPreferences; this.summariesRepository = summariesRepository; this.chatModel = chatModel; - this.defaultSummarizationAlgorithm = defaultSummarizationAlgorithm; + this.defaultSummarizator = defaultSummarizator; this.shutdownSignal = shutdownSignal; this.filePreferences = filePreferences; this.taskExecutor = taskExecutor; @@ -89,7 +89,7 @@ public EntriesChangedListener(BibDatabaseContext bibDatabaseContext) { public void listen(EntriesAddedEvent e) { e.getBibEntries().forEach(entry -> { if (aiPreferences.getAutoGenerateSummaries()) { - summarize(defaultSummarizationAlgorithm, entry, bibDatabaseContext); + summarize(defaultSummarizator, entry, bibDatabaseContext); } }); } @@ -97,7 +97,7 @@ public void listen(EntriesAddedEvent e) { @Subscribe public void listen(FieldChangedEvent e) { if (e.getField() == StandardField.FILE && aiPreferences.getAutoGenerateSummaries()) { - summarize(defaultSummarizationAlgorithm, e.getBibEntry(), bibDatabaseContext); + summarize(defaultSummarizator, e.getBibEntry(), bibDatabaseContext); } } } @@ -108,11 +108,11 @@ public void listen(FieldChangedEvent e) { * Returned {@link ProcessingInfo} is related to the passed {@link BibEntry}, so if you call this method twice * on the same {@link BibEntry}, the method will return the same {@link ProcessingInfo}. */ - public ProcessingInfo summarize(SummarizationAlgorithm summarizationAlgorithm, BibEntry bibEntry, BibDatabaseContext bibDatabaseContext) { + public ProcessingInfo summarize(Summarizator summarizator, BibEntry bibEntry, BibDatabaseContext bibDatabaseContext) { ProcessingInfo processingInfo = getProcessingInfo(bibEntry); if (processingInfo.getState() == ProcessingState.STOPPED) { - startSummarizationTask(summarizationAlgorithm, bibEntry, bibDatabaseContext, processingInfo); + startSummarizationTask(summarizator, bibEntry, bibDatabaseContext, processingInfo); } return processingInfo; @@ -127,7 +127,7 @@ public List> getProcessingInfo(List entries, BibDatabaseContext bibDatabaseContext @@ -142,7 +142,7 @@ public void summarize( List> needToProcess = result.stream().filter(processingInfo -> processingInfo.getState() == ProcessingState.STOPPED).toList(); startSummarizationTask( - summarizationAlgorithm, + summarizator, groupName, needToProcess, bibDatabaseContext @@ -150,7 +150,7 @@ public void summarize( } private void startSummarizationTask( - SummarizationAlgorithm summarizationAlgorithm, + Summarizator summarizator, BibEntry entry, BibDatabaseContext bibDatabaseContext, ProcessingInfo processingInfo @@ -161,7 +161,7 @@ private void startSummarizationTask( filePreferences, chatModel, summariesRepository, - summarizationAlgorithm, + summarizator, bibDatabaseContext, entry, shutdownSignal @@ -172,7 +172,7 @@ private void startSummarizationTask( } private void startSummarizationTask( - SummarizationAlgorithm summarizationAlgorithm, + Summarizator summarizator, StringProperty groupName, List> entries, BibDatabaseContext bibDatabaseContext @@ -184,7 +184,7 @@ private void startSummarizationTask( taskExecutor, chatModel, summariesRepository, - summarizationAlgorithm, + summarizator, bibDatabaseContext, groupName, entries, @@ -194,14 +194,14 @@ private void startSummarizationTask( } /** - * Method, similar to {@link #summarize(SummarizationAlgorithm, BibEntry, BibDatabaseContext)}, but it allows you to regenerate summary. + * Method, similar to {@link #summarize(Summarizator, BibEntry, BibDatabaseContext)}, but it allows you to regenerate summary. */ public void regenerateSummary( - SummarizationAlgorithm summarizationAlgorithm, + Summarizator summarizator, BibEntry bibEntry, BibDatabaseContext bibDatabaseContext ) { - ProcessingInfo processingInfo = summarize(summarizationAlgorithm, bibEntry, bibDatabaseContext); + ProcessingInfo processingInfo = summarize(summarizator, bibEntry, bibDatabaseContext); processingInfo.setState(ProcessingState.PROCESSING); if (bibDatabaseContext.getDatabasePath().isEmpty()) { @@ -212,6 +212,6 @@ public void regenerateSummary( summariesRepository.clear(bibDatabaseContext.getDatabasePath().get(), bibEntry.getCitationKey().get()); } - startSummarizationTask(summarizationAlgorithm, bibEntry, bibDatabaseContext, processingInfo); + startSummarizationTask(summarizator, bibEntry, bibDatabaseContext, processingInfo); } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/BibEntrySummarizer.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/BibEntrySummarizator.java similarity index 88% rename from jablib/src/main/java/org/jabref/logic/ai/summarization/logic/BibEntrySummarizer.java rename to jablib/src/main/java/org/jabref/logic/ai/summarization/logic/BibEntrySummarizator.java index a1b4deb6b8ce..d2b9624d1365 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/BibEntrySummarizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/BibEntrySummarizator.java @@ -10,8 +10,8 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.customimplementations.llms.ChatModel; -import org.jabref.logic.ai.rag.logic.parsing.UniversalFileParser; -import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; +import org.jabref.logic.ai.pipeline.logic.parsing.UniversalContentParser; +import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.Summarizator; import org.jabref.logic.ai.util.LongTaskInfo; import org.jabref.logic.l10n.Localization; import org.jabref.model.ai.summarization.BibEntrySummary; @@ -21,21 +21,21 @@ import org.slf4j.Logger; -public class BibEntrySummarizer { +public class BibEntrySummarizator { // TODO: Simplify this class. - private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(BibEntrySummarizer.class); + private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(BibEntrySummarizator.class); private final FilePreferences filePreferences; - private final SummarizationAlgorithm summarizationAlgorithm; - private final UniversalFileParser universalFileParser = new UniversalFileParser(); + private final Summarizator summarizator; + private final UniversalContentParser universalFileParser = new UniversalContentParser(); - public BibEntrySummarizer( + public BibEntrySummarizator( FilePreferences filePreferences, - SummarizationAlgorithm summarizationAlgorithm + Summarizator summarizator ) { this.filePreferences = filePreferences; - this.summarizationAlgorithm = summarizationAlgorithm; + this.summarizator = summarizator; } public BibEntrySummary summarize( @@ -90,7 +90,7 @@ public BibEntrySummary summarize( LocalDateTime.now(), chatModel.getAiProvider(), chatModel.getName(), - summarizationAlgorithm.getName(), + summarizator.getName(), finalSummary ); } @@ -120,7 +120,7 @@ private Optional generateSummary( return Optional.empty(); } - String linkedFileSummary = summarizationAlgorithm.summarize( + String linkedFileSummary = summarizator.summarize( chatModel, longTaskInfo, document.get() @@ -135,7 +135,7 @@ public String summarizeSeveralDocuments( LongTaskInfo longTaskInfo, Stream documents ) throws InterruptedException { - return summarizationAlgorithm.summarize( + return summarizator.summarize( chatModel, longTaskInfo, documents.collect(Collectors.joining("\n\n")) diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/PersistentBibEntrySummarizer.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/PersistentBibEntrySummarizator.java similarity index 87% rename from jablib/src/main/java/org/jabref/logic/ai/summarization/logic/PersistentBibEntrySummarizer.java rename to jablib/src/main/java/org/jabref/logic/ai/summarization/logic/PersistentBibEntrySummarizator.java index 207fef033e48..2ee779ee3431 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/PersistentBibEntrySummarizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/PersistentBibEntrySummarizator.java @@ -4,7 +4,7 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.customimplementations.llms.ChatModel; -import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; +import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.Summarizator; import org.jabref.logic.ai.summarization.repositories.SummariesRepository; import org.jabref.logic.ai.util.LongTaskInfo; import org.jabref.logic.util.CitationKeyCheck; @@ -15,23 +15,23 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class PersistentBibEntrySummarizer { - private static final Logger LOGGER = LoggerFactory.getLogger(PersistentBibEntrySummarizer.class); +public class PersistentBibEntrySummarizator { + private static final Logger LOGGER = LoggerFactory.getLogger(PersistentBibEntrySummarizator.class); private final SummariesRepository summariesRepository; - private final BibEntrySummarizer bibEntrySummarizer; + private final BibEntrySummarizator bibEntrySummarizator; - public PersistentBibEntrySummarizer( + public PersistentBibEntrySummarizator( FilePreferences filePreferences, SummariesRepository summariesRepository, - SummarizationAlgorithm summarizationAlgorithm + Summarizator summarizator ) { this.summariesRepository = summariesRepository; - this.bibEntrySummarizer = new BibEntrySummarizer( + this.bibEntrySummarizator = new BibEntrySummarizator( filePreferences, - summarizationAlgorithm + summarizator ); } @@ -60,7 +60,7 @@ public BibEntrySummary summarize( bibEntrySummary = savedSummary.get(); } else { try { - bibEntrySummary = bibEntrySummarizer.summarize( + bibEntrySummary = bibEntrySummarizator.summarize( chatModel, longTaskInfo, bibDatabaseContext, diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/ChunkedSummarizationAlgorithm.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/ChunkedSummarizator.java similarity index 94% rename from jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/ChunkedSummarizationAlgorithm.java rename to jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/ChunkedSummarizator.java index 8ead55def29b..001ea543e7cb 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/ChunkedSummarizationAlgorithm.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/ChunkedSummarizator.java @@ -9,7 +9,7 @@ import org.jabref.logic.ai.summarization.templates.SummarizationCombineSystemMessageTemplate; import org.jabref.logic.ai.summarization.templates.SummarizationCombineUserMessageTemplate; import org.jabref.logic.ai.util.LongTaskInfo; -import org.jabref.model.ai.summarization.SummarizationAlgorithmName; +import org.jabref.model.ai.summarization.SummarizatorKind; import dev.langchain4j.data.document.DefaultDocument; import dev.langchain4j.data.document.DocumentSplitter; @@ -20,8 +20,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class ChunkedSummarizationAlgorithm implements SummarizationAlgorithm { - private static final Logger LOGGER = LoggerFactory.getLogger(ChunkedSummarizationAlgorithm.class); +public class ChunkedSummarizator implements Summarizator { + private static final Logger LOGGER = LoggerFactory.getLogger(ChunkedSummarizator.class); // TODO: Make a parameter? private static final int MAX_OVERLAP_SIZE_IN_CHARS = 100; @@ -31,7 +31,7 @@ public class ChunkedSummarizationAlgorithm implements SummarizationAlgorithm { private final SummarizationCombineSystemMessageTemplate summarizationCombineSystemMessageTemplate; private final SummarizationCombineUserMessageTemplate summarizationCombineUserMessageTemplate; - public ChunkedSummarizationAlgorithm( + public ChunkedSummarizator( SummarizationChunkSystemMessageTemplate summarizationChunkSystemMessageTemplate, SummarizationChunkUserMessageTemplate summarizationChunkUserMessageTemplate, SummarizationCombineSystemMessageTemplate summarizationCombineSystemMessageTemplate, @@ -121,7 +121,7 @@ public String summarize( } @Override - public SummarizationAlgorithmName getName() { - return SummarizationAlgorithmName.CHUNKED; + public SummarizatorKind getName() { + return SummarizatorKind.CHUNKED; } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/SummarizationAlgorithm.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/Summarizator.java similarity index 69% rename from jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/SummarizationAlgorithm.java rename to jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/Summarizator.java index 58e444902cc2..d07640952419 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/SummarizationAlgorithm.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/Summarizator.java @@ -2,14 +2,14 @@ import org.jabref.logic.ai.customimplementations.llms.ChatModel; import org.jabref.logic.ai.util.LongTaskInfo; -import org.jabref.model.ai.summarization.SummarizationAlgorithmName; +import org.jabref.model.ai.summarization.SummarizatorKind; -public interface SummarizationAlgorithm { +public interface Summarizator { String summarize( ChatModel chatModel, LongTaskInfo longTaskInfo, String text ) throws InterruptedException; - SummarizationAlgorithmName getName(); + SummarizatorKind getName(); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java index 149764903de7..ee63533aea03 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java @@ -11,7 +11,7 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.customimplementations.llms.ChatModel; -import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; +import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.Summarizator; import org.jabref.logic.ai.summarization.repositories.SummariesRepository; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; @@ -39,7 +39,7 @@ public class GenerateSummaryForSeveralTask extends BackgroundTask { private final BibDatabaseContext bibDatabaseContext; private final SummariesRepository summariesRepository; private final ChatModel chatModel; - private final SummarizationAlgorithm summarizationAlgorithm; + private final Summarizator summarizator; private final ReadOnlyBooleanProperty shutdownSignal; private final FilePreferences filePreferences; private final TaskExecutor taskExecutor; @@ -53,7 +53,7 @@ public GenerateSummaryForSeveralTask( TaskExecutor taskExecutor, ChatModel chatModel, SummariesRepository summariesRepository, - SummarizationAlgorithm summarizationAlgorithm, + Summarizator summarizator, BibDatabaseContext bibDatabaseContext, StringProperty groupName, List> entries, @@ -64,7 +64,7 @@ public GenerateSummaryForSeveralTask( this.bibDatabaseContext = bibDatabaseContext; this.summariesRepository = summariesRepository; this.chatModel = chatModel; - this.summarizationAlgorithm = summarizationAlgorithm; + this.summarizator = summarizator; this.shutdownSignal = shutdownSignal; this.filePreferences = filePreferences; this.taskExecutor = taskExecutor; @@ -97,7 +97,7 @@ public Void call() throws ExecutionException, InterruptedException { filePreferences, chatModel, summariesRepository, - summarizationAlgorithm, + summarizator, bibDatabaseContext, processingInfo.getObject(), shutdownSignal diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java index 57f381b11ffe..986ed530c20a 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java @@ -5,8 +5,8 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.customimplementations.llms.ChatModel; import org.jabref.logic.ai.summarization.SummariesService; -import org.jabref.logic.ai.summarization.logic.PersistentBibEntrySummarizer; -import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.SummarizationAlgorithm; +import org.jabref.logic.ai.summarization.logic.PersistentBibEntrySummarizator; +import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.Summarizator; import org.jabref.logic.ai.summarization.repositories.SummariesRepository; import org.jabref.logic.ai.util.LongTaskInfo; import org.jabref.logic.l10n.Localization; @@ -35,7 +35,7 @@ public class GenerateSummaryTask extends BackgroundTask { private final String citationKey; private final ReadOnlyBooleanProperty shutdownSignal; - private final PersistentBibEntrySummarizer persistentBibEntrySummarizer; + private final PersistentBibEntrySummarizator persistentBibEntrySummarizator; private final ProgressCounter progressCounter = new ProgressCounter(); @@ -43,7 +43,7 @@ public GenerateSummaryTask( FilePreferences filePreferences, ChatModel chatModel, SummariesRepository summariesRepository, - SummarizationAlgorithm summarizationAlgorithm, + Summarizator summarizator, BibDatabaseContext bibDatabaseContext, BibEntry entry, ReadOnlyBooleanProperty shutdownSignal @@ -54,10 +54,10 @@ public GenerateSummaryTask( this.citationKey = entry.getCitationKey().orElse(""); this.shutdownSignal = shutdownSignal; - this.persistentBibEntrySummarizer = new PersistentBibEntrySummarizer( + this.persistentBibEntrySummarizator = new PersistentBibEntrySummarizator( filePreferences, summariesRepository, - summarizationAlgorithm + summarizator ); configure(); @@ -79,7 +79,7 @@ public BibEntrySummary call() { shutdownSignal ); - BibEntrySummary bibEntrySummary = persistentBibEntrySummarizer.summarize( + BibEntrySummary bibEntrySummary = persistentBibEntrySummarizator.summarize( chatModel, longTaskInfo, bibDatabaseContext, diff --git a/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java b/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java index da30afcf649e..cfcac442e74d 100644 --- a/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java @@ -99,11 +99,11 @@ import org.jabref.logic.util.strings.StringUtil; import org.jabref.logic.xmp.XmpPreferences; import org.jabref.model.ai.chatting.AiProvider; -import org.jabref.model.ai.embeddings.EmbeddingModel; -import org.jabref.model.ai.rag.DocumentSplittingStrategy; -import org.jabref.model.ai.summarization.SummarizationAlgorithmName; +import org.jabref.model.ai.embeddings.EmbeddingModelEnumeration; +import org.jabref.model.ai.rag.DocumentSplitterKind; +import org.jabref.model.ai.summarization.SummarizatorKind; import org.jabref.model.ai.templating.AiTemplate; -import org.jabref.model.ai.tokenization.TokenEstimationStrategy; +import org.jabref.model.ai.tokenization.TokenEstimatorKind; import org.jabref.model.database.BibDatabaseMode; import org.jabref.model.entry.BibEntryPreferences; import org.jabref.model.entry.BibEntryType; @@ -2050,12 +2050,12 @@ public AiPreferences getAiPreferences() { get(AI_GEMINI_API_BASE_URL), get(AI_HUGGING_FACE_API_BASE_URL), get(AI_GPT_4_ALL_API_BASE_URL), - SummarizationAlgorithmName.valueOf(get(AI_SUMMARIZATION_ALGORITHM)), - TokenEstimationStrategy.valueOf(get(AI_TOKEN_ESTIMATION_ALGORITHM)), - EmbeddingModel.valueOf(get(AI_EMBEDDING_MODEL)), + SummarizatorKind.valueOf(get(AI_SUMMARIZATION_ALGORITHM)), + TokenEstimatorKind.valueOf(get(AI_TOKEN_ESTIMATION_ALGORITHM)), + EmbeddingModelEnumeration.valueOf(get(AI_EMBEDDING_MODEL)), getDouble(AI_TEMPERATURE), getInt(AI_CONTEXT_WINDOW_SIZE), - DocumentSplittingStrategy.valueOf(get(AI_DOCUMENT_SPLITTING_STRATEGY)), + DocumentSplitterKind.valueOf(get(AI_DOCUMENT_SPLITTING_STRATEGY)), getInt(AI_DOCUMENT_SPLITTER_CHUNK_SIZE), getInt(AI_DOCUMENT_SPLITTER_OVERLAP_SIZE), getInt(AI_RAG_MAX_RESULTS_COUNT), diff --git a/jablib/src/main/java/org/jabref/model/ai/embeddings/EmbeddingModel.java b/jablib/src/main/java/org/jabref/model/ai/embeddings/EmbeddingModelEnumeration.java similarity index 99% rename from jablib/src/main/java/org/jabref/model/ai/embeddings/EmbeddingModel.java rename to jablib/src/main/java/org/jabref/model/ai/embeddings/EmbeddingModelEnumeration.java index 1447992948df..d0f703dded6e 100644 --- a/jablib/src/main/java/org/jabref/model/ai/embeddings/EmbeddingModel.java +++ b/jablib/src/main/java/org/jabref/model/ai/embeddings/EmbeddingModelEnumeration.java @@ -6,7 +6,7 @@ * This enumeration was formed by listing available embeddings model from Model Zoo of DJL * and by using a script that would rewrite the output in Java syntax + calculate size information by finding the models on Hugging Face. */ -public enum EmbeddingModel { +public enum EmbeddingModelEnumeration { ZEROHELL_TINYDPR_ACC_0_315_BS_307("zerohell/tinydpr-acc_0.315-bs_307", 45821255L), OMARELSAYEED_SEARCH_MODEL_PRECHATS_AUGMENTED("omarelsayeed/Search_Model_PRECHATS_AUGMENTED", 47407976L), DDOBOKKI_ELECTRA_SMALL_NLI_STS("ddobokki/electra-small-nli-sts", 55621715L), @@ -321,7 +321,7 @@ public enum EmbeddingModel { private final String name; private final long sizeInBytes; - EmbeddingModel(String name, long sizeInBytes) { + EmbeddingModelEnumeration(String name, long sizeInBytes) { this.name = name; this.sizeInBytes = sizeInBytes; } diff --git a/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingInfo.java b/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingInfo.java index 9810a695c321..b9cdc048c85d 100644 --- a/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingInfo.java +++ b/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingInfo.java @@ -8,6 +8,7 @@ import org.jspecify.annotations.Nullable; +@Deprecated public class ProcessingInfo { private final O object; private final ObjectProperty state; diff --git a/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingState.java b/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingState.java index 9c0ad6c11234..60f557d6e4e5 100644 --- a/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingState.java +++ b/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingState.java @@ -1,5 +1,6 @@ package org.jabref.model.ai.processingstatus; +@Deprecated public enum ProcessingState { PROCESSING, SUCCESS, diff --git a/jablib/src/main/java/org/jabref/model/ai/rag/DocumentSplittingStrategy.java b/jablib/src/main/java/org/jabref/model/ai/rag/DocumentSplitterKind.java similarity index 58% rename from jablib/src/main/java/org/jabref/model/ai/rag/DocumentSplittingStrategy.java rename to jablib/src/main/java/org/jabref/model/ai/rag/DocumentSplitterKind.java index dd46aeff10f8..2d576783c59a 100644 --- a/jablib/src/main/java/org/jabref/model/ai/rag/DocumentSplittingStrategy.java +++ b/jablib/src/main/java/org/jabref/model/ai/rag/DocumentSplitterKind.java @@ -1,5 +1,5 @@ package org.jabref.model.ai.rag; -public enum DocumentSplittingStrategy { +public enum DocumentSplitterKind { SLIDING_WINDOW; } diff --git a/jablib/src/main/java/org/jabref/model/ai/summarization/BibEntrySummary.java b/jablib/src/main/java/org/jabref/model/ai/summarization/BibEntrySummary.java index 595dcd1f9310..6aca736ca404 100644 --- a/jablib/src/main/java/org/jabref/model/ai/summarization/BibEntrySummary.java +++ b/jablib/src/main/java/org/jabref/model/ai/summarization/BibEntrySummary.java @@ -9,7 +9,7 @@ public record BibEntrySummary( LocalDateTime timestamp, AiProvider aiProvider, String model, - SummarizationAlgorithmName summarizationAlgorithm, + SummarizatorKind summarizationAlgorithm, String content ) implements Serializable { } diff --git a/jablib/src/main/java/org/jabref/model/ai/summarization/SummarizationAlgorithmName.java b/jablib/src/main/java/org/jabref/model/ai/summarization/SummarizatorKind.java similarity index 76% rename from jablib/src/main/java/org/jabref/model/ai/summarization/SummarizationAlgorithmName.java rename to jablib/src/main/java/org/jabref/model/ai/summarization/SummarizatorKind.java index 8d064cb9a500..b8c203a7b4db 100644 --- a/jablib/src/main/java/org/jabref/model/ai/summarization/SummarizationAlgorithmName.java +++ b/jablib/src/main/java/org/jabref/model/ai/summarization/SummarizatorKind.java @@ -2,12 +2,12 @@ import org.jabref.logic.l10n.Localization; -public enum SummarizationAlgorithmName { +public enum SummarizatorKind { CHUNKED(Localization.lang("Chunked Summarization")); private final String displayName; - SummarizationAlgorithmName(String displayName) { + SummarizatorKind(String displayName) { this.displayName = displayName; } diff --git a/jablib/src/main/java/org/jabref/model/ai/tokenization/TokenEstimationStrategy.java b/jablib/src/main/java/org/jabref/model/ai/tokenization/TokenEstimatorKind.java similarity index 71% rename from jablib/src/main/java/org/jabref/model/ai/tokenization/TokenEstimationStrategy.java rename to jablib/src/main/java/org/jabref/model/ai/tokenization/TokenEstimatorKind.java index 90a2f7c300e1..156ce13ca29c 100644 --- a/jablib/src/main/java/org/jabref/model/ai/tokenization/TokenEstimationStrategy.java +++ b/jablib/src/main/java/org/jabref/model/ai/tokenization/TokenEstimatorKind.java @@ -1,7 +1,7 @@ package org.jabref.model.ai.tokenization; -/// Idea took from: . -public enum TokenEstimationStrategy { +/// Idea taken from: . +public enum TokenEstimatorKind { /// Average between (word count / 0.75) and (character count / 4). AVERAGE, diff --git a/jablib/src/test/java/org/jabref/logic/ai/ingestion/FullyIngestedDocumentsRepositoryTest.java b/jablib/src/test/java/org/jabref/logic/ai/ingestion/IngestedDocumentsRepositoryTest.java similarity index 78% rename from jablib/src/test/java/org/jabref/logic/ai/ingestion/FullyIngestedDocumentsRepositoryTest.java rename to jablib/src/test/java/org/jabref/logic/ai/ingestion/IngestedDocumentsRepositoryTest.java index b0e661d98b53..4cfebcfa9e59 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/ingestion/FullyIngestedDocumentsRepositoryTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/ingestion/IngestedDocumentsRepositoryTest.java @@ -3,7 +3,7 @@ import java.nio.file.Path; import java.util.Optional; -import org.jabref.logic.ai.rag.repositories.FullyIngestedDocumentsRepository; +import org.jabref.logic.ai.pipeline.repositories.IngestedDocumentsRepository; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -12,14 +12,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -abstract class FullyIngestedDocumentsRepositoryTest { +abstract class IngestedDocumentsRepositoryTest { @TempDir Path tempDir; - private FullyIngestedDocumentsRepository tracker; + private IngestedDocumentsRepository tracker; - abstract FullyIngestedDocumentsRepository makeTracker(Path path); + abstract IngestedDocumentsRepository makeTracker(Path path); - abstract void close(FullyIngestedDocumentsRepository tracker); + abstract void close(IngestedDocumentsRepository tracker); @BeforeEach void setUp() { diff --git a/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsRepositoryTest.java b/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsRepositoryTest.java deleted file mode 100644 index f0ccb775d4d7..000000000000 --- a/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreFullyIngestedDocumentsRepositoryTest.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.jabref.logic.ai.ingestion; - -import java.nio.file.Path; - -import org.jabref.logic.ai.rag.repositories.FullyIngestedDocumentsRepository; -import org.jabref.logic.ai.rag.repositories.MVStoreFullyIngestedDocumentsRepository; -import org.jabref.logic.util.NotificationService; - -import static org.mockito.Mockito.mock; - -class MVStoreFullyIngestedDocumentsRepositoryTest extends FullyIngestedDocumentsRepositoryTest { - @Override - FullyIngestedDocumentsRepository makeTracker(Path path) { - return new MVStoreFullyIngestedDocumentsRepository(mock(NotificationService.class), path); - } - - @Override - void close(FullyIngestedDocumentsRepository tracker) { - ((MVStoreFullyIngestedDocumentsRepository) tracker).close(); - } -} diff --git a/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreIngestedDocumentsRepositoryTest.java b/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreIngestedDocumentsRepositoryTest.java new file mode 100644 index 000000000000..0add39e9965f --- /dev/null +++ b/jablib/src/test/java/org/jabref/logic/ai/ingestion/MVStoreIngestedDocumentsRepositoryTest.java @@ -0,0 +1,21 @@ +package org.jabref.logic.ai.ingestion; + +import java.nio.file.Path; + +import org.jabref.logic.ai.pipeline.repositories.IngestedDocumentsRepository; +import org.jabref.logic.ai.pipeline.repositories.MVStoreIngestedDocumentsRepository; +import org.jabref.logic.util.NotificationService; + +import static org.mockito.Mockito.mock; + +class MVStoreIngestedDocumentsRepositoryTest extends IngestedDocumentsRepositoryTest { + @Override + IngestedDocumentsRepository makeTracker(Path path) { + return new MVStoreIngestedDocumentsRepository(mock(NotificationService.class), path); + } + + @Override + void close(IngestedDocumentsRepository tracker) { + ((MVStoreIngestedDocumentsRepository) tracker).close(); + } +} From 40fb890b643f63c25725364e50e57a972438fefc Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Fri, 28 Nov 2025 21:24:15 +0100 Subject: [PATCH 030/243] Renaming + introduce AnswerEngine --- .../components/summary/SummaryComponent.java | 8 +-- .../util/EmbeddingModelGuardedComponent.java | 6 +- .../jabref/gui/groups/GroupTreeViewModel.java | 2 +- jablib/src/main/java/module-info.java | 3 +- .../java/org/jabref/logic/ai/AiService.java | 69 ++++++++++--------- .../logic/ai/chatting/logic/AiChatLogic.java | 8 +-- .../ChattingUserMessageTemplate.java | 4 +- .../logic/ai/current/CurrentAnswerEngine.java | 61 ++++++++++++++++ ...del.java => CurrentChatLanguageModel.java} | 16 ++--- ...tter.java => CurrentDocumentSplitter.java} | 17 ++--- ...gModel.java => CurrentEmbeddingModel.java} | 6 +- ...arizator.java => CurrentSummarizator.java} | 17 ++--- ...rategy.java => CurrentTokenEstimator.java} | 47 ++++++------- .../customimplementations/llms/ChatModel.java | 4 +- ...enizer.java => AverageTokenEstimator.java} | 6 +- ...er.java => ByCharacterTokenEstimator.java} | 2 +- ...enizer.java => ByWordsTokenEstimator.java} | 2 +- ...enizer.java => MaximumTokenEstimator.java} | 6 +- ...enizer.java => MinimumTokenEstimator.java} | 6 +- .../{Tokenizer.java => TokenEstimator.java} | 2 +- .../documentsplitting/DocumentSplitter.java | 2 +- .../SlidingWindowDocumentSplitter.java | 2 +- .../ai/pipeline/logic/rag/AnswerEngine.java | 13 ++++ .../rag/EmbeddingsSearchAnswerEngine.java | 27 ++++++++ .../logic/rag/FullDocumentAnswerEngine.java | 19 +++++ .../preferences/AiDefaultExpertSettings.java | 12 ++-- .../logic/ai/preferences/AiPreferences.java | 68 +++++++++++------- .../logic/BibEntrySummarizator.java | 2 +- .../ChunkedSummarizator.java | 2 +- .../summarizationalgorithms/Summarizator.java | 2 +- .../preferences/JabRefCliPreferences.java | 33 +++++---- .../model/ai/pipeline/AnswerEngineKind.java | 18 +++++ .../ai/pipeline/DocumentSplitterKind.java | 17 +++++ .../ai/pipeline/RelevantInformation.java | 7 ++ .../model/ai/rag/DocumentSplitterKind.java | 5 -- .../org/jabref/model/ai/rag/PaperExcerpt.java | 4 -- 36 files changed, 349 insertions(+), 176 deletions(-) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/current/CurrentAnswerEngine.java rename jablib/src/main/java/org/jabref/logic/ai/current/{CurrentlySelectedChatLanguageModel.java => CurrentChatLanguageModel.java} (93%) rename jablib/src/main/java/org/jabref/logic/ai/current/{CurrentlySelectedDocumentSplitter.java => CurrentDocumentSplitter.java} (77%) rename jablib/src/main/java/org/jabref/logic/ai/current/{CurrentlySelectedEmbeddingModel.java => CurrentEmbeddingModel.java} (96%) rename jablib/src/main/java/org/jabref/logic/ai/current/{CurrentlySelectedSummarizator.java => CurrentSummarizator.java} (86%) rename jablib/src/main/java/org/jabref/logic/ai/current/{CurrentlySelectedTokenEstimationStrategy.java => CurrentTokenEstimator.java} (53%) rename jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/{AverageTokenizer.java => AverageTokenEstimator.java} (78%) rename jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/{ByCharacterTokenizer.java => ByCharacterTokenEstimator.java} (93%) rename jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/{ByWordsTokenizer.java => ByWordsTokenEstimator.java} (94%) rename jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/{MaximumTokenizer.java => MaximumTokenEstimator.java} (78%) rename jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/{MinimumTokenizer.java => MinimumTokenEstimator.java} (78%) rename jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/{Tokenizer.java => TokenEstimator.java} (91%) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/AnswerEngine.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/EmbeddingsSearchAnswerEngine.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/FullDocumentAnswerEngine.java create mode 100644 jablib/src/main/java/org/jabref/model/ai/pipeline/AnswerEngineKind.java create mode 100644 jablib/src/main/java/org/jabref/model/ai/pipeline/DocumentSplitterKind.java create mode 100644 jablib/src/main/java/org/jabref/model/ai/pipeline/RelevantInformation.java delete mode 100644 jablib/src/main/java/org/jabref/model/ai/rag/DocumentSplitterKind.java delete mode 100644 jablib/src/main/java/org/jabref/model/ai/rag/PaperExcerpt.java diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java index 95f5d6fb4cf5..ed8b1fe4fc69 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryComponent.java @@ -52,7 +52,7 @@ public SummaryComponent(BibDatabaseContext bibDatabaseContext, this.aiPreferences = aiPreferences; aiService.getSummariesService().summarize( - aiService.getSummarizationAlgorithm(), + aiService.getSummarizator(), entry, bibDatabaseContext ).stateProperty().addListener(o -> rebuildUi()); @@ -108,7 +108,7 @@ private Node tryToGenerateCitationKeyThenBind(BibEntry entry) { } private Node tryToShowSummary() { - ProcessingInfo processingInfo = aiService.getSummariesService().summarize(aiService.getSummarizationAlgorithm(), entry, bibDatabaseContext); + ProcessingInfo processingInfo = aiService.getSummariesService().summarize(aiService.getSummarizator(), entry, bibDatabaseContext); return switch (processingInfo.getState()) { case SUCCESS -> { @@ -133,7 +133,7 @@ private Node showErrorWhileSummarizing(ProcessingInfo Localization.lang("Got error while processing the file:"), processingInfo.getException().get().getLocalizedMessage(), Localization.lang("Regenerate"), - () -> aiService.getSummariesService().regenerateSummary(aiService.getSummarizationAlgorithm(), entry, bibDatabaseContext) + () -> aiService.getSummariesService().regenerateSummary(aiService.getSummarizator(), entry, bibDatabaseContext) ); } @@ -156,7 +156,7 @@ private Node showSummary(BibEntrySummary bibEntrySummary) { return; } - aiService.getSummariesService().regenerateSummary(aiService.getSummarizationAlgorithm(), entry, bibDatabaseContext); + aiService.getSummariesService().regenerateSummary(aiService.getSummarizator(), entry, bibDatabaseContext); // No need to rebuildUi(), because this class listens to the state of ProcessingInfo of the bibEntrySummary. }); } diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java index e2da5bbc4793..c07012b5928a 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/util/EmbeddingModelGuardedComponent.java @@ -9,7 +9,7 @@ import org.jabref.gui.frame.ExternalApplicationsPreferences; import org.jabref.gui.util.UiTaskExecutor; import org.jabref.logic.ai.AiService; -import org.jabref.logic.ai.current.CurrentlySelectedEmbeddingModel; +import org.jabref.logic.ai.current.CurrentEmbeddingModel; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.l10n.Localization; @@ -68,12 +68,12 @@ public Node showBuildingEmbeddingModel() { } @Subscribe - public void listen(CurrentlySelectedEmbeddingModel.EmbeddingModelBuiltEvent event) { + public void listen(CurrentEmbeddingModel.EmbeddingModelBuiltEvent event) { UiTaskExecutor.runInJavaFXThread(EmbeddingModelGuardedComponent.this::rebuildUi); } @Subscribe - public void listen(CurrentlySelectedEmbeddingModel.EmbeddingModelBuildingErrorEvent event) { + public void listen(CurrentEmbeddingModel.EmbeddingModelBuildingErrorEvent event) { UiTaskExecutor.runInJavaFXThread(EmbeddingModelGuardedComponent.this::rebuildUi); } } diff --git a/jabgui/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java b/jabgui/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java index b9f5dc07c650..7e51a86b832b 100644 --- a/jabgui/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java +++ b/jabgui/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java @@ -534,7 +534,7 @@ public void generateSummaries(GroupNodeViewModel groupNode) { .toList(); aiService.getSummariesService().summarize( - aiService.getSummarizationAlgorithm(), + aiService.getSummarizator(), group.nameProperty(), entries, currentDatabase.get() diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index be5da2c8110e..0144c43324d8 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -142,7 +142,7 @@ exports org.jabref.logic.ai.preferences; exports org.jabref.model.ai.chatting; exports org.jabref.model.ai.templating; - exports org.jabref.model.ai.rag; + exports org.jabref.model.ai.pipeline; exports org.jabref.model.ai.embeddings; exports org.jabref.model.ai.summarization; exports org.jabref.model.ai.identifiers; @@ -295,5 +295,6 @@ requires org.jooq.jool; requires org.libreoffice.uno; requires transitive org.jspecify; + requires org.jabref.jablib; // endregion } diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiService.java b/jablib/src/main/java/org/jabref/logic/ai/AiService.java index 1a46cf87fefb..c949d5e37afd 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiService.java @@ -10,11 +10,12 @@ import org.jabref.logic.ai.chatting.ChatHistoryService; import org.jabref.logic.ai.chatting.repositories.MVStoreChatHistoryRepository; import org.jabref.logic.ai.current.CurrentAiTemplates; -import org.jabref.logic.ai.current.CurrentlySelectedChatLanguageModel; -import org.jabref.logic.ai.current.CurrentlySelectedDocumentSplitter; -import org.jabref.logic.ai.current.CurrentlySelectedEmbeddingModel; -import org.jabref.logic.ai.current.CurrentlySelectedSummarizator; -import org.jabref.logic.ai.current.CurrentlySelectedTokenEstimationStrategy; +import org.jabref.logic.ai.current.CurrentAnswerEngine; +import org.jabref.logic.ai.current.CurrentChatLanguageModel; +import org.jabref.logic.ai.current.CurrentDocumentSplitter; +import org.jabref.logic.ai.current.CurrentEmbeddingModel; +import org.jabref.logic.ai.current.CurrentSummarizator; +import org.jabref.logic.ai.current.CurrentTokenEstimator; import org.jabref.logic.ai.customimplementations.embeddingstores.MVStoreEmbeddingStore; import org.jabref.logic.ai.pipeline.IngestionService; import org.jabref.logic.ai.pipeline.repositories.MVStoreIngestedDocumentsRepository; @@ -61,11 +62,12 @@ public class AiService implements AutoCloseable { private final CurrentAiTemplates currentAiTemplates; private final ChatHistoryService chatHistoryService; - private final CurrentlySelectedDocumentSplitter currentlySelectedDocumentSplitter; - private final CurrentlySelectedTokenEstimationStrategy currentlySelectedTokenEstimationStrategy; - private final CurrentlySelectedChatLanguageModel currentlySelectedChatLanguageModel; - private final CurrentlySelectedEmbeddingModel currentlySelectedEmbeddingModel; - private final CurrentlySelectedSummarizator currentlySelectedSummarizationAlgorithm; + private final CurrentDocumentSplitter currentDocumentSplitter; + private final CurrentTokenEstimator currentTokenEstimator; + private final CurrentChatLanguageModel currentChatLanguageModel; + private final CurrentEmbeddingModel currentEmbeddingModel; + private final CurrentSummarizator currentSummarizator; + private final CurrentAnswerEngine currentAnswerEngine; private final IngestionService ingestionService; private final SummariesService summariesService; @@ -85,19 +87,20 @@ public AiService( this.currentAiTemplates = new CurrentAiTemplates(aiPreferences); this.chatHistoryService = new ChatHistoryService(citationKeyPatternPreferences, mvStoreChatHistoryStorage); - this.currentlySelectedDocumentSplitter = new CurrentlySelectedDocumentSplitter(aiPreferences); - this.currentlySelectedTokenEstimationStrategy = new CurrentlySelectedTokenEstimationStrategy(aiPreferences); - this.currentlySelectedChatLanguageModel = new CurrentlySelectedChatLanguageModel(aiPreferences, currentlySelectedTokenEstimationStrategy); - this.currentlySelectedEmbeddingModel = new CurrentlySelectedEmbeddingModel(aiPreferences, notificationService, taskExecutor); - this.currentlySelectedSummarizationAlgorithm = new CurrentlySelectedSummarizator(aiPreferences, currentAiTemplates); + this.currentDocumentSplitter = new CurrentDocumentSplitter(aiPreferences); + this.currentTokenEstimator = new CurrentTokenEstimator(aiPreferences); + this.currentChatLanguageModel = new CurrentChatLanguageModel(aiPreferences, currentTokenEstimator); + this.currentEmbeddingModel = new CurrentEmbeddingModel(aiPreferences, notificationService, taskExecutor); + this.currentSummarizator = new CurrentSummarizator(aiPreferences, currentAiTemplates); + this.currentAnswerEngine = new CurrentAnswerEngine(aiPreferences); this.ingestionService = new IngestionService( aiPreferences, filePreferences, taskExecutor, - currentlySelectedEmbeddingModel, + currentEmbeddingModel, mvStoreEmbeddingStore, - currentlySelectedDocumentSplitter, + currentDocumentSplitter, mvStoreFullyIngestedDocumentsTracker, shutdownSignal ); @@ -106,23 +109,23 @@ public AiService( aiPreferences, filePreferences, taskExecutor, - currentlySelectedChatLanguageModel, - currentlySelectedSummarizationAlgorithm, + currentChatLanguageModel, + currentSummarizator, mvStoreSummariesStorage, shutdownSignal ); } - public CurrentlySelectedDocumentSplitter getDocumentSplitter() { - return currentlySelectedDocumentSplitter; + public CurrentDocumentSplitter getDocumentSplitter() { + return currentDocumentSplitter; } - public CurrentlySelectedChatLanguageModel getChatLanguageModel() { - return currentlySelectedChatLanguageModel; + public CurrentChatLanguageModel getChatLanguageModel() { + return currentChatLanguageModel; } - public CurrentlySelectedEmbeddingModel getEmbeddingModel() { - return currentlySelectedEmbeddingModel; + public CurrentEmbeddingModel getEmbeddingModel() { + return currentEmbeddingModel; } public ChatHistoryService getChatHistoryService() { @@ -145,12 +148,16 @@ public EmbeddingStore getEmbeddingStore() { return mvStoreEmbeddingStore; } - public Summarizator getSummarizationAlgorithm() { - return currentlySelectedSummarizationAlgorithm; + public Summarizator getSummarizator() { + return currentSummarizator; } - public CurrentlySelectedTokenEstimationStrategy getTokenEstimationStrategy() { - return currentlySelectedTokenEstimationStrategy; + public CurrentTokenEstimator getTokenEstimator() { + return currentTokenEstimator; + } + + public CurrentAnswerEngine getAnswerEngine() { + return currentAnswerEngine; } public void setupDatabase(BibDatabaseContext context) { @@ -164,8 +171,8 @@ public void close() { shutdownSignal.set(true); cachedThreadPool.shutdownNow(); - currentlySelectedChatLanguageModel.close(); - currentlySelectedEmbeddingModel.close(); + currentChatLanguageModel.close(); + currentEmbeddingModel.close(); mvStoreChatHistoryStorage.close(); mvStoreFullyIngestedDocumentsTracker.close(); diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java index 9f42a4f37459..d93a6f623236 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java @@ -12,7 +12,7 @@ import org.jabref.logic.ai.pipeline.logic.EmbeddingsCleaner; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.model.ai.chatting.ErrorMessage; -import org.jabref.model.ai.rag.PaperExcerpt; +import org.jabref.model.ai.pipeline.RelevantInformation; import org.jabref.model.ai.templating.AiTemplate; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -161,7 +161,7 @@ public AiMessage execute(UserMessage message) { EmbeddingSearchResult embeddingSearchResult = embeddingStore.search(embeddingSearchRequest); - List excerpts = embeddingSearchResult + List excerpts = embeddingSearchResult .matches() .stream() .map(EmbeddingMatch::embedded) @@ -169,9 +169,9 @@ public AiMessage execute(UserMessage message) { String link = textSegment.metadata().getString(EmbeddingsCleaner.LINK_METADATA_KEY); if (link == null) { - return new PaperExcerpt("", textSegment.text()); + return new RelevantInformation(List.of(), textSegment.text()); } else { - return new PaperExcerpt(findEntryByLink(link).flatMap(BibEntry::getCitationKey).orElse(""), textSegment.text()); + return new RelevantInformation(List.of(findEntryByLink(link).flatMap(BibEntry::getCitationKey).orElse("")), textSegment.text()); } }) .toList(); diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/templates/ChattingUserMessageTemplate.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/templates/ChattingUserMessageTemplate.java index 21141d6e350a..04b886d059a7 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/templates/ChattingUserMessageTemplate.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/templates/ChattingUserMessageTemplate.java @@ -4,7 +4,7 @@ import java.util.function.Supplier; import org.jabref.logic.ai.templates.Template; -import org.jabref.model.ai.rag.PaperExcerpt; +import org.jabref.model.ai.pipeline.RelevantInformation; import org.jabref.model.ai.templating.AiTemplate; import org.jabref.model.entry.BibEntry; @@ -15,7 +15,7 @@ public ChattingUserMessageTemplate(Supplier source) { super(source); } - public String render(List entries, String message, List excerpts) { + public String render(List entries, String message, List excerpts) { VelocityContext context = makeContext(); context.put("entries", entries); diff --git a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentAnswerEngine.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentAnswerEngine.java new file mode 100644 index 000000000000..9dc5d6e013b6 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentAnswerEngine.java @@ -0,0 +1,61 @@ +package org.jabref.logic.ai.current; + +import java.util.List; + +import org.jabref.logic.ai.pipeline.logic.rag.AnswerEngine; +import org.jabref.logic.ai.pipeline.logic.rag.EmbeddingsSearchAnswerEngine; +import org.jabref.logic.ai.pipeline.logic.rag.FullDocumentAnswerEngine; +import org.jabref.logic.ai.preferences.AiPreferences; +import org.jabref.model.ai.pipeline.AnswerEngineKind; +import org.jabref.model.ai.pipeline.RelevantInformation; +import org.jabref.model.entry.BibEntry; + +import jakarta.annotation.Nullable; + +public class CurrentAnswerEngine implements AnswerEngine { + private final AiPreferences aiPreferences; + + @Nullable + private AnswerEngine answerEngine = null; + + public CurrentAnswerEngine(AiPreferences aiPreferences) { + this.aiPreferences = aiPreferences; + + update(); + configure(); + } + + private void update() { + switch (aiPreferences.getAnswerEngineKind()) { + case AnswerEngineKind.EMBEDDINGS_SEARCH -> + answerEngine = new EmbeddingsSearchAnswerEngine( + aiPreferences.getRagMinScore(), + aiPreferences.getRagMaxResultsCount() + ); + + case FULL_DOCUMENT -> + answerEngine = new FullDocumentAnswerEngine(); + } + } + + private void configure() { + aiPreferences.customizeExpertSettingsProperty().addListener(_ -> update()); + aiPreferences.answerEngineKindProperty().addListener(_ -> update()); + aiPreferences.ragMaxResultsCountProperty().addListener(_ -> update()); + aiPreferences.ragMinScoreProperty().addListener(_ -> update()); + } + + @Override + public List process(String query, List bibEntriesFilter) { + if (answerEngine == null) { + return List.of(); + } else { + return answerEngine.process(query, bibEntriesFilter); + } + } + + @Override + public AnswerEngineKind getKind() { + return aiPreferences.getAnswerEngineKind(); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedChatLanguageModel.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentChatLanguageModel.java similarity index 93% rename from jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedChatLanguageModel.java rename to jablib/src/main/java/org/jabref/logic/ai/current/CurrentChatLanguageModel.java index 3a4813f5561f..e49d2e1a3184 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedChatLanguageModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentChatLanguageModel.java @@ -11,7 +11,7 @@ import org.jabref.logic.ai.customimplementations.llms.ChatModel; import org.jabref.logic.ai.customimplementations.llms.Gpt4AllModel; import org.jabref.logic.ai.customimplementations.llms.JvmOpenAiChatLanguageModel; -import org.jabref.logic.ai.customimplementations.tokenization.algorithms.Tokenizer; +import org.jabref.logic.ai.customimplementations.tokenization.algorithms.TokenEstimator; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.l10n.Localization; import org.jabref.model.ai.chatting.AiProvider; @@ -31,12 +31,12 @@ * Notice, that the real chat model is created lazily, when it's needed. This is done, so API key is fetched only, * when user wants to chat with AI. */ -public class CurrentlySelectedChatLanguageModel implements ChatModel, AutoCloseable { +public class CurrentChatLanguageModel implements ChatModel, AutoCloseable { private static final Duration CONNECTION_TIMEOUT = Duration.ofSeconds(5); private final AiPreferences aiPreferences; - private final CurrentlySelectedTokenEstimationStrategy currentlySelectedTokenEstimationStrategy; + private final CurrentTokenEstimator currentTokenEstimator; private final HttpClient httpClient; private final ExecutorService executorService = Executors.newSingleThreadExecutor( @@ -46,13 +46,13 @@ public class CurrentlySelectedChatLanguageModel implements ChatModel, AutoClosea @Nullable private dev.langchain4j.model.chat.ChatModel langchainChatModel = null; - public CurrentlySelectedChatLanguageModel( + public CurrentChatLanguageModel( AiPreferences aiPreferences, - CurrentlySelectedTokenEstimationStrategy currentlySelectedTokenEstimationStrategy + CurrentTokenEstimator currentTokenEstimator ) { this.aiPreferences = aiPreferences; - this.currentlySelectedTokenEstimationStrategy = currentlySelectedTokenEstimationStrategy; + this.currentTokenEstimator = currentTokenEstimator; this.httpClient = HttpClient.newBuilder().connectTimeout(CONNECTION_TIMEOUT).executor(executorService).build(); @@ -157,8 +157,8 @@ public void close() { } @Override - public Tokenizer getTokenizer() { - return currentlySelectedTokenEstimationStrategy; + public TokenEstimator getTokenizer() { + return currentTokenEstimator; } @Override diff --git a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedDocumentSplitter.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentDocumentSplitter.java similarity index 77% rename from jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedDocumentSplitter.java rename to jablib/src/main/java/org/jabref/logic/ai/current/CurrentDocumentSplitter.java index 051f3730eb57..76330ed6ee11 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedDocumentSplitter.java +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentDocumentSplitter.java @@ -6,17 +6,17 @@ import org.jabref.logic.ai.pipeline.logic.documentsplitting.SlidingWindowDocumentSplitter; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.util.LongTaskInfo; -import org.jabref.model.ai.rag.DocumentSplitterKind; +import org.jabref.model.ai.pipeline.DocumentSplitterKind; import org.jspecify.annotations.Nullable; -public class CurrentlySelectedDocumentSplitter implements DocumentSplitter { +public class CurrentDocumentSplitter implements DocumentSplitter { private final AiPreferences aiPreferences; @Nullable private DocumentSplitter documentSplitter = null; - public CurrentlySelectedDocumentSplitter(AiPreferences aiPreferences) { + public CurrentDocumentSplitter(AiPreferences aiPreferences) { this.aiPreferences = aiPreferences; update(); @@ -25,7 +25,7 @@ public CurrentlySelectedDocumentSplitter(AiPreferences aiPreferences) { private void configure() { aiPreferences.customizeExpertSettingsProperty().addListener(_ -> update()); - aiPreferences.documentSplittingStrategyProperty().addListener(_ -> update()); + aiPreferences.documentSplitterKindProperty().addListener(_ -> update()); aiPreferences.documentSplitterChunkSizeProperty().addListener(_ -> update()); aiPreferences.documentSplitterOverlapSizeProperty().addListener(_ -> update()); } @@ -33,7 +33,7 @@ private void configure() { private void update() { // Because in the future there will be more strategies. //noinspection SwitchStatementWithTooFewBranches - switch (aiPreferences.getDocumentSplittingStrategy()) { + switch (aiPreferences.getDocumentSplitterKind()) { case DocumentSplitterKind.SLIDING_WINDOW -> { documentSplitter = new SlidingWindowDocumentSplitter( aiPreferences.getDocumentSplitterChunkSize(), @@ -54,11 +54,6 @@ public Stream split(LongTaskInfo longTaskInfo, String text) throws Inter @Override public DocumentSplitterKind getKind() { - if (documentSplitter == null) { - // Unfortunately. - return null; - } - - return documentSplitter.getKind(); + aiPreferences.getDocumentSplitterKind(); } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedEmbeddingModel.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentEmbeddingModel.java similarity index 96% rename from jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedEmbeddingModel.java rename to jablib/src/main/java/org/jabref/logic/ai/current/CurrentEmbeddingModel.java index 180cebb23324..431f7eecde60 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedEmbeddingModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentEmbeddingModel.java @@ -29,8 +29,8 @@ *

* This class listens to preferences changes. */ -public class CurrentlySelectedEmbeddingModel implements EmbeddingModel, AutoCloseable { - private static final Logger LOGGER = LoggerFactory.getLogger(CurrentlySelectedEmbeddingModel.class); +public class CurrentEmbeddingModel implements EmbeddingModel, AutoCloseable { + private static final Logger LOGGER = LoggerFactory.getLogger(CurrentEmbeddingModel.class); private final AiPreferences aiPreferences; private final NotificationService notificationService; @@ -54,7 +54,7 @@ public static class EmbeddingModelBuildingErrorEvent { // Empty if there is no error. private String errorWhileBuildingModel = ""; - public CurrentlySelectedEmbeddingModel( + public CurrentEmbeddingModel( AiPreferences aiPreferences, NotificationService notificationService, TaskExecutor taskExecutor diff --git a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedSummarizator.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentSummarizator.java similarity index 86% rename from jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedSummarizator.java rename to jablib/src/main/java/org/jabref/logic/ai/current/CurrentSummarizator.java index 727772080dc9..dadbc7777c7e 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedSummarizator.java +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentSummarizator.java @@ -10,14 +10,14 @@ import org.jspecify.annotations.Nullable; -public class CurrentlySelectedSummarizator implements Summarizator { +public class CurrentSummarizator implements Summarizator { private final AiPreferences aiPreferences; private final CurrentAiTemplates currentAiTemplates; @Nullable private Summarizator summarizator = null; - public CurrentlySelectedSummarizator( + public CurrentSummarizator( AiPreferences aiPreferences, CurrentAiTemplates currentAiTemplates ) { @@ -29,7 +29,7 @@ public CurrentlySelectedSummarizator( } private void setupListeningToPreferences() { - aiPreferences.defaultSummarizationAlgorithmProperty().addListener(_ -> { + aiPreferences.summarizatorKindProperty().addListener(_ -> { updateAlgorithm(); }); @@ -51,7 +51,7 @@ private void setupListeningToPreferences() { private void updateAlgorithm() { // Because in the future there will be more strategies. //noinspection SwitchStatementWithTooFewBranches - switch (aiPreferences.getDefaultSummarizationAlgorithm()) { + switch (aiPreferences.getSummarizatorKind()) { case SummarizatorKind.CHUNKED -> { summarizator = createChunkedSummarizationAlgorithm(); } @@ -77,12 +77,7 @@ public String summarize(ChatModel chatModel, LongTaskInfo longTaskInfo, String t } @Override - public SummarizatorKind getName() { - if (summarizator == null) { - // Sadly. - return null; - } - - return summarizator.getName(); + public SummarizatorKind getKind() { + return aiPreferences.getSummarizatorKind(); } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedTokenEstimationStrategy.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentTokenEstimator.java similarity index 53% rename from jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedTokenEstimationStrategy.java rename to jablib/src/main/java/org/jabref/logic/ai/current/CurrentTokenEstimator.java index ca57019dd77b..5018d5684787 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentlySelectedTokenEstimationStrategy.java +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentTokenEstimator.java @@ -2,25 +2,25 @@ import java.util.List; -import org.jabref.logic.ai.customimplementations.tokenization.algorithms.AverageTokenizer; -import org.jabref.logic.ai.customimplementations.tokenization.algorithms.ByCharacterTokenizer; -import org.jabref.logic.ai.customimplementations.tokenization.algorithms.ByWordsTokenizer; -import org.jabref.logic.ai.customimplementations.tokenization.algorithms.MaximumTokenizer; -import org.jabref.logic.ai.customimplementations.tokenization.algorithms.MinimumTokenizer; -import org.jabref.logic.ai.customimplementations.tokenization.algorithms.Tokenizer; +import org.jabref.logic.ai.customimplementations.tokenization.algorithms.AverageTokenEstimator; +import org.jabref.logic.ai.customimplementations.tokenization.algorithms.ByCharacterTokenEstimator; +import org.jabref.logic.ai.customimplementations.tokenization.algorithms.ByWordsTokenEstimator; +import org.jabref.logic.ai.customimplementations.tokenization.algorithms.MaximumTokenEstimator; +import org.jabref.logic.ai.customimplementations.tokenization.algorithms.MinimumTokenEstimator; +import org.jabref.logic.ai.customimplementations.tokenization.algorithms.TokenEstimator; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.model.ai.tokenization.TokenEstimatorKind; import dev.langchain4j.data.message.ChatMessage; import org.jspecify.annotations.Nullable; -public class CurrentlySelectedTokenEstimationStrategy implements Tokenizer { +public class CurrentTokenEstimator implements TokenEstimator { private final AiPreferences aiPreferences; @Nullable - private Tokenizer tokenizer; + private TokenEstimator tokenEstimator; - public CurrentlySelectedTokenEstimationStrategy( + public CurrentTokenEstimator( AiPreferences aiPreferences ) { this.aiPreferences = aiPreferences; @@ -30,46 +30,41 @@ public CurrentlySelectedTokenEstimationStrategy( } private void setupListeningToPreferences() { - aiPreferences.tokenEstimationStrategyProperty().addListener(_ -> { + aiPreferences.tokenEstimatorKindProperty().addListener(_ -> { createTokenizer(); }); } private void createTokenizer() { - switch (aiPreferences.getTokenEstimationStrategy()) { - case TokenEstimatorKind.AVERAGE -> tokenizer = new AverageTokenizer(); - case TokenEstimatorKind.MAX -> tokenizer = new MaximumTokenizer(); - case TokenEstimatorKind.MIN -> tokenizer = new MinimumTokenizer(); - case TokenEstimatorKind.CHARS -> tokenizer = new ByCharacterTokenizer(); - case TokenEstimatorKind.WORDS -> tokenizer = new ByWordsTokenizer(); + switch (aiPreferences.getTokenEstimatorKind()) { + case TokenEstimatorKind.AVERAGE -> tokenEstimator = new AverageTokenEstimator(); + case TokenEstimatorKind.MAX -> tokenEstimator = new MaximumTokenEstimator(); + case TokenEstimatorKind.MIN -> tokenEstimator = new MinimumTokenEstimator(); + case TokenEstimatorKind.CHARS -> tokenEstimator = new ByCharacterTokenEstimator(); + case TokenEstimatorKind.WORDS -> tokenEstimator = new ByWordsTokenEstimator(); } } @Override public int estimate(ChatMessage message) { - if (tokenizer == null) { + if (tokenEstimator == null) { return 0; } - return tokenizer.estimate(message); + return tokenEstimator.estimate(message); } @Override public int estimate(List messages) { - if (tokenizer == null) { + if (tokenEstimator == null) { return 0; } - return tokenizer.estimate(messages); + return tokenEstimator.estimate(messages); } @Override public TokenEstimatorKind getKind() { - // Sadly. - if (tokenizer == null) { - return null; - } - - return tokenizer.getKind(); + return aiPreferences.getTokenEstimatorKind(); } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/ChatModel.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/ChatModel.java index bd39593da802..33cc259d6924 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/ChatModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/ChatModel.java @@ -1,10 +1,10 @@ package org.jabref.logic.ai.customimplementations.llms; -import org.jabref.logic.ai.customimplementations.tokenization.algorithms.Tokenizer; +import org.jabref.logic.ai.customimplementations.tokenization.algorithms.TokenEstimator; import org.jabref.model.ai.chatting.AiProvider; public interface ChatModel extends dev.langchain4j.model.chat.ChatModel { - Tokenizer getTokenizer(); + TokenEstimator getTokenizer(); AiProvider getAiProvider(); String getName(); int getContextWindowSize(); diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/AverageTokenizer.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/AverageTokenEstimator.java similarity index 78% rename from jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/AverageTokenizer.java rename to jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/AverageTokenEstimator.java index 87eacf80a679..da7bd81125fb 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/AverageTokenizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/AverageTokenEstimator.java @@ -6,9 +6,9 @@ import dev.langchain4j.data.message.ChatMessage; -public class AverageTokenizer implements Tokenizer { - private final ByCharacterTokenizer byCharacterTokenizer = new ByCharacterTokenizer(); - private final ByWordsTokenizer byWordsTokenizer = new ByWordsTokenizer(); +public class AverageTokenEstimator implements TokenEstimator { + private final ByCharacterTokenEstimator byCharacterTokenizer = new ByCharacterTokenEstimator(); + private final ByWordsTokenEstimator byWordsTokenizer = new ByWordsTokenEstimator(); @Override public int estimate(ChatMessage message) { diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByCharacterTokenizer.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByCharacterTokenEstimator.java similarity index 93% rename from jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByCharacterTokenizer.java rename to jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByCharacterTokenEstimator.java index 46dd849f5e6c..e97a1074b701 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByCharacterTokenizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByCharacterTokenEstimator.java @@ -9,7 +9,7 @@ import dev.langchain4j.data.message.ChatMessage; -public class ByCharacterTokenizer implements Tokenizer { +public class ByCharacterTokenEstimator implements TokenEstimator { private static final float CHAR_FACTOR = 4; @Override diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByWordsTokenizer.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByWordsTokenEstimator.java similarity index 94% rename from jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByWordsTokenizer.java rename to jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByWordsTokenEstimator.java index 9a535357c183..a24152d7a9ff 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByWordsTokenizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/ByWordsTokenEstimator.java @@ -9,7 +9,7 @@ import dev.langchain4j.data.message.ChatMessage; -public class ByWordsTokenizer implements Tokenizer { +public class ByWordsTokenEstimator implements TokenEstimator { private static final float WORD_FACTOR = 0.75f; @Override diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MaximumTokenizer.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MaximumTokenEstimator.java similarity index 78% rename from jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MaximumTokenizer.java rename to jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MaximumTokenEstimator.java index 4e7bef244464..892d0030860a 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MaximumTokenizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MaximumTokenEstimator.java @@ -6,9 +6,9 @@ import dev.langchain4j.data.message.ChatMessage; -public class MaximumTokenizer implements Tokenizer { - private final ByCharacterTokenizer byCharacterTokenizer = new ByCharacterTokenizer(); - private final ByWordsTokenizer byWordsTokenizer = new ByWordsTokenizer(); +public class MaximumTokenEstimator implements TokenEstimator { + private final ByCharacterTokenEstimator byCharacterTokenizer = new ByCharacterTokenEstimator(); + private final ByWordsTokenEstimator byWordsTokenizer = new ByWordsTokenEstimator(); @Override public int estimate(ChatMessage message) { diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MinimumTokenizer.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MinimumTokenEstimator.java similarity index 78% rename from jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MinimumTokenizer.java rename to jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MinimumTokenEstimator.java index 71a84a55934f..7c5887548e65 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MinimumTokenizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/MinimumTokenEstimator.java @@ -6,9 +6,9 @@ import dev.langchain4j.data.message.ChatMessage; -public class MinimumTokenizer implements Tokenizer { - private final ByCharacterTokenizer byCharacterTokenizer = new ByCharacterTokenizer(); - private final ByWordsTokenizer byWordsTokenizer = new ByWordsTokenizer(); +public class MinimumTokenEstimator implements TokenEstimator { + private final ByCharacterTokenEstimator byCharacterTokenizer = new ByCharacterTokenEstimator(); + private final ByWordsTokenEstimator byWordsTokenizer = new ByWordsTokenEstimator(); @Override public int estimate(ChatMessage message) { diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/Tokenizer.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/TokenEstimator.java similarity index 91% rename from jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/Tokenizer.java rename to jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/TokenEstimator.java index 194047208dfb..5c9b7765732f 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/Tokenizer.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/tokenization/algorithms/TokenEstimator.java @@ -6,7 +6,7 @@ import dev.langchain4j.data.message.ChatMessage; -public interface Tokenizer { +public interface TokenEstimator { int estimate(ChatMessage message); int estimate(List messages); diff --git a/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/documentsplitting/DocumentSplitter.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/documentsplitting/DocumentSplitter.java index 4d1d787b57cf..bf9c5f66def7 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/documentsplitting/DocumentSplitter.java +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/documentsplitting/DocumentSplitter.java @@ -3,7 +3,7 @@ import java.util.stream.Stream; import org.jabref.logic.ai.util.LongTaskInfo; -import org.jabref.model.ai.rag.DocumentSplitterKind; +import org.jabref.model.ai.pipeline.DocumentSplitterKind; public interface DocumentSplitter { Stream split(LongTaskInfo longTaskInfo, String text) throws InterruptedException; diff --git a/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/documentsplitting/SlidingWindowDocumentSplitter.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/documentsplitting/SlidingWindowDocumentSplitter.java index 8635e4fadbe6..6b8f9a8f7deb 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/documentsplitting/SlidingWindowDocumentSplitter.java +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/documentsplitting/SlidingWindowDocumentSplitter.java @@ -3,7 +3,7 @@ import java.util.stream.Stream; import org.jabref.logic.ai.util.LongTaskInfo; -import org.jabref.model.ai.rag.DocumentSplitterKind; +import org.jabref.model.ai.pipeline.DocumentSplitterKind; import dev.langchain4j.data.document.DefaultDocument; import dev.langchain4j.data.document.splitter.DocumentSplitters; diff --git a/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/AnswerEngine.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/AnswerEngine.java new file mode 100644 index 000000000000..bba786abc9fd --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/AnswerEngine.java @@ -0,0 +1,13 @@ +package org.jabref.logic.ai.pipeline.logic.rag; + +import java.util.List; + +import org.jabref.model.ai.pipeline.AnswerEngineKind; +import org.jabref.model.ai.pipeline.RelevantInformation; +import org.jabref.model.entry.BibEntry; + +public interface AnswerEngine { + List process(String query, List bibEntriesFilter); + + AnswerEngineKind getKind(); +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/EmbeddingsSearchAnswerEngine.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/EmbeddingsSearchAnswerEngine.java new file mode 100644 index 000000000000..e1c1b09a1686 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/EmbeddingsSearchAnswerEngine.java @@ -0,0 +1,27 @@ +package org.jabref.logic.ai.pipeline.logic.rag; + +import java.util.List; + +import org.jabref.model.ai.pipeline.AnswerEngineKind; +import org.jabref.model.ai.pipeline.RelevantInformation; +import org.jabref.model.entry.BibEntry; + +public class EmbeddingsSearchAnswerEngine implements AnswerEngine { + private final double minimumScore; + private final int maximumResultsCount; + + public EmbeddingsSearchAnswerEngine(double minimumScore, int maximumResultsCount) { + this.minimumScore = minimumScore; + this.maximumResultsCount = maximumResultsCount; + } + + @Override + public List process(String query, List bibEntriesFilter) { + return List.of(); + } + + @Override + public AnswerEngineKind getKind() { + return AnswerEngineKind.EMBEDDINGS_SEARCH; + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/FullDocumentAnswerEngine.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/FullDocumentAnswerEngine.java new file mode 100644 index 000000000000..68a7f876f367 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/FullDocumentAnswerEngine.java @@ -0,0 +1,19 @@ +package org.jabref.logic.ai.pipeline.logic.rag; + +import java.util.List; + +import org.jabref.model.ai.pipeline.AnswerEngineKind; +import org.jabref.model.ai.pipeline.RelevantInformation; +import org.jabref.model.entry.BibEntry; + +public class FullDocumentAnswerEngine implements AnswerEngine { + @Override + public List process(String query, List bibEntriesFilter) { + return List.of(); + } + + @Override + public AnswerEngineKind getKind() { + return AnswerEngineKind.FULL_DOCUMENT; + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultExpertSettings.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultExpertSettings.java index b2726cc6b696..fa5afa59b335 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultExpertSettings.java +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiDefaultExpertSettings.java @@ -1,7 +1,8 @@ package org.jabref.logic.ai.preferences; import org.jabref.model.ai.embeddings.EmbeddingModelEnumeration; -import org.jabref.model.ai.rag.DocumentSplitterKind; +import org.jabref.model.ai.pipeline.AnswerEngineKind; +import org.jabref.model.ai.pipeline.DocumentSplitterKind; import org.jabref.model.ai.summarization.SummarizatorKind; import org.jabref.model.ai.tokenization.TokenEstimatorKind; @@ -11,13 +12,16 @@ /// There are facilities in JabRef codebase to reset either all settings or 1 section, but not a part of a section. public class AiDefaultExpertSettings { public static final EmbeddingModelEnumeration EMBEDDING_MODEL = EmbeddingModelEnumeration.SENTENCE_TRANSFORMERS_ALL_MINILM_L12_V2; - public static final SummarizatorKind SUMMARIZATION_ALGORITHM_NAME = SummarizatorKind.CHUNKED; - public static final TokenEstimatorKind TOKEN_ESTIMATION_STRATEGY = TokenEstimatorKind.MAX; + public static final SummarizatorKind SUMMARIZATOR_KIND = SummarizatorKind.CHUNKED; + public static final TokenEstimatorKind TOKEN_ESTIMATOR_KIND = TokenEstimatorKind.MAX; public static final float TEMPERATURE = 0.7f; public static final int CONTEXT_WINDOW_SIZE = 8192; - public static final DocumentSplitterKind DOCUMENT_SPLITTING_STRATEGY = DocumentSplitterKind.SLIDING_WINDOW; + + public static final DocumentSplitterKind DOCUMENT_SPLITTER_KIND = DocumentSplitterKind.SLIDING_WINDOW; public static final int DOCUMENT_SPLITTER_CHUNK_SIZE = 300; public static final int DOCUMENT_SPLITTER_OVERLAP_SIZE = 100; + + public static final AnswerEngineKind ANSWER_ENGINE_KIND = AnswerEngineKind.EMBEDDINGS_SEARCH; public static final int RAG_MAX_RESULTS_COUNT = 10; public static final float RAG_MIN_SCORE = 0.3f; } diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java index 67c8f0be1dc8..f7bfd6cfee67 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java @@ -19,7 +19,8 @@ import org.jabref.logic.util.strings.StringUtil; import org.jabref.model.ai.chatting.AiProvider; import org.jabref.model.ai.embeddings.EmbeddingModelEnumeration; -import org.jabref.model.ai.rag.DocumentSplitterKind; +import org.jabref.model.ai.pipeline.AnswerEngineKind; +import org.jabref.model.ai.pipeline.DocumentSplitterKind; import org.jabref.model.ai.summarization.SummarizatorKind; import org.jabref.model.ai.templating.AiTemplate; import org.jabref.model.ai.tokenization.TokenEstimatorKind; @@ -55,16 +56,17 @@ public class AiPreferences { private final StringProperty huggingFaceApiBaseUrl; private final StringProperty gpt4AllApiBaseUrl; - private final ObjectProperty defaultSummarizationAlgorithm; - private final ObjectProperty tokenEstimationStrategy; + private final ObjectProperty summarizatorKind; + private final ObjectProperty tokenEstimatorKind; private final ObjectProperty embeddingModel; private final DoubleProperty temperature; private final IntegerProperty contextWindowSize; - private final ObjectProperty documentSplittingStrategy; + private final ObjectProperty documentSplitterKind; private final IntegerProperty documentSplitterChunkSize; private final IntegerProperty documentSplitterOverlapSize; + private final ObjectProperty answerEngineKind; private final IntegerProperty ragMaxResultsCount; private final DoubleProperty ragMinScore; @@ -88,7 +90,7 @@ public AiPreferences( String geminiApiBaseUrl, String huggingFaceApiBaseUrl, String gpt4AllApiBaseUrl, - SummarizatorKind defaultSummarizationAlgorithm, + SummarizatorKind summarizatorKind, TokenEstimatorKind tokenEstimatorKind, EmbeddingModelEnumeration embeddingModel, double temperature, @@ -96,6 +98,7 @@ public AiPreferences( DocumentSplitterKind documentSplitterKind, int documentSplitterChunkSize, int documentSplitterOverlapSize, + AnswerEngineKind answerEngineKind, int ragMaxResultsCount, double ragMinScore, Map templates @@ -120,16 +123,17 @@ public AiPreferences( this.huggingFaceApiBaseUrl = new SimpleStringProperty(huggingFaceApiBaseUrl); this.gpt4AllApiBaseUrl = new SimpleStringProperty(gpt4AllApiBaseUrl); - this.defaultSummarizationAlgorithm = new SimpleObjectProperty<>(defaultSummarizationAlgorithm); - this.tokenEstimationStrategy = new SimpleObjectProperty<>(tokenEstimatorKind); + this.summarizatorKind = new SimpleObjectProperty<>(summarizatorKind); + this.tokenEstimatorKind = new SimpleObjectProperty<>(tokenEstimatorKind); this.embeddingModel = new SimpleObjectProperty<>(embeddingModel); this.temperature = new SimpleDoubleProperty(temperature); this.contextWindowSize = new SimpleIntegerProperty(contextWindowSize); - this.documentSplittingStrategy = new SimpleObjectProperty<>(documentSplitterKind); + this.documentSplitterKind = new SimpleObjectProperty<>(documentSplitterKind); this.documentSplitterChunkSize = new SimpleIntegerProperty(documentSplitterChunkSize); this.documentSplitterOverlapSize = new SimpleIntegerProperty(documentSplitterOverlapSize); + this.answerEngineKind = new SimpleObjectProperty<>(answerEngineKind); this.ragMaxResultsCount = new SimpleIntegerProperty(ragMaxResultsCount); this.ragMinScore = new SimpleDoubleProperty(ragMinScore); @@ -296,28 +300,28 @@ public void setCustomizeExpertSettings(boolean customizeExpertSettings) { this.customizeExpertSettings.set(customizeExpertSettings); } - public ObjectProperty defaultSummarizationAlgorithmProperty() { - return defaultSummarizationAlgorithm; + public ObjectProperty summarizatorKindProperty() { + return summarizatorKind; } - public SummarizatorKind getDefaultSummarizationAlgorithm() { - return defaultSummarizationAlgorithm.get(); + public SummarizatorKind getSummarizatorKind() { + return summarizatorKind.get(); } - public void setDefaultSummarizationAlgorithm(SummarizatorKind defaultSummarizationAlgorithm) { - this.defaultSummarizationAlgorithm.set(defaultSummarizationAlgorithm); + public void setSummarizatorKind(SummarizatorKind summarizatorKind) { + this.summarizatorKind.set(summarizatorKind); } - public ObjectProperty tokenEstimationStrategyProperty() { - return tokenEstimationStrategy; + public ObjectProperty tokenEstimatorKindProperty() { + return tokenEstimatorKind; } - public TokenEstimatorKind getTokenEstimationStrategy() { - return tokenEstimationStrategy.get(); + public TokenEstimatorKind getTokenEstimatorKind() { + return tokenEstimatorKind.get(); } - public void setTokenEstimationStrategy(TokenEstimatorKind tokenEstimatorKind) { - this.tokenEstimationStrategy.set(tokenEstimatorKind); + public void setTokenEstimatorKind(TokenEstimatorKind tokenEstimatorKind) { + this.tokenEstimatorKind.set(tokenEstimatorKind); } public ObjectProperty embeddingModelProperty() { @@ -439,16 +443,16 @@ public void setContextWindowSize(int contextWindowSize) { this.contextWindowSize.set(contextWindowSize); } - public ObjectProperty documentSplittingStrategyProperty() { - return documentSplittingStrategy; + public ObjectProperty documentSplitterKindProperty() { + return documentSplitterKind; } - public DocumentSplitterKind getDocumentSplittingStrategy() { - return documentSplittingStrategy.get(); + public DocumentSplitterKind getDocumentSplitterKind() { + return documentSplitterKind.get(); } - public void setDocumentSplittingStrategy(DocumentSplitterKind documentSplitterKind) { - this.documentSplittingStrategy.set(documentSplitterKind); + public void setDocumentSplitterKind(DocumentSplitterKind documentSplitterKind) { + this.documentSplitterKind.set(documentSplitterKind); } public IntegerProperty documentSplitterChunkSizeProperty() { @@ -483,6 +487,18 @@ public void setDocumentSplitterOverlapSize(int documentSplitterOverlapSize) { this.documentSplitterOverlapSize.set(documentSplitterOverlapSize); } + public ObjectProperty answerEngineKindProperty() { + return answerEngineKind; + } + + public AnswerEngineKind getAnswerEngineKind() { + return answerEngineKind.get(); + } + + public void setAnswerEngineKind(AnswerEngineKind answerEngineKind) { + this.answerEngineKind.set(answerEngineKind); + } + public IntegerProperty ragMaxResultsCountProperty() { return ragMaxResultsCount; } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/BibEntrySummarizator.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/BibEntrySummarizator.java index d2b9624d1365..ece4285f31bc 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/BibEntrySummarizator.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/BibEntrySummarizator.java @@ -90,7 +90,7 @@ public BibEntrySummary summarize( LocalDateTime.now(), chatModel.getAiProvider(), chatModel.getName(), - summarizator.getName(), + summarizator.getKind(), finalSummary ); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/ChunkedSummarizator.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/ChunkedSummarizator.java index 001ea543e7cb..65969c687ea9 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/ChunkedSummarizator.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/ChunkedSummarizator.java @@ -121,7 +121,7 @@ public String summarize( } @Override - public SummarizatorKind getName() { + public SummarizatorKind getKind() { return SummarizatorKind.CHUNKED; } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/Summarizator.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/Summarizator.java index d07640952419..5877c2396a32 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/Summarizator.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/Summarizator.java @@ -11,5 +11,5 @@ String summarize( String text ) throws InterruptedException; - SummarizatorKind getName(); + SummarizatorKind getKind(); } diff --git a/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java b/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java index cfcac442e74d..29cd658b2ef6 100644 --- a/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java @@ -100,7 +100,8 @@ import org.jabref.logic.xmp.XmpPreferences; import org.jabref.model.ai.chatting.AiProvider; import org.jabref.model.ai.embeddings.EmbeddingModelEnumeration; -import org.jabref.model.ai.rag.DocumentSplitterKind; +import org.jabref.model.ai.pipeline.AnswerEngineKind; +import org.jabref.model.ai.pipeline.DocumentSplitterKind; import org.jabref.model.ai.summarization.SummarizatorKind; import org.jabref.model.ai.templating.AiTemplate; import org.jabref.model.ai.tokenization.TokenEstimatorKind; @@ -398,13 +399,14 @@ public class JabRefCliPreferences implements CliPreferences { private static final String AI_GEMINI_API_BASE_URL = "aiGeminiApiBaseUrl"; private static final String AI_HUGGING_FACE_API_BASE_URL = "aiHuggingFaceApiBaseUrl"; private static final String AI_GPT_4_ALL_API_BASE_URL = "aiGpt4AllApiBaseUrl"; - private static final String AI_SUMMARIZATION_ALGORITHM = "aiSummarizationAlgorithm"; - private static final String AI_TOKEN_ESTIMATION_ALGORITHM = "aiTokenEstimationAlgorithm"; + private static final String AI_SUMMARIZATOR_KIND = "aiSummarizatorKind"; + private static final String AI_TOKEN_ESTIMATOR_KIND = "aiTokenEstimatorKind"; private static final String AI_TEMPERATURE = "aiTemperature"; private static final String AI_CONTEXT_WINDOW_SIZE = "aiMessageWindowSize"; - private static final String AI_DOCUMENT_SPLITTING_STRATEGY = "aiDocumentSplittingStrategy"; + private static final String AI_DOCUMENT_SPLITTER_KIND = "aiDocumentSplitterKind"; private static final String AI_DOCUMENT_SPLITTER_CHUNK_SIZE = "aiDocumentSplitterChunkSize"; private static final String AI_DOCUMENT_SPLITTER_OVERLAP_SIZE = "aiDocumentSplitterOverlapSize"; + private static final String AI_ANSWER_ENGINE_KIND = "aiAnswerEngineKind"; private static final String AI_RAG_MAX_RESULTS_COUNT = "aiRagMaxResultsCount"; private static final String AI_RAG_MIN_SCORE = "aiRagMinScore"; @@ -749,8 +751,8 @@ public JabRefCliPreferences() { defaults.put(AI_GEMINI_API_BASE_URL, AiProvider.GEMINI.getApiUrl()); defaults.put(AI_HUGGING_FACE_API_BASE_URL, AiProvider.HUGGING_FACE.getApiUrl()); defaults.put(AI_GPT_4_ALL_API_BASE_URL, AiProvider.GPT4ALL.getApiUrl()); - defaults.put(AI_SUMMARIZATION_ALGORITHM, AiDefaultExpertSettings.SUMMARIZATION_ALGORITHM_NAME.name()); - defaults.put(AI_TOKEN_ESTIMATION_ALGORITHM, AiDefaultExpertSettings.TOKEN_ESTIMATION_STRATEGY.name()); + defaults.put(AI_SUMMARIZATOR_KIND, AiDefaultExpertSettings.SUMMARIZATOR_KIND.name()); + defaults.put(AI_TOKEN_ESTIMATOR_KIND, AiDefaultExpertSettings.TOKEN_ESTIMATOR_KIND.name()); defaults.put(AI_TEMPERATURE, AiDefaultExpertSettings.TEMPERATURE); defaults.put(AI_CONTEXT_WINDOW_SIZE, PredefinedChatModel.getContextWindowSize( @@ -758,9 +760,10 @@ public JabRefCliPreferences() { AiProviderDefaultChatModels.getDefaultChatModel(AiProvider.valueOf((String) defaults.get(AI_PROVIDER))).getName() ) ); - defaults.put(AI_DOCUMENT_SPLITTING_STRATEGY, AiDefaultExpertSettings.DOCUMENT_SPLITTING_STRATEGY.name()); + defaults.put(AI_DOCUMENT_SPLITTER_KIND, AiDefaultExpertSettings.DOCUMENT_SPLITTER_KIND.name()); defaults.put(AI_DOCUMENT_SPLITTER_CHUNK_SIZE, AiDefaultExpertSettings.DOCUMENT_SPLITTER_CHUNK_SIZE); defaults.put(AI_DOCUMENT_SPLITTER_OVERLAP_SIZE, AiDefaultExpertSettings.DOCUMENT_SPLITTER_OVERLAP_SIZE); + defaults.put(AI_ANSWER_ENGINE_KIND, AiDefaultExpertSettings.ANSWER_ENGINE_KIND.name()); defaults.put(AI_RAG_MAX_RESULTS_COUNT, AiDefaultExpertSettings.RAG_MAX_RESULTS_COUNT); defaults.put(AI_RAG_MIN_SCORE, AiDefaultExpertSettings.RAG_MIN_SCORE); @@ -2050,14 +2053,15 @@ public AiPreferences getAiPreferences() { get(AI_GEMINI_API_BASE_URL), get(AI_HUGGING_FACE_API_BASE_URL), get(AI_GPT_4_ALL_API_BASE_URL), - SummarizatorKind.valueOf(get(AI_SUMMARIZATION_ALGORITHM)), - TokenEstimatorKind.valueOf(get(AI_TOKEN_ESTIMATION_ALGORITHM)), + SummarizatorKind.valueOf(get(AI_SUMMARIZATOR_KIND)), + TokenEstimatorKind.valueOf(get(AI_TOKEN_ESTIMATOR_KIND)), EmbeddingModelEnumeration.valueOf(get(AI_EMBEDDING_MODEL)), getDouble(AI_TEMPERATURE), getInt(AI_CONTEXT_WINDOW_SIZE), - DocumentSplitterKind.valueOf(get(AI_DOCUMENT_SPLITTING_STRATEGY)), + DocumentSplitterKind.valueOf(get(AI_DOCUMENT_SPLITTER_KIND)), getInt(AI_DOCUMENT_SPLITTER_CHUNK_SIZE), getInt(AI_DOCUMENT_SPLITTER_OVERLAP_SIZE), + AnswerEngineKind.valueOf(get(AI_ANSWER_ENGINE_KIND)), getInt(AI_RAG_MAX_RESULTS_COUNT), getDouble(AI_RAG_MIN_SCORE), Map.of( @@ -2091,14 +2095,17 @@ AiTemplate.CITATION_PARSING_USER_MESSAGE, get(AI_CITATION_PARSING_USER_MESSAGE_T EasyBind.listen(aiPreferences.huggingFaceApiBaseUrlProperty(), (_, _, newValue) -> put(AI_HUGGING_FACE_API_BASE_URL, newValue)); EasyBind.listen(aiPreferences.gpt4AllApiBaseUrlProperty(), (_, _, newValue) -> put(AI_GPT_4_ALL_API_BASE_URL, newValue)); - EasyBind.listen(aiPreferences.defaultSummarizationAlgorithmProperty(), (_, _, newValue) -> put(AI_SUMMARIZATION_ALGORITHM, newValue.name())); - EasyBind.listen(aiPreferences.tokenEstimationStrategyProperty(), (_, _, newValue) -> put(AI_TOKEN_ESTIMATION_ALGORITHM, newValue.name())); + EasyBind.listen(aiPreferences.summarizatorKindProperty(), (_, _, newValue) -> put(AI_SUMMARIZATOR_KIND, newValue.name())); + EasyBind.listen(aiPreferences.tokenEstimatorKindProperty(), (_, _, newValue) -> put(AI_TOKEN_ESTIMATOR_KIND, newValue.name())); EasyBind.listen(aiPreferences.embeddingModelProperty(), (_, _, newValue) -> put(AI_EMBEDDING_MODEL, newValue.name())); EasyBind.listen(aiPreferences.temperatureProperty(), (_, _, newValue) -> putDouble(AI_TEMPERATURE, newValue.doubleValue())); EasyBind.listen(aiPreferences.contextWindowSizeProperty(), (_, _, newValue) -> putInt(AI_CONTEXT_WINDOW_SIZE, newValue)); - EasyBind.listen(aiPreferences.documentSplittingStrategyProperty(), (_, _, newValue) -> put(AI_DOCUMENT_SPLITTING_STRATEGY, newValue.name())); + + EasyBind.listen(aiPreferences.documentSplitterKindProperty(), (_, _, newValue) -> put(AI_DOCUMENT_SPLITTER_KIND, newValue.name())); EasyBind.listen(aiPreferences.documentSplitterChunkSizeProperty(), (_, _, newValue) -> putInt(AI_DOCUMENT_SPLITTER_CHUNK_SIZE, newValue)); EasyBind.listen(aiPreferences.documentSplitterOverlapSizeProperty(), (_, _, newValue) -> putInt(AI_DOCUMENT_SPLITTER_OVERLAP_SIZE, newValue)); + + EasyBind.listen(aiPreferences.answerEngineKindProperty(), (_, _, newValue) -> put(AI_ANSWER_ENGINE_KIND, newValue.name())); EasyBind.listen(aiPreferences.ragMaxResultsCountProperty(), (_, _, newValue) -> putInt(AI_RAG_MAX_RESULTS_COUNT, newValue)); EasyBind.listen(aiPreferences.ragMinScoreProperty(), (_, _, newValue) -> putDouble(AI_RAG_MIN_SCORE, newValue.doubleValue())); diff --git a/jablib/src/main/java/org/jabref/model/ai/pipeline/AnswerEngineKind.java b/jablib/src/main/java/org/jabref/model/ai/pipeline/AnswerEngineKind.java new file mode 100644 index 000000000000..24bb488459e1 --- /dev/null +++ b/jablib/src/main/java/org/jabref/model/ai/pipeline/AnswerEngineKind.java @@ -0,0 +1,18 @@ +package org.jabref.model.ai.pipeline; + +import org.jabref.logic.l10n.Localization; + +public enum AnswerEngineKind { + EMBEDDINGS_SEARCH(Localization.lang("Embeddings Search")), + FULL_DOCUMENT(Localization.lang("Full Document")); + + private final String displayName; + + AnswerEngineKind(String displayName) { + this.displayName = displayName; + } + + public String getDisplayName() { + return displayName; + } +} diff --git a/jablib/src/main/java/org/jabref/model/ai/pipeline/DocumentSplitterKind.java b/jablib/src/main/java/org/jabref/model/ai/pipeline/DocumentSplitterKind.java new file mode 100644 index 000000000000..0b84a84fb500 --- /dev/null +++ b/jablib/src/main/java/org/jabref/model/ai/pipeline/DocumentSplitterKind.java @@ -0,0 +1,17 @@ +package org.jabref.model.ai.pipeline; + +import org.jabref.logic.l10n.Localization; + +public enum DocumentSplitterKind { + SLIDING_WINDOW(Localization.lang("Sliding Window")); + + private final String displayName; + + DocumentSplitterKind(String displayName) { + this.displayName = displayName; + } + + public String getDisplayName() { + return displayName; + } +} diff --git a/jablib/src/main/java/org/jabref/model/ai/pipeline/RelevantInformation.java b/jablib/src/main/java/org/jabref/model/ai/pipeline/RelevantInformation.java new file mode 100644 index 000000000000..d3aad66d7079 --- /dev/null +++ b/jablib/src/main/java/org/jabref/model/ai/pipeline/RelevantInformation.java @@ -0,0 +1,7 @@ +package org.jabref.model.ai.pipeline; + +import java.util.List; + +// Sources are citation key. +public record RelevantInformation(List sources, String text) { +} diff --git a/jablib/src/main/java/org/jabref/model/ai/rag/DocumentSplitterKind.java b/jablib/src/main/java/org/jabref/model/ai/rag/DocumentSplitterKind.java deleted file mode 100644 index 2d576783c59a..000000000000 --- a/jablib/src/main/java/org/jabref/model/ai/rag/DocumentSplitterKind.java +++ /dev/null @@ -1,5 +0,0 @@ -package org.jabref.model.ai.rag; - -public enum DocumentSplitterKind { - SLIDING_WINDOW; -} diff --git a/jablib/src/main/java/org/jabref/model/ai/rag/PaperExcerpt.java b/jablib/src/main/java/org/jabref/model/ai/rag/PaperExcerpt.java deleted file mode 100644 index 3f054da0a834..000000000000 --- a/jablib/src/main/java/org/jabref/model/ai/rag/PaperExcerpt.java +++ /dev/null @@ -1,4 +0,0 @@ -package org.jabref.model.ai.rag; - -public record PaperExcerpt(String citationKey, String text) { -} From 6b2662b48eb775d979e516c377fafaea7b374d7e Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Fri, 28 Nov 2025 22:10:49 +0100 Subject: [PATCH 031/243] Fix errors + finish FullDocumentAnswerEngine --- .../ai/components/aichat/AiChatComponent.java | 5 +- .../org/jabref/gui/preferences/ai/AiTab.java | 2 +- jablib/src/main/java/module-info.java | 4 +- .../java/org/jabref/logic/ai/AiService.java | 2 +- .../logic/ai/chatting/logic/AiChatLogic.java | 91 +++------------ .../logic/ai/current/CurrentAnswerEngine.java | 34 +++++- .../ai/current/CurrentDocumentSplitter.java | 2 +- .../ai/pipeline/logic/rag/AnswerEngine.java | 9 +- .../rag/EmbeddingsSearchAnswerEngine.java | 110 +++++++++++++++++- .../logic/rag/FullDocumentAnswerEngine.java | 40 ++++++- .../identifiers/FullBibEntryAiIdentifier.java | 7 ++ 11 files changed, 212 insertions(+), 94 deletions(-) create mode 100644 jablib/src/main/java/org/jabref/model/ai/identifiers/FullBibEntryAiIdentifier.java diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java index df4f2e197b5d..ca7f835d2c33 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java @@ -102,14 +102,13 @@ public AiChatComponent( this.aiChatLogic = new AiChatLogic( aiPreferences, aiService.getChatLanguageModel(), - aiService.getEmbeddingModel(), - aiService.getEmbeddingStore(), aiService.getCurrentAiTemplates().getChattingSystemMessageTemplate(), aiService.getCurrentAiTemplates().getChattingUserMessageTemplate(), bibDatabaseContext, chatHistory, entries, - name + name, + aiService.getAnswerEngine() ); aiService.getIngestionService().ingest(name, ListUtil.getLinkedFiles(entries).toList(), bibDatabaseContext); diff --git a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java index 1cfa87a54fb4..72a4201ab9bd 100644 --- a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java +++ b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java @@ -23,7 +23,7 @@ import org.jabref.gui.util.ViewModelListCellFactory; import org.jabref.logic.help.HelpFile; import org.jabref.logic.l10n.Localization; -import org.jabref.model.ai.chatting.AiProviderType; +import org.jabref.model.ai.chatting.AiProvider; import org.jabref.model.ai.embeddings.EmbeddingModelEnumeration; import org.jabref.model.ai.templating.AiTemplate; diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index 0144c43324d8..18caeb01f8dd 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -124,6 +124,9 @@ exports org.jabref.logic.ai.customimplementations.llms; exports org.jabref.logic.ai.pipeline; exports org.jabref.logic.ai.pipeline.logic.documentsplitting; + exports org.jabref.logic.ai.pipeline.logic.ingestion; + exports org.jabref.logic.ai.pipeline.logic.parsing; + exports org.jabref.logic.ai.pipeline.logic.rag; exports org.jabref.logic.ai.summarization.tasks; exports org.jabref.logic.ai.summarization.repositories; exports org.jabref.logic.ai.summarization.templates; @@ -295,6 +298,5 @@ requires org.jooq.jool; requires org.libreoffice.uno; requires transitive org.jspecify; - requires org.jabref.jablib; // endregion } diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiService.java b/jablib/src/main/java/org/jabref/logic/ai/AiService.java index c949d5e37afd..f1bc5fa16a2f 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiService.java @@ -92,7 +92,7 @@ public AiService( this.currentChatLanguageModel = new CurrentChatLanguageModel(aiPreferences, currentTokenEstimator); this.currentEmbeddingModel = new CurrentEmbeddingModel(aiPreferences, notificationService, taskExecutor); this.currentSummarizator = new CurrentSummarizator(aiPreferences, currentAiTemplates); - this.currentAnswerEngine = new CurrentAnswerEngine(aiPreferences); + this.currentAnswerEngine = new CurrentAnswerEngine(aiPreferences, filePreferences, currentEmbeddingModel, mvStoreEmbeddingStore); this.ingestionService = new IngestionService( aiPreferences, diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java index d93a6f623236..bb124b8f53dc 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java @@ -1,41 +1,33 @@ package org.jabref.logic.ai.chatting.logic; import java.util.List; -import java.util.Optional; +import javafx.beans.property.SimpleBooleanProperty; import javafx.beans.property.StringProperty; -import javafx.collections.ListChangeListener; import javafx.collections.ObservableList; import org.jabref.logic.ai.chatting.templates.ChattingSystemMessageTemplate; import org.jabref.logic.ai.chatting.templates.ChattingUserMessageTemplate; -import org.jabref.logic.ai.pipeline.logic.EmbeddingsCleaner; +import org.jabref.logic.ai.pipeline.logic.rag.AnswerEngine; import org.jabref.logic.ai.preferences.AiPreferences; +import org.jabref.logic.ai.util.LongTaskInfo; +import org.jabref.logic.util.ProgressCounter; import org.jabref.model.ai.chatting.ErrorMessage; +import org.jabref.model.ai.identifiers.FullBibEntryAiIdentifier; import org.jabref.model.ai.pipeline.RelevantInformation; import org.jabref.model.ai.templating.AiTemplate; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.LinkedFile; -import org.jabref.model.util.ListUtil; import dev.langchain4j.data.message.AiMessage; import dev.langchain4j.data.message.ChatMessage; import dev.langchain4j.data.message.SystemMessage; import dev.langchain4j.data.message.UserMessage; -import dev.langchain4j.data.segment.TextSegment; import dev.langchain4j.memory.ChatMemory; import dev.langchain4j.memory.chat.TokenWindowChatMemory; import dev.langchain4j.model.chat.ChatModel; -import dev.langchain4j.model.embedding.EmbeddingModel; import dev.langchain4j.model.openai.OpenAiChatModelName; import dev.langchain4j.model.openai.OpenAiTokenCountEstimator; -import dev.langchain4j.store.embedding.EmbeddingMatch; -import dev.langchain4j.store.embedding.EmbeddingSearchRequest; -import dev.langchain4j.store.embedding.EmbeddingSearchResult; -import dev.langchain4j.store.embedding.EmbeddingStore; -import dev.langchain4j.store.embedding.filter.Filter; -import dev.langchain4j.store.embedding.filter.MetadataFilterBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,8 +36,6 @@ public class AiChatLogic { private final AiPreferences aiPreferences; private final ChatModel chatLanguageModel; - private final EmbeddingModel embeddingModel; - private final EmbeddingStore embeddingStore; private final ChattingSystemMessageTemplate chattingSystemMessageTemplate; private final ChattingUserMessageTemplate chattingUserMessageTemplate; @@ -56,32 +46,28 @@ public class AiChatLogic { private ChatMemory chatMemory; - private Optional filter = Optional.empty(); + private final AnswerEngine answerEngine; public AiChatLogic( AiPreferences aiPreferences, ChatModel chatLanguageModel, - EmbeddingModel embeddingModel, - EmbeddingStore embeddingStore, ChattingSystemMessageTemplate chattingSystemMessageTemplate, ChattingUserMessageTemplate chattingUserMessageTemplate, BibDatabaseContext bibDatabaseContext, ObservableList chatHistory, ObservableList entries, - StringProperty name + StringProperty name, + AnswerEngine answerEngine ) { this.aiPreferences = aiPreferences; this.chatLanguageModel = chatLanguageModel; - this.embeddingModel = embeddingModel; - this.embeddingStore = embeddingStore; this.chattingSystemMessageTemplate = chattingSystemMessageTemplate; this.chattingUserMessageTemplate = chattingUserMessageTemplate; this.chatHistory = chatHistory; this.entries = entries; this.name = name; this.bibDatabaseContext = bibDatabaseContext; - - this.entries.addListener((ListChangeListener) change -> rebuildFilter()); + this.answerEngine = answerEngine; setupListeningToPreferencesChanges(); rebuildFull(chatHistory); @@ -98,10 +84,10 @@ private void setupListeningToPreferencesChanges() { private void rebuildFull(List chatMessages) { rebuildChatMemory(chatMessages); - rebuildFilter(); } private void rebuildChatMemory(List chatMessages) { + // TODO: remove this. No algorithm for squashing the conversation. // Because we can't get a tokenizer for each model, {@link AiChatLogic} assumes that // every text is tokenized like it's tokenized for OpenAI's GPT-4o-mini model. // @@ -121,22 +107,6 @@ private void rebuildChatMemory(List chatMessages) { setSystemMessage(chattingSystemMessageTemplate.render(entries)); } - private void rebuildFilter() { - List linkedFiles = ListUtil.getLinkedFiles(entries).toList(); - - if (linkedFiles.isEmpty()) { - filter = Optional.empty(); - } else { - filter = Optional.of(MetadataFilterBuilder - .metadataKey(EmbeddingsCleaner.LINK_METADATA_KEY) - .isIn(linkedFiles - .stream() - .map(LinkedFile::getLink) - .toList() - )); - } - } - private void setSystemMessage(String systemMessage) { chatMemory.add(new SystemMessage(systemMessage)); } @@ -151,32 +121,6 @@ public AiMessage execute(UserMessage message) { name.get(), message.singleText()); - EmbeddingSearchRequest embeddingSearchRequest = EmbeddingSearchRequest - .builder() - .maxResults(aiPreferences.getRagMaxResultsCount()) - .minScore(aiPreferences.getRagMinScore()) - .filter(filter.orElse(null)) - .queryEmbedding(embeddingModel.embed(message.singleText()).content()) - .build(); - - EmbeddingSearchResult embeddingSearchResult = embeddingStore.search(embeddingSearchRequest); - - List excerpts = embeddingSearchResult - .matches() - .stream() - .map(EmbeddingMatch::embedded) - .map(textSegment -> { - String link = textSegment.metadata().getString(EmbeddingsCleaner.LINK_METADATA_KEY); - - if (link == null) { - return new RelevantInformation(List.of(), textSegment.text()); - } else { - return new RelevantInformation(List.of(findEntryByLink(link).flatMap(BibEntry::getCitationKey).orElse("")), textSegment.text()); - } - }) - .toList(); - - LOGGER.debug("Found excerpts for the message: {}", excerpts); // This is crazy, but langchain4j {@link ChatMemory} does not allow to remove single messages. ChatMemory tempChatMemory = TokenWindowChatMemory @@ -186,6 +130,13 @@ public AiMessage execute(UserMessage message) { chatMemory.messages().forEach(tempChatMemory::add); + List excerpts = answerEngine.process( + // TODO: think about this. + new LongTaskInfo(new ProgressCounter(), new SimpleBooleanProperty(false)), + message.singleText(), + entries.stream().map(entry -> new FullBibEntryAiIdentifier(bibDatabaseContext, entry)).toList() + ); + tempChatMemory.add(new UserMessage(chattingUserMessageTemplate.render(entries, message.singleText(), excerpts))); chatMemory.add(message); @@ -199,14 +150,6 @@ public AiMessage execute(UserMessage message) { return aiMessage; } - private Optional findEntryByLink(String link) { - return bibDatabaseContext - .getEntries() - .stream() - .filter(entry -> entry.getFiles().stream().anyMatch(file -> file.getLink().equals(link))) - .findFirst(); - } - public ObservableList getChatHistory() { return chatHistory; } diff --git a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentAnswerEngine.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentAnswerEngine.java index 9dc5d6e013b6..0ba3e4562149 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentAnswerEngine.java +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentAnswerEngine.java @@ -2,24 +2,42 @@ import java.util.List; +import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.pipeline.logic.rag.AnswerEngine; import org.jabref.logic.ai.pipeline.logic.rag.EmbeddingsSearchAnswerEngine; import org.jabref.logic.ai.pipeline.logic.rag.FullDocumentAnswerEngine; import org.jabref.logic.ai.preferences.AiPreferences; +import org.jabref.logic.ai.util.LongTaskInfo; +import org.jabref.model.ai.identifiers.FullBibEntryAiIdentifier; import org.jabref.model.ai.pipeline.AnswerEngineKind; import org.jabref.model.ai.pipeline.RelevantInformation; -import org.jabref.model.entry.BibEntry; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.store.embedding.EmbeddingStore; import jakarta.annotation.Nullable; public class CurrentAnswerEngine implements AnswerEngine { private final AiPreferences aiPreferences; + private final FilePreferences filePreferences; + + private final EmbeddingModel embeddingModel; + private final EmbeddingStore embeddingStore; @Nullable private AnswerEngine answerEngine = null; - public CurrentAnswerEngine(AiPreferences aiPreferences) { + public CurrentAnswerEngine( + AiPreferences aiPreferences, + FilePreferences filePreferences, + EmbeddingModel embeddingModel, + EmbeddingStore embeddingStore + ) { this.aiPreferences = aiPreferences; + this.filePreferences = filePreferences; + + this.embeddingModel = embeddingModel; + this.embeddingStore = embeddingStore; update(); configure(); @@ -29,12 +47,14 @@ private void update() { switch (aiPreferences.getAnswerEngineKind()) { case AnswerEngineKind.EMBEDDINGS_SEARCH -> answerEngine = new EmbeddingsSearchAnswerEngine( + embeddingModel, + embeddingStore, aiPreferences.getRagMinScore(), aiPreferences.getRagMaxResultsCount() ); case FULL_DOCUMENT -> - answerEngine = new FullDocumentAnswerEngine(); + answerEngine = new FullDocumentAnswerEngine(filePreferences); } } @@ -46,11 +66,15 @@ private void configure() { } @Override - public List process(String query, List bibEntriesFilter) { + public List process( + LongTaskInfo longTaskInfo, + String query, + List entriesFilter + ) { if (answerEngine == null) { return List.of(); } else { - return answerEngine.process(query, bibEntriesFilter); + return answerEngine.process(longTaskInfo, query, entriesFilter); } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentDocumentSplitter.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentDocumentSplitter.java index 76330ed6ee11..21a1a0c4178a 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentDocumentSplitter.java +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentDocumentSplitter.java @@ -54,6 +54,6 @@ public Stream split(LongTaskInfo longTaskInfo, String text) throws Inter @Override public DocumentSplitterKind getKind() { - aiPreferences.getDocumentSplitterKind(); + return aiPreferences.getDocumentSplitterKind(); } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/AnswerEngine.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/AnswerEngine.java index bba786abc9fd..7cc6ed34fd5e 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/AnswerEngine.java +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/AnswerEngine.java @@ -2,12 +2,17 @@ import java.util.List; +import org.jabref.logic.ai.util.LongTaskInfo; +import org.jabref.model.ai.identifiers.FullBibEntryAiIdentifier; import org.jabref.model.ai.pipeline.AnswerEngineKind; import org.jabref.model.ai.pipeline.RelevantInformation; -import org.jabref.model.entry.BibEntry; public interface AnswerEngine { - List process(String query, List bibEntriesFilter); + List process( + LongTaskInfo longTaskInfo, + String query, + List entriesFilter + ); AnswerEngineKind getKind(); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/EmbeddingsSearchAnswerEngine.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/EmbeddingsSearchAnswerEngine.java index e1c1b09a1686..07c01f18c686 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/EmbeddingsSearchAnswerEngine.java +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/EmbeddingsSearchAnswerEngine.java @@ -1,23 +1,127 @@ package org.jabref.logic.ai.pipeline.logic.rag; import java.util.List; +import java.util.Optional; +import org.jabref.logic.ai.pipeline.logic.EmbeddingsCleaner; +import org.jabref.logic.ai.util.LongTaskInfo; +import org.jabref.model.ai.identifiers.FullBibEntryAiIdentifier; import org.jabref.model.ai.pipeline.AnswerEngineKind; import org.jabref.model.ai.pipeline.RelevantInformation; import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.LinkedFile; +import org.jabref.model.util.ListUtil; + +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.store.embedding.EmbeddingMatch; +import dev.langchain4j.store.embedding.EmbeddingSearchRequest; +import dev.langchain4j.store.embedding.EmbeddingSearchResult; +import dev.langchain4j.store.embedding.EmbeddingStore; +import dev.langchain4j.store.embedding.filter.Filter; +import dev.langchain4j.store.embedding.filter.MetadataFilterBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class EmbeddingsSearchAnswerEngine implements AnswerEngine { + private static final Logger LOGGER = LoggerFactory.getLogger(EmbeddingsSearchAnswerEngine.class); + + private final EmbeddingModel embeddingModel; + private final EmbeddingStore embeddingStore; private final double minimumScore; private final int maximumResultsCount; - public EmbeddingsSearchAnswerEngine(double minimumScore, int maximumResultsCount) { + public EmbeddingsSearchAnswerEngine( + EmbeddingModel embeddingModel, + EmbeddingStore embeddingStore, + double minimumScore, + int maximumResultsCount + ) { + this.embeddingModel = embeddingModel; + this.embeddingStore = embeddingStore; this.minimumScore = minimumScore; this.maximumResultsCount = maximumResultsCount; } @Override - public List process(String query, List bibEntriesFilter) { - return List.of(); + public List process( + LongTaskInfo longTaskInfo, + String query, + List entriesFilter + ) { + // TODO: Simplify. + + List entries = entriesFilter + .stream() + .map(FullBibEntryAiIdentifier::entry) + .toList(); + + List linkedFiles = ListUtil.getLinkedFiles(entries).toList(); + + Optional filter; + if (linkedFiles.isEmpty()) { + filter = Optional.empty(); + } else { + filter = Optional.of(MetadataFilterBuilder + .metadataKey(EmbeddingsCleaner.LINK_METADATA_KEY) + .isIn(linkedFiles + .stream() + .map(LinkedFile::getLink) + .toList() + )); + } + + EmbeddingSearchRequest embeddingSearchRequest = EmbeddingSearchRequest + .builder() + .maxResults(maximumResultsCount) + .minScore(minimumScore) + .filter(filter.orElse(null)) + .queryEmbedding(embeddingModel.embed(query).content()) + .build(); + + EmbeddingSearchResult embeddingSearchResult = embeddingStore.search(embeddingSearchRequest); + + List excerpts = embeddingSearchResult + .matches() + .stream() + .map(EmbeddingMatch::embedded) + .map(textSegment -> { + String link = textSegment.metadata().getString(EmbeddingsCleaner.LINK_METADATA_KEY); + + if (link == null) { + return new RelevantInformation(List.of(), textSegment.text()); + } else { + return new RelevantInformation(List.of(findEntryByLink(entriesFilter, link).flatMap(BibEntry::getCitationKey).orElse("")), textSegment.text()); + } + }) + .toList(); + + LOGGER.debug("Found excerpts for the message: {}", excerpts); + + return excerpts; + } + + private Optional findEntryByLink(List entries, String link) { + // TODO: Simplify?? + return entries + .stream() + .flatMap(identifier -> + identifier + .databaseContext() + .getEntries() + .stream() + .filter(entry -> + entry + .getFiles() + .stream() + .anyMatch(file -> + file + .getLink(). + equals(link) + ) + ) + ) + .findFirst(); } @Override diff --git a/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/FullDocumentAnswerEngine.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/FullDocumentAnswerEngine.java index 68a7f876f367..1f2f767e2bc3 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/FullDocumentAnswerEngine.java +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/rag/FullDocumentAnswerEngine.java @@ -1,15 +1,49 @@ package org.jabref.logic.ai.pipeline.logic.rag; import java.util.List; +import java.util.Optional; +import org.jabref.logic.FilePreferences; +import org.jabref.logic.ai.pipeline.logic.parsing.UniversalContentParser; +import org.jabref.logic.ai.util.LongTaskInfo; +import org.jabref.model.ai.identifiers.FullBibEntryAiIdentifier; import org.jabref.model.ai.pipeline.AnswerEngineKind; import org.jabref.model.ai.pipeline.RelevantInformation; -import org.jabref.model.entry.BibEntry; public class FullDocumentAnswerEngine implements AnswerEngine { + private final FilePreferences filePreferences; + + // TODO: Add dependency on parsing. + private final UniversalContentParser universalContentParser = new UniversalContentParser(); + + public FullDocumentAnswerEngine(FilePreferences filePreferences) { + this.filePreferences = filePreferences; + } + @Override - public List process(String query, List bibEntriesFilter) { - return List.of(); + public List process( + LongTaskInfo longTaskInfo, + String query, + List entriesFilter + ) { + // Look at this! + return entriesFilter + .stream() + .flatMap(entryIdentifier -> + entryIdentifier + .entry() + .getFiles() + .stream() + .map(linkedFile -> + linkedFile + .findIn(entryIdentifier.databaseContext(), filePreferences) + .flatMap(p -> universalContentParser.parse(longTaskInfo, p)) + .map(c -> new RelevantInformation(List.of(linkedFile.getLink()), c)) + ) + .filter(Optional::isPresent) + .map(Optional::get) + ) + .toList(); } @Override diff --git a/jablib/src/main/java/org/jabref/model/ai/identifiers/FullBibEntryAiIdentifier.java b/jablib/src/main/java/org/jabref/model/ai/identifiers/FullBibEntryAiIdentifier.java new file mode 100644 index 000000000000..dcdd4c420310 --- /dev/null +++ b/jablib/src/main/java/org/jabref/model/ai/identifiers/FullBibEntryAiIdentifier.java @@ -0,0 +1,7 @@ +package org.jabref.model.ai.identifiers; + +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; + +public record FullBibEntryAiIdentifier(BibDatabaseContext databaseContext, BibEntry entry) { +} From e47819d212107069e9f1fb8aeb2de6548e38c747 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Fri, 28 Nov 2025 22:53:55 +0100 Subject: [PATCH 032/243] Refactor AiChatLogic.java --- .../logic/ai/chatting/logic/AiChatLogic.java | 68 +++++-------------- 1 file changed, 18 insertions(+), 50 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java index bb124b8f53dc..89803d6d09aa 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java @@ -1,5 +1,6 @@ package org.jabref.logic.ai.chatting.logic; +import java.util.ArrayList; import java.util.List; import javafx.beans.property.SimpleBooleanProperty; @@ -23,11 +24,7 @@ import dev.langchain4j.data.message.ChatMessage; import dev.langchain4j.data.message.SystemMessage; import dev.langchain4j.data.message.UserMessage; -import dev.langchain4j.memory.ChatMemory; -import dev.langchain4j.memory.chat.TokenWindowChatMemory; import dev.langchain4j.model.chat.ChatModel; -import dev.langchain4j.model.openai.OpenAiChatModelName; -import dev.langchain4j.model.openai.OpenAiTokenCountEstimator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,7 +41,7 @@ public class AiChatLogic { private final StringProperty name; private final BibDatabaseContext bibDatabaseContext; - private ChatMemory chatMemory; + private final List chatMemory; private final AnswerEngine answerEngine; @@ -69,8 +66,11 @@ public AiChatLogic( this.bibDatabaseContext = bibDatabaseContext; this.answerEngine = answerEngine; + this.chatMemory = new ArrayList<>(); + chatHistory.stream().filter(chatMessage -> !(chatMessage instanceof ErrorMessage)).forEach(chatMemory::add); + setSystemMessage(chattingSystemMessageTemplate.render(entries)); + setupListeningToPreferencesChanges(); - rebuildFull(chatHistory); } private void setupListeningToPreferencesChanges() { @@ -78,57 +78,25 @@ private void setupListeningToPreferencesChanges() { .templateProperty(AiTemplate.CHATTING_SYSTEM_MESSAGE) .addListener(obs -> setSystemMessage(chattingSystemMessageTemplate.render(entries))); - - aiPreferences.contextWindowSizeProperty().addListener(obs -> rebuildFull(chatMemory.messages())); - } - - private void rebuildFull(List chatMessages) { - rebuildChatMemory(chatMessages); - } - - private void rebuildChatMemory(List chatMessages) { - // TODO: remove this. No algorithm for squashing the conversation. - // Because we can't get a tokenizer for each model, {@link AiChatLogic} assumes that - // every text is tokenized like it's tokenized for OpenAI's GPT-4o-mini model. - // - // Reasons why we can't get tokenizer for each model: - // - Some tokenizers might not be available in langchain4j. - // - User may use a custom model, but there is no way to supply a custom tokenizer. - // - OpenAI API (and compatible ones) doesn't have an endpoint for tokenizing text. - // - // This is another dark workaround of AI integration. But it works "good-enough" for now. - this.chatMemory = TokenWindowChatMemory - .builder() - .maxTokens(aiPreferences.getContextWindowSize(), new OpenAiTokenCountEstimator(OpenAiChatModelName.GPT_4_O_MINI)) - .build(); - - chatMessages.stream().filter(chatMessage -> !(chatMessage instanceof ErrorMessage)).forEach(chatMemory::add); - - setSystemMessage(chattingSystemMessageTemplate.render(entries)); } private void setSystemMessage(String systemMessage) { - chatMemory.add(new SystemMessage(systemMessage)); + if (chatMemory.isEmpty()) { + chatMemory.add(new SystemMessage(systemMessage)); + } else { + chatMemory.set(0, new SystemMessage(systemMessage)); + } } public AiMessage execute(UserMessage message) { - // Message will be automatically added to ChatMemory through ConversationalRetrievalChain. - chatHistory.add(message); - LOGGER.info("Sending message to AI provider ({}) for answering in {}: {}", + LOGGER.info( + "Sending message to AI provider ({}) for answering in {}: {}", aiPreferences.getAiProvider().getApiUrl(), name.get(), - message.singleText()); - - - // This is crazy, but langchain4j {@link ChatMemory} does not allow to remove single messages. - ChatMemory tempChatMemory = TokenWindowChatMemory - .builder() - .maxTokens(aiPreferences.getContextWindowSize(), new OpenAiTokenCountEstimator(OpenAiChatModelName.GPT_4_O_MINI)) - .build(); - - chatMemory.messages().forEach(tempChatMemory::add); + message.singleText() + ); List excerpts = answerEngine.process( // TODO: think about this. @@ -137,11 +105,11 @@ public AiMessage execute(UserMessage message) { entries.stream().map(entry -> new FullBibEntryAiIdentifier(bibDatabaseContext, entry)).toList() ); - tempChatMemory.add(new UserMessage(chattingUserMessageTemplate.render(entries, message.singleText(), excerpts))); - chatMemory.add(message); + chatMemory.add(new UserMessage(chattingUserMessageTemplate.render(entries, message.singleText(), excerpts))); - AiMessage aiMessage = chatLanguageModel.chat(tempChatMemory.messages()).aiMessage(); + AiMessage aiMessage = chatLanguageModel.chat(chatMemory).aiMessage(); + chatMemory.set(chatMemory.size() - 1, message); // Removing excerpts from the chat history. chatMemory.add(aiMessage); chatHistory.add(aiMessage); From eb525ef97e8ff30ec28c448d2ffa70b094a04c2b Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Fri, 28 Nov 2025 23:23:40 +0100 Subject: [PATCH 033/243] Refactor a bit chat history --- .../org/jabref/gui/entryeditor/AiChatTab.java | 4 +- .../jabref/gui/groups/GroupTreeViewModel.java | 2 +- .../java/org/jabref/logic/ai/AiService.java | 20 ++- .../chatting/ChatHistoryManagementRecord.java | 18 +++ ...vice.java => EntryChatHistoryService.java} | 148 ++++-------------- .../ai/chatting/GroupChatHistoryService.java | 127 +++++++++++++++ ...y.java => EntryChatHistoryRepository.java} | 7 +- .../GroupChatHistoryRepository.java | 13 ++ .../MVStoreChatHistoryRepository.java | 2 +- .../ai/current/CurrentChatLanguageModel.java | 4 +- .../logic/ai/pipeline/IngestionService.java | 2 +- ...va => EntryChatHistoryRepositoryTest.java} | 10 +- .../MVStoreChatHistoryRepositoryTest.java | 3 +- 13 files changed, 218 insertions(+), 142 deletions(-) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryManagementRecord.java rename jablib/src/main/java/org/jabref/logic/ai/chatting/{ChatHistoryService.java => EntryChatHistoryService.java} (52%) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChatHistoryService.java rename jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/{ChatHistoryRepository.java => EntryChatHistoryRepository.java} (55%) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepository.java rename jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/{ChatHistoryRepositoryTest.java => EntryChatHistoryRepositoryTest.java} (85%) diff --git a/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java b/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java index 2db36b1ae91f..0617b6b8e627 100644 --- a/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java +++ b/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java @@ -74,7 +74,7 @@ public boolean shouldShow(BibEntry entry) { */ @Override protected void bindToEntry(BibEntry entry) { - previousBibEntry.ifPresent(previousBibEntry -> aiService.getChatHistoryService().closeChatHistoryForEntry(previousBibEntry)); + previousBibEntry.ifPresent(previousBibEntry -> aiService.getEntryChatHistoryService().closeChatHistory(previousBibEntry)); previousBibEntry = Optional.of(entry); BibDatabaseContext bibDatabaseContext = stateManager.getActiveDatabase().orElse(new BibDatabaseContext()); @@ -135,7 +135,7 @@ private void showChatPanel(BibDatabaseContext bibDatabaseContext, BibEntry entry setContent(new AiChatGuardedComponent( chatName, - aiService.getChatHistoryService().getChatHistoryForEntry(bibDatabaseContext, entry), + aiService.getEntryChatHistoryService().getChatHistory(bibDatabaseContext, entry), bibDatabaseContext, FXCollections.observableArrayList(new ArrayList<>(List.of(entry))), aiService, diff --git a/jabgui/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java b/jabgui/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java index 7e51a86b832b..40d48750b9e7 100644 --- a/jabgui/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java +++ b/jabgui/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java @@ -465,7 +465,7 @@ public void chatWithGroup(GroupNodeViewModel group) { StringProperty nameProperty = new SimpleStringProperty(Localization.lang("Group %0", groupNameProperty.get())); groupNameProperty.addListener((obs, oldValue, newValue) -> nameProperty.setValue(Localization.lang("Group %0", groupNameProperty.get()))); - ObservableList chatHistory = aiService.getChatHistoryService().getChatHistoryForGroup(currentDatabase.get(), group.getGroupNode()); + ObservableList chatHistory = aiService.getGroupChatHistoryService().getChatHistory(currentDatabase.get(), group.getGroupNode()); ObservableList bibEntries = FXCollections.observableArrayList(group.getGroupNode().findMatches(currentDatabase.get().getDatabase())); openAiChat(nameProperty, chatHistory, currentDatabase.get(), bibEntries); diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiService.java b/jablib/src/main/java/org/jabref/logic/ai/AiService.java index f1bc5fa16a2f..62e3c58c699d 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiService.java @@ -7,7 +7,8 @@ import javafx.beans.property.SimpleBooleanProperty; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.chatting.ChatHistoryService; +import org.jabref.logic.ai.chatting.EntryChatHistoryService; +import org.jabref.logic.ai.chatting.GroupChatHistoryService; import org.jabref.logic.ai.chatting.repositories.MVStoreChatHistoryRepository; import org.jabref.logic.ai.current.CurrentAiTemplates; import org.jabref.logic.ai.current.CurrentAnswerEngine; @@ -61,7 +62,8 @@ public class AiService implements AutoCloseable { private final MVStoreSummariesRepository mvStoreSummariesStorage; private final CurrentAiTemplates currentAiTemplates; - private final ChatHistoryService chatHistoryService; + private final EntryChatHistoryService entryChatHistoryService; + private final GroupChatHistoryService groupChatHistoryService; private final CurrentDocumentSplitter currentDocumentSplitter; private final CurrentTokenEstimator currentTokenEstimator; private final CurrentChatLanguageModel currentChatLanguageModel; @@ -85,7 +87,8 @@ public AiService( this.mvStoreSummariesStorage = new MVStoreSummariesRepository(notificationService, Directories.getAiFilesDirectory().resolve(SUMMARIES_FILE_NAME)); this.currentAiTemplates = new CurrentAiTemplates(aiPreferences); - this.chatHistoryService = new ChatHistoryService(citationKeyPatternPreferences, mvStoreChatHistoryStorage); + this.entryChatHistoryService = new EntryChatHistoryService(citationKeyPatternPreferences, mvStoreChatHistoryStorage); + this.groupChatHistoryService = new GroupChatHistoryService(mvStoreChatHistoryStorage); this.currentDocumentSplitter = new CurrentDocumentSplitter(aiPreferences); this.currentTokenEstimator = new CurrentTokenEstimator(aiPreferences); @@ -128,8 +131,12 @@ public CurrentEmbeddingModel getEmbeddingModel() { return currentEmbeddingModel; } - public ChatHistoryService getChatHistoryService() { - return chatHistoryService; + public EntryChatHistoryService getEntryChatHistoryService() { + return entryChatHistoryService; + } + + public GroupChatHistoryService getGroupChatHistoryService() { + return groupChatHistoryService; } public IngestionService getIngestionService() { @@ -161,7 +168,8 @@ public CurrentAnswerEngine getAnswerEngine() { } public void setupDatabase(BibDatabaseContext context) { - chatHistoryService.setupDatabase(context); + entryChatHistoryService.setupDatabase(context); + groupChatHistoryService.setupDatabase(context); ingestionService.setupDatabase(context); summariesService.setupDatabase(context); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryManagementRecord.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryManagementRecord.java new file mode 100644 index 000000000000..4d7219f94013 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryManagementRecord.java @@ -0,0 +1,18 @@ +package org.jabref.logic.ai.chatting; + +import java.util.Optional; + +import javafx.collections.ObservableList; + +import org.jabref.model.database.BibDatabaseContext; + +import dev.langchain4j.data.message.ChatMessage; + +// Note about `Optional`: it was necessary in a previous version, but currently we never save an `Optional.empty()`. +// However, we decided to leave it here: to reduce migrations and to make it possible to chat with a {@link BibEntry} without {@link BibDatabaseContext} +// ({@link BibDatabaseContext} is required only for load/store of the chat). +public record ChatHistoryManagementRecord( + Optional bibDatabaseContext, + ObservableList chatHistory +) { +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChatHistoryService.java similarity index 52% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChatHistoryService.java index cd826b0b1c15..8fae96ec508c 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChatHistoryService.java @@ -9,7 +9,7 @@ import javafx.collections.FXCollections; import javafx.collections.ObservableList; -import org.jabref.logic.ai.chatting.repositories.ChatHistoryRepository; +import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepository; import org.jabref.logic.citationkeypattern.CitationKeyGenerator; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; import org.jabref.logic.util.CitationKeyCheck; @@ -34,7 +34,7 @@ /// [BibEntry] and [org.jabref.model.groups.AbstractGroup]. The chat history is stored in runtime. /// /// To save and load chat history, [BibEntry] and [org.jabref.model.groups.AbstractGroup] must satisfy several constraints. -/// Serialization and deserialization is handled in [ChatHistoryRepository]. +/// Serialization and deserialization is handled in [EntryChatHistoryRepository]. /// /// Constraints for serialization and deserialization of a chat history of a [BibEntry]: /// 1. There should exist an associated [BibDatabaseContext] for the [BibEntry]. @@ -46,55 +46,37 @@ /// 2. The database path of the associated [BibDatabaseContext] must be set. /// 3. The name of an [GroupTreeNode] must be set and unique (this requirement is possibly already satisfied in /// JabRef, but for [BibEntry] it is definitely not). -public class ChatHistoryService implements AutoCloseable { - private static final Logger LOGGER = LoggerFactory.getLogger(ChatHistoryService.class); +public class EntryChatHistoryService implements AutoCloseable { + private static final Logger LOGGER = LoggerFactory.getLogger(EntryChatHistoryService.class); private final CitationKeyPatternPreferences citationKeyPatternPreferences; - private final ChatHistoryRepository implementation; - - // Note about `Optional`: it was necessary in previous version, but currently we never save an `Optional.empty()`. - // However, we decided to left it here: to reduce migrations and to make possible to chat with a {@link BibEntry} without {@link BibDatabaseContext} - // ({@link BibDatabaseContext} is required only for load/store of the chat). - private record ChatHistoryManagementRecord(Optional bibDatabaseContext, ObservableList chatHistory) { - } + private final EntryChatHistoryRepository entryChatHistoryRepository; // We use a {@link TreeMap} here to store {@link BibEntry} chat histories by their id. // When you compare {@link BibEntry} instances, they are compared by value, not by reference. // And when you store {@link BibEntry} instances in a {@link HashMap}, an old hash may be stored when the {@link BibEntry} is changed. // See also ADR-38. - private final TreeMap bibEntriesChatHistory = new TreeMap<>(Comparator.comparing(BibEntry::getId)); - - // We use {@link TreeMap} for group chat history for the same reason as for {@link BibEntry}ies. - private final TreeMap groupsChatHistory = new TreeMap<>(Comparator.comparing(GroupTreeNode::getName)); + private final TreeMap bibEntriesChatHistory = + new TreeMap<>(Comparator.comparing(BibEntry::getId)); - public ChatHistoryService( + public EntryChatHistoryService( CitationKeyPatternPreferences citationKeyPatternPreferences, - ChatHistoryRepository implementation + EntryChatHistoryRepository entryChatHistoryRepository ) { this.citationKeyPatternPreferences = citationKeyPatternPreferences; - this.implementation = implementation; + this.entryChatHistoryRepository = entryChatHistoryRepository; } public void setupDatabase(BibDatabaseContext bibDatabaseContext) { - bibDatabaseContext.getMetaData().getGroups().ifPresent(rootGroupTreeNode -> - rootGroupTreeNode.iterateOverTree().forEach(groupNode -> { - groupNode.getGroup().nameProperty().addListener((observable, oldValue, newValue) -> { - if (newValue != null && oldValue != null) { - transferGroupHistory(bibDatabaseContext, groupNode, oldValue, newValue); - } - }); - - groupNode.getGroupProperty().addListener((obs, oldValue, newValue) -> { - if (oldValue != null && newValue != null) { - transferGroupHistory(bibDatabaseContext, groupNode, oldValue.getName(), newValue.getName()); - } - }); - })); - - bibDatabaseContext.getDatabase().getEntries().forEach(entry -> entry.registerListener(new CitationKeyChangeListener(bibDatabaseContext))); + bibDatabaseContext + .getDatabase() + .getEntries() + .forEach(entry -> + entry.registerListener(new CitationKeyChangeListener(bibDatabaseContext)) + ); } - public ObservableList getChatHistoryForEntry(BibDatabaseContext bibDatabaseContext, BibEntry entry) { + public ObservableList getChatHistory(BibDatabaseContext bibDatabaseContext, BibEntry entry) { return bibEntriesChatHistory.computeIfAbsent(entry, entryArg -> { ObservableList chatHistory; @@ -102,24 +84,24 @@ public ObservableList getChatHistoryForEntry(BibDatabaseContext bib chatHistory = FXCollections.observableArrayList(); } else { BibEntryAiIdentifier identifier = new BibEntryAiIdentifier(bibDatabaseContext.getDatabasePath().get(), entry.getCitationKey().get()); - List chatMessagesList = implementation.loadMessagesForEntry(identifier); + List chatMessagesList = entryChatHistoryRepository.loadMessagesForEntry(identifier); chatHistory = FXCollections.observableArrayList(chatMessagesList); } return new ChatHistoryManagementRecord(Optional.of(bibDatabaseContext), chatHistory); - }).chatHistory; + }).chatHistory(); } /** * Removes the chat history for the given {@link BibEntry} from the internal RAM map. * If the {@link BibEntry} satisfies requirements for serialization and deserialization of chat history (see - * the docstring for the {@link ChatHistoryService}), then the chat history will be stored via the - * {@link ChatHistoryRepository}. + * the docstring for the {@link EntryChatHistoryService}), then the chat history will be stored via the + * {@link EntryChatHistoryRepository}. *

- * It is not necessary to call this method (everything will be stored in {@link ChatHistoryService#close()}, + * It is not necessary to call this method (everything will be stored in {@link EntryChatHistoryService#close()}, * but it's best to call it when the chat history {@link BibEntry} is no longer needed. */ - public void closeChatHistoryForEntry(BibEntry entry) { + public void closeChatHistory(BibEntry entry) { ChatHistoryManagementRecord chatHistoryManagementRecord = bibEntriesChatHistory.get(entry); if (chatHistoryManagementRecord == null) { return; @@ -131,7 +113,7 @@ public void closeChatHistoryForEntry(BibEntry entry) { // Method `correctCitationKey` will already check `entry.getCitationKey().isPresent()`, but it is still // there, to suppress warning from IntelliJ IDEA on `entry.getCitationKey().get()`. BibEntryAiIdentifier identifier = new BibEntryAiIdentifier(bibDatabaseContext.get().getDatabasePath().get(), entry.getCitationKey().get()); - implementation.storeMessagesForEntry( + entryChatHistoryRepository.storeMessagesForEntry( identifier, chatHistoryManagementRecord.chatHistory() ); @@ -141,54 +123,6 @@ public void closeChatHistoryForEntry(BibEntry entry) { bibEntriesChatHistory.remove(entry); } - public ObservableList getChatHistoryForGroup(BibDatabaseContext bibDatabaseContext, GroupTreeNode group) { - return groupsChatHistory.computeIfAbsent(group, groupArg -> { - ObservableList chatHistory; - - if (bibDatabaseContext.getDatabasePath().isEmpty()) { - chatHistory = FXCollections.observableArrayList(); - } else { - GroupAiIdentifier identifier = new GroupAiIdentifier(bibDatabaseContext.getDatabasePath().get(), group.getGroup().getName()); - List chatMessagesList = implementation.loadMessagesForGroup( - identifier - ); - - chatHistory = FXCollections.observableArrayList(chatMessagesList); - } - - return new ChatHistoryManagementRecord(Optional.of(bibDatabaseContext), chatHistory); - }).chatHistory; - } - - /** - * Removes the chat history for the given {@link GroupTreeNode} from the internal RAM map. - * If the {@link GroupTreeNode} satisfies requirements for serialization and deserialization of chat history (see - * the docstring for the {@link ChatHistoryService}), then the chat history will be stored via the - * {@link ChatHistoryRepository}. - *

- * It is not necessary to call this method (everything will be stored in {@link ChatHistoryService#close()}, - * but it's best to call it when the chat history {@link GroupTreeNode} is no longer needed. - */ - public void closeChatHistoryForGroup(GroupTreeNode group) { - ChatHistoryManagementRecord chatHistoryManagementRecord = groupsChatHistory.get(group); - if (chatHistoryManagementRecord == null) { - return; - } - - Optional bibDatabaseContext = chatHistoryManagementRecord.bibDatabaseContext(); - - if (bibDatabaseContext.isPresent() && bibDatabaseContext.get().getDatabasePath().isPresent()) { - GroupAiIdentifier identifier = new GroupAiIdentifier(bibDatabaseContext.get().getDatabasePath().get(), group.getGroup().getName()); - implementation.storeMessagesForGroup( - identifier, - chatHistoryManagementRecord.chatHistory() - ); - } - - // TODO: What if there is two AI chats for the same entry? And one is closed and one is not? - groupsChatHistory.remove(group); - } - private boolean correctCitationKey(BibDatabaseContext bibDatabaseContext, BibEntry bibEntry) { if (!CitationKeyCheck.citationKeyIsPresentAndUnique(bibDatabaseContext, bibEntry)) { tryToGenerateCitationKey(bibDatabaseContext, bibEntry); @@ -204,31 +138,13 @@ private void tryToGenerateCitationKey(BibDatabaseContext bibDatabaseContext, Bib @Override public void close() throws Exception { // We need to clone `bibEntriesChatHistory.keySet()` because closeChatHistoryForEntry() modifies the `bibEntriesChatHistory` map. - new HashSet<>(bibEntriesChatHistory.keySet()).forEach(this::closeChatHistoryForEntry); - - // Clone is for the same reason, as written above. - new HashSet<>(groupsChatHistory.keySet()).forEach(this::closeChatHistoryForGroup); - - implementation.close(); - } - - private void transferGroupHistory(BibDatabaseContext bibDatabaseContext, GroupTreeNode groupTreeNode, String oldName, String newName) { - if (bibDatabaseContext.getDatabasePath().isEmpty()) { - LOGGER.warn("Could not transfer chat history of group {} (old name: {}): database path is empty.", newName, oldName); - return; - } - - List chatMessages = groupsChatHistory.computeIfAbsent(groupTreeNode, - e -> new ChatHistoryManagementRecord(Optional.of(bibDatabaseContext), FXCollections.observableArrayList())).chatHistory; - - GroupAiIdentifier oldIdentifier = new GroupAiIdentifier(bibDatabaseContext.getDatabasePath().get(), oldName); - GroupAiIdentifier newIdentifier = new GroupAiIdentifier(bibDatabaseContext.getDatabasePath().get(), newName); + new HashSet<>(bibEntriesChatHistory.keySet()).forEach(this::closeChatHistory); - implementation.storeMessagesForGroup(oldIdentifier, List.of()); - implementation.storeMessagesForGroup(newIdentifier, chatMessages); + // TODO: IT DOES NOT HAVE THE RIGHT TO CLOSE THIS. + entryChatHistoryRepository.close(); } - private void transferEntryHistory(BibDatabaseContext bibDatabaseContext, BibEntry entry, String oldCitationKey, String newCitationKey) { + private void transferHistory(BibDatabaseContext bibDatabaseContext, BibEntry entry, String oldCitationKey, String newCitationKey) { // TODO: This method does not check if the citation key is valid. if (bibDatabaseContext.getDatabasePath().isEmpty()) { @@ -237,14 +153,14 @@ private void transferEntryHistory(BibDatabaseContext bibDatabaseContext, BibEntr } List chatMessages = bibEntriesChatHistory.computeIfAbsent(entry, - e -> new ChatHistoryManagementRecord(Optional.of(bibDatabaseContext), FXCollections.observableArrayList())).chatHistory; + e -> new ChatHistoryManagementRecord(Optional.of(bibDatabaseContext), FXCollections.observableArrayList())).chatHistory(); GroupAiIdentifier groupIdentifier = new GroupAiIdentifier(bibDatabaseContext.getDatabasePath().get(), oldCitationKey); BibEntryAiIdentifier bibEntryIdentifier = new BibEntryAiIdentifier(bibDatabaseContext.getDatabasePath().get(), newCitationKey); // TODO: Why group here? - implementation.storeMessagesForGroup(groupIdentifier, List.of()); - implementation.storeMessagesForEntry(bibEntryIdentifier, chatMessages); + // implementation.storeMessagesForGroup(groupIdentifier, List.of()); + entryChatHistoryRepository.storeMessagesForEntry(bibEntryIdentifier, chatMessages); } private class CitationKeyChangeListener { @@ -260,7 +176,7 @@ void listen(FieldChangedEvent e) { return; } - transferEntryHistory(bibDatabaseContext, e.getBibEntry(), e.getOldValue(), e.getNewValue()); + transferHistory(bibDatabaseContext, e.getBibEntry(), e.getOldValue(), e.getNewValue()); } } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChatHistoryService.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChatHistoryService.java new file mode 100644 index 000000000000..fb0358866e7c --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChatHistoryService.java @@ -0,0 +1,127 @@ +package org.jabref.logic.ai.chatting; + +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.TreeMap; + +import javafx.collections.FXCollections; +import javafx.collections.ObservableList; + +import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepository; +import org.jabref.logic.ai.chatting.repositories.GroupChatHistoryRepository; +import org.jabref.model.ai.identifiers.GroupAiIdentifier; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.groups.GroupTreeNode; + +import dev.langchain4j.data.message.ChatMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GroupChatHistoryService implements AutoCloseable { + private final Logger LOGGER = LoggerFactory.getLogger(GroupChatHistoryService.class); + + private final GroupChatHistoryRepository groupChatHistoryRepository; + + // We use {@link TreeMap} for group chat history for the same reason as for {@link BibEntry}ies. + private final TreeMap groupsChatHistory = + new TreeMap<>(Comparator.comparing(GroupTreeNode::getName)); + + public GroupChatHistoryService( + GroupChatHistoryRepository groupChatHistoryRepository + ) { + this.groupChatHistoryRepository = groupChatHistoryRepository; + } + + public void setupDatabase(BibDatabaseContext bibDatabaseContext) { + bibDatabaseContext.getMetaData().getGroups().ifPresent(rootGroupTreeNode -> + rootGroupTreeNode.iterateOverTree().forEach(groupNode -> { + groupNode.getGroup().nameProperty().addListener((observable, oldValue, newValue) -> { + if (newValue != null && oldValue != null) { + transferHistory(bibDatabaseContext, groupNode, oldValue, newValue); + } + }); + + groupNode.getGroupProperty().addListener((obs, oldValue, newValue) -> { + if (oldValue != null && newValue != null) { + transferHistory(bibDatabaseContext, groupNode, oldValue.getName(), newValue.getName()); + } + }); + })); + } + + private void transferHistory(BibDatabaseContext bibDatabaseContext, GroupTreeNode groupTreeNode, String oldName, String newName) { + if (bibDatabaseContext.getDatabasePath().isEmpty()) { + LOGGER.warn("Could not transfer chat history of group {} (old name: {}): database path is empty.", newName, oldName); + return; + } + + List chatMessages = groupsChatHistory.computeIfAbsent(groupTreeNode, + e -> new ChatHistoryManagementRecord(Optional.of(bibDatabaseContext), FXCollections.observableArrayList())).chatHistory(); + + GroupAiIdentifier oldIdentifier = new GroupAiIdentifier(bibDatabaseContext.getDatabasePath().get(), oldName); + GroupAiIdentifier newIdentifier = new GroupAiIdentifier(bibDatabaseContext.getDatabasePath().get(), newName); + + groupChatHistoryRepository.storeMessagesForGroup(oldIdentifier, List.of()); + groupChatHistoryRepository.storeMessagesForGroup(newIdentifier, chatMessages); + } + + public ObservableList getChatHistory(BibDatabaseContext bibDatabaseContext, GroupTreeNode group) { + return groupsChatHistory.computeIfAbsent(group, groupArg -> { + ObservableList chatHistory; + + if (bibDatabaseContext.getDatabasePath().isEmpty()) { + chatHistory = FXCollections.observableArrayList(); + } else { + GroupAiIdentifier identifier = new GroupAiIdentifier(bibDatabaseContext.getDatabasePath().get(), group.getGroup().getName()); + List chatMessagesList = groupChatHistoryRepository.loadMessagesForGroup( + identifier + ); + + chatHistory = FXCollections.observableArrayList(chatMessagesList); + } + + return new ChatHistoryManagementRecord(Optional.of(bibDatabaseContext), chatHistory); + }).chatHistory(); + } + + /** + * Removes the chat history for the given {@link GroupTreeNode} from the internal RAM map. + * If the {@link GroupTreeNode} satisfies requirements for serialization and deserialization of chat history (see + * the docstring for the {@link EntryChatHistoryService}), then the chat history will be stored via the + * {@link EntryChatHistoryRepository}. + *

+ * It is not necessary to call this method (everything will be stored in {@link EntryChatHistoryService#close()}, + * but it's best to call it when the chat history {@link GroupTreeNode} is no longer needed. + */ + public void closeChatHistory(GroupTreeNode group) { + ChatHistoryManagementRecord chatHistoryManagementRecord = groupsChatHistory.get(group); + if (chatHistoryManagementRecord == null) { + return; + } + + Optional bibDatabaseContext = chatHistoryManagementRecord.bibDatabaseContext(); + + if (bibDatabaseContext.isPresent() && bibDatabaseContext.get().getDatabasePath().isPresent()) { + GroupAiIdentifier identifier = new GroupAiIdentifier(bibDatabaseContext.get().getDatabasePath().get(), group.getGroup().getName()); + groupChatHistoryRepository.storeMessagesForGroup( + identifier, + chatHistoryManagementRecord.chatHistory() + ); + } + + // TODO: What if there is two AI chats for the same entry? And one is closed and one is not? + groupsChatHistory.remove(group); + } + + + @Override + public void close() throws Exception { + // Clone is for the same reason, as written above. + new HashSet<>(groupsChatHistory.keySet()).forEach(this::closeChatHistory); + + // TODO: IT DOES NOT HAVE THE RIGHT TO CLOSE THIS. + groupChatHistoryRepository.close(); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/ChatHistoryRepository.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/EntryChatHistoryRepository.java similarity index 55% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/ChatHistoryRepository.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/EntryChatHistoryRepository.java index 66eb6a8021c7..93f0425e19a3 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/ChatHistoryRepository.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/EntryChatHistoryRepository.java @@ -3,16 +3,11 @@ import java.util.List; import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; -import org.jabref.model.ai.identifiers.GroupAiIdentifier; import dev.langchain4j.data.message.ChatMessage; -public interface ChatHistoryRepository extends AutoCloseable { +public interface EntryChatHistoryRepository extends AutoCloseable { List loadMessagesForEntry(BibEntryAiIdentifier identifier); void storeMessagesForEntry(BibEntryAiIdentifier identifier, List messages); - - List loadMessagesForGroup(GroupAiIdentifier identifier); - - void storeMessagesForGroup(GroupAiIdentifier identifier, List messages); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepository.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepository.java new file mode 100644 index 000000000000..da1b25c1b0b5 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepository.java @@ -0,0 +1,13 @@ +package org.jabref.logic.ai.chatting.repositories; + +import java.util.List; + +import org.jabref.model.ai.identifiers.GroupAiIdentifier; + +import dev.langchain4j.data.message.ChatMessage; + +public interface GroupChatHistoryRepository extends AutoCloseable { + List loadMessagesForGroup(GroupAiIdentifier identifier); + + void storeMessagesForGroup(GroupAiIdentifier identifier, List messages); +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepository.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepository.java index 2230835d5904..295d13fc13c2 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepository.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepository.java @@ -20,7 +20,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class MVStoreChatHistoryRepository extends MVStoreBase implements ChatHistoryRepository { +public class MVStoreChatHistoryRepository extends MVStoreBase implements EntryChatHistoryRepository, GroupChatHistoryRepository { private static final String ENTRY_CHAT_HISTORY_PREFIX = "entry"; private static final String GROUP_CHAT_HISTORY_PREFIX = "group"; diff --git a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentChatLanguageModel.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentChatLanguageModel.java index e49d2e1a3184..9c431ef6419d 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentChatLanguageModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentChatLanguageModel.java @@ -7,7 +7,7 @@ import java.util.concurrent.Executors; import org.jabref.logic.ai.chatting.logic.AiChatLogic; -import org.jabref.logic.ai.chatting.repositories.ChatHistoryRepository; +import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepository; import org.jabref.logic.ai.customimplementations.llms.ChatModel; import org.jabref.logic.ai.customimplementations.llms.Gpt4AllModel; import org.jabref.logic.ai.customimplementations.llms.JvmOpenAiChatLanguageModel; @@ -63,7 +63,7 @@ public CurrentChatLanguageModel( * Update the underlying {@link dev.langchain4j.model.chat.ChatModel} by current {@link AiPreferences} parameters. * When the model is updated, the chat messages are not lost. * See {@link AiChatLogic}, where messages are stored in {@link ChatMemory}, - * and see {@link ChatHistoryRepository}. + * and see {@link EntryChatHistoryRepository}. */ private void rebuild() { String apiKey = aiPreferences.getApiKeyForAiProvider(aiPreferences.getAiProvider()); diff --git a/jablib/src/main/java/org/jabref/logic/ai/pipeline/IngestionService.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/IngestionService.java index bf149764e4d9..fff97ef54fc2 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/pipeline/IngestionService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/IngestionService.java @@ -34,7 +34,7 @@ * Use this class in the logic and UI. */ public class IngestionService { - // We use a {@link TreeMap} here for the same reasons we use it in {@link ChatHistoryService}. + // We use a {@link TreeMap} here for the same reasons we use it in {@link EntryChatHistoryService}. private final TreeMap> ingestionStatusMap = new TreeMap<>(Comparator.comparing(LinkedFile::getLink)); private final List> listsUnderIngestion = new ArrayList<>(); diff --git a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryRepositoryTest.java b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/EntryChatHistoryRepositoryTest.java similarity index 85% rename from jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryRepositoryTest.java rename to jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/EntryChatHistoryRepositoryTest.java index eb9a87544d0c..c614facfb263 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/ChatHistoryRepositoryTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/EntryChatHistoryRepositoryTest.java @@ -3,7 +3,7 @@ import java.nio.file.Path; import java.util.List; -import org.jabref.logic.ai.chatting.repositories.ChatHistoryRepository; +import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepository; import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; import org.jabref.model.ai.identifiers.GroupAiIdentifier; @@ -17,14 +17,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -abstract class ChatHistoryRepositoryTest { +abstract class EntryChatHistoryRepositoryTest { @TempDir Path tempDir; - private ChatHistoryRepository storage; + private EntryChatHistoryRepository storage; - abstract ChatHistoryRepository makeStorage(Path path); + abstract EntryChatHistoryRepository makeStorage(Path path); - abstract void close(ChatHistoryRepository storage); + abstract void close(EntryChatHistoryRepository storage); @BeforeEach void setUp() { diff --git a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryTest.java b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryTest.java index ad77ee96b1a6..54631e17b78b 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryTest.java @@ -2,13 +2,12 @@ import java.nio.file.Path; -import org.jabref.logic.ai.chatting.repositories.ChatHistoryRepository; import org.jabref.logic.ai.chatting.repositories.MVStoreChatHistoryRepository; import org.jabref.logic.util.NotificationService; import static org.mockito.Mockito.mock; -class MVStoreChatHistoryRepositoryTest extends ChatHistoryRepositoryTest { +class MVStoreChatHistoryRepositoryTest extends EntryChatHistoryRepositoryTest { @Override ChatHistoryRepository makeStorage(Path path) { return new MVStoreChatHistoryRepository(mock(NotificationService.class), path); From 6d212be184b6c8a821d02ad2d4c831e719237eed Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Fri, 28 Nov 2025 23:50:31 +0100 Subject: [PATCH 034/243] Fix chat history refactoring --- jablib/src/main/java/org/jabref/logic/ai/AiService.java | 5 ++++- .../jabref/logic/ai/chatting/EntryChatHistoryService.java | 3 --- .../jabref/logic/ai/chatting/GroupChatHistoryService.java | 3 --- .../logic/importer/fileformat/pdf/CitationsFromPdf.java | 2 ++ 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiService.java b/jablib/src/main/java/org/jabref/logic/ai/AiService.java index 62e3c58c699d..476d1ab295ba 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiService.java @@ -175,13 +175,16 @@ public void setupDatabase(BibDatabaseContext context) { } @Override - public void close() { + public void close() throws Exception { shutdownSignal.set(true); cachedThreadPool.shutdownNow(); currentChatLanguageModel.close(); currentEmbeddingModel.close(); + entryChatHistoryService.close(); + groupChatHistoryService.close(); + mvStoreChatHistoryStorage.close(); mvStoreFullyIngestedDocumentsTracker.close(); mvStoreEmbeddingStore.close(); diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChatHistoryService.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChatHistoryService.java index 8fae96ec508c..c6c75fb00cf2 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChatHistoryService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChatHistoryService.java @@ -139,9 +139,6 @@ private void tryToGenerateCitationKey(BibDatabaseContext bibDatabaseContext, Bib public void close() throws Exception { // We need to clone `bibEntriesChatHistory.keySet()` because closeChatHistoryForEntry() modifies the `bibEntriesChatHistory` map. new HashSet<>(bibEntriesChatHistory.keySet()).forEach(this::closeChatHistory); - - // TODO: IT DOES NOT HAVE THE RIGHT TO CLOSE THIS. - entryChatHistoryRepository.close(); } private void transferHistory(BibDatabaseContext bibDatabaseContext, BibEntry entry, String oldCitationKey, String newCitationKey) { diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChatHistoryService.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChatHistoryService.java index fb0358866e7c..6133006d21ef 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChatHistoryService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChatHistoryService.java @@ -120,8 +120,5 @@ public void closeChatHistory(GroupTreeNode group) { public void close() throws Exception { // Clone is for the same reason, as written above. new HashSet<>(groupsChatHistory.keySet()).forEach(this::closeChatHistory); - - // TODO: IT DOES NOT HAVE THE RIGHT TO CLOSE THIS. - groupChatHistoryRepository.close(); } } diff --git a/jablib/src/main/java/org/jabref/logic/importer/fileformat/pdf/CitationsFromPdf.java b/jablib/src/main/java/org/jabref/logic/importer/fileformat/pdf/CitationsFromPdf.java index d31565b17f54..67aa3cb14565 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fileformat/pdf/CitationsFromPdf.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fileformat/pdf/CitationsFromPdf.java @@ -39,6 +39,8 @@ public static ParserResult extractCitationsUsingLLM(JabRefCliPreferences prefere ); return importer.importDatabase(path); + } catch (Exception e) { + throw new RuntimeException(e); } } } From 52ce8f1c9d8d074633997afa3ce4b09b774cbdf9 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Sat, 29 Nov 2025 15:39:37 +0100 Subject: [PATCH 035/243] Refactor chat history --- docs/code-howtos/ai.md | 15 ++ .../main/java/org/jabref/gui/JabRefGUI.java | 1 - .../ai/components/aichat/AiChatComponent.java | 59 +++--- .../aichat/AiChatGuardedComponent.java | 26 +-- .../ai/components/aichat/AiChatWindow.java | 23 ++- .../chathistory/ChatHistoryComponent.java | 38 ++-- .../chatmessage/ChatMessageComponent.java | 61 +++--- .../privacynotice/PrivacyNoticeComponent.java | 2 +- .../org/jabref/gui/entryeditor/AiChatTab.java | 13 +- .../jabref/gui/groups/GroupTreeViewModel.java | 10 +- .../org/jabref/gui/preferences/ai/AiTab.java | 2 +- .../gui/preferences/ai/AiTabViewModel.java | 2 +- .../aichat/AiChatComponentTest.java | 2 +- jablib/src/main/java/module-info.java | 2 + .../java/org/jabref/logic/ai/AiService.java | 66 ++++--- .../jabref/logic/ai/chatting/ChatHistory.java | 19 ++ .../chatting/CitationKeyChangeListener.java | 59 ++++++ .../logic/ai/chatting/EntryChatHistory.java | 50 +++++ .../ai/chatting/EntryChatHistoryService.java | 179 ------------------ .../EntryChattingDatabaseListener.java | 21 ++ .../logic/ai/chatting/GroupChatHistory.java | 50 +++++ .../ai/chatting/GroupChatHistoryService.java | 124 ------------ .../GroupChattingDatabaseListener.java | 56 ++++++ .../logic/ai/chatting/logic/AiChatLogic.java | 35 +++- .../logic/ChatHistoryRecordUtils.java | 28 +++ ...java => EntryChatHistoryRepositoryV1.java} | 3 +- .../EntryChatHistoryRepositoryV2.java | 20 ++ ...java => GroupChatHistoryRepositoryV1.java} | 3 +- .../GroupChatHistoryRepositoryV2.java | 20 ++ .../MVStoreChatHistoryRepository.java | 132 ------------- .../MVStoreChatHistoryRepositoryV1.java | 86 +++++++++ .../MVStoreEntryChatHistoryRepositoryV2.java | 69 +++++++ .../MVStoreGroupChatHistoryRepositoryV2.java | 69 +++++++ .../ai/current/CurrentChatLanguageModel.java | 6 +- .../customimplementations/llms/ChatModel.java | 2 +- .../logic/ai/preferences/AiPreferences.java | 2 +- .../AiProviderDefaultChatModels.java | 2 +- .../ai/preferences/PredefinedChatModel.java | 2 +- .../fileformat/pdf/CitationsFromPdf.java | 1 - .../preferences/JabRefCliPreferences.java | 2 +- .../ChatHistoryManagementRecordV1.java} | 5 +- .../ai/chatting/ChatHistoryRecordV1.java | 53 ++++++ .../ai/chatting/ChatHistoryRecordV2.java | 12 ++ .../jabref/model/ai/chatting/ChatType.java | 6 + .../chatting/{ => messages}/ErrorMessage.java | 2 +- .../ai/identifiers/BibEntryAiIdentifier.java | 6 + .../ai/identifiers/GroupAiIdentifier.java | 6 + .../ai/{chatting => llm}/AiProvider.java | 2 +- .../ai/summarization/BibEntrySummary.java | 2 +- ... => EntryChatHistoryRepositoryV1Test.java} | 10 +- ...> MVStoreChatHistoryRepositoryV1Test.java} | 8 +- .../SummariesRepositoryTest.java | 2 +- 52 files changed, 872 insertions(+), 604 deletions(-) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistory.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/CitationKeyChangeListener.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChatHistory.java delete mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChatHistoryService.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChattingDatabaseListener.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChatHistory.java delete mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChatHistoryService.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChattingDatabaseListener.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/logic/ChatHistoryRecordUtils.java rename jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/{EntryChatHistoryRepository.java => EntryChatHistoryRepositoryV1.java} (81%) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/EntryChatHistoryRepositoryV2.java rename jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/{GroupChatHistoryRepository.java => GroupChatHistoryRepositoryV1.java} (81%) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepositoryV2.java delete mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepository.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepositoryV1.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreEntryChatHistoryRepositoryV2.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreGroupChatHistoryRepositoryV2.java rename jablib/src/main/java/org/jabref/{logic/ai/chatting/ChatHistoryManagementRecord.java => model/ai/chatting/ChatHistoryManagementRecordV1.java} (87%) create mode 100644 jablib/src/main/java/org/jabref/model/ai/chatting/ChatHistoryRecordV1.java create mode 100644 jablib/src/main/java/org/jabref/model/ai/chatting/ChatHistoryRecordV2.java create mode 100644 jablib/src/main/java/org/jabref/model/ai/chatting/ChatType.java rename jablib/src/main/java/org/jabref/model/ai/chatting/{ => messages}/ErrorMessage.java (95%) rename jablib/src/main/java/org/jabref/model/ai/{chatting => llm}/AiProvider.java (96%) rename jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/{EntryChatHistoryRepositoryTest.java => EntryChatHistoryRepositoryV1Test.java} (88%) rename jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/{MVStoreChatHistoryRepositoryTest.java => MVStoreChatHistoryRepositoryV1Test.java} (60%) diff --git a/docs/code-howtos/ai.md b/docs/code-howtos/ai.md index 7729bee2ae7b..60c9e21387e5 100644 --- a/docs/code-howtos/ai.md +++ b/docs/code-howtos/ai.md @@ -6,3 +6,18 @@ - generic classess - long task and shutdown signal - ensure to use ChatModel from jabref + + +how to use MVStores: +general dissection: +1. MVStore - for different repositories. +2. MVMap - (for entry chats) - one for "databasePath + entryID". +3. Entry - specific chunks and chathistory records with their ID (UUID). +Messages/Chunks should be distinct/separate/atomic. E.g. we don't store a List of messages in an entry. + +I would like a forth layer: to distrinct database and entry, but it is what it is. + + + +explain how messages are stored (v1 and v2). + diff --git a/jabgui/src/main/java/org/jabref/gui/JabRefGUI.java b/jabgui/src/main/java/org/jabref/gui/JabRefGUI.java index 9d8e3eb29df3..bebfceaa69d9 100644 --- a/jabgui/src/main/java/org/jabref/gui/JabRefGUI.java +++ b/jabgui/src/main/java/org/jabref/gui/JabRefGUI.java @@ -213,7 +213,6 @@ public void initialize() { JabRefGUI.aiService = new AiService( preferences.getAiPreferences(), preferences.getFilePreferences(), - preferences.getCitationKeyPatternPreferences(), dialogService, taskExecutor); Injector.setModelOrService(AiService.class, aiService); diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java index ca7f835d2c33..b7216eb19c51 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java @@ -1,9 +1,12 @@ package org.jabref.gui.ai.components.aichat; import java.nio.file.Path; +import java.time.Instant; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.Optional; +import java.util.UUID; import java.util.stream.Stream; import javafx.beans.Observable; @@ -28,6 +31,7 @@ import org.jabref.gui.icon.IconTheme; import org.jabref.gui.util.UiTaskExecutor; import org.jabref.logic.ai.AiService; +import org.jabref.logic.ai.chatting.ChatHistory; import org.jabref.logic.ai.chatting.logic.AiChatLogic; import org.jabref.logic.ai.chatting.tasks.GenerateAiResponseTask; import org.jabref.logic.ai.preferences.AiPreferences; @@ -36,7 +40,8 @@ import org.jabref.logic.util.CitationKeyCheck; import org.jabref.logic.util.TaskExecutor; import org.jabref.logic.util.io.FileUtil; -import org.jabref.model.ai.chatting.ErrorMessage; +import org.jabref.model.ai.chatting.ChatHistoryRecordV2; +import org.jabref.model.ai.chatting.messages.ErrorMessage; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.util.ListUtil; @@ -44,8 +49,6 @@ import com.airhacks.afterburner.views.ViewLoader; import com.google.common.annotations.VisibleForTesting; import dev.langchain4j.data.message.AiMessage; -import dev.langchain4j.data.message.ChatMessage; -import dev.langchain4j.data.message.ChatMessageType; import dev.langchain4j.data.message.UserMessage; import org.controlsfx.control.PopOver; import org.slf4j.Logger; @@ -61,6 +64,7 @@ public class AiChatComponent extends VBox { private final AiService aiService; private final ObservableList entries; + private final ChatHistory chatHistory; private final BibDatabaseContext bibDatabaseContext; private final AiPreferences aiPreferences; private final DialogService dialogService; @@ -85,7 +89,7 @@ public class AiChatComponent extends VBox { public AiChatComponent( AiService aiService, StringProperty name, - ObservableList chatHistory, + ChatHistory chatHistory, ObservableList entries, BibDatabaseContext bibDatabaseContext, AiPreferences aiPreferences, @@ -94,6 +98,7 @@ public AiChatComponent( ) { this.aiService = aiService; this.entries = entries; + this.chatHistory = chatHistory; this.bibDatabaseContext = bibDatabaseContext; this.aiPreferences = aiPreferences; this.dialogService = dialogService; @@ -120,7 +125,7 @@ public AiChatComponent( @FXML public void initialize() { - uiChatHistory.setItems(aiChatLogic.getChatHistory()); + uiChatHistory.updateMessages(chatHistory); initializeChatPrompt(); initializeNotice(); initializeNotifications(); @@ -207,17 +212,17 @@ private void initializeChatPrompt() { chatPrompt.setRegenerateCallback(() -> { setLoading(true); - Optional lastUserPrompt = Optional.empty(); - if (!aiChatLogic.getChatHistory().isEmpty()) { + Optional lastUserPrompt = Optional.empty(); + if (!chatHistory.isEmpty()) { lastUserPrompt = getLastUserMessage(); } if (lastUserPrompt.isPresent()) { - while (aiChatLogic.getChatHistory().getLast().type() != ChatMessageType.USER) { + while (!Objects.equals(chatHistory.getAllMessages().getLast().messageTypeClassName(), UserMessage.class.getName())) { deleteLastMessage(); } deleteLastMessage(); chatPrompt.switchToNormalState(); - onSendMessage(lastUserPrompt.get().singleText()); + onSendMessage(lastUserPrompt.get().content()); } }); @@ -320,7 +325,14 @@ private void onSendMessage(String userPrompt) { private void addError(String error) { ErrorMessage chatMessage = new ErrorMessage(error); - aiChatLogic.getChatHistory().add(chatMessage); + + chatHistory.addMessage(new ChatHistoryRecordV2( + UUID.randomUUID().toString(), + chatMessage.getClass().getName(), + chatMessage.getText(), + Instant.now() + )); + uiChatHistory.updateMessages(chatHistory); } private void updatePromptHistory() { @@ -329,11 +341,11 @@ private void updatePromptHistory() { } private Stream getReversedUserMessagesStream() { - return aiChatLogic - .getChatHistory() + return chatHistory + .getAllMessages() .reversed() .stream() - .filter(message -> message instanceof UserMessage) + .filter(message -> Objects.equals(message.messageTypeClassName(), UserMessage.class.getName())) .map(UserMessage.class::cast); } @@ -350,23 +362,26 @@ private void onClearChatHistory() { ); if (agreed) { - aiChatLogic.getChatHistory().clear(); + chatHistory.clear(); } } private void deleteLastMessage() { - if (!aiChatLogic.getChatHistory().isEmpty()) { - int index = aiChatLogic.getChatHistory().size() - 1; - aiChatLogic.getChatHistory().remove(index); + if (!chatHistory.isEmpty()) { + int index = chatHistory.size() - 1; + uiChatHistory.getMessage(index).map(component -> component.getChatMessage().id()).ifPresent(id -> { + chatHistory.deleteMessage(id); + uiChatHistory.updateMessages(chatHistory); + }); } } - private Optional getLastUserMessage() { - int messageIndex = aiChatLogic.getChatHistory().size() - 1; + private Optional getLastUserMessage() { + int messageIndex = chatHistory.size() - 1; while (messageIndex >= 0) { - ChatMessage chat = aiChatLogic.getChatHistory().get(messageIndex); - if (chat.type() == ChatMessageType.USER) { - return Optional.of((UserMessage) chat); + ChatHistoryRecordV2 chat = chatHistory.getAllMessages().get(messageIndex); + if (Objects.equals(chat.messageTypeClassName(), UserMessage.class.getName())) { + return Optional.of(chat); } messageIndex--; } diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatGuardedComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatGuardedComponent.java index d813c339496f..90071052d665 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatGuardedComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatGuardedComponent.java @@ -9,13 +9,12 @@ import org.jabref.gui.entryeditor.AdaptVisibleTabs; import org.jabref.gui.frame.ExternalApplicationsPreferences; import org.jabref.logic.ai.AiService; +import org.jabref.logic.ai.chatting.ChatHistory; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; -import dev.langchain4j.data.message.ChatMessage; - /** * Main class for AI chatting. It checks if the AI features are enabled and if the embedding model is properly set up. */ @@ -27,7 +26,7 @@ public class AiChatGuardedComponent extends EmbeddingModelGuardedComponent { /// this parameter. However, for group chat window, you should. private final StringProperty name; - private final ObservableList chatHistory; + private final ChatHistory chatHistory; private final BibDatabaseContext bibDatabaseContext; private final ObservableList entries; private final AiService aiService; @@ -35,16 +34,17 @@ public class AiChatGuardedComponent extends EmbeddingModelGuardedComponent { private final AiPreferences aiPreferences; private final TaskExecutor taskExecutor; - public AiChatGuardedComponent(StringProperty name, - ObservableList chatHistory, - BibDatabaseContext bibDatabaseContext, - ObservableList entries, - AiService aiService, - DialogService dialogService, - AiPreferences aiPreferences, - ExternalApplicationsPreferences externalApplicationsPreferences, - AdaptVisibleTabs adaptVisibleTabs, - TaskExecutor taskExecutor + public AiChatGuardedComponent( + StringProperty name, + ChatHistory chatHistory, + BibDatabaseContext bibDatabaseContext, + ObservableList entries, + AiService aiService, + DialogService dialogService, + AiPreferences aiPreferences, + ExternalApplicationsPreferences externalApplicationsPreferences, + AdaptVisibleTabs adaptVisibleTabs, + TaskExecutor taskExecutor ) { super(aiService, aiPreferences, externalApplicationsPreferences, dialogService, adaptVisibleTabs); diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatWindow.java b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatWindow.java index f0151f889a66..8df32fc5781e 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatWindow.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatWindow.java @@ -9,14 +9,13 @@ import org.jabref.gui.frame.ExternalApplicationsPreferences; import org.jabref.gui.util.BaseWindow; import org.jabref.logic.ai.AiService; +import org.jabref.logic.ai.chatting.ChatHistory; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; -import dev.langchain4j.data.message.ChatMessage; - public class AiChatWindow extends BaseWindow { private final AiService aiService; private final DialogService dialogService; @@ -28,12 +27,13 @@ public class AiChatWindow extends BaseWindow { // This field is used for finding an existing AI chat window when user wants to chat with the same group again. private String chatName; - public AiChatWindow(AiService aiService, - DialogService dialogService, - AiPreferences aiPreferences, - ExternalApplicationsPreferences externalApplicationsPreferences, - AdaptVisibleTabs adaptVisibleTabs, - TaskExecutor taskExecutor + public AiChatWindow( + AiService aiService, + DialogService dialogService, + AiPreferences aiPreferences, + ExternalApplicationsPreferences externalApplicationsPreferences, + AdaptVisibleTabs adaptVisibleTabs, + TaskExecutor taskExecutor ) { this.aiService = aiService; this.dialogService = dialogService; @@ -43,7 +43,12 @@ public AiChatWindow(AiService aiService, this.taskExecutor = taskExecutor; } - public void setChat(StringProperty name, ObservableList chatHistory, BibDatabaseContext bibDatabaseContext, ObservableList entries) { + public void setChat( + StringProperty name, + ChatHistory chatHistory, + BibDatabaseContext bibDatabaseContext, + ObservableList entries + ) { setTitle(Localization.lang("AI chat with %0", name.getValue())); chatName = name.getValue(); setScene( diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/chathistory/ChatHistoryComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/chathistory/ChatHistoryComponent.java index c99e125b8e7f..f7ef59d8ae5e 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/chathistory/ChatHistoryComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/chathistory/ChatHistoryComponent.java @@ -1,16 +1,16 @@ package org.jabref.gui.ai.components.aichat.chathistory; -import javafx.collections.ListChangeListener; -import javafx.collections.ObservableList; +import java.util.Optional; + import javafx.fxml.FXML; import javafx.scene.control.ScrollPane; import javafx.scene.layout.VBox; import org.jabref.gui.ai.components.aichat.chatmessage.ChatMessageComponent; import org.jabref.gui.util.UiTaskExecutor; +import org.jabref.logic.ai.chatting.ChatHistory; import com.airhacks.afterburner.views.ViewLoader; -import dev.langchain4j.data.message.ChatMessage; public class ChatHistoryComponent extends ScrollPane { @FXML private VBox vBox; @@ -27,22 +27,22 @@ public ChatHistoryComponent() { }); } - /** - * @implNote You must call this method only once. - */ - public void setItems(ObservableList items) { - fill(items); - items.addListener((ListChangeListener) obs -> fill(items)); - } - - private void fill(ObservableList items) { + public void updateMessages(ChatHistory chatHistory) { UiTaskExecutor.runInJavaFXThread(() -> { vBox.getChildren().clear(); - items.forEach(chatMessage -> - vBox.getChildren().add(new ChatMessageComponent(chatMessage, chatMessageComponent -> { - int index = vBox.getChildren().indexOf(chatMessageComponent); - items.remove(index); - }))); + // TODO: Inefficient. + chatHistory + .getAllMessages() + .forEach(chatMessage -> + vBox.getChildren().add(new ChatMessageComponent( + chatMessage, + chatMessageComponent -> { + // TODO: Mix of logic. + chatHistory.deleteMessage(chatMessageComponent.getChatMessage().id()); + updateMessages(chatHistory); + }) + ) + ); }); } @@ -50,4 +50,8 @@ public void scrollDown() { this.layout(); this.setVvalue(this.getVmax()); } + + public Optional getMessage(int index) { + return Optional.ofNullable(vBox.getChildren().get(index)).map(node -> (ChatMessageComponent) node); + } } diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/chatmessage/ChatMessageComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/chatmessage/ChatMessageComponent.java index 57ebe8916397..5a35e0a50edd 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/chatmessage/ChatMessageComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/chatmessage/ChatMessageComponent.java @@ -1,5 +1,6 @@ package org.jabref.gui.ai.components.aichat.chatmessage; +import java.util.Objects; import java.util.function.Consumer; import javafx.beans.property.ObjectProperty; @@ -15,11 +16,11 @@ import org.jabref.gui.util.MarkdownTextFlow; import org.jabref.logic.l10n.Localization; -import org.jabref.model.ai.chatting.ErrorMessage; +import org.jabref.model.ai.chatting.ChatHistoryRecordV2; +import org.jabref.model.ai.chatting.messages.ErrorMessage; import com.airhacks.afterburner.views.ViewLoader; import dev.langchain4j.data.message.AiMessage; -import dev.langchain4j.data.message.ChatMessage; import dev.langchain4j.data.message.UserMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,7 +28,7 @@ public class ChatMessageComponent extends HBox { private static final Logger LOGGER = LoggerFactory.getLogger(ChatMessageComponent.class); - private final ObjectProperty chatMessage = new SimpleObjectProperty<>(); + private final ObjectProperty chatMessage = new SimpleObjectProperty<>(); private final ObjectProperty> onDelete = new SimpleObjectProperty<>(); @FXML private HBox wrapperHBox; @@ -55,17 +56,17 @@ public ChatMessageComponent() { markdownContentPane.prefHeightProperty().bind(markdownTextFlow.heightProperty()); } - public ChatMessageComponent(ChatMessage chatMessage, Consumer onDeleteCallback) { + public ChatMessageComponent(ChatHistoryRecordV2 chatMessage, Consumer onDeleteCallback) { this(); setChatMessage(chatMessage); setOnDelete(onDeleteCallback); } - public void setChatMessage(ChatMessage chatMessage) { + public void setChatMessage(ChatHistoryRecordV2 chatMessage) { this.chatMessage.set(chatMessage); } - public ChatMessage getChatMessage() { + public ChatHistoryRecordV2 getChatMessage() { return chatMessage.get(); } @@ -74,32 +75,28 @@ public void setOnDelete(Consumer onDeleteCallback) { } private void loadChatMessage() { - switch (chatMessage.get()) { - case UserMessage userMessage -> { - setColor("-jr-ai-message-user", "-jr-ai-message-user-border"); - setNodeOrientation(NodeOrientation.RIGHT_TO_LEFT); - wrapperHBox.setAlignment(Pos.TOP_RIGHT); - sourceLabel.setText(Localization.lang("User")); - markdownTextFlow.setMarkdown(userMessage.singleText()); - } - - case AiMessage aiMessage -> { - setColor("-jr-ai-message-ai", "-jr-ai-message-ai-border"); - setNodeOrientation(NodeOrientation.LEFT_TO_RIGHT); - wrapperHBox.setAlignment(Pos.TOP_LEFT); - sourceLabel.setText(Localization.lang("AI")); - markdownTextFlow.setMarkdown(aiMessage.text()); - } - - case ErrorMessage errorMessage -> { - setColor("-jr-ai-message-error", "-jr-ai-message-error-border"); - setNodeOrientation(NodeOrientation.LEFT_TO_RIGHT); - sourceLabel.setText(Localization.lang("Error")); - markdownTextFlow.setMarkdown(errorMessage.getText()); - } - - default -> - LOGGER.error("ChatMessageComponent supports only user, AI, or error messages, but other type was passed: {}", chatMessage.get().type().name()); + String type = chatMessage.get().messageTypeClassName(); + String content = chatMessage.get().content(); + + if (Objects.equals(type, UserMessage.class.getName())) { + setColor("-jr-ai-message-user", "-jr-ai-message-user-border"); + setNodeOrientation(NodeOrientation.RIGHT_TO_LEFT); + wrapperHBox.setAlignment(Pos.TOP_RIGHT); + sourceLabel.setText(Localization.lang("User")); + markdownTextFlow.setMarkdown(content); + } else if (type == AiMessage.class.getName()) { + setColor("-jr-ai-message-ai", "-jr-ai-message-ai-border"); + setNodeOrientation(NodeOrientation.LEFT_TO_RIGHT); + wrapperHBox.setAlignment(Pos.TOP_LEFT); + sourceLabel.setText(Localization.lang("AI")); + markdownTextFlow.setMarkdown(content); + } else if (Objects.equals(type, ErrorMessage.class.getName())) { + setColor("-jr-ai-message-error", "-jr-ai-message-error-border"); + setNodeOrientation(NodeOrientation.LEFT_TO_RIGHT); + sourceLabel.setText(Localization.lang("Error")); + markdownTextFlow.setMarkdown(content); + } else { + LOGGER.error("ChatMessageComponent supports only user, AI, or error messages, but other type was passed: {}", type); } } diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/PrivacyNoticeComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/PrivacyNoticeComponent.java index a5c0fe0b0cdd..016915364d41 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/PrivacyNoticeComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/PrivacyNoticeComponent.java @@ -19,7 +19,7 @@ import org.jabref.gui.frame.ExternalApplicationsPreferences; import org.jabref.gui.preferences.GuiPreferences; import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.model.ai.chatting.AiProvider; +import org.jabref.model.ai.llm.AiProvider; import com.airhacks.afterburner.views.ViewLoader; import jakarta.inject.Inject; diff --git a/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java b/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java index 0617b6b8e627..b3661ae0318c 100644 --- a/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java +++ b/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java @@ -18,6 +18,7 @@ import org.jabref.gui.frame.ExternalApplicationsPreferences; import org.jabref.gui.preferences.GuiPreferences; import org.jabref.logic.ai.AiService; +import org.jabref.logic.ai.chatting.EntryChatHistory; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.citationkeypattern.CitationKeyGenerator; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; @@ -25,6 +26,7 @@ import org.jabref.logic.util.CitationKeyCheck; import org.jabref.logic.util.TaskExecutor; import org.jabref.logic.util.io.FileUtil; +import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; @@ -40,8 +42,6 @@ public class AiChatTab extends EntryEditorTab { private final AdaptVisibleTabs adaptVisibleTabs; private final CitationKeyPatternPreferences citationKeyPatternPreferences; - private Optional previousBibEntry = Optional.empty(); - public AiChatTab(AiService aiService, DialogService dialogService, GuiPreferences preferences, @@ -74,8 +74,6 @@ public boolean shouldShow(BibEntry entry) { */ @Override protected void bindToEntry(BibEntry entry) { - previousBibEntry.ifPresent(previousBibEntry -> aiService.getEntryChatHistoryService().closeChatHistory(previousBibEntry)); - previousBibEntry = Optional.of(entry); BibDatabaseContext bibDatabaseContext = stateManager.getActiveDatabase().orElse(new BibDatabaseContext()); if (!aiPreferences.getEnableAi()) { @@ -133,9 +131,14 @@ private void showChatPanel(BibDatabaseContext bibDatabaseContext, BibEntry entry StringProperty chatName = new SimpleStringProperty("entry " + entry.getCitationKey().orElse("")); entry.getCiteKeyBinding().addListener((observable, oldValue, newValue) -> chatName.setValue("entry " + newValue)); + Optional databasePath = bibDatabaseContext.getDatabasePath(); + Optional citationKey = entry.getCitationKey(); + assert databasePath.isPresent(); // TODO: WTF THINK ABOUT THIS. + assert citationKey.isPresent(); // TODO: WTF THINK ABOUT THIS. + setContent(new AiChatGuardedComponent( chatName, - aiService.getEntryChatHistoryService().getChatHistory(bibDatabaseContext, entry), + new EntryChatHistory(aiService.getEntryChatHistoryRepository(), new BibEntryAiIdentifier(databasePath.get(), citationKey.get())), bibDatabaseContext, FXCollections.observableArrayList(new ArrayList<>(List.of(entry))), aiService, diff --git a/jabgui/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java b/jabgui/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java index 40d48750b9e7..554e965523d0 100644 --- a/jabgui/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java +++ b/jabgui/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java @@ -28,8 +28,11 @@ import org.jabref.gui.preferences.GuiPreferences; import org.jabref.gui.util.CustomLocalDragboard; import org.jabref.logic.ai.AiService; +import org.jabref.logic.ai.chatting.ChatHistory; +import org.jabref.logic.ai.chatting.GroupChatHistory; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.TaskExecutor; +import org.jabref.model.ai.identifiers.GroupAiIdentifier; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; @@ -47,7 +50,6 @@ import org.jabref.model.metadata.MetaData; import com.tobiasdiez.easybind.EasyBind; -import dev.langchain4j.data.message.ChatMessage; import org.jspecify.annotations.NonNull; public class GroupTreeViewModel extends AbstractViewModel { @@ -465,13 +467,15 @@ public void chatWithGroup(GroupNodeViewModel group) { StringProperty nameProperty = new SimpleStringProperty(Localization.lang("Group %0", groupNameProperty.get())); groupNameProperty.addListener((obs, oldValue, newValue) -> nameProperty.setValue(Localization.lang("Group %0", groupNameProperty.get()))); - ObservableList chatHistory = aiService.getGroupChatHistoryService().getChatHistory(currentDatabase.get(), group.getGroupNode()); + // TODO: Unchecked get. + ChatHistory chatHistory = + new GroupChatHistory(aiService.getGroupChatHistoryRepository(), new GroupAiIdentifier(currentDatabase.get().getDatabasePath().get(), group.getGroupNode().getGroup().nameProperty().get())); ObservableList bibEntries = FXCollections.observableArrayList(group.getGroupNode().findMatches(currentDatabase.get().getDatabase())); openAiChat(nameProperty, chatHistory, currentDatabase.get(), bibEntries); } - private void openAiChat(StringProperty name, ObservableList chatHistory, BibDatabaseContext bibDatabaseContext, ObservableList entries) { + private void openAiChat(StringProperty name, ChatHistory chatHistory, BibDatabaseContext bibDatabaseContext, ObservableList entries) { Optional existingWindow = stateManager.getAiChatWindows().stream().filter(window -> window.getChatName().equals(name.get())).findFirst(); if (existingWindow.isPresent()) { diff --git a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java index 72a4201ab9bd..f082ad1d7da6 100644 --- a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java +++ b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java @@ -23,8 +23,8 @@ import org.jabref.gui.util.ViewModelListCellFactory; import org.jabref.logic.help.HelpFile; import org.jabref.logic.l10n.Localization; -import org.jabref.model.ai.chatting.AiProvider; import org.jabref.model.ai.embeddings.EmbeddingModelEnumeration; +import org.jabref.model.ai.llm.AiProvider; import org.jabref.model.ai.templating.AiTemplate; import com.airhacks.afterburner.views.ViewLoader; diff --git a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java index 35be8dd8932e..064c55d3357b 100644 --- a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java +++ b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java @@ -28,8 +28,8 @@ import org.jabref.logic.util.LocalizedNumbers; import org.jabref.logic.util.OptionalObjectProperty; import org.jabref.logic.util.strings.StringUtil; -import org.jabref.model.ai.chatting.AiProvider; import org.jabref.model.ai.embeddings.EmbeddingModelEnumeration; +import org.jabref.model.ai.llm.AiProvider; import org.jabref.model.ai.templating.AiTemplate; import de.saxsys.mvvmfx.utils.validation.FunctionBasedValidator; diff --git a/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java b/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java index 30469f7dade3..05ce26fb92cb 100644 --- a/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java +++ b/jabgui/src/test/java/org/jabref/gui/ai/components/aichat/AiChatComponentTest.java @@ -16,7 +16,7 @@ import org.jabref.logic.l10n.Language; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.TaskExecutor; -import org.jabref.model.ai.chatting.AiProvider; +import org.jabref.model.ai.llm.AiProvider; import org.jabref.model.database.BibDatabaseContext; import org.junit.jupiter.api.BeforeEach; diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index 18caeb01f8dd..d3d8b78e4ec7 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -151,6 +151,7 @@ exports org.jabref.model.ai.identifiers; exports org.jabref.model.ai.tokenization; exports org.jabref.logic.ai.current; + exports org.jabref.model.ai.llm; // endregion requires java.base; @@ -298,5 +299,6 @@ requires org.jooq.jool; requires org.libreoffice.uno; requires transitive org.jspecify; + requires org.jetbrains.annotations; // endregion } diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiService.java b/jablib/src/main/java/org/jabref/logic/ai/AiService.java index 476d1ab295ba..c1448f3d8b14 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiService.java @@ -7,9 +7,13 @@ import javafx.beans.property.SimpleBooleanProperty; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.chatting.EntryChatHistoryService; -import org.jabref.logic.ai.chatting.GroupChatHistoryService; -import org.jabref.logic.ai.chatting.repositories.MVStoreChatHistoryRepository; +import org.jabref.logic.ai.chatting.EntryChattingDatabaseListener; +import org.jabref.logic.ai.chatting.GroupChattingDatabaseListener; +import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepositoryV2; +import org.jabref.logic.ai.chatting.repositories.GroupChatHistoryRepositoryV2; +import org.jabref.logic.ai.chatting.repositories.MVStoreChatHistoryRepositoryV1; +import org.jabref.logic.ai.chatting.repositories.MVStoreEntryChatHistoryRepositoryV2; +import org.jabref.logic.ai.chatting.repositories.MVStoreGroupChatHistoryRepositoryV2; import org.jabref.logic.ai.current.CurrentAiTemplates; import org.jabref.logic.ai.current.CurrentAnswerEngine; import org.jabref.logic.ai.current.CurrentChatLanguageModel; @@ -24,7 +28,6 @@ import org.jabref.logic.ai.summarization.SummariesService; import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.Summarizator; import org.jabref.logic.ai.summarization.repositories.MVStoreSummariesRepository; -import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; import org.jabref.logic.util.Directories; import org.jabref.logic.util.NotificationService; import org.jabref.logic.util.TaskExecutor; @@ -40,12 +43,14 @@ * Holds all the AI components: LLM and embedding model, chat history and embeddings cache. */ public class AiService implements AutoCloseable { - public static final String VERSION = "1"; + public static final String VERSION = "2"; private static final String EMBEDDINGS_FILE_NAME = "embeddings.mv"; private static final String FULLY_INGESTED_FILE_NAME = "fully-ingested.mv"; private static final String SUMMARIES_FILE_NAME = "summaries.mv"; - private static final String CHAT_HISTORY_FILE_NAME = "chat-histories.mv"; + private static final String CHAT_HISTORY_FILE_NAME = "chat-histories.mv"; // v1 + private static final String ENTRY_CHAT_HISTORY_FILE_NAME = "entries-chat-histories.mv"; // v2 + private static final String GROUP_CHAT_HISTORY_FILE_NAME = "groups-chat-histories.mv"; // v2 // This field is used to shut down AI-related background tasks. // If a background task processes a big document and has a loop, then the task should check the status @@ -56,40 +61,46 @@ public class AiService implements AutoCloseable { new ThreadFactoryBuilder().setNameFormat("ai-retrieval-pool-%d").build() ); - private final MVStoreChatHistoryRepository mvStoreChatHistoryStorage; + private final MVStoreChatHistoryRepositoryV1 mvStoreChatHistoryStorage; + + private final MVStoreEntryChatHistoryRepositoryV2 mvStoreEntryChatHistoryStorage; + private final MVStoreGroupChatHistoryRepositoryV2 mvStoreGroupChatHistoryStorage; + private final MVStoreEmbeddingStore mvStoreEmbeddingStore; private final MVStoreIngestedDocumentsRepository mvStoreFullyIngestedDocumentsTracker; private final MVStoreSummariesRepository mvStoreSummariesStorage; private final CurrentAiTemplates currentAiTemplates; - private final EntryChatHistoryService entryChatHistoryService; - private final GroupChatHistoryService groupChatHistoryService; private final CurrentDocumentSplitter currentDocumentSplitter; private final CurrentTokenEstimator currentTokenEstimator; private final CurrentChatLanguageModel currentChatLanguageModel; private final CurrentEmbeddingModel currentEmbeddingModel; private final CurrentSummarizator currentSummarizator; private final CurrentAnswerEngine currentAnswerEngine; + private final IngestionService ingestionService; private final SummariesService summariesService; + private final EntryChattingDatabaseListener entryChattingDatabaseListener; + private final GroupChattingDatabaseListener groupChattingDatabaseListener; + public AiService( AiPreferences aiPreferences, FilePreferences filePreferences, - CitationKeyPatternPreferences citationKeyPatternPreferences, NotificationService notificationService, TaskExecutor taskExecutor ) { - this.mvStoreChatHistoryStorage = new MVStoreChatHistoryRepository(notificationService, Directories.getAiFilesDirectory().resolve(CHAT_HISTORY_FILE_NAME)); + this.mvStoreChatHistoryStorage = new MVStoreChatHistoryRepositoryV1(notificationService, Directories.getAiFilesDirectory().resolve(CHAT_HISTORY_FILE_NAME)); + + this.mvStoreEntryChatHistoryStorage = new MVStoreEntryChatHistoryRepositoryV2(Directories.getAiFilesDirectory().resolve(ENTRY_CHAT_HISTORY_FILE_NAME), notificationService); + this.mvStoreGroupChatHistoryStorage = new MVStoreGroupChatHistoryRepositoryV2(Directories.getAiFilesDirectory().resolve(GROUP_CHAT_HISTORY_FILE_NAME), notificationService); + this.mvStoreEmbeddingStore = new MVStoreEmbeddingStore(Directories.getAiFilesDirectory().resolve(EMBEDDINGS_FILE_NAME), notificationService); this.mvStoreFullyIngestedDocumentsTracker = new MVStoreIngestedDocumentsRepository(notificationService, Directories.getAiFilesDirectory().resolve(FULLY_INGESTED_FILE_NAME)); this.mvStoreSummariesStorage = new MVStoreSummariesRepository(notificationService, Directories.getAiFilesDirectory().resolve(SUMMARIES_FILE_NAME)); this.currentAiTemplates = new CurrentAiTemplates(aiPreferences); - this.entryChatHistoryService = new EntryChatHistoryService(citationKeyPatternPreferences, mvStoreChatHistoryStorage); - this.groupChatHistoryService = new GroupChatHistoryService(mvStoreChatHistoryStorage); - this.currentDocumentSplitter = new CurrentDocumentSplitter(aiPreferences); this.currentTokenEstimator = new CurrentTokenEstimator(aiPreferences); this.currentChatLanguageModel = new CurrentChatLanguageModel(aiPreferences, currentTokenEstimator); @@ -117,6 +128,9 @@ public AiService( mvStoreSummariesStorage, shutdownSignal ); + + this.entryChattingDatabaseListener = new EntryChattingDatabaseListener(mvStoreEntryChatHistoryStorage); + this.groupChattingDatabaseListener = new GroupChattingDatabaseListener(mvStoreGroupChatHistoryStorage); } public CurrentDocumentSplitter getDocumentSplitter() { @@ -131,14 +145,6 @@ public CurrentEmbeddingModel getEmbeddingModel() { return currentEmbeddingModel; } - public EntryChatHistoryService getEntryChatHistoryService() { - return entryChatHistoryService; - } - - public GroupChatHistoryService getGroupChatHistoryService() { - return groupChatHistoryService; - } - public IngestionService getIngestionService() { return ingestionService; } @@ -167,9 +173,18 @@ public CurrentAnswerEngine getAnswerEngine() { return currentAnswerEngine; } + public EntryChatHistoryRepositoryV2 getEntryChatHistoryRepository() { + return mvStoreEntryChatHistoryStorage; + } + + public GroupChatHistoryRepositoryV2 getGroupChatHistoryRepository() { + return mvStoreGroupChatHistoryStorage; + } + public void setupDatabase(BibDatabaseContext context) { - entryChatHistoryService.setupDatabase(context); - groupChatHistoryService.setupDatabase(context); + entryChattingDatabaseListener.setupDatabase(context); + groupChattingDatabaseListener.setupDatabase(context); + ingestionService.setupDatabase(context); summariesService.setupDatabase(context); } @@ -182,9 +197,6 @@ public void close() throws Exception { currentChatLanguageModel.close(); currentEmbeddingModel.close(); - entryChatHistoryService.close(); - groupChatHistoryService.close(); - mvStoreChatHistoryStorage.close(); mvStoreFullyIngestedDocumentsTracker.close(); mvStoreEmbeddingStore.close(); diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistory.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistory.java new file mode 100644 index 000000000000..8256d6b64046 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistory.java @@ -0,0 +1,19 @@ +package org.jabref.logic.ai.chatting; + +import java.util.List; + +import org.jabref.model.ai.chatting.ChatHistoryRecordV2; + +public interface ChatHistory { + void addMessage(ChatHistoryRecordV2 chatHistoryRecord); + + void deleteMessage(String id); + + void clear(); + + List getAllMessages(); + + boolean isEmpty(); + + int size(); +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/CitationKeyChangeListener.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/CitationKeyChangeListener.java new file mode 100644 index 000000000000..245161a474a3 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/CitationKeyChangeListener.java @@ -0,0 +1,59 @@ +package org.jabref.logic.ai.chatting; + +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; + +import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepositoryV2; +import org.jabref.model.ai.chatting.ChatHistoryRecordV2; +import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.event.FieldChangedEvent; +import org.jabref.model.entry.field.InternalField; + +import com.google.common.eventbus.Subscribe; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CitationKeyChangeListener { + private static final Logger LOGGER = LoggerFactory.getLogger(CitationKeyChangeListener.class); + + private final EntryChatHistoryRepositoryV2 entryChatHistoryRepository; + private final BibDatabaseContext bibDatabaseContext; + + public CitationKeyChangeListener(EntryChatHistoryRepositoryV2 entryChatHistoryRepository, BibDatabaseContext bibDatabaseContext) { + this.entryChatHistoryRepository = entryChatHistoryRepository; + this.bibDatabaseContext = bibDatabaseContext; + } + + @Subscribe + void listen(FieldChangedEvent e) { + if (e.getField() != InternalField.KEY_FIELD) { + return; + } + + transferHistory(bibDatabaseContext, e.getBibEntry(), e.getOldValue(), e.getNewValue()); + } + + private void transferHistory(BibDatabaseContext bibDatabaseContext, BibEntry entry, String oldCitationKey, String newCitationKey) { + // TODO: This method does not check if the citation key is valid. + + Optional databasePath = bibDatabaseContext.getDatabasePath(); + + if (databasePath.isEmpty()) { + LOGGER.warn("Could not transfer chat history of entry {} (old key: {}): database path is empty.", newCitationKey, oldCitationKey); + return; + } + + BibEntryAiIdentifier oldIdentifier = new BibEntryAiIdentifier(databasePath.get(), oldCitationKey); + BibEntryAiIdentifier newIdentifier = new BibEntryAiIdentifier(databasePath.get(), newCitationKey); + + List chatHistory = entryChatHistoryRepository.getAllMessages(oldIdentifier); + + entryChatHistoryRepository.clear(oldIdentifier); + entryChatHistoryRepository.clear(newIdentifier); + + chatHistory.forEach(record -> entryChatHistoryRepository.addMessage(newIdentifier, record)); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChatHistory.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChatHistory.java new file mode 100644 index 000000000000..7e1de7a9d90b --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChatHistory.java @@ -0,0 +1,50 @@ +package org.jabref.logic.ai.chatting; + +import java.util.List; + +import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepositoryV2; +import org.jabref.model.ai.chatting.ChatHistoryRecordV2; +import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; + +public class EntryChatHistory implements ChatHistory { + private final EntryChatHistoryRepositoryV2 entryChatHistoryRepository; + private final BibEntryAiIdentifier identifier; + + public EntryChatHistory( + EntryChatHistoryRepositoryV2 entryChatHistoryRepository, + BibEntryAiIdentifier identifier + ) { + this.entryChatHistoryRepository = entryChatHistoryRepository; + this.identifier = identifier; + } + + @Override + public void addMessage(ChatHistoryRecordV2 chatHistoryRecord) { + entryChatHistoryRepository.addMessage(identifier, chatHistoryRecord); + } + + @Override + public void deleteMessage(String id) { + entryChatHistoryRepository.deleteMessage(identifier, id); + } + + @Override + public void clear() { + entryChatHistoryRepository.clear(identifier); + } + + @Override + public List getAllMessages() { + return entryChatHistoryRepository.getAllMessages(identifier); + } + + @Override + public boolean isEmpty() { + return entryChatHistoryRepository.isEmpty(identifier); + } + + @Override + public int size() { + return entryChatHistoryRepository.size(identifier); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChatHistoryService.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChatHistoryService.java deleted file mode 100644 index c6c75fb00cf2..000000000000 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChatHistoryService.java +++ /dev/null @@ -1,179 +0,0 @@ -package org.jabref.logic.ai.chatting; - -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.TreeMap; - -import javafx.collections.FXCollections; -import javafx.collections.ObservableList; - -import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepository; -import org.jabref.logic.citationkeypattern.CitationKeyGenerator; -import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; -import org.jabref.logic.util.CitationKeyCheck; -import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; -import org.jabref.model.ai.identifiers.GroupAiIdentifier; -import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.event.FieldChangedEvent; -import org.jabref.model.entry.field.InternalField; -import org.jabref.model.groups.GroupTreeNode; - -import com.google.common.eventbus.Subscribe; -import dev.langchain4j.data.message.ChatMessage; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/// Main class for getting and storing chat history for entries and groups. -/// Use this class in logic and UI. -/// It currently resides in the UI package because it relies on the `org.jabref.gui.StateManager` to get the open databases and to find the correct [BibDatabaseContext] based on an entry. -/// -/// The returned chat history is a [ObservableList]. So chat history exists for every possible -/// [BibEntry] and [org.jabref.model.groups.AbstractGroup]. The chat history is stored in runtime. -/// -/// To save and load chat history, [BibEntry] and [org.jabref.model.groups.AbstractGroup] must satisfy several constraints. -/// Serialization and deserialization is handled in [EntryChatHistoryRepository]. -/// -/// Constraints for serialization and deserialization of a chat history of a [BibEntry]: -/// 1. There should exist an associated [BibDatabaseContext] for the [BibEntry]. -/// 2. The database path of the associated [BibDatabaseContext] must be set. -/// 3. The citation key of the [BibEntry] must be set and unique. -/// -/// Constraints for serialization and deserialization of a chat history of an [GroupTreeNode]: -/// 1. There should exist an associated [BibDatabaseContext] for the [GroupTreeNode]. -/// 2. The database path of the associated [BibDatabaseContext] must be set. -/// 3. The name of an [GroupTreeNode] must be set and unique (this requirement is possibly already satisfied in -/// JabRef, but for [BibEntry] it is definitely not). -public class EntryChatHistoryService implements AutoCloseable { - private static final Logger LOGGER = LoggerFactory.getLogger(EntryChatHistoryService.class); - - private final CitationKeyPatternPreferences citationKeyPatternPreferences; - private final EntryChatHistoryRepository entryChatHistoryRepository; - - // We use a {@link TreeMap} here to store {@link BibEntry} chat histories by their id. - // When you compare {@link BibEntry} instances, they are compared by value, not by reference. - // And when you store {@link BibEntry} instances in a {@link HashMap}, an old hash may be stored when the {@link BibEntry} is changed. - // See also ADR-38. - private final TreeMap bibEntriesChatHistory = - new TreeMap<>(Comparator.comparing(BibEntry::getId)); - - public EntryChatHistoryService( - CitationKeyPatternPreferences citationKeyPatternPreferences, - EntryChatHistoryRepository entryChatHistoryRepository - ) { - this.citationKeyPatternPreferences = citationKeyPatternPreferences; - this.entryChatHistoryRepository = entryChatHistoryRepository; - } - - public void setupDatabase(BibDatabaseContext bibDatabaseContext) { - bibDatabaseContext - .getDatabase() - .getEntries() - .forEach(entry -> - entry.registerListener(new CitationKeyChangeListener(bibDatabaseContext)) - ); - } - - public ObservableList getChatHistory(BibDatabaseContext bibDatabaseContext, BibEntry entry) { - return bibEntriesChatHistory.computeIfAbsent(entry, entryArg -> { - ObservableList chatHistory; - - if (entry.getCitationKey().isEmpty() || !correctCitationKey(bibDatabaseContext, entry) || bibDatabaseContext.getDatabasePath().isEmpty()) { - chatHistory = FXCollections.observableArrayList(); - } else { - BibEntryAiIdentifier identifier = new BibEntryAiIdentifier(bibDatabaseContext.getDatabasePath().get(), entry.getCitationKey().get()); - List chatMessagesList = entryChatHistoryRepository.loadMessagesForEntry(identifier); - chatHistory = FXCollections.observableArrayList(chatMessagesList); - } - - return new ChatHistoryManagementRecord(Optional.of(bibDatabaseContext), chatHistory); - }).chatHistory(); - } - - /** - * Removes the chat history for the given {@link BibEntry} from the internal RAM map. - * If the {@link BibEntry} satisfies requirements for serialization and deserialization of chat history (see - * the docstring for the {@link EntryChatHistoryService}), then the chat history will be stored via the - * {@link EntryChatHistoryRepository}. - *

- * It is not necessary to call this method (everything will be stored in {@link EntryChatHistoryService#close()}, - * but it's best to call it when the chat history {@link BibEntry} is no longer needed. - */ - public void closeChatHistory(BibEntry entry) { - ChatHistoryManagementRecord chatHistoryManagementRecord = bibEntriesChatHistory.get(entry); - if (chatHistoryManagementRecord == null) { - return; - } - - Optional bibDatabaseContext = chatHistoryManagementRecord.bibDatabaseContext(); - - if (bibDatabaseContext.isPresent() && entry.getCitationKey().isPresent() && correctCitationKey(bibDatabaseContext.get(), entry) && bibDatabaseContext.get().getDatabasePath().isPresent()) { - // Method `correctCitationKey` will already check `entry.getCitationKey().isPresent()`, but it is still - // there, to suppress warning from IntelliJ IDEA on `entry.getCitationKey().get()`. - BibEntryAiIdentifier identifier = new BibEntryAiIdentifier(bibDatabaseContext.get().getDatabasePath().get(), entry.getCitationKey().get()); - entryChatHistoryRepository.storeMessagesForEntry( - identifier, - chatHistoryManagementRecord.chatHistory() - ); - } - - // TODO: What if there is two AI chats for the same entry? And one is closed and one is not? - bibEntriesChatHistory.remove(entry); - } - - private boolean correctCitationKey(BibDatabaseContext bibDatabaseContext, BibEntry bibEntry) { - if (!CitationKeyCheck.citationKeyIsPresentAndUnique(bibDatabaseContext, bibEntry)) { - tryToGenerateCitationKey(bibDatabaseContext, bibEntry); - } - - return CitationKeyCheck.citationKeyIsPresentAndUnique(bibDatabaseContext, bibEntry); - } - - private void tryToGenerateCitationKey(BibDatabaseContext bibDatabaseContext, BibEntry bibEntry) { - new CitationKeyGenerator(bibDatabaseContext, citationKeyPatternPreferences).generateAndSetKey(bibEntry); - } - - @Override - public void close() throws Exception { - // We need to clone `bibEntriesChatHistory.keySet()` because closeChatHistoryForEntry() modifies the `bibEntriesChatHistory` map. - new HashSet<>(bibEntriesChatHistory.keySet()).forEach(this::closeChatHistory); - } - - private void transferHistory(BibDatabaseContext bibDatabaseContext, BibEntry entry, String oldCitationKey, String newCitationKey) { - // TODO: This method does not check if the citation key is valid. - - if (bibDatabaseContext.getDatabasePath().isEmpty()) { - LOGGER.warn("Could not transfer chat history of entry {} (old key: {}): database path is empty.", newCitationKey, oldCitationKey); - return; - } - - List chatMessages = bibEntriesChatHistory.computeIfAbsent(entry, - e -> new ChatHistoryManagementRecord(Optional.of(bibDatabaseContext), FXCollections.observableArrayList())).chatHistory(); - - GroupAiIdentifier groupIdentifier = new GroupAiIdentifier(bibDatabaseContext.getDatabasePath().get(), oldCitationKey); - BibEntryAiIdentifier bibEntryIdentifier = new BibEntryAiIdentifier(bibDatabaseContext.getDatabasePath().get(), newCitationKey); - - // TODO: Why group here? - // implementation.storeMessagesForGroup(groupIdentifier, List.of()); - entryChatHistoryRepository.storeMessagesForEntry(bibEntryIdentifier, chatMessages); - } - - private class CitationKeyChangeListener { - private final BibDatabaseContext bibDatabaseContext; - - public CitationKeyChangeListener(BibDatabaseContext bibDatabaseContext) { - this.bibDatabaseContext = bibDatabaseContext; - } - - @Subscribe - void listen(FieldChangedEvent e) { - if (e.getField() != InternalField.KEY_FIELD) { - return; - } - - transferHistory(bibDatabaseContext, e.getBibEntry(), e.getOldValue(), e.getNewValue()); - } - } -} diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChattingDatabaseListener.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChattingDatabaseListener.java new file mode 100644 index 000000000000..628a8168a5af --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChattingDatabaseListener.java @@ -0,0 +1,21 @@ +package org.jabref.logic.ai.chatting; + +import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepositoryV2; +import org.jabref.model.database.BibDatabaseContext; + +public class EntryChattingDatabaseListener { + private final EntryChatHistoryRepositoryV2 entryChatHistoryRepository; + + public EntryChattingDatabaseListener(EntryChatHistoryRepositoryV2 entryChatHistoryRepository) { + this.entryChatHistoryRepository = entryChatHistoryRepository; + } + + public void setupDatabase(BibDatabaseContext databaseContext) { + databaseContext + .getDatabase() + .getEntries() + .forEach(entry -> + entry.registerListener(new CitationKeyChangeListener(entryChatHistoryRepository, databaseContext)) + ); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChatHistory.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChatHistory.java new file mode 100644 index 000000000000..836d85053714 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChatHistory.java @@ -0,0 +1,50 @@ +package org.jabref.logic.ai.chatting; + +import java.util.List; + +import org.jabref.logic.ai.chatting.repositories.GroupChatHistoryRepositoryV2; +import org.jabref.model.ai.chatting.ChatHistoryRecordV2; +import org.jabref.model.ai.identifiers.GroupAiIdentifier; + +public class GroupChatHistory implements ChatHistory { + private final GroupChatHistoryRepositoryV2 groupChatHistoryRepository; + private final GroupAiIdentifier identifier; + + public GroupChatHistory( + GroupChatHistoryRepositoryV2 groupChatHistoryRepository, + GroupAiIdentifier identifier + ) { + this.groupChatHistoryRepository = groupChatHistoryRepository; + this.identifier = identifier; + } + + @Override + public void addMessage(ChatHistoryRecordV2 chatHistoryRecord) { + groupChatHistoryRepository.addMessage(identifier, chatHistoryRecord); + } + + @Override + public void deleteMessage(String id) { + groupChatHistoryRepository.deleteMessage(identifier, id); + } + + @Override + public void clear() { + groupChatHistoryRepository.clear(identifier); + } + + @Override + public List getAllMessages() { + return groupChatHistoryRepository.getAllMessages(identifier); + } + + @Override + public boolean isEmpty() { + return groupChatHistoryRepository.isEmpty(identifier); + } + + @Override + public int size() { + return groupChatHistoryRepository.size(identifier); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChatHistoryService.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChatHistoryService.java deleted file mode 100644 index 6133006d21ef..000000000000 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChatHistoryService.java +++ /dev/null @@ -1,124 +0,0 @@ -package org.jabref.logic.ai.chatting; - -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.TreeMap; - -import javafx.collections.FXCollections; -import javafx.collections.ObservableList; - -import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepository; -import org.jabref.logic.ai.chatting.repositories.GroupChatHistoryRepository; -import org.jabref.model.ai.identifiers.GroupAiIdentifier; -import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.groups.GroupTreeNode; - -import dev.langchain4j.data.message.ChatMessage; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class GroupChatHistoryService implements AutoCloseable { - private final Logger LOGGER = LoggerFactory.getLogger(GroupChatHistoryService.class); - - private final GroupChatHistoryRepository groupChatHistoryRepository; - - // We use {@link TreeMap} for group chat history for the same reason as for {@link BibEntry}ies. - private final TreeMap groupsChatHistory = - new TreeMap<>(Comparator.comparing(GroupTreeNode::getName)); - - public GroupChatHistoryService( - GroupChatHistoryRepository groupChatHistoryRepository - ) { - this.groupChatHistoryRepository = groupChatHistoryRepository; - } - - public void setupDatabase(BibDatabaseContext bibDatabaseContext) { - bibDatabaseContext.getMetaData().getGroups().ifPresent(rootGroupTreeNode -> - rootGroupTreeNode.iterateOverTree().forEach(groupNode -> { - groupNode.getGroup().nameProperty().addListener((observable, oldValue, newValue) -> { - if (newValue != null && oldValue != null) { - transferHistory(bibDatabaseContext, groupNode, oldValue, newValue); - } - }); - - groupNode.getGroupProperty().addListener((obs, oldValue, newValue) -> { - if (oldValue != null && newValue != null) { - transferHistory(bibDatabaseContext, groupNode, oldValue.getName(), newValue.getName()); - } - }); - })); - } - - private void transferHistory(BibDatabaseContext bibDatabaseContext, GroupTreeNode groupTreeNode, String oldName, String newName) { - if (bibDatabaseContext.getDatabasePath().isEmpty()) { - LOGGER.warn("Could not transfer chat history of group {} (old name: {}): database path is empty.", newName, oldName); - return; - } - - List chatMessages = groupsChatHistory.computeIfAbsent(groupTreeNode, - e -> new ChatHistoryManagementRecord(Optional.of(bibDatabaseContext), FXCollections.observableArrayList())).chatHistory(); - - GroupAiIdentifier oldIdentifier = new GroupAiIdentifier(bibDatabaseContext.getDatabasePath().get(), oldName); - GroupAiIdentifier newIdentifier = new GroupAiIdentifier(bibDatabaseContext.getDatabasePath().get(), newName); - - groupChatHistoryRepository.storeMessagesForGroup(oldIdentifier, List.of()); - groupChatHistoryRepository.storeMessagesForGroup(newIdentifier, chatMessages); - } - - public ObservableList getChatHistory(BibDatabaseContext bibDatabaseContext, GroupTreeNode group) { - return groupsChatHistory.computeIfAbsent(group, groupArg -> { - ObservableList chatHistory; - - if (bibDatabaseContext.getDatabasePath().isEmpty()) { - chatHistory = FXCollections.observableArrayList(); - } else { - GroupAiIdentifier identifier = new GroupAiIdentifier(bibDatabaseContext.getDatabasePath().get(), group.getGroup().getName()); - List chatMessagesList = groupChatHistoryRepository.loadMessagesForGroup( - identifier - ); - - chatHistory = FXCollections.observableArrayList(chatMessagesList); - } - - return new ChatHistoryManagementRecord(Optional.of(bibDatabaseContext), chatHistory); - }).chatHistory(); - } - - /** - * Removes the chat history for the given {@link GroupTreeNode} from the internal RAM map. - * If the {@link GroupTreeNode} satisfies requirements for serialization and deserialization of chat history (see - * the docstring for the {@link EntryChatHistoryService}), then the chat history will be stored via the - * {@link EntryChatHistoryRepository}. - *

- * It is not necessary to call this method (everything will be stored in {@link EntryChatHistoryService#close()}, - * but it's best to call it when the chat history {@link GroupTreeNode} is no longer needed. - */ - public void closeChatHistory(GroupTreeNode group) { - ChatHistoryManagementRecord chatHistoryManagementRecord = groupsChatHistory.get(group); - if (chatHistoryManagementRecord == null) { - return; - } - - Optional bibDatabaseContext = chatHistoryManagementRecord.bibDatabaseContext(); - - if (bibDatabaseContext.isPresent() && bibDatabaseContext.get().getDatabasePath().isPresent()) { - GroupAiIdentifier identifier = new GroupAiIdentifier(bibDatabaseContext.get().getDatabasePath().get(), group.getGroup().getName()); - groupChatHistoryRepository.storeMessagesForGroup( - identifier, - chatHistoryManagementRecord.chatHistory() - ); - } - - // TODO: What if there is two AI chats for the same entry? And one is closed and one is not? - groupsChatHistory.remove(group); - } - - - @Override - public void close() throws Exception { - // Clone is for the same reason, as written above. - new HashSet<>(groupsChatHistory.keySet()).forEach(this::closeChatHistory); - } -} diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChattingDatabaseListener.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChattingDatabaseListener.java new file mode 100644 index 000000000000..5da2b7964f38 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChattingDatabaseListener.java @@ -0,0 +1,56 @@ +package org.jabref.logic.ai.chatting; + +import java.util.List; + +import org.jabref.logic.ai.chatting.repositories.GroupChatHistoryRepositoryV2; +import org.jabref.model.ai.chatting.ChatHistoryRecordV2; +import org.jabref.model.ai.identifiers.GroupAiIdentifier; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.groups.GroupTreeNode; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GroupChattingDatabaseListener { + private static final Logger LOGGER = LoggerFactory.getLogger(GroupChattingDatabaseListener.class); + + private final GroupChatHistoryRepositoryV2 groupChatHistoryRepository; + + public GroupChattingDatabaseListener(GroupChatHistoryRepositoryV2 groupChatHistoryRepository) { + this.groupChatHistoryRepository = groupChatHistoryRepository; + } + + public void setupDatabase(BibDatabaseContext databaseContext) { + databaseContext.getMetaData().getGroups().ifPresent(rootGroupTreeNode -> + rootGroupTreeNode.iterateOverTree().forEach(groupNode -> { + groupNode.getGroup().nameProperty().addListener((observable, oldValue, newValue) -> { + if (newValue != null && oldValue != null) { + transferHistory(databaseContext, groupNode, oldValue, newValue); + } + }); + + groupNode.getGroupProperty().addListener((obs, oldValue, newValue) -> { + if (oldValue != null && newValue != null) { + transferHistory(databaseContext, groupNode, oldValue.getName(), newValue.getName()); + } + }); + })); + } + + private void transferHistory(BibDatabaseContext bibDatabaseContext, GroupTreeNode groupTreeNode, String oldName, String newName) { + if (bibDatabaseContext.getDatabasePath().isEmpty()) { + LOGGER.warn("Could not transfer chat history of group {} (old name: {}): database path is empty.", newName, oldName); + return; + } + + GroupAiIdentifier oldIdentifier = new GroupAiIdentifier(bibDatabaseContext.getDatabasePath().get(), oldName); + GroupAiIdentifier newIdentifier = new GroupAiIdentifier(bibDatabaseContext.getDatabasePath().get(), newName); + + List chatHistory = groupChatHistoryRepository.getAllMessages(oldIdentifier); + + groupChatHistoryRepository.clear(oldIdentifier); + groupChatHistoryRepository.clear(newIdentifier); + + chatHistory.forEach(record -> groupChatHistoryRepository.addMessage(newIdentifier, record)); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java index 89803d6d09aa..d66f91df5960 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java @@ -1,19 +1,24 @@ package org.jabref.logic.ai.chatting.logic; +import java.time.Instant; import java.util.ArrayList; import java.util.List; +import java.util.Objects; +import java.util.UUID; import javafx.beans.property.SimpleBooleanProperty; import javafx.beans.property.StringProperty; import javafx.collections.ObservableList; +import org.jabref.logic.ai.chatting.ChatHistory; import org.jabref.logic.ai.chatting.templates.ChattingSystemMessageTemplate; import org.jabref.logic.ai.chatting.templates.ChattingUserMessageTemplate; import org.jabref.logic.ai.pipeline.logic.rag.AnswerEngine; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.util.LongTaskInfo; import org.jabref.logic.util.ProgressCounter; -import org.jabref.model.ai.chatting.ErrorMessage; +import org.jabref.model.ai.chatting.ChatHistoryRecordV2; +import org.jabref.model.ai.chatting.messages.ErrorMessage; import org.jabref.model.ai.identifiers.FullBibEntryAiIdentifier; import org.jabref.model.ai.pipeline.RelevantInformation; import org.jabref.model.ai.templating.AiTemplate; @@ -36,7 +41,7 @@ public class AiChatLogic { private final ChattingSystemMessageTemplate chattingSystemMessageTemplate; private final ChattingUserMessageTemplate chattingUserMessageTemplate; - private final ObservableList chatHistory; + private final ChatHistory chatHistory; private final ObservableList entries; private final StringProperty name; private final BibDatabaseContext bibDatabaseContext; @@ -51,7 +56,7 @@ public AiChatLogic( ChattingSystemMessageTemplate chattingSystemMessageTemplate, ChattingUserMessageTemplate chattingUserMessageTemplate, BibDatabaseContext bibDatabaseContext, - ObservableList chatHistory, + ChatHistory chatHistory, ObservableList entries, StringProperty name, AnswerEngine answerEngine @@ -67,7 +72,11 @@ public AiChatLogic( this.answerEngine = answerEngine; this.chatMemory = new ArrayList<>(); - chatHistory.stream().filter(chatMessage -> !(chatMessage instanceof ErrorMessage)).forEach(chatMemory::add); + chatHistory + .getAllMessages() + .stream() + .filter(chatMessage -> !Objects.equals(chatMessage.messageTypeClassName(), ErrorMessage.class.getName())) + .forEach(record -> chatMemory.add(ChatHistoryRecordUtils.toLangchainMessage(record))); setSystemMessage(chattingSystemMessageTemplate.render(entries)); setupListeningToPreferencesChanges(); @@ -89,7 +98,12 @@ private void setSystemMessage(String systemMessage) { } public AiMessage execute(UserMessage message) { - chatHistory.add(message); + chatHistory.addMessage(new ChatHistoryRecordV2( + UUID.randomUUID().toString(), + message.getClass().getName(), + message.singleText(), + Instant.now() + )); LOGGER.info( "Sending message to AI provider ({}) for answering in {}: {}", @@ -111,14 +125,15 @@ public AiMessage execute(UserMessage message) { chatMemory.set(chatMemory.size() - 1, message); // Removing excerpts from the chat history. chatMemory.add(aiMessage); - chatHistory.add(aiMessage); + chatHistory.addMessage(new ChatHistoryRecordV2( + UUID.randomUUID().toString(), + aiMessage.getClass().getName(), + aiMessage.text(), + Instant.now() + )); LOGGER.debug("Message was answered by the AI provider for {}: {}", name.get(), aiMessage.text()); return aiMessage; } - - public ObservableList getChatHistory() { - return chatHistory; - } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/ChatHistoryRecordUtils.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/ChatHistoryRecordUtils.java new file mode 100644 index 000000000000..0b2ab305ae49 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/ChatHistoryRecordUtils.java @@ -0,0 +1,28 @@ +package org.jabref.logic.ai.chatting.logic; + +import org.jabref.model.ai.chatting.ChatHistoryRecordV2; +import org.jabref.model.ai.chatting.messages.ErrorMessage; + +import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.data.message.ChatMessage; +import dev.langchain4j.data.message.UserMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ChatHistoryRecordUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(ChatHistoryRecordUtils.class); + + public static ChatMessage toLangchainMessage(ChatHistoryRecordV2 record) { + if (record.messageTypeClassName().equals(AiMessage.class.getName())) { + return new AiMessage(record.content()); + } else if (record.messageTypeClassName().equals(UserMessage.class.getName())) { + return new UserMessage(record.content()); + } else if (record.messageTypeClassName().equals(ErrorMessage.class.getName())) { + return new ErrorMessage(record.content()); + } else { + LOGGER.warn("ChatHistoryRecordV2 supports only AI and user messages, but retrieved message has other type: {}. Will treat as an AI message.", record.messageTypeClassName()); + return new AiMessage(record.content()); + } + + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/EntryChatHistoryRepository.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/EntryChatHistoryRepositoryV1.java similarity index 81% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/EntryChatHistoryRepository.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/EntryChatHistoryRepositoryV1.java index 93f0425e19a3..525d2157f5af 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/EntryChatHistoryRepository.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/EntryChatHistoryRepositoryV1.java @@ -6,7 +6,8 @@ import dev.langchain4j.data.message.ChatMessage; -public interface EntryChatHistoryRepository extends AutoCloseable { +@Deprecated +public interface EntryChatHistoryRepositoryV1 extends AutoCloseable { List loadMessagesForEntry(BibEntryAiIdentifier identifier); void storeMessagesForEntry(BibEntryAiIdentifier identifier, List messages); diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/EntryChatHistoryRepositoryV2.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/EntryChatHistoryRepositoryV2.java new file mode 100644 index 000000000000..87f8dbf3baaf --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/EntryChatHistoryRepositoryV2.java @@ -0,0 +1,20 @@ +package org.jabref.logic.ai.chatting.repositories; + +import java.util.List; + +import org.jabref.model.ai.chatting.ChatHistoryRecordV2; +import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; + +public interface EntryChatHistoryRepositoryV2 { + void addMessage(BibEntryAiIdentifier identifier, ChatHistoryRecordV2 chatHistoryRecord); + + void deleteMessage(BibEntryAiIdentifier identifier, String id); + + void clear(BibEntryAiIdentifier identifier); + + List getAllMessages(BibEntryAiIdentifier identifier); + + boolean isEmpty(BibEntryAiIdentifier identifier); + + int size(BibEntryAiIdentifier identifier); +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepository.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepositoryV1.java similarity index 81% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepository.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepositoryV1.java index da1b25c1b0b5..821f8b76c714 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepository.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepositoryV1.java @@ -6,7 +6,8 @@ import dev.langchain4j.data.message.ChatMessage; -public interface GroupChatHistoryRepository extends AutoCloseable { +@Deprecated +public interface GroupChatHistoryRepositoryV1 extends AutoCloseable { List loadMessagesForGroup(GroupAiIdentifier identifier); void storeMessagesForGroup(GroupAiIdentifier identifier, List messages); diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepositoryV2.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepositoryV2.java new file mode 100644 index 000000000000..7ac64eaac4b1 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepositoryV2.java @@ -0,0 +1,20 @@ +package org.jabref.logic.ai.chatting.repositories; + +import java.util.List; + +import org.jabref.model.ai.chatting.ChatHistoryRecordV2; +import org.jabref.model.ai.identifiers.GroupAiIdentifier; + +public interface GroupChatHistoryRepositoryV2 { + void addMessage(GroupAiIdentifier identifier, ChatHistoryRecordV2 chatHistoryRecord); + + void deleteMessage(GroupAiIdentifier identifier, String id); + + void clear(GroupAiIdentifier identifier); + + List getAllMessages(GroupAiIdentifier identifier); + + boolean isEmpty(GroupAiIdentifier identifier); + + int size(GroupAiIdentifier identifier); +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepository.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepository.java deleted file mode 100644 index 295d13fc13c2..000000000000 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepository.java +++ /dev/null @@ -1,132 +0,0 @@ -package org.jabref.logic.ai.chatting.repositories; - -import java.io.Serializable; -import java.nio.file.Path; -import java.util.Comparator; -import java.util.List; -import java.util.Map; - -import org.jabref.logic.ai.util.MVStoreBase; -import org.jabref.logic.l10n.Localization; -import org.jabref.logic.util.NotificationService; -import org.jabref.model.ai.chatting.ErrorMessage; -import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; -import org.jabref.model.ai.identifiers.GroupAiIdentifier; - -import dev.langchain4j.data.message.AiMessage; -import dev.langchain4j.data.message.ChatMessage; -import dev.langchain4j.data.message.UserMessage; -import kotlin.ranges.IntRange; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class MVStoreChatHistoryRepository extends MVStoreBase implements EntryChatHistoryRepository, GroupChatHistoryRepository { - private static final String ENTRY_CHAT_HISTORY_PREFIX = "entry"; - private static final String GROUP_CHAT_HISTORY_PREFIX = "group"; - - private record ChatHistoryRecord(String className, String content) implements Serializable { - private static final Logger LOGGER = LoggerFactory.getLogger(ChatHistoryRecord.class); - - public static ChatHistoryRecord fromLangchainMessage(ChatMessage chatMessage) { - String className = chatMessage.getClass().getName(); - String content = getContentFromLangchainMessage(chatMessage); - return new ChatHistoryRecord(className, content); - } - - private static String getContentFromLangchainMessage(ChatMessage chatMessage) { - String content; - - switch (chatMessage) { - case AiMessage aiMessage -> - content = aiMessage.text(); - case UserMessage userMessage -> - content = userMessage.singleText(); - case ErrorMessage errorMessage -> - content = errorMessage.getText(); - default -> { - LOGGER.warn("ChatHistoryRecord supports only AI, user. and error messages, but added message has other type: {}", chatMessage.type().name()); - return ""; - } - } - - return content; - } - - public ChatMessage toLangchainMessage() { - if (className.equals(AiMessage.class.getName())) { - return new AiMessage(content); - } else if (className.equals(UserMessage.class.getName())) { - return new UserMessage(content); - } else if (className.equals(ErrorMessage.class.getName())) { - return new ErrorMessage(content); - } else { - LOGGER.warn("ChatHistoryRecord supports only AI and user messages, but retrieved message has other type: {}. Will treat as an AI message.", className); - return new AiMessage(content); - } - } - } - - public MVStoreChatHistoryRepository(NotificationService dialogService, Path path) { - super(path, dialogService); - } - - @Override - public List loadMessagesForEntry(BibEntryAiIdentifier identifier) { - return loadMessagesFromMap(getMapForEntry(identifier)); - } - - @Override - public void storeMessagesForEntry(BibEntryAiIdentifier identifier, List messages) { - storeMessagesForMap(getMapForEntry(identifier), messages); - } - - @Override - public List loadMessagesForGroup(GroupAiIdentifier identifier) { - return loadMessagesFromMap(getMapForGroup(identifier)); - } - - @Override - public void storeMessagesForGroup(GroupAiIdentifier identifier, List messages) { - storeMessagesForMap(getMapForGroup(identifier), messages); - } - - private List loadMessagesFromMap(Map map) { - return map - .entrySet() - // We need to check all keys, because upon deletion, there can be "holes" in the integer. - .stream() - .sorted(Comparator.comparingInt(Map.Entry::getKey)) - .map(entry -> entry.getValue().toLangchainMessage()) - .toList(); - } - - private void storeMessagesForMap(Map map, List messages) { - map.clear(); - - new IntRange(0, messages.size() - 1).forEach(i -> - map.put(i, ChatHistoryRecord.fromLangchainMessage(messages.get(i))) - ); - } - - private Map getMapForEntry(BibEntryAiIdentifier identifier) { - return getMap(identifier.databasePath(), ENTRY_CHAT_HISTORY_PREFIX, identifier.citationKey()); - } - - private Map getMapForGroup(GroupAiIdentifier identifier) { - return getMap(identifier.databasePath(), GROUP_CHAT_HISTORY_PREFIX, identifier.groupName()); - } - - private Map getMap(Path bibDatabasePath, String type, String name) { - return mvStore.openMap(bibDatabasePath + "-" + type + "-" + name); - } - - @Override - protected String errorMessageForOpening() { - return "An error occurred while opening chat history storage. Chat history of entries and groups will not be stored in the next session."; - } - - @Override - protected String errorMessageForOpeningLocalized() { - return Localization.lang("An error occurred while opening chat history storage. Chat history of entries and groups will not be stored in the next session."); - } -} diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepositoryV1.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepositoryV1.java new file mode 100644 index 000000000000..cc53ecbb7b8a --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepositoryV1.java @@ -0,0 +1,86 @@ +package org.jabref.logic.ai.chatting.repositories; + +import java.nio.file.Path; +import java.util.Comparator; +import java.util.List; +import java.util.Map; + +import org.jabref.logic.ai.util.MVStoreBase; +import org.jabref.logic.l10n.Localization; +import org.jabref.logic.util.NotificationService; +import org.jabref.model.ai.chatting.ChatHistoryRecordV1; +import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; +import org.jabref.model.ai.identifiers.GroupAiIdentifier; + +import dev.langchain4j.data.message.ChatMessage; +import kotlin.ranges.IntRange; + +@Deprecated +public class MVStoreChatHistoryRepositoryV1 extends MVStoreBase implements EntryChatHistoryRepositoryV1, GroupChatHistoryRepositoryV1 { + private static final String ENTRY_CHAT_HISTORY_PREFIX = "entry"; + private static final String GROUP_CHAT_HISTORY_PREFIX = "group"; + + public MVStoreChatHistoryRepositoryV1(NotificationService dialogService, Path path) { + super(path, dialogService); + } + + @Override + public List loadMessagesForEntry(BibEntryAiIdentifier identifier) { + return loadMessagesFromMap(getMapForEntry(identifier)); + } + + @Override + public void storeMessagesForEntry(BibEntryAiIdentifier identifier, List messages) { + storeMessagesForMap(getMapForEntry(identifier), messages); + } + + @Override + public List loadMessagesForGroup(GroupAiIdentifier identifier) { + return loadMessagesFromMap(getMapForGroup(identifier)); + } + + @Override + public void storeMessagesForGroup(GroupAiIdentifier identifier, List messages) { + storeMessagesForMap(getMapForGroup(identifier), messages); + } + + private List loadMessagesFromMap(Map map) { + return map + .entrySet() + // We need to check all keys, because upon deletion, there can be "holes" in the integer. + .stream() + .sorted(Comparator.comparingInt(Map.Entry::getKey)) + .map(entry -> entry.getValue().toLangchainMessage()) + .toList(); + } + + private void storeMessagesForMap(Map map, List messages) { + map.clear(); + + new IntRange(0, messages.size() - 1).forEach(i -> + map.put(i, ChatHistoryRecordV1.fromLangchainMessage(messages.get(i))) + ); + } + + private Map getMapForEntry(BibEntryAiIdentifier identifier) { + return getMap(identifier.databasePath(), ENTRY_CHAT_HISTORY_PREFIX, identifier.citationKey()); + } + + private Map getMapForGroup(GroupAiIdentifier identifier) { + return getMap(identifier.databasePath(), GROUP_CHAT_HISTORY_PREFIX, identifier.groupName()); + } + + private Map getMap(Path bibDatabasePath, String type, String name) { + return mvStore.openMap(bibDatabasePath + "-" + type + "-" + name); + } + + @Override + protected String errorMessageForOpening() { + return "An error occurred while opening chat history storage. Chat history of entries and groups will not be stored in the next session."; + } + + @Override + protected String errorMessageForOpeningLocalized() { + return Localization.lang("An error occurred while opening chat history storage. Chat history of entries and groups will not be stored in the next session."); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreEntryChatHistoryRepositoryV2.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreEntryChatHistoryRepositoryV2.java new file mode 100644 index 000000000000..90c5815b11a7 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreEntryChatHistoryRepositoryV2.java @@ -0,0 +1,69 @@ +package org.jabref.logic.ai.chatting.repositories; + +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import org.jabref.logic.ai.util.MVStoreBase; +import org.jabref.logic.l10n.Localization; +import org.jabref.logic.util.NotificationService; +import org.jabref.model.ai.chatting.ChatHistoryRecordV2; +import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; + +import org.jspecify.annotations.NonNull; + +public class MVStoreEntryChatHistoryRepositoryV2 extends MVStoreBase implements EntryChatHistoryRepositoryV2 { + public MVStoreEntryChatHistoryRepositoryV2(@NonNull Path path, NotificationService dialogService) { + super(path, dialogService); + } + + @Override + protected String errorMessageForOpening() { + return "An error occurred while opening chat history storage. Chat history of entries will not be stored in the next session."; + } + + @Override + protected String errorMessageForOpeningLocalized() { + return Localization.lang("An error occurred while opening chat history storage. Chat history of entries will not be stored in the next session."); + } + + @Override + public void addMessage(BibEntryAiIdentifier identifier, ChatHistoryRecordV2 chatHistoryRecord) { + Map map = openMap(identifier); + map.put(chatHistoryRecord.id(), chatHistoryRecord); + } + + @Override + public void deleteMessage(BibEntryAiIdentifier identifier, String id) { + Map map = openMap(identifier); + map.remove(id); + } + + @Override + public void clear(BibEntryAiIdentifier identifier) { + Map map = openMap(identifier); + map.clear(); + } + + @Override + public List getAllMessages(BibEntryAiIdentifier identifier) { + Map map = openMap(identifier); + return map.values().stream().toList(); + } + + @Override + public boolean isEmpty(BibEntryAiIdentifier identifier) { + Map map = openMap(identifier); + return map.isEmpty(); + } + + @Override + public int size(BibEntryAiIdentifier identifier) { + Map map = openMap(identifier); + return map.size(); + } + + private Map openMap(BibEntryAiIdentifier identifier) { + return mvStore.openMap(identifier.toString()); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreGroupChatHistoryRepositoryV2.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreGroupChatHistoryRepositoryV2.java new file mode 100644 index 000000000000..b8304f97e35e --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreGroupChatHistoryRepositoryV2.java @@ -0,0 +1,69 @@ +package org.jabref.logic.ai.chatting.repositories; + +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import org.jabref.logic.ai.util.MVStoreBase; +import org.jabref.logic.l10n.Localization; +import org.jabref.logic.util.NotificationService; +import org.jabref.model.ai.chatting.ChatHistoryRecordV2; +import org.jabref.model.ai.identifiers.GroupAiIdentifier; + +import org.jspecify.annotations.NonNull; + +public class MVStoreGroupChatHistoryRepositoryV2 extends MVStoreBase implements GroupChatHistoryRepositoryV2 { + public MVStoreGroupChatHistoryRepositoryV2(@NonNull Path path, NotificationService dialogService) { + super(path, dialogService); + } + + @Override + protected String errorMessageForOpening() { + return "An error occurred while opening chat history storage. Chat history of entries will not be stored in the next session."; + } + + @Override + protected String errorMessageForOpeningLocalized() { + return Localization.lang("An error occurred while opening chat history storage. Chat history of entries will not be stored in the next session."); + } + + @Override + public void addMessage(GroupAiIdentifier identifier, ChatHistoryRecordV2 chatHistoryRecord) { + Map map = openMap(identifier); + map.put(chatHistoryRecord.id(), chatHistoryRecord); + } + + @Override + public void deleteMessage(GroupAiIdentifier identifier, String id) { + Map map = openMap(identifier); + map.remove(id); + } + + @Override + public void clear(GroupAiIdentifier identifier) { + Map map = openMap(identifier); + map.clear(); + } + + @Override + public List getAllMessages(GroupAiIdentifier identifier) { + Map map = openMap(identifier); + return map.values().stream().toList(); + } + + @Override + public boolean isEmpty(GroupAiIdentifier identifier) { + Map map = openMap(identifier); + return map.isEmpty(); + } + + @Override + public int size(GroupAiIdentifier identifier) { + Map map = openMap(identifier); + return map.size(); + } + + private Map openMap(GroupAiIdentifier identifier) { + return mvStore.openMap(identifier.toString()); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentChatLanguageModel.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentChatLanguageModel.java index 9c431ef6419d..38b5897015fc 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentChatLanguageModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentChatLanguageModel.java @@ -7,14 +7,14 @@ import java.util.concurrent.Executors; import org.jabref.logic.ai.chatting.logic.AiChatLogic; -import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepository; +import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepositoryV1; import org.jabref.logic.ai.customimplementations.llms.ChatModel; import org.jabref.logic.ai.customimplementations.llms.Gpt4AllModel; import org.jabref.logic.ai.customimplementations.llms.JvmOpenAiChatLanguageModel; import org.jabref.logic.ai.customimplementations.tokenization.algorithms.TokenEstimator; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.l10n.Localization; -import org.jabref.model.ai.chatting.AiProvider; +import org.jabref.model.ai.llm.AiProvider; import com.google.common.util.concurrent.ThreadFactoryBuilder; import dev.langchain4j.data.message.ChatMessage; @@ -63,7 +63,7 @@ public CurrentChatLanguageModel( * Update the underlying {@link dev.langchain4j.model.chat.ChatModel} by current {@link AiPreferences} parameters. * When the model is updated, the chat messages are not lost. * See {@link AiChatLogic}, where messages are stored in {@link ChatMemory}, - * and see {@link EntryChatHistoryRepository}. + * and see {@link EntryChatHistoryRepositoryV1}. */ private void rebuild() { String apiKey = aiPreferences.getApiKeyForAiProvider(aiPreferences.getAiProvider()); diff --git a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/ChatModel.java b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/ChatModel.java index 33cc259d6924..15463c7e775e 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/ChatModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/customimplementations/llms/ChatModel.java @@ -1,7 +1,7 @@ package org.jabref.logic.ai.customimplementations.llms; import org.jabref.logic.ai.customimplementations.tokenization.algorithms.TokenEstimator; -import org.jabref.model.ai.chatting.AiProvider; +import org.jabref.model.ai.llm.AiProvider; public interface ChatModel extends dev.langchain4j.model.chat.ChatModel { TokenEstimator getTokenizer(); diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java index f7bfd6cfee67..538604e87737 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java @@ -17,8 +17,8 @@ import javafx.beans.property.StringProperty; import org.jabref.logic.util.strings.StringUtil; -import org.jabref.model.ai.chatting.AiProvider; import org.jabref.model.ai.embeddings.EmbeddingModelEnumeration; +import org.jabref.model.ai.llm.AiProvider; import org.jabref.model.ai.pipeline.AnswerEngineKind; import org.jabref.model.ai.pipeline.DocumentSplitterKind; import org.jabref.model.ai.summarization.SummarizatorKind; diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiProviderDefaultChatModels.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiProviderDefaultChatModels.java index 59542c659d7f..37de452572be 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiProviderDefaultChatModels.java +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiProviderDefaultChatModels.java @@ -2,7 +2,7 @@ import java.util.Map; -import org.jabref.model.ai.chatting.AiProvider; +import org.jabref.model.ai.llm.AiProvider; public class AiProviderDefaultChatModels { private static final Map CHAT_MODELS = Map.of( diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/PredefinedChatModel.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/PredefinedChatModel.java index 3c097bfc294d..b2388b345c0b 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/preferences/PredefinedChatModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/PredefinedChatModel.java @@ -3,7 +3,7 @@ import java.util.Arrays; import java.util.List; -import org.jabref.model.ai.chatting.AiProvider; +import org.jabref.model.ai.llm.AiProvider; public enum PredefinedChatModel { GPT_4O_MINI(AiProvider.OPEN_AI, "gpt-4o-mini", 128000), diff --git a/jablib/src/main/java/org/jabref/logic/importer/fileformat/pdf/CitationsFromPdf.java b/jablib/src/main/java/org/jabref/logic/importer/fileformat/pdf/CitationsFromPdf.java index 67aa3cb14565..2b8ec9f26107 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fileformat/pdf/CitationsFromPdf.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fileformat/pdf/CitationsFromPdf.java @@ -29,7 +29,6 @@ public static ParserResult extractCitationsUsingLLM(JabRefCliPreferences prefere try (AiService aiService = new AiService( preferences.getAiPreferences(), preferences.getFilePreferences(), - preferences.getCitationKeyPatternPreferences(), notificationService, new CurrentThreadTaskExecutor())) { LlmPlainCitationParser importer = new LlmPlainCitationParser( diff --git a/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java b/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java index 29cd658b2ef6..9956481add45 100644 --- a/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/preferences/JabRefCliPreferences.java @@ -98,8 +98,8 @@ import org.jabref.logic.util.io.FileHistory; import org.jabref.logic.util.strings.StringUtil; import org.jabref.logic.xmp.XmpPreferences; -import org.jabref.model.ai.chatting.AiProvider; import org.jabref.model.ai.embeddings.EmbeddingModelEnumeration; +import org.jabref.model.ai.llm.AiProvider; import org.jabref.model.ai.pipeline.AnswerEngineKind; import org.jabref.model.ai.pipeline.DocumentSplitterKind; import org.jabref.model.ai.summarization.SummarizatorKind; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryManagementRecord.java b/jablib/src/main/java/org/jabref/model/ai/chatting/ChatHistoryManagementRecordV1.java similarity index 87% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryManagementRecord.java rename to jablib/src/main/java/org/jabref/model/ai/chatting/ChatHistoryManagementRecordV1.java index 4d7219f94013..5f65f2875307 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistoryManagementRecord.java +++ b/jablib/src/main/java/org/jabref/model/ai/chatting/ChatHistoryManagementRecordV1.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.chatting; +package org.jabref.model.ai.chatting; import java.util.Optional; @@ -11,7 +11,8 @@ // Note about `Optional`: it was necessary in a previous version, but currently we never save an `Optional.empty()`. // However, we decided to leave it here: to reduce migrations and to make it possible to chat with a {@link BibEntry} without {@link BibDatabaseContext} // ({@link BibDatabaseContext} is required only for load/store of the chat). -public record ChatHistoryManagementRecord( +@Deprecated +public record ChatHistoryManagementRecordV1( Optional bibDatabaseContext, ObservableList chatHistory ) { diff --git a/jablib/src/main/java/org/jabref/model/ai/chatting/ChatHistoryRecordV1.java b/jablib/src/main/java/org/jabref/model/ai/chatting/ChatHistoryRecordV1.java new file mode 100644 index 000000000000..8ed40e3e407c --- /dev/null +++ b/jablib/src/main/java/org/jabref/model/ai/chatting/ChatHistoryRecordV1.java @@ -0,0 +1,53 @@ +package org.jabref.model.ai.chatting; + +import java.io.Serializable; + +import org.jabref.model.ai.chatting.messages.ErrorMessage; + +import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.data.message.ChatMessage; +import dev.langchain4j.data.message.UserMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public record ChatHistoryRecordV1(String className, String content) implements Serializable { + private static final Logger LOGGER = LoggerFactory.getLogger(ChatHistoryRecordV1.class); + + public static ChatHistoryRecordV1 fromLangchainMessage(ChatMessage chatMessage) { + String className = chatMessage.getClass().getName(); + String content = getContentFromLangchainMessage(chatMessage); + return new ChatHistoryRecordV1(className, content); + } + + private static String getContentFromLangchainMessage(ChatMessage chatMessage) { + String content; + + switch (chatMessage) { + case AiMessage aiMessage -> + content = aiMessage.text(); + case UserMessage userMessage -> + content = userMessage.singleText(); + case ErrorMessage errorMessageV1 -> + content = errorMessageV1.getText(); + default -> { + LOGGER.warn("ChatHistoryRecordV1 supports only AI, user. and error messages, but added message has other type: {}", chatMessage.type().name()); + return ""; + } + } + + return content; + } + + public ChatMessage toLangchainMessage() { + if (className.equals(AiMessage.class.getName())) { + return new AiMessage(content); + } else if (className.equals(UserMessage.class.getName())) { + return new UserMessage(content); + } else if (className.equals(ErrorMessage.class.getName())) { + return new ErrorMessage(content); + } else { + LOGGER.warn("ChatHistoryRecordV1 supports only AI and user messages, but retrieved message has other type: {}. Will treat as an AI message.", className); + return new AiMessage(content); + } + } +} diff --git a/jablib/src/main/java/org/jabref/model/ai/chatting/ChatHistoryRecordV2.java b/jablib/src/main/java/org/jabref/model/ai/chatting/ChatHistoryRecordV2.java new file mode 100644 index 000000000000..4fae071fe59e --- /dev/null +++ b/jablib/src/main/java/org/jabref/model/ai/chatting/ChatHistoryRecordV2.java @@ -0,0 +1,12 @@ +package org.jabref.model.ai.chatting; + +import java.io.Serializable; +import java.time.Instant; + +public record ChatHistoryRecordV2( + String id, + String messageTypeClassName, + String content, + Instant createdAt +) implements Serializable { +} diff --git a/jablib/src/main/java/org/jabref/model/ai/chatting/ChatType.java b/jablib/src/main/java/org/jabref/model/ai/chatting/ChatType.java new file mode 100644 index 000000000000..bddf5a17d2d0 --- /dev/null +++ b/jablib/src/main/java/org/jabref/model/ai/chatting/ChatType.java @@ -0,0 +1,6 @@ +package org.jabref.model.ai.chatting; + +public enum ChatType { + WITH_ENTRY, + WITH_GROUP +} diff --git a/jablib/src/main/java/org/jabref/model/ai/chatting/ErrorMessage.java b/jablib/src/main/java/org/jabref/model/ai/chatting/messages/ErrorMessage.java similarity index 95% rename from jablib/src/main/java/org/jabref/model/ai/chatting/ErrorMessage.java rename to jablib/src/main/java/org/jabref/model/ai/chatting/messages/ErrorMessage.java index cf13b2413838..2163627e5e4e 100644 --- a/jablib/src/main/java/org/jabref/model/ai/chatting/ErrorMessage.java +++ b/jablib/src/main/java/org/jabref/model/ai/chatting/messages/ErrorMessage.java @@ -1,4 +1,4 @@ -package org.jabref.model.ai.chatting; +package org.jabref.model.ai.chatting.messages; import dev.langchain4j.data.message.ChatMessage; import dev.langchain4j.data.message.ChatMessageType; diff --git a/jablib/src/main/java/org/jabref/model/ai/identifiers/BibEntryAiIdentifier.java b/jablib/src/main/java/org/jabref/model/ai/identifiers/BibEntryAiIdentifier.java index 38edc938e9f7..4daa6e599cfb 100644 --- a/jablib/src/main/java/org/jabref/model/ai/identifiers/BibEntryAiIdentifier.java +++ b/jablib/src/main/java/org/jabref/model/ai/identifiers/BibEntryAiIdentifier.java @@ -2,5 +2,11 @@ import java.nio.file.Path; +import org.jetbrains.annotations.NotNull; + public record BibEntryAiIdentifier(Path databasePath, String citationKey) { + @Override + public @NotNull String toString() { + return databasePath.toString() + "/" + citationKey; + } } diff --git a/jablib/src/main/java/org/jabref/model/ai/identifiers/GroupAiIdentifier.java b/jablib/src/main/java/org/jabref/model/ai/identifiers/GroupAiIdentifier.java index e7bd8eb49fb9..7b481b0158ba 100644 --- a/jablib/src/main/java/org/jabref/model/ai/identifiers/GroupAiIdentifier.java +++ b/jablib/src/main/java/org/jabref/model/ai/identifiers/GroupAiIdentifier.java @@ -2,5 +2,11 @@ import java.nio.file.Path; +import org.jetbrains.annotations.NotNull; + public record GroupAiIdentifier(Path databasePath, String groupName) { + @Override + public @NotNull String toString() { + return databasePath.toString() + "/" + groupName; + } } diff --git a/jablib/src/main/java/org/jabref/model/ai/chatting/AiProvider.java b/jablib/src/main/java/org/jabref/model/ai/llm/AiProvider.java similarity index 96% rename from jablib/src/main/java/org/jabref/model/ai/chatting/AiProvider.java rename to jablib/src/main/java/org/jabref/model/ai/llm/AiProvider.java index 01c76eb2b56d..dc2d0192e343 100644 --- a/jablib/src/main/java/org/jabref/model/ai/chatting/AiProvider.java +++ b/jablib/src/main/java/org/jabref/model/ai/llm/AiProvider.java @@ -1,4 +1,4 @@ -package org.jabref.model.ai.chatting; +package org.jabref.model.ai.llm; import java.io.Serializable; diff --git a/jablib/src/main/java/org/jabref/model/ai/summarization/BibEntrySummary.java b/jablib/src/main/java/org/jabref/model/ai/summarization/BibEntrySummary.java index 6aca736ca404..144e043a1007 100644 --- a/jablib/src/main/java/org/jabref/model/ai/summarization/BibEntrySummary.java +++ b/jablib/src/main/java/org/jabref/model/ai/summarization/BibEntrySummary.java @@ -3,7 +3,7 @@ import java.io.Serializable; import java.time.LocalDateTime; -import org.jabref.model.ai.chatting.AiProvider; +import org.jabref.model.ai.llm.AiProvider; public record BibEntrySummary( LocalDateTime timestamp, diff --git a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/EntryChatHistoryRepositoryTest.java b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/EntryChatHistoryRepositoryV1Test.java similarity index 88% rename from jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/EntryChatHistoryRepositoryTest.java rename to jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/EntryChatHistoryRepositoryV1Test.java index c614facfb263..7d33494faa42 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/EntryChatHistoryRepositoryTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/EntryChatHistoryRepositoryV1Test.java @@ -3,7 +3,7 @@ import java.nio.file.Path; import java.util.List; -import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepository; +import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepositoryV1; import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; import org.jabref.model.ai.identifiers.GroupAiIdentifier; @@ -17,14 +17,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -abstract class EntryChatHistoryRepositoryTest { +abstract class EntryChatHistoryRepositoryV1Test { @TempDir Path tempDir; - private EntryChatHistoryRepository storage; + private EntryChatHistoryRepositoryV1 storage; - abstract EntryChatHistoryRepository makeStorage(Path path); + abstract EntryChatHistoryRepositoryV1 makeStorage(Path path); - abstract void close(EntryChatHistoryRepository storage); + abstract void close(EntryChatHistoryRepositoryV1 storage); @BeforeEach void setUp() { diff --git a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryTest.java b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryV1Test.java similarity index 60% rename from jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryTest.java rename to jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryV1Test.java index 54631e17b78b..151be546128c 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryV1Test.java @@ -2,19 +2,19 @@ import java.nio.file.Path; -import org.jabref.logic.ai.chatting.repositories.MVStoreChatHistoryRepository; +import org.jabref.logic.ai.chatting.repositories.MVStoreChatHistoryRepositoryV1; import org.jabref.logic.util.NotificationService; import static org.mockito.Mockito.mock; -class MVStoreChatHistoryRepositoryTest extends EntryChatHistoryRepositoryTest { +class MVStoreChatHistoryRepositoryV1Test extends EntryChatHistoryRepositoryV1Test { @Override ChatHistoryRepository makeStorage(Path path) { - return new MVStoreChatHistoryRepository(mock(NotificationService.class), path); + return new MVStoreChatHistoryRepositoryV1(mock(NotificationService.class), path); } @Override void close(ChatHistoryRepository storage) { - ((MVStoreChatHistoryRepository) storage).close(); + ((MVStoreChatHistoryRepositoryV1) storage).close(); } } diff --git a/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesRepositoryTest.java b/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesRepositoryTest.java index 35ab11c4eace..3a8b7afb8b31 100644 --- a/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesRepositoryTest.java +++ b/jablib/src/test/java/org/jabref/logic/ai/summarization/SummariesRepositoryTest.java @@ -5,7 +5,7 @@ import java.util.Optional; import org.jabref.logic.ai.summarization.repositories.SummariesRepository; -import org.jabref.model.ai.chatting.AiProvider; +import org.jabref.model.ai.llm.AiProvider; import org.jabref.model.ai.summarization.BibEntrySummary; import org.junit.jupiter.api.AfterEach; From 0803345115fc32d06388304111910efca8852c7c Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Sat, 29 Nov 2025 16:58:05 +0100 Subject: [PATCH 036/243] refactor chat history + start working on UI --- .../ai/components/aichat/AiChatComponent.java | 3 +- .../aichat/AiChatGuardedComponent.java | 2 +- .../ai/components/aichat/AiChatWindow.java | 2 +- .../chathistory/ChatHistoryComponent.java | 2 +- .../org/jabref/gui/entryeditor/AiChatTab.java | 2 +- .../entryeditor/aisummary/AiSummaryView.java | 14 +++ .../aisummary/AiSummaryViewModel.java | 101 ++++++++++++++++++ .../jabref/gui/groups/GroupTreeViewModel.java | 4 +- .../entryeditor/aisummary/AiSummaryView.fxml | 11 ++ jablib/src/main/java/module-info.java | 3 + .../java/org/jabref/logic/ai/AiService.java | 31 +++--- .../CitationKeyChangeListener.java | 8 +- .../EntryChattingDatabaseListener.java | 8 +- .../GroupChattingDatabaseListener.java | 8 +- .../logic/ai/chatting/logic/AiChatLogic.java | 3 +- ...2.java => EntryChatHistoryRepository.java} | 2 +- .../EntryChatHistoryRepositoryV1.java | 14 --- ...2.java => GroupChatHistoryRepository.java} | 2 +- .../GroupChatHistoryRepositoryV1.java | 14 --- .../MVStoreChatHistoryRepositoryV1.java | 86 --------------- ...=> MVStoreEntryChatHistoryRepository.java} | 4 +- ...=> MVStoreGroupChatHistoryRepository.java} | 4 +- .../ai/chatting/{ => util}/ChatHistory.java | 2 +- .../ChatHistoryRecordUtils.java | 2 +- .../chatting/{ => util}/EntryChatHistory.java | 8 +- .../chatting/{ => util}/GroupChatHistory.java | 8 +- .../ai/current/CurrentChatLanguageModel.java | 1 - .../logic/ai/preferences/AiPreferences.java | 2 + .../logic/util/OptionalObjectProperty.java | 2 +- .../EntryChatHistoryRepositoryV1Test.java | 76 ------------- .../MVStoreChatHistoryRepositoryV1Test.java | 20 ---- 31 files changed, 182 insertions(+), 267 deletions(-) create mode 100644 jabgui/src/main/java/org/jabref/gui/entryeditor/aisummary/AiSummaryView.java create mode 100644 jabgui/src/main/java/org/jabref/gui/entryeditor/aisummary/AiSummaryViewModel.java create mode 100644 jabgui/src/main/resources/org/jabref/gui/entryeditor/aisummary/AiSummaryView.fxml rename jablib/src/main/java/org/jabref/logic/ai/chatting/{ => listeners}/CitationKeyChangeListener.java (89%) rename jablib/src/main/java/org/jabref/logic/ai/chatting/{ => listeners}/EntryChattingDatabaseListener.java (79%) rename jablib/src/main/java/org/jabref/logic/ai/chatting/{ => listeners}/GroupChattingDatabaseListener.java (93%) rename jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/{EntryChatHistoryRepositoryV2.java => EntryChatHistoryRepository.java} (92%) delete mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/EntryChatHistoryRepositoryV1.java rename jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/{GroupChatHistoryRepositoryV2.java => GroupChatHistoryRepository.java} (92%) delete mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepositoryV1.java delete mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepositoryV1.java rename jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/{MVStoreEntryChatHistoryRepositoryV2.java => MVStoreEntryChatHistoryRepository.java} (91%) rename jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/{MVStoreGroupChatHistoryRepositoryV2.java => MVStoreGroupChatHistoryRepository.java} (91%) rename jablib/src/main/java/org/jabref/logic/ai/chatting/{ => util}/ChatHistory.java (88%) rename jablib/src/main/java/org/jabref/logic/ai/chatting/{logic => util}/ChatHistoryRecordUtils.java (96%) rename jablib/src/main/java/org/jabref/logic/ai/chatting/{ => util}/EntryChatHistory.java (86%) rename jablib/src/main/java/org/jabref/logic/ai/chatting/{ => util}/GroupChatHistory.java (86%) delete mode 100644 jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/EntryChatHistoryRepositoryV1Test.java delete mode 100644 jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryV1Test.java diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java index b7216eb19c51..2da2c3a9adfb 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java @@ -31,9 +31,9 @@ import org.jabref.gui.icon.IconTheme; import org.jabref.gui.util.UiTaskExecutor; import org.jabref.logic.ai.AiService; -import org.jabref.logic.ai.chatting.ChatHistory; import org.jabref.logic.ai.chatting.logic.AiChatLogic; import org.jabref.logic.ai.chatting.tasks.GenerateAiResponseTask; +import org.jabref.logic.ai.chatting.util.ChatHistory; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; @@ -303,6 +303,7 @@ private void onSendMessage(String userPrompt) { .onSuccess(aiMessage -> { setLoading(false); chatPrompt.requestPromptFocus(); + uiChatHistory.updateMessages(chatHistory); }) .onFailure(e -> { LOGGER.error("Got an error while sending a message to AI", e); diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatGuardedComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatGuardedComponent.java index 90071052d665..b4b1dfb53767 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatGuardedComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatGuardedComponent.java @@ -9,7 +9,7 @@ import org.jabref.gui.entryeditor.AdaptVisibleTabs; import org.jabref.gui.frame.ExternalApplicationsPreferences; import org.jabref.logic.ai.AiService; -import org.jabref.logic.ai.chatting.ChatHistory; +import org.jabref.logic.ai.chatting.util.ChatHistory; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.database.BibDatabaseContext; diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatWindow.java b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatWindow.java index 8df32fc5781e..25a18bb5b4d8 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatWindow.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatWindow.java @@ -9,7 +9,7 @@ import org.jabref.gui.frame.ExternalApplicationsPreferences; import org.jabref.gui.util.BaseWindow; import org.jabref.logic.ai.AiService; -import org.jabref.logic.ai.chatting.ChatHistory; +import org.jabref.logic.ai.chatting.util.ChatHistory; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.TaskExecutor; diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/chathistory/ChatHistoryComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/chathistory/ChatHistoryComponent.java index f7ef59d8ae5e..8a09dde3c0f7 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/chathistory/ChatHistoryComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/chathistory/ChatHistoryComponent.java @@ -8,7 +8,7 @@ import org.jabref.gui.ai.components.aichat.chatmessage.ChatMessageComponent; import org.jabref.gui.util.UiTaskExecutor; -import org.jabref.logic.ai.chatting.ChatHistory; +import org.jabref.logic.ai.chatting.util.ChatHistory; import com.airhacks.afterburner.views.ViewLoader; diff --git a/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java b/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java index b3661ae0318c..e1dbb7db3edc 100644 --- a/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java +++ b/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java @@ -18,7 +18,7 @@ import org.jabref.gui.frame.ExternalApplicationsPreferences; import org.jabref.gui.preferences.GuiPreferences; import org.jabref.logic.ai.AiService; -import org.jabref.logic.ai.chatting.EntryChatHistory; +import org.jabref.logic.ai.chatting.util.EntryChatHistory; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.citationkeypattern.CitationKeyGenerator; import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; diff --git a/jabgui/src/main/java/org/jabref/gui/entryeditor/aisummary/AiSummaryView.java b/jabgui/src/main/java/org/jabref/gui/entryeditor/aisummary/AiSummaryView.java new file mode 100644 index 000000000000..5f933c015d24 --- /dev/null +++ b/jabgui/src/main/java/org/jabref/gui/entryeditor/aisummary/AiSummaryView.java @@ -0,0 +1,14 @@ +package org.jabref.gui.entryeditor.aisummary; + +import javafx.scene.layout.Pane; + +import com.airhacks.afterburner.views.ViewLoader; + +public class AiSummaryView extends Pane { + public AiSummaryView() { + ViewLoader + .view(this) + .root(this) + .load(); + } +} diff --git a/jabgui/src/main/java/org/jabref/gui/entryeditor/aisummary/AiSummaryViewModel.java b/jabgui/src/main/java/org/jabref/gui/entryeditor/aisummary/AiSummaryViewModel.java new file mode 100644 index 000000000000..6a97182708c7 --- /dev/null +++ b/jabgui/src/main/java/org/jabref/gui/entryeditor/aisummary/AiSummaryViewModel.java @@ -0,0 +1,101 @@ +package org.jabref.gui.entryeditor.aisummary; + +import java.time.LocalDateTime; +import java.util.Optional; + +import javafx.beans.property.BooleanProperty; +import javafx.beans.property.ObjectProperty; +import javafx.beans.property.SimpleBooleanProperty; +import javafx.beans.property.SimpleObjectProperty; +import javafx.beans.property.SimpleStringProperty; +import javafx.beans.property.StringProperty; + +import org.jabref.gui.AbstractViewModel; +import org.jabref.logic.ai.preferences.AiPreferences; +import org.jabref.logic.util.OptionalObjectProperty; +import org.jabref.model.ai.llm.AiProvider; + +public class AiSummaryViewModel extends AbstractViewModel { + public enum State { + PROCESSING, + DONE, + ERROR + } + + // Global properties. + private final BooleanProperty showAiPrivacyPolicyGuard = new SimpleBooleanProperty(true); + private final ObjectProperty state = new SimpleObjectProperty<>(State.PROCESSING); + + // Error state properties. + private final OptionalObjectProperty error = new OptionalObjectProperty<>(Optional.empty()); + + // Processing state properties. + private final ObjectProperty processingAiProvider = new SimpleObjectProperty<>(); + private final StringProperty processingLlmName = new SimpleStringProperty(""); + + // Done state properties. + private final StringProperty summary = new SimpleStringProperty(""); + private final BooleanProperty summaryRenderMarkdown = new SimpleBooleanProperty(false); + private final ObjectProperty summaryTimestamp = new SimpleObjectProperty<>(); + private final ObjectProperty summaryAiProvider = new SimpleObjectProperty<>(); + private final StringProperty summaryLlmName = new SimpleStringProperty(""); + + public AiSummaryViewModel( + AiPreferences aiPreferences + ) { + showAiPrivacyPolicyGuard.bind(aiPreferences.enableAiProperty()); + aiPreferences.aiProviderProperty().bind(processingAiProvider); + aiPreferences.addListenerToChatModels(() -> processingLlmName.set(aiPreferences.getSelectedChatModel())); + + startSummarization(); + } + + public void regenerate() { + clearSummary(); + startSummarization(); + } + + private void clearSummary() { + + } + + public BooleanProperty showAiPrivacyPolicyGuardProperty() { + return showAiPrivacyPolicyGuard; + } + + public ObjectProperty stateProperty() { + return state; + } + + public OptionalObjectProperty errorProperty() { + return error; + } + + public ObjectProperty processingAiProviderProperty() { + return processingAiProvider; + } + + public StringProperty processingLlmNameProperty() { + return processingLlmName; + } + + public StringProperty summaryProperty() { + return summary; + } + + public BooleanProperty summaryRenderMarkdownProperty() { + return summaryRenderMarkdown; + } + + public ObjectProperty summaryTimestampProperty() { + return summaryTimestamp; + } + + public ObjectProperty summaryAiProviderProperty() { + return summaryAiProvider; + } + + public StringProperty summaryLlmNameProperty() { + return summaryLlmName; + } +} diff --git a/jabgui/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java b/jabgui/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java index 554e965523d0..026132726379 100644 --- a/jabgui/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java +++ b/jabgui/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java @@ -28,8 +28,8 @@ import org.jabref.gui.preferences.GuiPreferences; import org.jabref.gui.util.CustomLocalDragboard; import org.jabref.logic.ai.AiService; -import org.jabref.logic.ai.chatting.ChatHistory; -import org.jabref.logic.ai.chatting.GroupChatHistory; +import org.jabref.logic.ai.chatting.util.ChatHistory; +import org.jabref.logic.ai.chatting.util.GroupChatHistory; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.ai.identifiers.GroupAiIdentifier; diff --git a/jabgui/src/main/resources/org/jabref/gui/entryeditor/aisummary/AiSummaryView.fxml b/jabgui/src/main/resources/org/jabref/gui/entryeditor/aisummary/AiSummaryView.fxml new file mode 100644 index 000000000000..0f973e7fd7d7 --- /dev/null +++ b/jabgui/src/main/resources/org/jabref/gui/entryeditor/aisummary/AiSummaryView.fxml @@ -0,0 +1,11 @@ + + + + + + diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index d3d8b78e4ec7..a5eb5e2b8b71 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -152,6 +152,9 @@ exports org.jabref.model.ai.tokenization; exports org.jabref.logic.ai.current; exports org.jabref.model.ai.llm; + exports org.jabref.model.ai.chatting.messages; + exports org.jabref.logic.ai.chatting.listeners; + exports org.jabref.logic.ai.chatting.util; // endregion requires java.base; diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiService.java b/jablib/src/main/java/org/jabref/logic/ai/AiService.java index c1448f3d8b14..048e00adabc5 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiService.java @@ -7,13 +7,12 @@ import javafx.beans.property.SimpleBooleanProperty; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.chatting.EntryChattingDatabaseListener; -import org.jabref.logic.ai.chatting.GroupChattingDatabaseListener; -import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepositoryV2; -import org.jabref.logic.ai.chatting.repositories.GroupChatHistoryRepositoryV2; -import org.jabref.logic.ai.chatting.repositories.MVStoreChatHistoryRepositoryV1; -import org.jabref.logic.ai.chatting.repositories.MVStoreEntryChatHistoryRepositoryV2; -import org.jabref.logic.ai.chatting.repositories.MVStoreGroupChatHistoryRepositoryV2; +import org.jabref.logic.ai.chatting.listeners.EntryChattingDatabaseListener; +import org.jabref.logic.ai.chatting.listeners.GroupChattingDatabaseListener; +import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepository; +import org.jabref.logic.ai.chatting.repositories.GroupChatHistoryRepository; +import org.jabref.logic.ai.chatting.repositories.MVStoreEntryChatHistoryRepository; +import org.jabref.logic.ai.chatting.repositories.MVStoreGroupChatHistoryRepository; import org.jabref.logic.ai.current.CurrentAiTemplates; import org.jabref.logic.ai.current.CurrentAnswerEngine; import org.jabref.logic.ai.current.CurrentChatLanguageModel; @@ -61,10 +60,8 @@ public class AiService implements AutoCloseable { new ThreadFactoryBuilder().setNameFormat("ai-retrieval-pool-%d").build() ); - private final MVStoreChatHistoryRepositoryV1 mvStoreChatHistoryStorage; - - private final MVStoreEntryChatHistoryRepositoryV2 mvStoreEntryChatHistoryStorage; - private final MVStoreGroupChatHistoryRepositoryV2 mvStoreGroupChatHistoryStorage; + private final MVStoreEntryChatHistoryRepository mvStoreEntryChatHistoryStorage; + private final MVStoreGroupChatHistoryRepository mvStoreGroupChatHistoryStorage; private final MVStoreEmbeddingStore mvStoreEmbeddingStore; private final MVStoreIngestedDocumentsRepository mvStoreFullyIngestedDocumentsTracker; @@ -90,11 +87,8 @@ public AiService( NotificationService notificationService, TaskExecutor taskExecutor ) { - - this.mvStoreChatHistoryStorage = new MVStoreChatHistoryRepositoryV1(notificationService, Directories.getAiFilesDirectory().resolve(CHAT_HISTORY_FILE_NAME)); - - this.mvStoreEntryChatHistoryStorage = new MVStoreEntryChatHistoryRepositoryV2(Directories.getAiFilesDirectory().resolve(ENTRY_CHAT_HISTORY_FILE_NAME), notificationService); - this.mvStoreGroupChatHistoryStorage = new MVStoreGroupChatHistoryRepositoryV2(Directories.getAiFilesDirectory().resolve(GROUP_CHAT_HISTORY_FILE_NAME), notificationService); + this.mvStoreEntryChatHistoryStorage = new MVStoreEntryChatHistoryRepository(Directories.getAiFilesDirectory().resolve(ENTRY_CHAT_HISTORY_FILE_NAME), notificationService); + this.mvStoreGroupChatHistoryStorage = new MVStoreGroupChatHistoryRepository(Directories.getAiFilesDirectory().resolve(GROUP_CHAT_HISTORY_FILE_NAME), notificationService); this.mvStoreEmbeddingStore = new MVStoreEmbeddingStore(Directories.getAiFilesDirectory().resolve(EMBEDDINGS_FILE_NAME), notificationService); this.mvStoreFullyIngestedDocumentsTracker = new MVStoreIngestedDocumentsRepository(notificationService, Directories.getAiFilesDirectory().resolve(FULLY_INGESTED_FILE_NAME)); @@ -173,11 +167,11 @@ public CurrentAnswerEngine getAnswerEngine() { return currentAnswerEngine; } - public EntryChatHistoryRepositoryV2 getEntryChatHistoryRepository() { + public EntryChatHistoryRepository getEntryChatHistoryRepository() { return mvStoreEntryChatHistoryStorage; } - public GroupChatHistoryRepositoryV2 getGroupChatHistoryRepository() { + public GroupChatHistoryRepository getGroupChatHistoryRepository() { return mvStoreGroupChatHistoryStorage; } @@ -197,7 +191,6 @@ public void close() throws Exception { currentChatLanguageModel.close(); currentEmbeddingModel.close(); - mvStoreChatHistoryStorage.close(); mvStoreFullyIngestedDocumentsTracker.close(); mvStoreEmbeddingStore.close(); mvStoreSummariesStorage.close(); diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/CitationKeyChangeListener.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/listeners/CitationKeyChangeListener.java similarity index 89% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/CitationKeyChangeListener.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/listeners/CitationKeyChangeListener.java index 245161a474a3..40746b1759fe 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/CitationKeyChangeListener.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/listeners/CitationKeyChangeListener.java @@ -1,10 +1,10 @@ -package org.jabref.logic.ai.chatting; +package org.jabref.logic.ai.chatting.listeners; import java.nio.file.Path; import java.util.List; import java.util.Optional; -import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepositoryV2; +import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepository; import org.jabref.model.ai.chatting.ChatHistoryRecordV2; import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; import org.jabref.model.database.BibDatabaseContext; @@ -19,10 +19,10 @@ public class CitationKeyChangeListener { private static final Logger LOGGER = LoggerFactory.getLogger(CitationKeyChangeListener.class); - private final EntryChatHistoryRepositoryV2 entryChatHistoryRepository; + private final EntryChatHistoryRepository entryChatHistoryRepository; private final BibDatabaseContext bibDatabaseContext; - public CitationKeyChangeListener(EntryChatHistoryRepositoryV2 entryChatHistoryRepository, BibDatabaseContext bibDatabaseContext) { + public CitationKeyChangeListener(EntryChatHistoryRepository entryChatHistoryRepository, BibDatabaseContext bibDatabaseContext) { this.entryChatHistoryRepository = entryChatHistoryRepository; this.bibDatabaseContext = bibDatabaseContext; } diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChattingDatabaseListener.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/listeners/EntryChattingDatabaseListener.java similarity index 79% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChattingDatabaseListener.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/listeners/EntryChattingDatabaseListener.java index 628a8168a5af..73c686c933d7 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChattingDatabaseListener.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/listeners/EntryChattingDatabaseListener.java @@ -1,12 +1,12 @@ -package org.jabref.logic.ai.chatting; +package org.jabref.logic.ai.chatting.listeners; -import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepositoryV2; +import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepository; import org.jabref.model.database.BibDatabaseContext; public class EntryChattingDatabaseListener { - private final EntryChatHistoryRepositoryV2 entryChatHistoryRepository; + private final EntryChatHistoryRepository entryChatHistoryRepository; - public EntryChattingDatabaseListener(EntryChatHistoryRepositoryV2 entryChatHistoryRepository) { + public EntryChattingDatabaseListener(EntryChatHistoryRepository entryChatHistoryRepository) { this.entryChatHistoryRepository = entryChatHistoryRepository; } diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChattingDatabaseListener.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/listeners/GroupChattingDatabaseListener.java similarity index 93% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChattingDatabaseListener.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/listeners/GroupChattingDatabaseListener.java index 5da2b7964f38..10bf37b3038c 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChattingDatabaseListener.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/listeners/GroupChattingDatabaseListener.java @@ -1,8 +1,8 @@ -package org.jabref.logic.ai.chatting; +package org.jabref.logic.ai.chatting.listeners; import java.util.List; -import org.jabref.logic.ai.chatting.repositories.GroupChatHistoryRepositoryV2; +import org.jabref.logic.ai.chatting.repositories.GroupChatHistoryRepository; import org.jabref.model.ai.chatting.ChatHistoryRecordV2; import org.jabref.model.ai.identifiers.GroupAiIdentifier; import org.jabref.model.database.BibDatabaseContext; @@ -14,9 +14,9 @@ public class GroupChattingDatabaseListener { private static final Logger LOGGER = LoggerFactory.getLogger(GroupChattingDatabaseListener.class); - private final GroupChatHistoryRepositoryV2 groupChatHistoryRepository; + private final GroupChatHistoryRepository groupChatHistoryRepository; - public GroupChattingDatabaseListener(GroupChatHistoryRepositoryV2 groupChatHistoryRepository) { + public GroupChattingDatabaseListener(GroupChatHistoryRepository groupChatHistoryRepository) { this.groupChatHistoryRepository = groupChatHistoryRepository; } diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java index d66f91df5960..a98f556170ea 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/AiChatLogic.java @@ -10,9 +10,10 @@ import javafx.beans.property.StringProperty; import javafx.collections.ObservableList; -import org.jabref.logic.ai.chatting.ChatHistory; import org.jabref.logic.ai.chatting.templates.ChattingSystemMessageTemplate; import org.jabref.logic.ai.chatting.templates.ChattingUserMessageTemplate; +import org.jabref.logic.ai.chatting.util.ChatHistory; +import org.jabref.logic.ai.chatting.util.ChatHistoryRecordUtils; import org.jabref.logic.ai.pipeline.logic.rag.AnswerEngine; import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.util.LongTaskInfo; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/EntryChatHistoryRepositoryV2.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/EntryChatHistoryRepository.java similarity index 92% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/EntryChatHistoryRepositoryV2.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/EntryChatHistoryRepository.java index 87f8dbf3baaf..4a863fe19fb2 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/EntryChatHistoryRepositoryV2.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/EntryChatHistoryRepository.java @@ -5,7 +5,7 @@ import org.jabref.model.ai.chatting.ChatHistoryRecordV2; import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; -public interface EntryChatHistoryRepositoryV2 { +public interface EntryChatHistoryRepository { void addMessage(BibEntryAiIdentifier identifier, ChatHistoryRecordV2 chatHistoryRecord); void deleteMessage(BibEntryAiIdentifier identifier, String id); diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/EntryChatHistoryRepositoryV1.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/EntryChatHistoryRepositoryV1.java deleted file mode 100644 index 525d2157f5af..000000000000 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/EntryChatHistoryRepositoryV1.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.jabref.logic.ai.chatting.repositories; - -import java.util.List; - -import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; - -import dev.langchain4j.data.message.ChatMessage; - -@Deprecated -public interface EntryChatHistoryRepositoryV1 extends AutoCloseable { - List loadMessagesForEntry(BibEntryAiIdentifier identifier); - - void storeMessagesForEntry(BibEntryAiIdentifier identifier, List messages); -} diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepositoryV2.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepository.java similarity index 92% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepositoryV2.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepository.java index 7ac64eaac4b1..734a9677c20d 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepositoryV2.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepository.java @@ -5,7 +5,7 @@ import org.jabref.model.ai.chatting.ChatHistoryRecordV2; import org.jabref.model.ai.identifiers.GroupAiIdentifier; -public interface GroupChatHistoryRepositoryV2 { +public interface GroupChatHistoryRepository { void addMessage(GroupAiIdentifier identifier, ChatHistoryRecordV2 chatHistoryRecord); void deleteMessage(GroupAiIdentifier identifier, String id); diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepositoryV1.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepositoryV1.java deleted file mode 100644 index 821f8b76c714..000000000000 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/GroupChatHistoryRepositoryV1.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.jabref.logic.ai.chatting.repositories; - -import java.util.List; - -import org.jabref.model.ai.identifiers.GroupAiIdentifier; - -import dev.langchain4j.data.message.ChatMessage; - -@Deprecated -public interface GroupChatHistoryRepositoryV1 extends AutoCloseable { - List loadMessagesForGroup(GroupAiIdentifier identifier); - - void storeMessagesForGroup(GroupAiIdentifier identifier, List messages); -} diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepositoryV1.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepositoryV1.java deleted file mode 100644 index cc53ecbb7b8a..000000000000 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreChatHistoryRepositoryV1.java +++ /dev/null @@ -1,86 +0,0 @@ -package org.jabref.logic.ai.chatting.repositories; - -import java.nio.file.Path; -import java.util.Comparator; -import java.util.List; -import java.util.Map; - -import org.jabref.logic.ai.util.MVStoreBase; -import org.jabref.logic.l10n.Localization; -import org.jabref.logic.util.NotificationService; -import org.jabref.model.ai.chatting.ChatHistoryRecordV1; -import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; -import org.jabref.model.ai.identifiers.GroupAiIdentifier; - -import dev.langchain4j.data.message.ChatMessage; -import kotlin.ranges.IntRange; - -@Deprecated -public class MVStoreChatHistoryRepositoryV1 extends MVStoreBase implements EntryChatHistoryRepositoryV1, GroupChatHistoryRepositoryV1 { - private static final String ENTRY_CHAT_HISTORY_PREFIX = "entry"; - private static final String GROUP_CHAT_HISTORY_PREFIX = "group"; - - public MVStoreChatHistoryRepositoryV1(NotificationService dialogService, Path path) { - super(path, dialogService); - } - - @Override - public List loadMessagesForEntry(BibEntryAiIdentifier identifier) { - return loadMessagesFromMap(getMapForEntry(identifier)); - } - - @Override - public void storeMessagesForEntry(BibEntryAiIdentifier identifier, List messages) { - storeMessagesForMap(getMapForEntry(identifier), messages); - } - - @Override - public List loadMessagesForGroup(GroupAiIdentifier identifier) { - return loadMessagesFromMap(getMapForGroup(identifier)); - } - - @Override - public void storeMessagesForGroup(GroupAiIdentifier identifier, List messages) { - storeMessagesForMap(getMapForGroup(identifier), messages); - } - - private List loadMessagesFromMap(Map map) { - return map - .entrySet() - // We need to check all keys, because upon deletion, there can be "holes" in the integer. - .stream() - .sorted(Comparator.comparingInt(Map.Entry::getKey)) - .map(entry -> entry.getValue().toLangchainMessage()) - .toList(); - } - - private void storeMessagesForMap(Map map, List messages) { - map.clear(); - - new IntRange(0, messages.size() - 1).forEach(i -> - map.put(i, ChatHistoryRecordV1.fromLangchainMessage(messages.get(i))) - ); - } - - private Map getMapForEntry(BibEntryAiIdentifier identifier) { - return getMap(identifier.databasePath(), ENTRY_CHAT_HISTORY_PREFIX, identifier.citationKey()); - } - - private Map getMapForGroup(GroupAiIdentifier identifier) { - return getMap(identifier.databasePath(), GROUP_CHAT_HISTORY_PREFIX, identifier.groupName()); - } - - private Map getMap(Path bibDatabasePath, String type, String name) { - return mvStore.openMap(bibDatabasePath + "-" + type + "-" + name); - } - - @Override - protected String errorMessageForOpening() { - return "An error occurred while opening chat history storage. Chat history of entries and groups will not be stored in the next session."; - } - - @Override - protected String errorMessageForOpeningLocalized() { - return Localization.lang("An error occurred while opening chat history storage. Chat history of entries and groups will not be stored in the next session."); - } -} diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreEntryChatHistoryRepositoryV2.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreEntryChatHistoryRepository.java similarity index 91% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreEntryChatHistoryRepositoryV2.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreEntryChatHistoryRepository.java index 90c5815b11a7..11f19d53032d 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreEntryChatHistoryRepositoryV2.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreEntryChatHistoryRepository.java @@ -12,8 +12,8 @@ import org.jspecify.annotations.NonNull; -public class MVStoreEntryChatHistoryRepositoryV2 extends MVStoreBase implements EntryChatHistoryRepositoryV2 { - public MVStoreEntryChatHistoryRepositoryV2(@NonNull Path path, NotificationService dialogService) { +public class MVStoreEntryChatHistoryRepository extends MVStoreBase implements EntryChatHistoryRepository { + public MVStoreEntryChatHistoryRepository(@NonNull Path path, NotificationService dialogService) { super(path, dialogService); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreGroupChatHistoryRepositoryV2.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreGroupChatHistoryRepository.java similarity index 91% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreGroupChatHistoryRepositoryV2.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreGroupChatHistoryRepository.java index b8304f97e35e..9379b8de7d09 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreGroupChatHistoryRepositoryV2.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/repositories/MVStoreGroupChatHistoryRepository.java @@ -12,8 +12,8 @@ import org.jspecify.annotations.NonNull; -public class MVStoreGroupChatHistoryRepositoryV2 extends MVStoreBase implements GroupChatHistoryRepositoryV2 { - public MVStoreGroupChatHistoryRepositoryV2(@NonNull Path path, NotificationService dialogService) { +public class MVStoreGroupChatHistoryRepository extends MVStoreBase implements GroupChatHistoryRepository { + public MVStoreGroupChatHistoryRepository(@NonNull Path path, NotificationService dialogService) { super(path, dialogService); } diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistory.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/util/ChatHistory.java similarity index 88% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistory.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/util/ChatHistory.java index 8256d6b64046..480295c4d50e 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/ChatHistory.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/util/ChatHistory.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.chatting; +package org.jabref.logic.ai.chatting.util; import java.util.List; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/ChatHistoryRecordUtils.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/util/ChatHistoryRecordUtils.java similarity index 96% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/logic/ChatHistoryRecordUtils.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/util/ChatHistoryRecordUtils.java index 0b2ab305ae49..eee417f72b38 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/logic/ChatHistoryRecordUtils.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/util/ChatHistoryRecordUtils.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.chatting.logic; +package org.jabref.logic.ai.chatting.util; import org.jabref.model.ai.chatting.ChatHistoryRecordV2; import org.jabref.model.ai.chatting.messages.ErrorMessage; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChatHistory.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/util/EntryChatHistory.java similarity index 86% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChatHistory.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/util/EntryChatHistory.java index 7e1de7a9d90b..fb1a66b0bdbe 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/EntryChatHistory.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/util/EntryChatHistory.java @@ -1,17 +1,17 @@ -package org.jabref.logic.ai.chatting; +package org.jabref.logic.ai.chatting.util; import java.util.List; -import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepositoryV2; +import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepository; import org.jabref.model.ai.chatting.ChatHistoryRecordV2; import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; public class EntryChatHistory implements ChatHistory { - private final EntryChatHistoryRepositoryV2 entryChatHistoryRepository; + private final EntryChatHistoryRepository entryChatHistoryRepository; private final BibEntryAiIdentifier identifier; public EntryChatHistory( - EntryChatHistoryRepositoryV2 entryChatHistoryRepository, + EntryChatHistoryRepository entryChatHistoryRepository, BibEntryAiIdentifier identifier ) { this.entryChatHistoryRepository = entryChatHistoryRepository; diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChatHistory.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/util/GroupChatHistory.java similarity index 86% rename from jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChatHistory.java rename to jablib/src/main/java/org/jabref/logic/ai/chatting/util/GroupChatHistory.java index 836d85053714..0965bb308629 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/GroupChatHistory.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/util/GroupChatHistory.java @@ -1,17 +1,17 @@ -package org.jabref.logic.ai.chatting; +package org.jabref.logic.ai.chatting.util; import java.util.List; -import org.jabref.logic.ai.chatting.repositories.GroupChatHistoryRepositoryV2; +import org.jabref.logic.ai.chatting.repositories.GroupChatHistoryRepository; import org.jabref.model.ai.chatting.ChatHistoryRecordV2; import org.jabref.model.ai.identifiers.GroupAiIdentifier; public class GroupChatHistory implements ChatHistory { - private final GroupChatHistoryRepositoryV2 groupChatHistoryRepository; + private final GroupChatHistoryRepository groupChatHistoryRepository; private final GroupAiIdentifier identifier; public GroupChatHistory( - GroupChatHistoryRepositoryV2 groupChatHistoryRepository, + GroupChatHistoryRepository groupChatHistoryRepository, GroupAiIdentifier identifier ) { this.groupChatHistoryRepository = groupChatHistoryRepository; diff --git a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentChatLanguageModel.java b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentChatLanguageModel.java index 38b5897015fc..41e6d1cd36db 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/current/CurrentChatLanguageModel.java +++ b/jablib/src/main/java/org/jabref/logic/ai/current/CurrentChatLanguageModel.java @@ -7,7 +7,6 @@ import java.util.concurrent.Executors; import org.jabref.logic.ai.chatting.logic.AiChatLogic; -import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepositoryV1; import org.jabref.logic.ai.customimplementations.llms.ChatModel; import org.jabref.logic.ai.customimplementations.llms.Gpt4AllModel; import org.jabref.logic.ai.customimplementations.llms.JvmOpenAiChatLanguageModel; diff --git a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java index 538604e87737..9a59be84b3b1 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/ai/preferences/AiPreferences.java @@ -42,6 +42,8 @@ public class AiPreferences { private final ObjectProperty aiProvider; + // TODO: Add chat model property. + private final StringProperty openAiChatModel; private final StringProperty mistralAiChatModel; private final StringProperty geminiChatModel; diff --git a/jablib/src/main/java/org/jabref/logic/util/OptionalObjectProperty.java b/jablib/src/main/java/org/jabref/logic/util/OptionalObjectProperty.java index 7d447808e704..386fd16ff152 100644 --- a/jablib/src/main/java/org/jabref/logic/util/OptionalObjectProperty.java +++ b/jablib/src/main/java/org/jabref/logic/util/OptionalObjectProperty.java @@ -13,7 +13,7 @@ */ public class OptionalObjectProperty extends SimpleObjectProperty> { - private OptionalObjectProperty(Optional initialValue) { + public OptionalObjectProperty(Optional initialValue) { super(initialValue); } diff --git a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/EntryChatHistoryRepositoryV1Test.java b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/EntryChatHistoryRepositoryV1Test.java deleted file mode 100644 index 7d33494faa42..000000000000 --- a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/EntryChatHistoryRepositoryV1Test.java +++ /dev/null @@ -1,76 +0,0 @@ -package org.jabref.logic.ai.chatting.chathistory; - -import java.nio.file.Path; -import java.util.List; - -import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepositoryV1; -import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; -import org.jabref.model.ai.identifiers.GroupAiIdentifier; - -import dev.langchain4j.data.message.AiMessage; -import dev.langchain4j.data.message.ChatMessage; -import dev.langchain4j.data.message.UserMessage; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -abstract class EntryChatHistoryRepositoryV1Test { - @TempDir Path tempDir; - - private EntryChatHistoryRepositoryV1 storage; - - abstract EntryChatHistoryRepositoryV1 makeStorage(Path path); - - abstract void close(EntryChatHistoryRepositoryV1 storage); - - @BeforeEach - void setUp() { - storage = makeStorage(tempDir.resolve("test.bib")); - } - - private void reopen() { - close(storage); - setUp(); - } - - @AfterEach - void tearDown() { - close(storage); - } - - @Test - void entryChatHistory() { - List messages = List.of( - new UserMessage("hi!"), - new AiMessage("hello!") - ); - - BibEntryAiIdentifier identifier = new BibEntryAiIdentifier( - tempDir.resolve("test.bib"), - "citationKey" - ); - storage.storeMessagesForEntry(identifier, messages); - - reopen(); - - assertEquals(messages, storage.loadMessagesForEntry(identifier)); - } - - @Test - void groupChatHistory() { - List messages = List.of( - new UserMessage("hi!"), - new AiMessage("hello!") - ); - - GroupAiIdentifier identifier = new GroupAiIdentifier(tempDir.resolve("test.bib"), "group"); - storage.storeMessagesForGroup(identifier, messages); - - reopen(); - - assertEquals(messages, storage.loadMessagesForGroup(identifier)); - } -} diff --git a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryV1Test.java b/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryV1Test.java deleted file mode 100644 index 151be546128c..000000000000 --- a/jablib/src/test/java/org/jabref/logic/ai/chatting/chathistory/MVStoreChatHistoryRepositoryV1Test.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.jabref.logic.ai.chatting.chathistory; - -import java.nio.file.Path; - -import org.jabref.logic.ai.chatting.repositories.MVStoreChatHistoryRepositoryV1; -import org.jabref.logic.util.NotificationService; - -import static org.mockito.Mockito.mock; - -class MVStoreChatHistoryRepositoryV1Test extends EntryChatHistoryRepositoryV1Test { - @Override - ChatHistoryRepository makeStorage(Path path) { - return new MVStoreChatHistoryRepositoryV1(mock(NotificationService.class), path); - } - - @Override - void close(ChatHistoryRepository storage) { - ((MVStoreChatHistoryRepositoryV1) storage).close(); - } -} From 79a6df38eb7fd2e3f75d1f7284b8874db5400bbc Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Sat, 29 Nov 2025 17:00:01 +0100 Subject: [PATCH 037/243] initroduce 1 class. idk why --- .../chatting/ChattingDatabaseListeners.java | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/ChattingDatabaseListeners.java diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/ChattingDatabaseListeners.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/ChattingDatabaseListeners.java new file mode 100644 index 000000000000..4e6f9e9b2841 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/ChattingDatabaseListeners.java @@ -0,0 +1,29 @@ +package org.jabref.logic.ai.chatting; + +import org.jabref.logic.ai.chatting.listeners.EntryChattingDatabaseListener; +import org.jabref.logic.ai.chatting.listeners.GroupChattingDatabaseListener; +import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepository; +import org.jabref.logic.ai.chatting.repositories.GroupChatHistoryRepository; +import org.jabref.model.database.BibDatabaseContext; + +public class ChattingDatabaseListeners { + private final EntryChattingDatabaseListener entryChattingDatabaseListener; + private final GroupChattingDatabaseListener groupChattingDatabaseListener; + + public ChattingDatabaseListeners( + EntryChatHistoryRepository entryChatHistoryRepository, + GroupChatHistoryRepository groupChatHistoryRepository + ) { + this.entryChattingDatabaseListener = new EntryChattingDatabaseListener( + entryChatHistoryRepository + ); + this.groupChattingDatabaseListener = new GroupChattingDatabaseListener( + groupChatHistoryRepository + ); + } + + public void setupDatabase(BibDatabaseContext databaseContext) { + entryChattingDatabaseListener.setupDatabase(databaseContext); + groupChattingDatabaseListener.setupDatabase(databaseContext); + } +} From 5c7d11a31614597ceffdbef7f35b51f7048b3987 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Sat, 29 Nov 2025 17:08:28 +0100 Subject: [PATCH 038/243] Move out database listeners --- jablib/src/main/java/module-info.java | 1 + .../jabref/logic/ai/AiDatabaseListeners.java | 31 ++++++++++ .../java/org/jabref/logic/ai/AiService.java | 21 +++---- .../listeners/CitationKeyChangeListener.java | 59 ------------------- .../EntryChattingDatabaseListener.java | 55 +++++++++++++++++ .../ai/summarization/SummariesService.java | 41 ------------- .../SummarizationDatabaseListeners.java | 26 ++++++++ .../GenerateSummaryDatabaseListener.java | 56 ++++++++++++++++++ 8 files changed, 178 insertions(+), 112 deletions(-) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/AiDatabaseListeners.java delete mode 100644 jablib/src/main/java/org/jabref/logic/ai/chatting/listeners/CitationKeyChangeListener.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationDatabaseListeners.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/listeners/GenerateSummaryDatabaseListener.java diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index a5eb5e2b8b71..8c125d22bec2 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -303,5 +303,6 @@ requires org.libreoffice.uno; requires transitive org.jspecify; requires org.jetbrains.annotations; + requires org.jabref.jablib; // endregion } diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiDatabaseListeners.java b/jablib/src/main/java/org/jabref/logic/ai/AiDatabaseListeners.java new file mode 100644 index 000000000000..3fa7448495ae --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/AiDatabaseListeners.java @@ -0,0 +1,31 @@ +package org.jabref.logic.ai; + +import org.jabref.logic.ai.chatting.ChattingDatabaseListeners; +import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepository; +import org.jabref.logic.ai.chatting.repositories.GroupChatHistoryRepository; +import org.jabref.logic.ai.preferences.AiPreferences; +import org.jabref.logic.ai.summarization.SummariesService; +import org.jabref.logic.ai.summarization.SummarizationDatabaseListeners; +import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.Summarizator; +import org.jabref.model.database.BibDatabaseContext; + +public class AiDatabaseListeners { + private final ChattingDatabaseListeners chattingDatabaseListeners; + private final SummarizationDatabaseListeners summarizationDatabaseListeners; + + public AiDatabaseListeners( + EntryChatHistoryRepository entryChatHistoryRepository, + GroupChatHistoryRepository groupChatHistoryRepository, + AiPreferences aiPreferences, + SummariesService summariesService, + Summarizator summarizator + ) { + this.chattingDatabaseListeners = new ChattingDatabaseListeners(entryChatHistoryRepository, groupChatHistoryRepository); + this.summarizationDatabaseListeners = new SummarizationDatabaseListeners(aiPreferences, summariesService, summarizator); + } + + public void setupDatabase(BibDatabaseContext databaseContext) { + chattingDatabaseListeners.setupDatabase(databaseContext); + summarizationDatabaseListeners.setupDatabase(databaseContext); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/AiService.java b/jablib/src/main/java/org/jabref/logic/ai/AiService.java index 048e00adabc5..b6f04c2ffd07 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/AiService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/AiService.java @@ -7,8 +7,6 @@ import javafx.beans.property.SimpleBooleanProperty; import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.chatting.listeners.EntryChattingDatabaseListener; -import org.jabref.logic.ai.chatting.listeners.GroupChattingDatabaseListener; import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepository; import org.jabref.logic.ai.chatting.repositories.GroupChatHistoryRepository; import org.jabref.logic.ai.chatting.repositories.MVStoreEntryChatHistoryRepository; @@ -78,8 +76,7 @@ public class AiService implements AutoCloseable { private final IngestionService ingestionService; private final SummariesService summariesService; - private final EntryChattingDatabaseListener entryChattingDatabaseListener; - private final GroupChattingDatabaseListener groupChattingDatabaseListener; + private final AiDatabaseListeners aiDatabaseListeners; public AiService( AiPreferences aiPreferences, @@ -114,17 +111,20 @@ public AiService( ); this.summariesService = new SummariesService( - aiPreferences, filePreferences, taskExecutor, currentChatLanguageModel, - currentSummarizator, mvStoreSummariesStorage, shutdownSignal ); - this.entryChattingDatabaseListener = new EntryChattingDatabaseListener(mvStoreEntryChatHistoryStorage); - this.groupChattingDatabaseListener = new GroupChattingDatabaseListener(mvStoreGroupChatHistoryStorage); + this.aiDatabaseListeners = new AiDatabaseListeners( + mvStoreEntryChatHistoryStorage, + mvStoreGroupChatHistoryStorage, + aiPreferences, + summariesService, + currentSummarizator + ); } public CurrentDocumentSplitter getDocumentSplitter() { @@ -176,11 +176,8 @@ public GroupChatHistoryRepository getGroupChatHistoryRepository() { } public void setupDatabase(BibDatabaseContext context) { - entryChattingDatabaseListener.setupDatabase(context); - groupChattingDatabaseListener.setupDatabase(context); - + aiDatabaseListeners.setupDatabase(context); ingestionService.setupDatabase(context); - summariesService.setupDatabase(context); } @Override diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/listeners/CitationKeyChangeListener.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/listeners/CitationKeyChangeListener.java deleted file mode 100644 index 40746b1759fe..000000000000 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/listeners/CitationKeyChangeListener.java +++ /dev/null @@ -1,59 +0,0 @@ -package org.jabref.logic.ai.chatting.listeners; - -import java.nio.file.Path; -import java.util.List; -import java.util.Optional; - -import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepository; -import org.jabref.model.ai.chatting.ChatHistoryRecordV2; -import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; -import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.event.FieldChangedEvent; -import org.jabref.model.entry.field.InternalField; - -import com.google.common.eventbus.Subscribe; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class CitationKeyChangeListener { - private static final Logger LOGGER = LoggerFactory.getLogger(CitationKeyChangeListener.class); - - private final EntryChatHistoryRepository entryChatHistoryRepository; - private final BibDatabaseContext bibDatabaseContext; - - public CitationKeyChangeListener(EntryChatHistoryRepository entryChatHistoryRepository, BibDatabaseContext bibDatabaseContext) { - this.entryChatHistoryRepository = entryChatHistoryRepository; - this.bibDatabaseContext = bibDatabaseContext; - } - - @Subscribe - void listen(FieldChangedEvent e) { - if (e.getField() != InternalField.KEY_FIELD) { - return; - } - - transferHistory(bibDatabaseContext, e.getBibEntry(), e.getOldValue(), e.getNewValue()); - } - - private void transferHistory(BibDatabaseContext bibDatabaseContext, BibEntry entry, String oldCitationKey, String newCitationKey) { - // TODO: This method does not check if the citation key is valid. - - Optional databasePath = bibDatabaseContext.getDatabasePath(); - - if (databasePath.isEmpty()) { - LOGGER.warn("Could not transfer chat history of entry {} (old key: {}): database path is empty.", newCitationKey, oldCitationKey); - return; - } - - BibEntryAiIdentifier oldIdentifier = new BibEntryAiIdentifier(databasePath.get(), oldCitationKey); - BibEntryAiIdentifier newIdentifier = new BibEntryAiIdentifier(databasePath.get(), newCitationKey); - - List chatHistory = entryChatHistoryRepository.getAllMessages(oldIdentifier); - - entryChatHistoryRepository.clear(oldIdentifier); - entryChatHistoryRepository.clear(newIdentifier); - - chatHistory.forEach(record -> entryChatHistoryRepository.addMessage(newIdentifier, record)); - } -} diff --git a/jablib/src/main/java/org/jabref/logic/ai/chatting/listeners/EntryChattingDatabaseListener.java b/jablib/src/main/java/org/jabref/logic/ai/chatting/listeners/EntryChattingDatabaseListener.java index 73c686c933d7..ba6115175db8 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/chatting/listeners/EntryChattingDatabaseListener.java +++ b/jablib/src/main/java/org/jabref/logic/ai/chatting/listeners/EntryChattingDatabaseListener.java @@ -1,7 +1,20 @@ package org.jabref.logic.ai.chatting.listeners; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; + import org.jabref.logic.ai.chatting.repositories.EntryChatHistoryRepository; +import org.jabref.model.ai.chatting.ChatHistoryRecordV2; +import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.event.FieldChangedEvent; +import org.jabref.model.entry.field.InternalField; + +import com.google.common.eventbus.Subscribe; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class EntryChattingDatabaseListener { private final EntryChatHistoryRepository entryChatHistoryRepository; @@ -18,4 +31,46 @@ public void setupDatabase(BibDatabaseContext databaseContext) { entry.registerListener(new CitationKeyChangeListener(entryChatHistoryRepository, databaseContext)) ); } + + static class CitationKeyChangeListener { + private static final Logger LOGGER = LoggerFactory.getLogger(CitationKeyChangeListener.class); + + private final EntryChatHistoryRepository entryChatHistoryRepository; + private final BibDatabaseContext bibDatabaseContext; + + public CitationKeyChangeListener(EntryChatHistoryRepository entryChatHistoryRepository, BibDatabaseContext bibDatabaseContext) { + this.entryChatHistoryRepository = entryChatHistoryRepository; + this.bibDatabaseContext = bibDatabaseContext; + } + + @Subscribe + void listen(FieldChangedEvent e) { + if (e.getField() != InternalField.KEY_FIELD) { + return; + } + + transferHistory(bibDatabaseContext, e.getBibEntry(), e.getOldValue(), e.getNewValue()); + } + + private void transferHistory(BibDatabaseContext bibDatabaseContext, BibEntry entry, String oldCitationKey, String newCitationKey) { + // TODO: This method does not check if the citation key is valid. + + Optional databasePath = bibDatabaseContext.getDatabasePath(); + + if (databasePath.isEmpty()) { + LOGGER.warn("Could not transfer chat history of entry {} (old key: {}): database path is empty.", newCitationKey, oldCitationKey); + return; + } + + BibEntryAiIdentifier oldIdentifier = new BibEntryAiIdentifier(databasePath.get(), oldCitationKey); + BibEntryAiIdentifier newIdentifier = new BibEntryAiIdentifier(databasePath.get(), newCitationKey); + + List chatHistory = entryChatHistoryRepository.getAllMessages(oldIdentifier); + + entryChatHistoryRepository.clear(oldIdentifier); + entryChatHistoryRepository.clear(newIdentifier); + + chatHistory.forEach(record -> entryChatHistoryRepository.addMessage(newIdentifier, record)); + } + } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java index 749115ed1751..181f59ba3c6b 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java @@ -10,7 +10,6 @@ import org.jabref.logic.FilePreferences; import org.jabref.logic.ai.customimplementations.llms.ChatModel; -import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.Summarizator; import org.jabref.logic.ai.summarization.repositories.SummariesRepository; import org.jabref.logic.ai.summarization.tasks.GenerateSummaryForSeveralTask; @@ -21,12 +20,8 @@ import org.jabref.model.ai.processingstatus.ProcessingState; import org.jabref.model.ai.summarization.BibEntrySummary; import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.database.event.EntriesAddedEvent; import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.event.FieldChangedEvent; -import org.jabref.model.entry.field.StandardField; -import com.google.common.eventbus.Subscribe; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,62 +41,26 @@ public class SummariesService { private final List> listsUnderSummarization = new ArrayList<>(); - private final AiPreferences aiPreferences; private final SummariesRepository summariesRepository; private final ChatModel chatModel; - private final Summarizator defaultSummarizator; private final BooleanProperty shutdownSignal; private final FilePreferences filePreferences; private final TaskExecutor taskExecutor; - // TODO: chat model should be argument. public SummariesService( - AiPreferences aiPreferences, FilePreferences filePreferences, TaskExecutor taskExecutor, ChatModel chatModel, - Summarizator defaultSummarizator, SummariesRepository summariesRepository, BooleanProperty shutdownSignal ) { - this.aiPreferences = aiPreferences; this.summariesRepository = summariesRepository; this.chatModel = chatModel; - this.defaultSummarizator = defaultSummarizator; this.shutdownSignal = shutdownSignal; this.filePreferences = filePreferences; this.taskExecutor = taskExecutor; } - public void setupDatabase(BibDatabaseContext bibDatabaseContext) { - // GC was eating the listeners, so we have to fall back to the event bus. - bibDatabaseContext.getDatabase().registerListener(new EntriesChangedListener(bibDatabaseContext)); - } - - private class EntriesChangedListener { - private final BibDatabaseContext bibDatabaseContext; - - public EntriesChangedListener(BibDatabaseContext bibDatabaseContext) { - this.bibDatabaseContext = bibDatabaseContext; - } - - @Subscribe - public void listen(EntriesAddedEvent e) { - e.getBibEntries().forEach(entry -> { - if (aiPreferences.getAutoGenerateSummaries()) { - summarize(defaultSummarizator, entry, bibDatabaseContext); - } - }); - } - - @Subscribe - public void listen(FieldChangedEvent e) { - if (e.getField() == StandardField.FILE && aiPreferences.getAutoGenerateSummaries()) { - summarize(defaultSummarizator, e.getBibEntry(), bibDatabaseContext); - } - } - } - /** * Start generating summary of a {@link BibEntry}, if it was already generated. * This method returns a {@link ProcessingInfo} that can be used for tracking state of the summarization. diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationDatabaseListeners.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationDatabaseListeners.java new file mode 100644 index 000000000000..fbcf34d1ce3a --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationDatabaseListeners.java @@ -0,0 +1,26 @@ +package org.jabref.logic.ai.summarization; + +import org.jabref.logic.ai.preferences.AiPreferences; +import org.jabref.logic.ai.summarization.listeners.GenerateSummaryDatabaseListener; +import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.Summarizator; +import org.jabref.model.database.BibDatabaseContext; + +public class SummarizationDatabaseListeners { + private final GenerateSummaryDatabaseListener generateSummaryDatabaseListener; + + public SummarizationDatabaseListeners( + AiPreferences aiPreferences, + SummariesService summariesService, + Summarizator summarizator + ) { + this.generateSummaryDatabaseListener = new GenerateSummaryDatabaseListener( + aiPreferences, + summariesService, + summarizator + ); + } + + public void setupDatabase(BibDatabaseContext databaseContext) { + generateSummaryDatabaseListener.setupDatabase(databaseContext); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/listeners/GenerateSummaryDatabaseListener.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/listeners/GenerateSummaryDatabaseListener.java new file mode 100644 index 000000000000..f08d314f30fa --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/listeners/GenerateSummaryDatabaseListener.java @@ -0,0 +1,56 @@ +package org.jabref.logic.ai.summarization.listeners; + +import org.jabref.logic.ai.preferences.AiPreferences; +import org.jabref.logic.ai.summarization.SummariesService; +import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.Summarizator; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.database.event.EntriesAddedEvent; +import org.jabref.model.entry.event.FieldChangedEvent; +import org.jabref.model.entry.field.StandardField; + +import com.google.common.eventbus.Subscribe; + +public class GenerateSummaryDatabaseListener { + private final AiPreferences aiPreferences; + private final SummariesService summariesService; + private final Summarizator summarizator; + + public GenerateSummaryDatabaseListener( + AiPreferences aiPreferences, + SummariesService summariesService, + Summarizator summarizator + ) { + this.aiPreferences = aiPreferences; + this.summariesService = summariesService; + this.summarizator = summarizator; + } + + public void setupDatabase(BibDatabaseContext bibDatabaseContext) { + // GC was eating the listeners, so we have to fall back to the event bus. + bibDatabaseContext.getDatabase().registerListener(new EntriesChangedListener(bibDatabaseContext)); + } + + private class EntriesChangedListener { + private final BibDatabaseContext bibDatabaseContext; + + public EntriesChangedListener(BibDatabaseContext bibDatabaseContext) { + this.bibDatabaseContext = bibDatabaseContext; + } + + @Subscribe + public void listen(EntriesAddedEvent e) { + e.getBibEntries().forEach(entry -> { + if (aiPreferences.getAutoGenerateSummaries()) { + summariesService.summarize(summarizator, entry, bibDatabaseContext); + } + }); + } + + @Subscribe + public void listen(FieldChangedEvent e) { + if (e.getField() == StandardField.FILE && aiPreferences.getAutoGenerateSummaries()) { + summariesService.summarize(summarizator, e.getBibEntry(), bibDatabaseContext); + } + } + } +} From bc5ebfeda8d87f22fa82c02b79649464ff04a57a Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Sat, 29 Nov 2025 17:44:25 +0100 Subject: [PATCH 039/243] middle work --- jablib/src/main/java/module-info.java | 2 + .../PersistentLinkedFileIngestor.java | 2 +- .../ai/summarization/SummariesService.java | 4 +- .../SummarizationTaskAggregator.java | 47 +++++++++++++++++++ .../GenerateSummaryTask.java | 2 +- .../GenerateSummaryTaskRequest.java | 21 +++++++++ .../GenerateSummaryForSeveralTask.java | 3 +- .../ai/processingstatus/ProcessingInfo.java | 1 - .../ai/processingstatus/ProcessingState.java | 1 - 9 files changed, 76 insertions(+), 7 deletions(-) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationTaskAggregator.java rename jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/{ => generatesummary}/GenerateSummaryTask.java (98%) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/generatesummary/GenerateSummaryTaskRequest.java rename jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/{ => generatesummaryforseveral}/GenerateSummaryForSeveralTask.java (97%) diff --git a/jablib/src/main/java/module-info.java b/jablib/src/main/java/module-info.java index 8c125d22bec2..0a82c484ba41 100644 --- a/jablib/src/main/java/module-info.java +++ b/jablib/src/main/java/module-info.java @@ -155,6 +155,8 @@ exports org.jabref.model.ai.chatting.messages; exports org.jabref.logic.ai.chatting.listeners; exports org.jabref.logic.ai.chatting.util; + exports org.jabref.logic.ai.summarization.tasks.generatesummary; + exports org.jabref.logic.ai.summarization.tasks.generatesummaryforseveral; // endregion requires java.base; diff --git a/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/ingestion/PersistentLinkedFileIngestor.java b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/ingestion/PersistentLinkedFileIngestor.java index e94ed887f70d..e5f282672210 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/ingestion/PersistentLinkedFileIngestor.java +++ b/jablib/src/main/java/org/jabref/logic/ai/pipeline/logic/ingestion/PersistentLinkedFileIngestor.java @@ -51,7 +51,7 @@ public void ingest( ) throws InterruptedException { // TODO: Simplify this method. // Rationale for RuntimeException here: - // See org.jabref.logic.ai.summarization.tasks.GenerateSummaryTask.summarizeAll + // See org.jabref.logic.ai.summarization.tasks.generatesummary.GenerateSummaryTask.summarizeAll LOGGER.debug("Generating embeddings for file \"{}\"", linkedFile.getLink()); diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java index 181f59ba3c6b..64fb9ecda566 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummariesService.java @@ -12,8 +12,8 @@ import org.jabref.logic.ai.customimplementations.llms.ChatModel; import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.Summarizator; import org.jabref.logic.ai.summarization.repositories.SummariesRepository; -import org.jabref.logic.ai.summarization.tasks.GenerateSummaryForSeveralTask; -import org.jabref.logic.ai.summarization.tasks.GenerateSummaryTask; +import org.jabref.logic.ai.summarization.tasks.generatesummary.GenerateSummaryTask; +import org.jabref.logic.ai.summarization.tasks.generatesummaryforseveral.GenerateSummaryForSeveralTask; import org.jabref.logic.util.CitationKeyCheck; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.ai.processingstatus.ProcessingInfo; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationTaskAggregator.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationTaskAggregator.java new file mode 100644 index 000000000000..162faba42bd9 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationTaskAggregator.java @@ -0,0 +1,47 @@ +package org.jabref.logic.ai.summarization; + +import java.util.Comparator; +import java.util.List; +import java.util.TreeMap; + +import org.jabref.logic.ai.summarization.tasks.generatesummary.GenerateSummaryTask; +import org.jabref.logic.ai.summarization.tasks.generatesummary.GenerateSummaryTaskRequest; +import org.jabref.logic.util.TaskExecutor; +import org.jabref.model.ai.processingstatus.ProcessingInfo; +import org.jabref.model.ai.summarization.BibEntrySummary; +import org.jabref.model.entry.BibEntry; + +public class SummarizationTaskAggregator { + private final TaskExecutor taskExecutor; + + private final TreeMap> generateSummaryTasks = + new TreeMap<>(Comparator.comparing(BibEntry::getId)); + + private final TreeMap, ProcessingInfo> generateSummaryForSeveralTasks = + new TreeMap<>(new Comparator>() { + @Override + public int compare(List list1, List list2) { + if (list1 == list2) return true; + if (list1 == null || list2 == null) return false; + if (list1.size() != list2.size()) return false; + + // 2. Map to IDs and compare + List ids1 = list1.stream().map(BibEntry::getId).toList(); + List ids2 = list2.stream().map(BibEntry::getId).toList(); + + return ids1.equals(ids2); + } + }); + + public SummarizationTaskAggregator(TaskExecutor taskExecutor) { + this.taskExecutor = taskExecutor; + } + + public synchronized GenerateSummaryTask start(GenerateSummaryTaskRequest request) { + return tasks.computeIfAbsent(request.entry(), _ -> { + GenerateSummaryTask task = new GenerateSummaryTask(request); + taskExecutor.execute(task); + return task; + }); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/generatesummary/GenerateSummaryTask.java similarity index 98% rename from jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java rename to jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/generatesummary/GenerateSummaryTask.java index 986ed530c20a..0d89771e17cf 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/generatesummary/GenerateSummaryTask.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.summarization.tasks; +package org.jabref.logic.ai.summarization.tasks.generatesummary; import javafx.beans.property.ReadOnlyBooleanProperty; diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/generatesummary/GenerateSummaryTaskRequest.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/generatesummary/GenerateSummaryTaskRequest.java new file mode 100644 index 000000000000..9b4e0261167b --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/generatesummary/GenerateSummaryTaskRequest.java @@ -0,0 +1,21 @@ +package org.jabref.logic.ai.summarization.tasks.generatesummary; + +import javafx.beans.property.ReadOnlyBooleanProperty; + +import org.jabref.logic.FilePreferences; +import org.jabref.logic.ai.customimplementations.llms.ChatModel; +import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.Summarizator; +import org.jabref.logic.ai.summarization.repositories.SummariesRepository; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; + +public record GenerateSummaryTaskRequest( + FilePreferences filePreferences, + ChatModel chatModel, + SummariesRepository summariesRepository, + Summarizator summarizator, + BibDatabaseContext bibDatabaseContext, + BibEntry entry, + ReadOnlyBooleanProperty shutdownSignal +) { +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/generatesummaryforseveral/GenerateSummaryForSeveralTask.java similarity index 97% rename from jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java rename to jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/generatesummaryforseveral/GenerateSummaryForSeveralTask.java index ee63533aea03..75ad1bd97fa2 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/GenerateSummaryForSeveralTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/generatesummaryforseveral/GenerateSummaryForSeveralTask.java @@ -1,4 +1,4 @@ -package org.jabref.logic.ai.summarization.tasks; +package org.jabref.logic.ai.summarization.tasks.generatesummaryforseveral; import java.util.ArrayList; import java.util.List; @@ -13,6 +13,7 @@ import org.jabref.logic.ai.customimplementations.llms.ChatModel; import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.Summarizator; import org.jabref.logic.ai.summarization.repositories.SummariesRepository; +import org.jabref.logic.ai.summarization.tasks.generatesummary.GenerateSummaryTask; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.BackgroundTask; import org.jabref.logic.util.ProgressCounter; diff --git a/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingInfo.java b/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingInfo.java index b9cdc048c85d..9810a695c321 100644 --- a/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingInfo.java +++ b/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingInfo.java @@ -8,7 +8,6 @@ import org.jspecify.annotations.Nullable; -@Deprecated public class ProcessingInfo { private final O object; private final ObjectProperty state; diff --git a/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingState.java b/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingState.java index 60f557d6e4e5..9c0ad6c11234 100644 --- a/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingState.java +++ b/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingState.java @@ -1,6 +1,5 @@ package org.jabref.model.ai.processingstatus; -@Deprecated public enum ProcessingState { PROCESSING, SUCCESS, From cb2b08dfed6efced560a3e76561637ea715f5e26 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Sat, 29 Nov 2025 17:47:18 +0100 Subject: [PATCH 040/243] middle work --- .../SummarizationTaskAggregator.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationTaskAggregator.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationTaskAggregator.java index 162faba42bd9..9816593cdee7 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationTaskAggregator.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationTaskAggregator.java @@ -18,20 +18,20 @@ public class SummarizationTaskAggregator { new TreeMap<>(Comparator.comparing(BibEntry::getId)); private final TreeMap, ProcessingInfo> generateSummaryForSeveralTasks = - new TreeMap<>(new Comparator>() { - @Override - public int compare(List list1, List list2) { - if (list1 == list2) return true; - if (list1 == null || list2 == null) return false; - if (list1.size() != list2.size()) return false; - - // 2. Map to IDs and compare - List ids1 = list1.stream().map(BibEntry::getId).toList(); - List ids2 = list2.stream().map(BibEntry::getId).toList(); - - return ids1.equals(ids2); - } - }); + new TreeMap<>((a, b) -> { + if (a.size() != b.size()) { + return Integer.compare(a.size(), b.size()); + } + + for (int i = 0; i < a.size(); i++) { + int cmp = a.get(i).getId().compareTo(b.get(i).getId()); + if (cmp != 0) { + return cmp; + } + } + + return 0; + }); public SummarizationTaskAggregator(TaskExecutor taskExecutor) { this.taskExecutor = taskExecutor; From 55bb1aafbfa5474e9b29618a335583d71d67a8c0 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Sat, 29 Nov 2025 17:47:28 +0100 Subject: [PATCH 041/243] middle work --- .../logic/ai/summarization/SummarizationTaskAggregator.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationTaskAggregator.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationTaskAggregator.java index 9816593cdee7..e5c13050956b 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationTaskAggregator.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationTaskAggregator.java @@ -1,11 +1,8 @@ package org.jabref.logic.ai.summarization; import java.util.Comparator; -import java.util.List; import java.util.TreeMap; -import org.jabref.logic.ai.summarization.tasks.generatesummary.GenerateSummaryTask; -import org.jabref.logic.ai.summarization.tasks.generatesummary.GenerateSummaryTaskRequest; import org.jabref.logic.util.TaskExecutor; import org.jabref.model.ai.processingstatus.ProcessingInfo; import org.jabref.model.ai.summarization.BibEntrySummary; @@ -17,6 +14,7 @@ public class SummarizationTaskAggregator { private final TreeMap> generateSummaryTasks = new TreeMap<>(Comparator.comparing(BibEntry::getId)); + /* private final TreeMap, ProcessingInfo> generateSummaryForSeveralTasks = new TreeMap<>((a, b) -> { if (a.size() != b.size()) { @@ -44,4 +42,6 @@ public synchronized GenerateSummaryTask start(GenerateSummaryTaskRequest request return task; }); } + + */ } From ea09dc68f798bfb8649a95055961a099b1aec76c Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Sat, 29 Nov 2025 17:47:42 +0100 Subject: [PATCH 042/243] middle work --- .../logic/ai/summarization/SummarizationTaskAggregator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationTaskAggregator.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationTaskAggregator.java index e5c13050956b..13e64a5c4782 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationTaskAggregator.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationTaskAggregator.java @@ -3,14 +3,14 @@ import java.util.Comparator; import java.util.TreeMap; -import org.jabref.logic.util.TaskExecutor; import org.jabref.model.ai.processingstatus.ProcessingInfo; import org.jabref.model.ai.summarization.BibEntrySummary; import org.jabref.model.entry.BibEntry; public class SummarizationTaskAggregator { + /* private final TaskExecutor taskExecutor; - + */ private final TreeMap> generateSummaryTasks = new TreeMap<>(Comparator.comparing(BibEntry::getId)); From a00b9ac601583000bfd466773cf0acd4f695c245 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Sat, 29 Nov 2025 19:48:35 +0100 Subject: [PATCH 043/243] Start working on tasks --- .../SummarizationTaskAggregator.java | 38 ++++-- .../generatesummary/GenerateSummaryTask.java | 62 +++------ .../GenerateSummaryForSeveralTask.java | 125 +++++------------- .../GenerateSummaryForSeveralTaskRequest.java | 41 ++++++ .../logic/ai/util/TrackedBackgroundTask.java | 84 ++++++++++++ .../ai/processingstatus/ProcessingInfo.java | 1 + .../ai/processingstatus/ProcessingState.java | 1 + 7 files changed, 204 insertions(+), 148 deletions(-) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/generatesummaryforseveral/GenerateSummaryForSeveralTaskRequest.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/util/TrackedBackgroundTask.java diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationTaskAggregator.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationTaskAggregator.java index 13e64a5c4782..442fa62b9ae5 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationTaskAggregator.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/SummarizationTaskAggregator.java @@ -1,21 +1,27 @@ package org.jabref.logic.ai.summarization; import java.util.Comparator; +import java.util.List; import java.util.TreeMap; +import java.util.concurrent.Future; -import org.jabref.model.ai.processingstatus.ProcessingInfo; +import javafx.util.Pair; + +import org.jabref.logic.ai.summarization.tasks.generatesummary.GenerateSummaryTask; +import org.jabref.logic.ai.summarization.tasks.generatesummary.GenerateSummaryTaskRequest; +import org.jabref.logic.ai.summarization.tasks.generatesummaryforseveral.GenerateSummaryForSeveralTask; +import org.jabref.logic.ai.summarization.tasks.generatesummaryforseveral.GenerateSummaryForSeveralTaskRequest; +import org.jabref.logic.util.TaskExecutor; import org.jabref.model.ai.summarization.BibEntrySummary; import org.jabref.model.entry.BibEntry; public class SummarizationTaskAggregator { - /* private final TaskExecutor taskExecutor; - */ - private final TreeMap> generateSummaryTasks = + + private final TreeMap, GenerateSummaryTask>> generateSummaryTasks = new TreeMap<>(Comparator.comparing(BibEntry::getId)); - /* - private final TreeMap, ProcessingInfo> generateSummaryForSeveralTasks = + private final TreeMap, Pair, GenerateSummaryForSeveralTask>> generateSummaryForSeveralTasks = new TreeMap<>((a, b) -> { if (a.size() != b.size()) { return Integer.compare(a.size(), b.size()); @@ -35,13 +41,23 @@ public SummarizationTaskAggregator(TaskExecutor taskExecutor) { this.taskExecutor = taskExecutor; } - public synchronized GenerateSummaryTask start(GenerateSummaryTaskRequest request) { - return tasks.computeIfAbsent(request.entry(), _ -> { + public synchronized Pair, GenerateSummaryTask> startWithFuture(GenerateSummaryTaskRequest request) { + return generateSummaryTasks.computeIfAbsent(request.entry(), _ -> { GenerateSummaryTask task = new GenerateSummaryTask(request); - taskExecutor.execute(task); - return task; + Future future = taskExecutor.execute(task); + return new Pair<>(future, task); }); } - */ + public synchronized GenerateSummaryTask start(GenerateSummaryTaskRequest request) { + return startWithFuture(request).getValue(); + } + + public synchronized Pair, GenerateSummaryForSeveralTask> start(GenerateSummaryForSeveralTaskRequest request) { + return generateSummaryForSeveralTasks.computeIfAbsent(request.entries(), _ -> { + GenerateSummaryForSeveralTask task = new GenerateSummaryForSeveralTask(request); + Future future = taskExecutor.execute(task); + return new Pair<>(future, task); + }); + } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/generatesummary/GenerateSummaryTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/generatesummary/GenerateSummaryTask.java index 0d89771e17cf..bb97f66b1c9b 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/generatesummary/GenerateSummaryTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/generatesummary/GenerateSummaryTask.java @@ -1,20 +1,12 @@ package org.jabref.logic.ai.summarization.tasks.generatesummary; -import javafx.beans.property.ReadOnlyBooleanProperty; - -import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.customimplementations.llms.ChatModel; import org.jabref.logic.ai.summarization.SummariesService; import org.jabref.logic.ai.summarization.logic.PersistentBibEntrySummarizator; -import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.Summarizator; import org.jabref.logic.ai.summarization.repositories.SummariesRepository; import org.jabref.logic.ai.util.LongTaskInfo; +import org.jabref.logic.ai.util.TrackedBackgroundTask; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.util.BackgroundTask; -import org.jabref.logic.util.ProgressCounter; import org.jabref.model.ai.summarization.BibEntrySummary; -import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.entry.BibEntry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,38 +18,21 @@ *

* This task is created in the {@link SummariesService}, and stored then in a {@link SummariesRepository}. */ -public class GenerateSummaryTask extends BackgroundTask { +public class GenerateSummaryTask extends TrackedBackgroundTask { private static final Logger LOGGER = LoggerFactory.getLogger(GenerateSummaryTask.class); - private final ChatModel chatModel; - private final BibDatabaseContext bibDatabaseContext; - private final BibEntry entry; - private final String citationKey; - private final ReadOnlyBooleanProperty shutdownSignal; - + private final GenerateSummaryTaskRequest request; + private final String citationKey; // Useful for logging. private final PersistentBibEntrySummarizator persistentBibEntrySummarizator; - private final ProgressCounter progressCounter = new ProgressCounter(); - - public GenerateSummaryTask( - FilePreferences filePreferences, - ChatModel chatModel, - SummariesRepository summariesRepository, - Summarizator summarizator, - BibDatabaseContext bibDatabaseContext, - BibEntry entry, - ReadOnlyBooleanProperty shutdownSignal - ) { - this.chatModel = chatModel; - this.bibDatabaseContext = bibDatabaseContext; - this.entry = entry; - this.citationKey = entry.getCitationKey().orElse(""); - this.shutdownSignal = shutdownSignal; + public GenerateSummaryTask(GenerateSummaryTaskRequest request) { + this.request = request; + this.citationKey = request.entry().getCitationKey().orElse(""); this.persistentBibEntrySummarizator = new PersistentBibEntrySummarizator( - filePreferences, - summariesRepository, - summarizator + request.filePreferences(), + request.summariesRepository(), + request.summarizator() ); configure(); @@ -66,24 +41,22 @@ public GenerateSummaryTask( private void configure() { showToUser(true); titleProperty().set(Localization.lang("Waiting summary for %0...", citationKey)); - - progressCounter.listenToAllProperties(this::updateProgress); } @Override - public BibEntrySummary call() { + public BibEntrySummary perform() { LOGGER.debug("Starting summarization task for entry {}", citationKey); LongTaskInfo longTaskInfo = new LongTaskInfo( progressCounter, - shutdownSignal + request.shutdownSignal() ); BibEntrySummary bibEntrySummary = persistentBibEntrySummarizator.summarize( - chatModel, + request.chatModel(), longTaskInfo, - bibDatabaseContext, - entry + request.bibDatabaseContext(), + request.entry() ); LOGGER.debug("Finished summarization task for entry {}", citationKey); @@ -91,9 +64,4 @@ public BibEntrySummary call() { return bibEntrySummary; } - - private void updateProgress() { - updateProgress(progressCounter.getWorkDone(), progressCounter.getWorkMax()); - updateMessage(progressCounter.getMessage()); - } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/generatesummaryforseveral/GenerateSummaryForSeveralTask.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/generatesummaryforseveral/GenerateSummaryForSeveralTask.java index 75ad1bd97fa2..1acabce2f227 100644 --- a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/generatesummaryforseveral/GenerateSummaryForSeveralTask.java +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/generatesummaryforseveral/GenerateSummaryForSeveralTask.java @@ -5,23 +5,11 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; -import javafx.beans.property.ReadOnlyBooleanProperty; -import javafx.beans.property.StringProperty; import javafx.util.Pair; -import org.jabref.logic.FilePreferences; -import org.jabref.logic.ai.customimplementations.llms.ChatModel; -import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.Summarizator; -import org.jabref.logic.ai.summarization.repositories.SummariesRepository; import org.jabref.logic.ai.summarization.tasks.generatesummary.GenerateSummaryTask; +import org.jabref.logic.ai.util.TrackedBackgroundTask; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.util.BackgroundTask; -import org.jabref.logic.util.ProgressCounter; -import org.jabref.logic.util.TaskExecutor; -import org.jabref.model.ai.processingstatus.ProcessingInfo; -import org.jabref.model.ai.processingstatus.ProcessingState; -import org.jabref.model.ai.summarization.BibEntrySummary; -import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.slf4j.Logger; @@ -32,98 +20,55 @@ * It will check if summaries were already generated. * And it also will store the summaries. */ -public class GenerateSummaryForSeveralTask extends BackgroundTask { +public class GenerateSummaryForSeveralTask extends TrackedBackgroundTask { private static final Logger LOGGER = LoggerFactory.getLogger(GenerateSummaryForSeveralTask.class); - private final StringProperty groupName; - private final List> entries; - private final BibDatabaseContext bibDatabaseContext; - private final SummariesRepository summariesRepository; - private final ChatModel chatModel; - private final Summarizator summarizator; - private final ReadOnlyBooleanProperty shutdownSignal; - private final FilePreferences filePreferences; - private final TaskExecutor taskExecutor; - - private final ProgressCounter progressCounter = new ProgressCounter(); - - private String currentFile = ""; - - public GenerateSummaryForSeveralTask( - FilePreferences filePreferences, - TaskExecutor taskExecutor, - ChatModel chatModel, - SummariesRepository summariesRepository, - Summarizator summarizator, - BibDatabaseContext bibDatabaseContext, - StringProperty groupName, - List> entries, - ReadOnlyBooleanProperty shutdownSignal - ) { - this.groupName = groupName; - this.entries = entries; - this.bibDatabaseContext = bibDatabaseContext; - this.summariesRepository = summariesRepository; - this.chatModel = chatModel; - this.summarizator = summarizator; - this.shutdownSignal = shutdownSignal; - this.filePreferences = filePreferences; - this.taskExecutor = taskExecutor; + private final GenerateSummaryForSeveralTaskRequest request; + + public GenerateSummaryForSeveralTask(GenerateSummaryForSeveralTaskRequest request) { + this.request = request; configure(); } private void configure() { showToUser(true); - titleProperty().set(Localization.lang("Generating summaries for %0", groupName.get())); - groupName.addListener((_, _, newValue) -> titleProperty().set(Localization.lang("Generating summaries for %0", newValue))); + titleProperty().set(Localization.lang("Generating summaries for %0", request.groupName().get())); + request.groupName().addListener((_, _, newValue) -> titleProperty().set(Localization.lang("Generating summaries for %0", newValue))); - progressCounter.increaseWorkMax(entries.size()); - progressCounter.listenToAllProperties(this::updateProgress); - updateProgress(); + progressCounter.increaseWorkMax(request.entries().size()); } @Override - public Void call() throws ExecutionException, InterruptedException { - LOGGER.debug("Starting summaries generation of several files for {}", groupName.get()); - - List, BibEntry>> futures = new ArrayList<>(); - - entries - .stream() - .map(processingInfo -> { - processingInfo.setState(ProcessingState.PROCESSING); - return new Pair<>( - new GenerateSummaryTask( - filePreferences, - chatModel, - summariesRepository, - summarizator, - bibDatabaseContext, - processingInfo.getObject(), - shutdownSignal - ) - .showToUser(false) - .onSuccess(processingInfo::setSuccess) - .onFailure(processingInfo::setException) - .onFinished(() -> progressCounter.increaseWorkDone(1)) - .executeWith(taskExecutor), - processingInfo.getObject()); - }) - .forEach(futures::add); - - for (Pair, BibEntry> pair : futures) { - currentFile = pair.getValue().getCitationKey().orElse(""); - pair.getKey().get(); + public Void perform() throws ExecutionException, InterruptedException { + LOGGER.debug("Starting summaries generation of several files for {}", request.groupName().get()); + + // We don't really care about the types. We just need to wait here. + List> futures = new ArrayList<>(); + + request.entries() + .stream() + .map(entry -> { + Pair, GenerateSummaryTask> pair = request.summarizationTaskAggregator().startWithFuture( + request.toSingle(entry) + ); + + pair.getValue().statusProperty().addListener((_, _, newValue) -> { + if (newValue.isFinished()) { + progressCounter.increaseWorkDone(1); + } + }); + + return pair.getKey(); + }) + .forEach(futures::add); + + for (Future future : futures) { + future.get(); } - LOGGER.debug("Finished embeddings generation task of several files for {}", groupName.get()); + LOGGER.debug("Finished summary generation task of several files for {}", request.groupName().get()); progressCounter.stop(); return null; } - - private void updateProgress() { - updateProgress(progressCounter.getWorkDone(), progressCounter.getWorkMax()); - updateMessage(progressCounter.getMessage() + " - " + currentFile + ", ..."); - } } diff --git a/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/generatesummaryforseveral/GenerateSummaryForSeveralTaskRequest.java b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/generatesummaryforseveral/GenerateSummaryForSeveralTaskRequest.java new file mode 100644 index 000000000000..571fb79f4eff --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/summarization/tasks/generatesummaryforseveral/GenerateSummaryForSeveralTaskRequest.java @@ -0,0 +1,41 @@ +package org.jabref.logic.ai.summarization.tasks.generatesummaryforseveral; + +import java.util.List; + +import javafx.beans.property.ReadOnlyBooleanProperty; +import javafx.beans.property.StringProperty; + +import org.jabref.logic.FilePreferences; +import org.jabref.logic.ai.customimplementations.llms.ChatModel; +import org.jabref.logic.ai.summarization.SummarizationTaskAggregator; +import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.Summarizator; +import org.jabref.logic.ai.summarization.repositories.SummariesRepository; +import org.jabref.logic.ai.summarization.tasks.generatesummary.GenerateSummaryTaskRequest; +import org.jabref.logic.util.TaskExecutor; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; + +public record GenerateSummaryForSeveralTaskRequest( + FilePreferences filePreferences, + SummarizationTaskAggregator summarizationTaskAggregator, + TaskExecutor taskExecutor, + ChatModel chatModel, + SummariesRepository summariesRepository, + Summarizator summarizator, + BibDatabaseContext bibDatabaseContext, + StringProperty groupName, + List entries, + ReadOnlyBooleanProperty shutdownSignals +) { + public GenerateSummaryTaskRequest toSingle(BibEntry entry) { + return new GenerateSummaryTaskRequest( + filePreferences, + chatModel, + summariesRepository, + summarizator, + bibDatabaseContext, + entry, + shutdownSignals + ); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/ai/util/TrackedBackgroundTask.java b/jablib/src/main/java/org/jabref/logic/ai/util/TrackedBackgroundTask.java new file mode 100644 index 000000000000..ef120033878c --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/ai/util/TrackedBackgroundTask.java @@ -0,0 +1,84 @@ +package org.jabref.logic.ai.util; + +import javafx.beans.property.ObjectProperty; +import javafx.beans.property.ReadOnlyObjectProperty; +import javafx.beans.property.SimpleObjectProperty; + +import org.jabref.logic.util.BackgroundTask; +import org.jabref.logic.util.ProgressCounter; + +public abstract class TrackedBackgroundTask extends BackgroundTask { + public enum Status { + PENDING, + PROCESSING, + SUCCESS, + ERROR, + CANCELLED; + + public boolean isFinished() { + return this == SUCCESS || this == ERROR || this == CANCELLED; + } + } + + private final ObjectProperty status = new SimpleObjectProperty<>(Status.PENDING); + private final ObjectProperty result = new SimpleObjectProperty<>(); + private final ObjectProperty exception = new SimpleObjectProperty<>(); + + protected final ProgressCounter progressCounter = new ProgressCounter(); + + public TrackedBackgroundTask() { + super(); + progressCounter.listenToAllProperties(this::updateProgress); + } + + public V call() throws Exception { + try { + status.set(Status.PROCESSING); + V output = perform(); + status.set(Status.SUCCESS); + result.set(output); + return output; + } catch (Exception e) { + status.set(Status.ERROR); + exception.set(e); + throw e; + } + } + + public ReadOnlyObjectProperty statusProperty() { + return status; + } + + public Status getStatus() { + return status.get(); + } + + public ReadOnlyObjectProperty resultProperty() { + return result; + } + + public V getResult() { + return result.get(); + } + + public ReadOnlyObjectProperty exceptionProperty() { + return exception; + } + + public Exception getException() { + return exception.get(); + } + + @Override + public void cancel() { + super.cancel(); + status.set(Status.CANCELLED); + } + + private void updateProgress() { + updateProgress(progressCounter.getWorkDone(), progressCounter.getWorkMax()); + updateMessage(progressCounter.getMessage()); + } + + protected abstract V perform() throws Exception; +} diff --git a/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingInfo.java b/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingInfo.java index 9810a695c321..b9cdc048c85d 100644 --- a/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingInfo.java +++ b/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingInfo.java @@ -8,6 +8,7 @@ import org.jspecify.annotations.Nullable; +@Deprecated public class ProcessingInfo { private final O object; private final ObjectProperty state; diff --git a/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingState.java b/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingState.java index 9c0ad6c11234..60f557d6e4e5 100644 --- a/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingState.java +++ b/jablib/src/main/java/org/jabref/model/ai/processingstatus/ProcessingState.java @@ -1,5 +1,6 @@ package org.jabref.model.ai.processingstatus; +@Deprecated public enum ProcessingState { PROCESSING, SUCCESS, From 02ff6ff4e2945cb75af0fe52a9bdd0b05a598f83 Mon Sep 17 00:00:00 2001 From: Ruslan Popov Date: Sun, 30 Nov 2025 16:23:29 +0100 Subject: [PATCH 044/243] middle work --- docs/code-howtos/ai.md | 3 + jabgui/src/main/java/module-info.java | 2 + .../jabref/gui/ai/AiPrivacyNoticeView.java | 89 +++++++++ .../gui/ai/AiPrivacyNoticeViewModel.java | 58 ++++++ .../ai/components/aichat/AiChatComponent.java | 2 +- .../AiPrivacyNoticeGuardedComponent.java | 3 - .../privacynotice/PrivacyNoticeComponent.java | 19 +- .../summary/SummaryShowingComponent.java | 2 +- .../ai/statuspane/ErrorStatusPaneView.java | 129 +++++++++++++ .../statuspane/ErrorStatusPaneViewModel.java | 69 +++++++ .../ai/statuspane/LoadingStatusPaneView.java | 56 ++++++ .../LoadingStatusPaneViewModel.java | 17 ++ .../ai/statuspane/SimpleStatusPaneView.java | 56 ++++++ .../statuspane/SimpleStatusPaneViewModel.java | 17 ++ .../org/jabref/gui/entryeditor/AiChatTab.java | 2 +- .../jabref/gui/entryeditor/AiSummaryTab.java | 49 ++--- .../jabref/gui/entryeditor/EntryEditor.java | 2 +- .../entryeditor/aisummary/AiSummaryView.java | 138 +++++++++++++- .../aisummary/AiSummaryViewModel.java | 180 ++++++++++++++++-- .../org/jabref/gui/preferences/ai/AiTab.java | 36 ++-- .../gui/preferences/ai/AiTabViewModel.java | 32 ++-- .../org/jabref/gui/ai/AiPrivacyNotice.fxml | 43 +++++ .../gui/ai/statuspane/ErrorStatusPane.fxml | 48 +++++ .../gui/ai/statuspane/LoadingStatusPane.fxml | 38 ++++ .../gui/ai/statuspane/SimpleStatusPane.fxml | 29 +++ .../gui/entryeditor/aisummary/AiSummary.fxml | 76 ++++++++ .../entryeditor/aisummary/AiSummaryView.fxml | 11 -- .../gui/ai/AiPrivacyNoticeViewModelTest.java | 32 ++++ .../aichat/AiChatComponentTest.java | 4 +- jablib/src/main/java/module-info.java | 6 +- .../java/org/jabref/logic/ai/AiService.java | 19 ++ .../logic/ai/chatting/logic/AiChatLogic.java | 16 +- ...a => ChattingSystemMessageAiTemplate.java} | 15 +- ...ava => ChattingUserMessageAiTemplate.java} | 15 +- .../logic/ParseCitationsWithLlm.java | 12 +- ...itationParsingSystemMessageAiTemplate.java | 29 +++ .../CitationParsingSystemMessageTemplate.java | 24 --- .../CitationParsingUserMessageAiTemplate.java | 32 ++++ .../CitationParsingUserMessageTemplate.java | 27 --- .../logic/ai/current/CurrentAiTemplates.java | 115 +++++++---- .../logic/ai/current/CurrentSummarizator.java | 50 ++--- .../logic/parsing/UniversalContentParser.java | 4 + .../ai/preferences/AiDefaultTemplates.java | 32 ++-- .../logic/ai/preferences/AiPreferences.java | 44 +++-- .../ai/summarization/SummariesService.java | 11 +- .../logic/PersistentBibEntrySummarizator.java | 16 +- .../logic/SummarizatorFactory.java | 42 ++++ .../ChunkedSummarizator.java | 24 +-- .../FullDocumentSummarizator.java | 58 ++++++ .../MVStoreSummariesRepository.java | 13 +- .../repositories/SummariesRepository.java | 8 +- .../generatesummary/GenerateSummaryTask.java | 3 +- .../GenerateSummaryTaskRequest.java | 1 + .../GenerateSummaryForSeveralTaskRequest.java | 2 + ...arizationChunkSystemMessageAiTemplate.java | 29 +++ ...mmarizationChunkSystemMessageTemplate.java | 24 --- ...mmarizationChunkUserMessageAiTemplate.java | 32 ++++ ...SummarizationChunkUserMessageTemplate.java | 27 --- ...izationCombineSystemMessageAiTemplate.java | 29 +++ ...arizationCombineSystemMessageTemplate.java | 24 --- ...arizationCombineUserMessageAiTemplate.java | 33 ++++ ...mmarizationCombineUserMessageTemplate.java | 28 --- ...onFullDocumentSystemMessageAiTemplate.java | 30 +++ ...tionFullDocumentUserMessageAiTemplate.java | 33 ++++ .../{Template.java => AiTemplate.java} | 7 +- .../ai/templates/AiTemplatesFactory.java | 27 +++ .../preferences/JabRefCliPreferences.java | 58 +++--- .../jabref/logic/util/CitationKeyCheck.java | 4 +- .../org/jabref/model/ai/llm/AiProvider.java | 24 +-- .../ai/summarization/SummarizatorKind.java | 3 +- .../{AiTemplate.java => AiTemplateKind.java} | 6 +- 71 files changed, 1789 insertions(+), 489 deletions(-) create mode 100644 jabgui/src/main/java/org/jabref/gui/ai/AiPrivacyNoticeView.java create mode 100644 jabgui/src/main/java/org/jabref/gui/ai/AiPrivacyNoticeViewModel.java create mode 100644 jabgui/src/main/java/org/jabref/gui/ai/statuspane/ErrorStatusPaneView.java create mode 100644 jabgui/src/main/java/org/jabref/gui/ai/statuspane/ErrorStatusPaneViewModel.java create mode 100644 jabgui/src/main/java/org/jabref/gui/ai/statuspane/LoadingStatusPaneView.java create mode 100644 jabgui/src/main/java/org/jabref/gui/ai/statuspane/LoadingStatusPaneViewModel.java create mode 100644 jabgui/src/main/java/org/jabref/gui/ai/statuspane/SimpleStatusPaneView.java create mode 100644 jabgui/src/main/java/org/jabref/gui/ai/statuspane/SimpleStatusPaneViewModel.java create mode 100644 jabgui/src/main/resources/org/jabref/gui/ai/AiPrivacyNotice.fxml create mode 100644 jabgui/src/main/resources/org/jabref/gui/ai/statuspane/ErrorStatusPane.fxml create mode 100644 jabgui/src/main/resources/org/jabref/gui/ai/statuspane/LoadingStatusPane.fxml create mode 100644 jabgui/src/main/resources/org/jabref/gui/ai/statuspane/SimpleStatusPane.fxml create mode 100644 jabgui/src/main/resources/org/jabref/gui/entryeditor/aisummary/AiSummary.fxml delete mode 100644 jabgui/src/main/resources/org/jabref/gui/entryeditor/aisummary/AiSummaryView.fxml create mode 100644 jabgui/src/test/java/org/jabref/gui/ai/AiPrivacyNoticeViewModelTest.java rename jablib/src/main/java/org/jabref/logic/ai/chatting/templates/{ChattingSystemMessageTemplate.java => ChattingSystemMessageAiTemplate.java} (52%) rename jablib/src/main/java/org/jabref/logic/ai/chatting/templates/{ChattingUserMessageTemplate.java => ChattingUserMessageAiTemplate.java} (61%) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/citationparsing/templates/CitationParsingSystemMessageAiTemplate.java delete mode 100644 jablib/src/main/java/org/jabref/logic/ai/citationparsing/templates/CitationParsingSystemMessageTemplate.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/citationparsing/templates/CitationParsingUserMessageAiTemplate.java delete mode 100644 jablib/src/main/java/org/jabref/logic/ai/citationparsing/templates/CitationParsingUserMessageTemplate.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/logic/SummarizatorFactory.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/logic/summarizationalgorithms/FullDocumentSummarizator.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationChunkSystemMessageAiTemplate.java delete mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationChunkSystemMessageTemplate.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationChunkUserMessageAiTemplate.java delete mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationChunkUserMessageTemplate.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationCombineSystemMessageAiTemplate.java delete mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationCombineSystemMessageTemplate.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationCombineUserMessageAiTemplate.java delete mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationCombineUserMessageTemplate.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationFullDocumentSystemMessageAiTemplate.java create mode 100644 jablib/src/main/java/org/jabref/logic/ai/summarization/templates/SummarizationFullDocumentUserMessageAiTemplate.java rename jablib/src/main/java/org/jabref/logic/ai/templates/{Template.java => AiTemplate.java} (85%) create mode 100644 jablib/src/main/java/org/jabref/logic/ai/templates/AiTemplatesFactory.java rename jablib/src/main/java/org/jabref/model/ai/templating/{AiTemplate.java => AiTemplateKind.java} (76%) diff --git a/docs/code-howtos/ai.md b/docs/code-howtos/ai.md index 60c9e21387e5..fbab579a9ef9 100644 --- a/docs/code-howtos/ai.md +++ b/docs/code-howtos/ai.md @@ -21,3 +21,6 @@ I would like a forth layer: to distrinct database and entry, but it is what it i explain how messages are stored (v1 and v2). + + +logical flaw in task aggregation: yoou need to add status listeners BEFORE executing. diff --git a/jabgui/src/main/java/module-info.java b/jabgui/src/main/java/module-info.java index 904087a567e2..14d517d61011 100644 --- a/jabgui/src/main/java/module-info.java +++ b/jabgui/src/main/java/module-info.java @@ -192,5 +192,7 @@ // requires mslinks; requires org.antlr.antlr4.runtime; requires org.libreoffice.uno; + requires org.glassfish.grizzly; + requires org.jabref; // endregion } diff --git a/jabgui/src/main/java/org/jabref/gui/ai/AiPrivacyNoticeView.java b/jabgui/src/main/java/org/jabref/gui/ai/AiPrivacyNoticeView.java new file mode 100644 index 000000000000..8940f3ac4db8 --- /dev/null +++ b/jabgui/src/main/java/org/jabref/gui/ai/AiPrivacyNoticeView.java @@ -0,0 +1,89 @@ +package org.jabref.gui.ai; + +import javafx.beans.binding.Bindings; +import javafx.beans.binding.DoubleBinding; +import javafx.fxml.FXML; +import javafx.scene.control.Button; +import javafx.scene.control.Hyperlink; +import javafx.scene.control.Label; +import javafx.scene.control.ScrollPane; +import javafx.scene.layout.GridPane; +import javafx.scene.layout.VBox; +import javafx.scene.text.Text; + +import org.jabref.gui.DialogService; +import org.jabref.gui.preferences.GuiPreferences; +import org.jabref.model.ai.llm.AiProvider; + +import com.airhacks.afterburner.views.ViewLoader; +import jakarta.inject.Inject; + +public class AiPrivacyNoticeView extends ScrollPane { + @FXML private VBox text; + @FXML private GridPane aiPolicies; + @FXML private Text embeddingModelText; + @FXML private Button agreeButton; + @FXML private Hyperlink djlLink; + @FXML private Button hideAiTabsButton; + + @Inject private GuiPreferences preferences; + @Inject private DialogService dialogService; + + private AiPrivacyNoticeViewModel viewModel; + + public AiPrivacyNoticeView() { + ViewLoader.view(this) + .root(this) + .load(); + } + + @FXML + private void initialize() { + viewModel = new AiPrivacyNoticeViewModel( + preferences, + dialogService + ); + + initializeUi(); + initializeBindings(); + } + + private void initializeUi() { + addPrivacyHyperlink(aiPolicies, AiProvider.OPEN_AI); + addPrivacyHyperlink(aiPolicies, AiProvider.MISTRAL_AI); + addPrivacyHyperlink(aiPolicies, AiProvider.GEMINI); + addPrivacyHyperlink(aiPolicies, AiProvider.HUGGING_FACE); + addPrivacyHyperlink(aiPolicies, AiProvider.GPT4ALL); + + String embeddingTemplate = embeddingModelText.getText(); + String replaced = embeddingTemplate.replaceAll("%0", viewModel.embeddingModelSizeProperty().get()); + embeddingModelText.setText(replaced); + + djlLink.setOnAction(_ -> viewModel.openBrowser("https://github.com/deepjavalibrary/djl/discussions/3370#discussioncomment-10233632")); + + agreeButton.setOnAction(_ -> viewModel.onPrivacyAgree()); + hideAiTabsButton.setOnAction(_ -> viewModel.hideAITabs()); + } + + private void initializeBindings() { + DoubleBinding textWidth = Bindings.subtract(this.widthProperty(), 88d); + text.getChildren().forEach(child -> { + if (child instanceof Text line) { + line.wrappingWidthProperty().bind(textWidth); + } + }); + aiPolicies.prefWidthProperty().bind(textWidth); + embeddingModelText.wrappingWidthProperty().bind(textWidth); + } + + private void addPrivacyHyperlink(GridPane gridPane, AiProvider aiProvider) { + int row = gridPane.getRowCount(); + Label aiName = new Label(aiProvider.getDisplayName()); + gridPane.add(aiName, 0, row); + + Hyperlink hyperlink = new Hyperlink(aiProvider.getPrivacyPolicyUrl()); + hyperlink.setWrapText(true); + hyperlink.setOnAction(_ -> viewModel.openBrowser(aiProvider.getApiUrl())); + gridPane.add(hyperlink, 1, row); + } +} diff --git a/jabgui/src/main/java/org/jabref/gui/ai/AiPrivacyNoticeViewModel.java b/jabgui/src/main/java/org/jabref/gui/ai/AiPrivacyNoticeViewModel.java new file mode 100644 index 000000000000..df26a204fca7 --- /dev/null +++ b/jabgui/src/main/java/org/jabref/gui/ai/AiPrivacyNoticeViewModel.java @@ -0,0 +1,58 @@ +package org.jabref.gui.ai; + +import java.io.IOException; + +import javafx.beans.property.SimpleStringProperty; +import javafx.beans.property.StringProperty; + +import org.jabref.gui.DialogService; +import org.jabref.gui.desktop.os.NativeDesktop; +import org.jabref.gui.preferences.GuiPreferences; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AiPrivacyNoticeViewModel { + private static final Logger LOGGER = LoggerFactory.getLogger(AiPrivacyNoticeViewModel.class); + + private final StringProperty embeddingModelSize = new SimpleStringProperty(""); + + private final GuiPreferences preferences; + private final DialogService dialogService; + + public AiPrivacyNoticeViewModel( + GuiPreferences guiPreferences, + DialogService dialogService + ) { + this.preferences = guiPreferences; + this.dialogService = dialogService; + + preferences.getAiPreferences().embeddingModelProperty().addListener((_, _, value) -> + embeddingModelSize.set(value.sizeInfo()) + ); + } + + public StringProperty embeddingModelSizeProperty() { + return embeddingModelSize; + } + + public void onPrivacyAgree() { + preferences.getAiPreferences().setEnableAi(true); + } + + public void openBrowser(String link) { + try { + NativeDesktop.openBrowser(link, preferences.getExternalApplicationsPreferences()); + } catch (IOException e) { + LOGGER.error("Error opening the browser to the Privacy Policy page of the AI provider.", e); + + dialogService.showErrorDialogAndWait(e); + } + } + + public void hideAITabs() { + var entryEditorPreferences = preferences.getEntryEditorPreferences(); + entryEditorPreferences.setShouldShowAiSummaryTab(false); + entryEditorPreferences.setShouldShowAiChatTab(false); + } +} diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java index 2da2c3a9adfb..52b8c4dab993 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/aichat/AiChatComponent.java @@ -148,7 +148,7 @@ private void initializeNotice() { @VisibleForTesting String computeNoticeText() { - String provider = aiPreferences.getAiProvider().getLabel(); + String provider = aiPreferences.getAiProvider().getDisplayName(); String model = aiPreferences.getSelectedChatModel(); return noticeTemplate.replace("%0", provider + " " + model); } diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/AiPrivacyNoticeGuardedComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/AiPrivacyNoticeGuardedComponent.java index 09e309fa2c24..b016a9e49849 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/AiPrivacyNoticeGuardedComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/AiPrivacyNoticeGuardedComponent.java @@ -34,10 +34,7 @@ public final void rebuildUi() { } else { setContent( new PrivacyNoticeComponent( - aiPreferences, this::rebuildUi, - externalApplicationsPreferences, - dialogService, adaptVisibleTabs ) ); diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/PrivacyNoticeComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/PrivacyNoticeComponent.java index 016915364d41..2b3e31b07a36 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/PrivacyNoticeComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/privacynotice/PrivacyNoticeComponent.java @@ -16,9 +16,7 @@ import org.jabref.gui.desktop.os.NativeDesktop; import org.jabref.gui.entryeditor.AdaptVisibleTabs; import org.jabref.gui.entryeditor.EntryEditorPreferences; -import org.jabref.gui.frame.ExternalApplicationsPreferences; import org.jabref.gui.preferences.GuiPreferences; -import org.jabref.logic.ai.preferences.AiPreferences; import org.jabref.model.ai.llm.AiProvider; import com.airhacks.afterburner.views.ViewLoader; @@ -33,19 +31,14 @@ public class PrivacyNoticeComponent extends ScrollPane { @FXML private GridPane aiPolicies; @FXML private Text embeddingModelText; - private final AiPreferences aiPreferences; private final Runnable onIAgreeButtonClickCallback; - private final DialogService dialogService; private final AdaptVisibleTabs adaptVisibleTabs; - private final ExternalApplicationsPreferences externalApplicationsPreferences; + @Inject private DialogService dialogService; @Inject private GuiPreferences preferences; - public PrivacyNoticeComponent(AiPreferences aiPreferences, Runnable onIAgreeButtonClickCallback, ExternalApplicationsPreferences externalApplicationsPreferences, DialogService dialogService, AdaptVisibleTabs adaptVisibleTabs) { - this.aiPreferences = aiPreferences; + public PrivacyNoticeComponent(Runnable onIAgreeButtonClickCallback, AdaptVisibleTabs adaptVisibleTabs) { this.onIAgreeButtonClickCallback = onIAgreeButtonClickCallback; - this.externalApplicationsPreferences = externalApplicationsPreferences; - this.dialogService = dialogService; this.adaptVisibleTabs = adaptVisibleTabs; ViewLoader.view(this) @@ -61,7 +54,7 @@ private void initialize() { addPrivacyHyperlink(aiPolicies, AiProvider.HUGGING_FACE); addPrivacyHyperlink(aiPolicies, AiProvider.GPT4ALL); - String newEmbeddingModelText = embeddingModelText.getText().replaceAll("%0", aiPreferences.getEmbeddingModel().sizeInfo()); + String newEmbeddingModelText = embeddingModelText.getText().replaceAll("%0", preferences.getAiPreferences().getEmbeddingModel().sizeInfo()); embeddingModelText.setText(newEmbeddingModelText); // Because of the https://bugs.openjdk.org/browse/JDK-8090400 bug, the text in the privacy policy cannot be @@ -79,7 +72,7 @@ private void initialize() { private void addPrivacyHyperlink(GridPane gridPane, AiProvider aiProvider) { int row = gridPane.getRowCount(); - Label aiName = new Label(aiProvider.getLabel()); + Label aiName = new Label(aiProvider.getDisplayName()); gridPane.add(aiName, 0, row); Hyperlink hyperlink = new Hyperlink(aiProvider.getPrivacyPolicyUrl()); @@ -91,7 +84,7 @@ private void addPrivacyHyperlink(GridPane gridPane, AiProvider aiProvider) { @FXML private void onIAgreeButtonClick() { - aiPreferences.setEnableAi(true); + preferences.getAiPreferences().setEnableAi(true); onIAgreeButtonClickCallback.run(); } @@ -102,7 +95,7 @@ private void onDjlPrivacyPolicyClick() { private void openBrowser(String link) { try { - NativeDesktop.openBrowser(link, externalApplicationsPreferences); + NativeDesktop.openBrowser(link, preferences.getExternalApplicationsPreferences()); } catch (IOException e) { LOGGER.error("Error opening the browser to the Privacy Policy page of the AI provider.", e); dialogService.showErrorDialogAndWait(e); diff --git a/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryShowingComponent.java b/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryShowingComponent.java index 7d759d76baec..737c0e3e2be7 100644 --- a/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryShowingComponent.java +++ b/jabgui/src/main/java/org/jabref/gui/ai/components/summary/SummaryShowingComponent.java @@ -68,7 +68,7 @@ private void updateInfoText() { String newInfo = summaryInfoText .getText() .replaceAll("%0", formatTimestamp(bibEntrySummary.timestamp())) - .replaceAll("%1", bibEntrySummary.aiProvider().getLabel() + " " + bibEntrySummary.model()); + .replaceAll("%1", bibEntrySummary.aiProvider().getDisplayName() + " " + bibEntrySummary.model()); summaryInfoText.setText(newInfo); } diff --git a/jabgui/src/main/java/org/jabref/gui/ai/statuspane/ErrorStatusPaneView.java b/jabgui/src/main/java/org/jabref/gui/ai/statuspane/ErrorStatusPaneView.java new file mode 100644 index 000000000000..fbd3e55ae999 --- /dev/null +++ b/jabgui/src/main/java/org/jabref/gui/ai/statuspane/ErrorStatusPaneView.java @@ -0,0 +1,129 @@ +package org.jabref.gui.ai.statuspane; + +import javafx.beans.property.ObjectProperty; +import javafx.beans.property.StringProperty; +import javafx.event.ActionEvent; +import javafx.event.EventHandler; +import javafx.fxml.FXML; +import javafx.scene.control.Button; +import javafx.scene.control.Label; +import javafx.scene.control.TextArea; +import javafx.scene.layout.BorderPane; + +import com.airhacks.afterburner.views.ViewLoader; + +/** + * Shows title, description, text area, and restart and cancel buttons. + */ +public class ErrorStatusPaneView extends BorderPane { + @FXML private Label titleLabel; + @FXML private Label descriptionLabel; + @FXML private TextArea textArea; + @FXML private Button restartButton; + @FXML private Button cancelButton; + + private ErrorStatusPaneViewModel viewModel; + + public ErrorStatusPaneView() { + ViewLoader.view(this) + .root(this) + .load(); + } + + @FXML + private void initialize() { + viewModel = new ErrorStatusPaneViewModel(); + + titleLabel.textProperty().bind(viewModel.titleProperty()); + descriptionLabel.textProperty().bind(viewModel.descriptionProperty()); + textArea.textProperty().bind(viewModel.exceptionStringProperty()); + restartButton.textProperty().bind(viewModel.restartButtonTextProperty()); + cancelButton.textProperty().bind(viewModel.cancelButtonTextProperty()); + } + + public StringProperty titleProperty() { + return viewModel.titleProperty(); + } + + public String getTitle() { + return viewModel.titleProperty().get(); + } + + public void setTitle(String title) { + viewModel.titleProperty().set(title); + } + + public StringProperty descriptionProperty() { + return viewModel.descriptionProperty(); + } + + public String getDescription() { + return viewModel.descriptionProperty().get(); + } + + public void setDescription(String description) { + viewModel.descriptionProperty().set(description); + } + + public ObjectProperty exceptionProperty() { + return viewModel.exceptionProperty(); + } + + public StringProperty restartButtonTextProperty() { + return viewModel.restartButtonTextProperty(); + } + + public String getRestartButtonText() { + return viewModel.restartButtonTextProperty().get(); + } + + public void setRestartButtonText(String restartButtonText) { + viewModel.restartButtonTextProperty().set(restartButtonText); + } + + public ObjectProperty> onRestartProperty() { + return viewModel.onRestartProperty(); + } + + public EventHandler getOnRestart() { + return viewModel.onRestartProperty().get(); + } + + public void setOnRestart(EventHandler onRestart) { + viewModel.onRestartProperty().set(onRestart); + } + + public StringProperty cancelButtonTextProperty() { + return viewModel.cancelButtonTextProperty(); + } + + public String getCancelButtonText() { + return viewModel.cancelButtonTextProperty().get(); + } + + public void setCancelButtonText(String cancelButtonText) { + viewModel.cancelButtonTextProperty().set(cancelButtonText); + } + + public ObjectProperty> onCancelProperty() { + return viewModel.onCancelProperty(); + } + + public EventHandler getOnCancel() { + return viewModel.onCancelProperty().get(); + } + + public void setOnCancel(EventHandler onCancel) { + viewModel.onCancelProperty().set(onCancel); + } + + @FXML + private void restart() { + viewModel.restart(); + } + + @FXML + private void cancel() { + viewModel.cancel(); + } +} diff --git a/jabgui/src/main/java/org/jabref/gui/ai/statuspane/ErrorStatusPaneViewModel.java b/jabgui/src/main/java/org/jabref/gui/ai/statuspane/ErrorStatusPaneViewModel.java new file mode 100644 index 000000000000..d489e6c8e335 --- /dev/null +++ b/jabgui/src/main/java/org/jabref/gui/ai/statuspane/ErrorStatusPaneViewModel.java @@ -0,0 +1,69 @@ +package org.jabref.gui.ai.statuspane; + +import javafx.beans.property.ObjectProperty; +import javafx.beans.property.SimpleObjectProperty; +import javafx.beans.property.SimpleStringProperty; +import javafx.beans.property.StringProperty; +import javafx.event.ActionEvent; +import javafx.event.EventHandler; + +public class ErrorStatusPaneViewModel { + private final StringProperty title = new SimpleStringProperty(""); + private final StringProperty description = new SimpleStringProperty(""); + + private final ObjectProperty exception = new SimpleObjectProperty<>(); + private final StringProperty exceptionString = new SimpleStringProperty(""); + + private final StringProperty restartButtonText = new SimpleStringProperty(""); + private final ObjectProperty> onRestart = new SimpleObjectProperty<>(); + private final StringProperty cancelButtonText = new SimpleStringProperty(""); + private final ObjectProperty> onCancel = new SimpleObjectProperty<>(); + + public ErrorStatusPaneViewModel() { + exception.addListener((_, _, value) -> exceptionString.set(value.getMessage())); + } + + public void restart() { + if (onRestart.get() != null) { + onRestart.get().handle(new ActionEvent()); + } + } + + public void cancel() { + if (onCancel.get() != null) { + onCancel.get().handle(new ActionEvent()); + } + } + + public StringProperty titleProperty() { + return title; + } + + public StringProperty descriptionProperty() { + return description; + } + + public ObjectProperty exceptionProperty() { + return exception; + } + + public StringProperty exceptionStringProperty() { + return exceptionString; + } + + public StringProperty restartButtonTextProperty() { + return restartButtonText; + } + + public ObjectProperty> onRestartProperty() { + return onRestart; + } + + public StringProperty cancelButtonTextProperty() { + return cancelButtonText; + } + + public ObjectProperty> onCancelProperty() { + return onCancel; + } +} diff --git a/jabgui/src/main/java/org/jabref/gui/ai/statuspane/LoadingStatusPaneView.java b/jabgui/src/main/java/org/jabref/gui/ai/statuspane/LoadingStatusPaneView.java new file mode 100644 index 000000000000..31d0ff6a4590 --- /dev/null +++ b/jabgui/src/main/java/org/jabref/gui/ai/statuspane/LoadingStatusPaneView.java @@ -0,0 +1,56 @@ +package org.jabref.gui.ai.statuspane; + +import javafx.beans.property.StringProperty; +import javafx.fxml.FXML; +import javafx.scene.control.Label; +import javafx.scene.layout.BorderPane; + +import com.airhacks.afterburner.views.ViewLoader; + +/** + * Shows title, description, and progress indicator. + */ +public class LoadingStatusPaneView extends BorderPane { + @FXML private Label titleLabel; + @FXML private Label descriptionLabel; + + private LoadingStatusPaneViewModel viewModel; + + public LoadingStatusPaneView() { + ViewLoader.view(this) + .root(this) + .load(); + } + + @FXML + private void initialize() { + viewModel = new LoadingStatusPaneViewModel(); + + titleLabel.textProperty().bind(viewModel.titleProperty()); + descriptionLabel.textProperty().bind(viewModel.descriptionProperty()); + } + + public StringProperty titleProperty() { + return viewModel.titleProperty(); + } + + public String getTitle() { + return viewModel.titleProperty().get(); + } + + public void setTitle(String title) { + viewModel.titleProperty().set(title); + } + + public StringProperty descriptionProperty() { + return viewModel.descriptionProperty(); + } + + public String getDescription() { + return viewModel.descriptionProperty().get(); + } + + public void setDescription(String description) { + viewModel.descriptionProperty().set(description); + } +} diff --git a/jabgui/src/main/java/org/jabref/gui/ai/statuspane/LoadingStatusPaneViewModel.java b/jabgui/src/main/java/org/jabref/gui/ai/statuspane/LoadingStatusPaneViewModel.java new file mode 100644 index 000000000000..50d7b243785d --- /dev/null +++ b/jabgui/src/main/java/org/jabref/gui/ai/statuspane/LoadingStatusPaneViewModel.java @@ -0,0 +1,17 @@ +package org.jabref.gui.ai.statuspane; + +import javafx.beans.property.SimpleStringProperty; +import javafx.beans.property.StringProperty; + +public class LoadingStatusPaneViewModel { + private final StringProperty title = new SimpleStringProperty(""); + private final StringProperty description = new SimpleStringProperty(""); + + public StringProperty titleProperty() { + return title; + } + + public StringProperty descriptionProperty() { + return description; + } +} diff --git a/jabgui/src/main/java/org/jabref/gui/ai/statuspane/SimpleStatusPaneView.java b/jabgui/src/main/java/org/jabref/gui/ai/statuspane/SimpleStatusPaneView.java new file mode 100644 index 000000000000..c698e9bac2bc --- /dev/null +++ b/jabgui/src/main/java/org/jabref/gui/ai/statuspane/SimpleStatusPaneView.java @@ -0,0 +1,56 @@ +package org.jabref.gui.ai.statuspane; + +import javafx.beans.property.StringProperty; +import javafx.fxml.FXML; +import javafx.scene.control.Label; +import javafx.scene.layout.BorderPane; + +import com.airhacks.afterburner.views.ViewLoader; + +/** + * Shows title and description. + */ +public class SimpleStatusPaneView extends BorderPane { + @FXML private Label titleLabel; + @FXML private Label descriptionLabel; + + private SimpleStatusPaneViewModel viewModel; + + public SimpleStatusPaneView() { + ViewLoader.view(this) + .root(this) + .load(); + } + + @FXML + private void initialize() { + viewModel = new SimpleStatusPaneViewModel(); + + titleLabel.textProperty().bind(viewModel.titleProperty()); + descriptionLabel.textProperty().bind(viewModel.descriptionProperty()); + } + + public StringProperty titleProperty() { + return viewModel.titleProperty(); + } + + public String getTitle() { + return viewModel.titleProperty().get(); + } + + public void setTitle(String title) { + viewModel.titleProperty().set(title); + } + + public StringProperty descriptionProperty() { + return viewModel.descriptionProperty(); + } + + public String getDescription() { + return viewModel.descriptionProperty().get(); + } + + public void setDescription(String description) { + viewModel.descriptionProperty().set(description); + } +} diff --git a/jabgui/src/main/java/org/jabref/gui/ai/statuspane/SimpleStatusPaneViewModel.java b/jabgui/src/main/java/org/jabref/gui/ai/statuspane/SimpleStatusPaneViewModel.java new file mode 100644 index 000000000000..5996d7051755 --- /dev/null +++ b/jabgui/src/main/java/org/jabref/gui/ai/statuspane/SimpleStatusPaneViewModel.java @@ -0,0 +1,17 @@ +package org.jabref.gui.ai.statuspane; + +import javafx.beans.property.SimpleStringProperty; +import javafx.beans.property.StringProperty; + +public class SimpleStatusPaneViewModel { + private final StringProperty title = new SimpleStringProperty(""); + private final StringProperty description = new SimpleStringProperty(""); + + public StringProperty titleProperty() { + return title; + } + + public StringProperty descriptionProperty() { + return description; + } +} diff --git a/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java b/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java index e1dbb7db3edc..5d445d52f117 100644 --- a/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java +++ b/jabgui/src/main/java/org/jabref/gui/entryeditor/AiChatTab.java @@ -90,7 +90,7 @@ protected void bindToEntry(BibEntry entry) { } private void showPrivacyNotice(BibEntry entry) { - setContent(new PrivacyNoticeComponent(aiPreferences, () -> bindToEntry(entry), externalApplicationsPreferences, dialogService, adaptVisibleTabs)); + setContent(new PrivacyNoticeComponent(() -> bindToEntry(entry), adaptVisibleTabs)); } private void showErrorNotPdfs() { diff --git a/jabgui/src/main/java/org/jabref/gui/entryeditor/AiSummaryTab.java b/jabgui/src/main/java/org/jabref/gui/entryeditor/AiSummaryTab.java index 310a76c00564..5747e5f9d274 100644 --- a/jabgui/src/main/java/org/jabref/gui/entryeditor/AiSummaryTab.java +++ b/jabgui/src/main/java/org/jabref/gui/entryeditor/AiSummaryTab.java @@ -2,50 +2,36 @@ import javafx.scene.control.Tooltip; -import org.jabref.gui.DialogService; import org.jabref.gui.StateManager; -import org.jabref.gui.ai.components.summary.SummaryComponent; -import org.jabref.gui.frame.ExternalApplicationsPreferences; +import org.jabref.gui.entryeditor.aisummary.AiSummaryView; import org.jabref.gui.preferences.GuiPreferences; -import org.jabref.logic.ai.AiService; -import org.jabref.logic.ai.preferences.AiPreferences; -import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; import org.jabref.logic.l10n.Localization; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; public class AiSummaryTab extends EntryEditorTab { - private final AiService aiService; - private final DialogService dialogService; + private final GuiPreferences preferences; private final StateManager stateManager; - private final AdaptVisibleTabs adaptVisibleTabs; - private final AiPreferences aiPreferences; - private final ExternalApplicationsPreferences externalApplicationsPreferences; - private final CitationKeyPatternPreferences citationKeyPatternPreferences; - private final EntryEditorPreferences entryEditorPreferences; - public AiSummaryTab(AiService aiService, - DialogService dialogService, - StateManager stateManager, - AdaptVisibleTabs adaptVisibleTabs, - GuiPreferences preferences) { - this.aiService = aiService; - this.dialogService = dialogService; + private final AiSummaryView aiSummaryView; + + public AiSummaryTab( + GuiPreferences preferences, + StateManager stateManager + ) { + this.preferences = preferences; this.stateManager = stateManager; - this.adaptVisibleTabs = adaptVisibleTabs; - this.aiPreferences = preferences.getAiPreferences(); - this.externalApplicationsPreferences = preferences.getExternalApplicationsPreferences(); - this.citationKeyPatternPreferences = preferences.getCitationKeyPatternPreferences(); - this.entryEditorPreferences = preferences.getEntryEditorPreferences(); + this.aiSummaryView = new AiSummaryView(); setText(Localization.lang("AI summary")); setTooltip(new Tooltip(Localization.lang("AI-generated summary of attached file(s)"))); + setContent(aiSummaryView); } @Override public boolean shouldShow(BibEntry entry) { - return entryEditorPreferences.shouldShowAiSummaryTab(); + return preferences.getEntryEditorPreferences().shouldShowAiSummaryTab(); } /** @@ -54,15 +40,6 @@ public boolean shouldShow(BibEntry entry) { @Override protected void bindToEntry(BibEntry entry) { BibDatabaseContext bibDatabaseContext = stateManager.getActiveDatabase().orElse(new BibDatabaseContext()); - setContent(new SummaryComponent( - bibDatabaseContext, - entry, - aiService, - aiPreferences, - externalApplicationsPreferences, - citationKeyPatternPreferences, - dialogService, - adaptVisibleTabs - )); + aiSummaryView.bind(bibDatabaseContext, entry); } } diff --git a/jabgui/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java b/jabgui/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java index 10d0969bfdb8..9c4bb63ac179 100644 --- a/jabgui/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java +++ b/jabgui/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java @@ -360,7 +360,7 @@ private List createTabs() { tabs.add(sourceTab); tabs.add(new LatexCitationsTab(preferences, dialogService, stateManager, directoryMonitor)); tabs.add(new FulltextSearchResultsTab(stateManager, preferences, dialogService, taskExecutor, this)); - tabs.add(new AiSummaryTab(aiService, dialogService, stateManager, this, preferences)); + tabs.add(new AiSummaryTab(preferences, stateManager)); tabs.add(new AiChatTab(aiService, dialogService, preferences, stateManager, this, taskExecutor)); return tabs; diff --git a/jabgui/src/main/java/org/jabref/gui/entryeditor/aisummary/AiSummaryView.java b/jabgui/src/main/java/org/jabref/gui/entryeditor/aisummary/AiSummaryView.java index 5f933c015d24..c05867b0000d 100644 --- a/jabgui/src/main/java/org/jabref/gui/entryeditor/aisummary/AiSummaryView.java +++ b/jabgui/src/main/java/org/jabref/gui/entryeditor/aisummary/AiSummaryView.java @@ -1,14 +1,140 @@ package org.jabref.gui.entryeditor.aisummary; -import javafx.scene.layout.Pane; +import javafx.beans.binding.Bindings; +import javafx.beans.property.ObjectProperty; +import javafx.fxml.FXML; +import javafx.scene.control.Button; +import javafx.scene.control.ComboBox; +import javafx.scene.control.Label; +import javafx.scene.layout.BorderPane; +import javafx.scene.layout.StackPane; + +import org.jabref.gui.ai.AiPrivacyNoticeView; +import org.jabref.gui.ai.statuspane.ErrorStatusPaneView; +import org.jabref.gui.ai.statuspane.LoadingStatusPaneView; +import org.jabref.gui.ai.statuspane.SimpleStatusPaneView; +import org.jabref.gui.preferences.GuiPreferences; +import org.jabref.logic.ai.AiService; +import org.jabref.logic.l10n.Localization; +import org.jabref.model.ai.identifiers.FullBibEntryAiIdentifier; +import org.jabref.model.ai.summarization.SummarizatorKind; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; import com.airhacks.afterburner.views.ViewLoader; +import jakarta.inject.Inject; + +public class AiSummaryView extends StackPane { + @FXML private AiPrivacyNoticeView privacyNotice; + + @FXML private BorderPane pendingPane; + @FXML private ComboBox summarizatorCombo; + @FXML private Button generateButton; + @FXML private Label pendingHint; + + @FXML private LoadingStatusPaneView processingPane; + @FXML private ErrorStatusPaneView errorPane; + + @FXML private SimpleStatusPaneView noDatabasePathPane; + @FXML private SimpleStatusPaneView noCitationKeyPane; + @FXML private SimpleStatusPaneView wrongCitationKeyPane; + @FXML private SimpleStatusPaneView noFilesPane; + @FXML private SimpleStatusPaneView noSupportedFileTypesPane; + + @Inject private GuiPreferences preferences; + @Inject private AiService aiService; + + private AiSummaryViewModel viewModel; -public class AiSummaryView extends Pane { public AiSummaryView() { - ViewLoader - .view(this) - .root(this) - .load(); + ViewLoader.view(this) + .root(this) + .load(); + } + + @FXML + private void initialize() { + viewModel = new AiSummaryViewModel( + preferences, + aiService + ); + + ObjectProperty state = viewModel.stateProperty(); + + privacyNotice.visibleProperty().bind(viewModel.showAiPrivacyPolicyGuardProperty()); + privacyNotice.managedProperty().bind(privacyNotice.visibleProperty()); + + pendingPane.visibleProperty().bind(state.isEqualTo(AiSummaryViewModel.State.PENDING).and(viewModel.showAiPrivacyPolicyGuardProperty().not())); + pendingPane.managedProperty().bind(pendingPane.visibleProperty()); + viewModel.processingAiProviderProperty().addListener(_ -> { + updatePendingHint(); + updateProcessingLabel(); + }); + viewModel.processingLlmNameProperty().addListener(_ -> { + updatePendingHint(); + updateProcessingLabel(); + }); + + processingPane.visibleProperty().bind(state.isEqualTo(AiSummaryViewModel.State.PROCESSING)); + processingPane.managedProperty().bind(processingPane.visibleProperty()); + viewModel.selectedSummarizatorKindProperty().addListener(_ -> updateProcessingLabel()); + + errorPane.visibleProperty().bind(state.isEqualTo(AiSummaryViewModel.State.ERROR_WHILE_GENERATING)); + errorPane.managedProperty().bind(errorPane.visibleProperty()); + + noDatabasePathPane.visibleProperty().bind(state.isEqualTo(AiSummaryViewModel.State.NO_DATABASE_PATH)); + noDatabasePathPane.managedProperty().bind(noDatabasePathPane.visibleProperty()); + + noCitationKeyPane.visibleProperty().bind(state.isEqualTo(AiSummaryViewModel.State.NO_CITATION_KEY)); + noCitationKeyPane.managedProperty().bind(noCitationKeyPane.visibleProperty()); + + wrongCitationKeyPane.visibleProperty().bind(state.isEqualTo(AiSummaryViewModel.State.WRONG_CITATION_KEY)); + wrongCitationKeyPane.managedProperty().bind(wrongCitationKeyPane.visibleProperty()); + + noFilesPane.visibleProperty().bind(state.isEqualTo(AiSummaryViewModel.State.NO_FILES)); + noFilesPane.managedProperty().bind(noFilesPane.visibleProperty()); + + noSupportedFileTypesPane.visibleProperty().bind(state.isEqualTo(AiSummaryViewModel.State.NO_SUPPORTED_FILE_TYPES)); + noSupportedFileTypesPane.managedProperty().bind(noSupportedFileTypesPane.visibleProperty()); + + summarizatorCombo.itemsProperty().bind(viewModel.summarizatorKindsProperty()); + summarizatorCombo.valueProperty().bindBidirectional(viewModel.selectedSummarizatorKindProperty()); + + errorSummarizatorCombo.itemsProperty().bind(viewModel.summarizatorKindsProperty()); + errorSummarizatorCombo.valueProperty().bindBidirectional(viewModel.selectedSummarizatorKindProperty()); + + generateButton.setOnAction(_ -> viewModel.generate()); + regenerateButton.setOnAction(_ -> viewModel.regenerate()); + + processingLabel.textProperty().bind(Bindings.concat("Processing with ", viewModel.processingAiProviderProperty().asString(), " ", viewModel.processingLlmNameProperty())); + + viewModel.errorProperty().addListener((_, _, value) -> { + if (value.isPresent()) { + errorText.setText(value.get().getLocalizedMessage()); + } else { + errorText.setText(""); + } + }); + } + + public void bind(BibDatabaseContext bibDatabaseContext, BibEntry entry) { + viewModel.bindEntry(new FullBibEntryAiIdentifier(bibDatabaseContext, entry)); + } + + private void updatePendingHint() { + pendingHint.setText(Localization.lang( + "Your entry will be processed by %0 %1", + viewModel.processingAiProviderProperty().get().getDisplayName(), + viewModel.processingLlmNameProperty().get()) + ); + } + + private void updateProcessingLabel() { + processingLabel.setText(Localization.lang( + "Your entry is being summarized by %0 %1 using algorithm %3", + viewModel.processingAiProviderProperty().get().getDisplayName(), + viewModel.processingLlmNameProperty().get(), + viewModel.selectedSummarizatorKindProperty().get().getDisplayName() + )); } } diff --git a/jabgui/src/main/java/org/jabref/gui/entryeditor/aisummary/AiSummaryViewModel.java b/jabgui/src/main/java/org/jabref/gui/entryeditor/aisummary/AiSummaryViewModel.java index 6a97182708c7..d5b41940e912 100644 --- a/jabgui/src/main/java/org/jabref/gui/entryeditor/aisummary/AiSummaryViewModel.java +++ b/jabgui/src/main/java/org/jabref/gui/entryeditor/aisummary/AiSummaryViewModel.java @@ -1,30 +1,70 @@ package org.jabref.gui.entryeditor.aisummary; +import java.nio.file.Path; import java.time.LocalDateTime; import java.util.Optional; import javafx.beans.property.BooleanProperty; +import javafx.beans.property.ListProperty; import javafx.beans.property.ObjectProperty; import javafx.beans.property.SimpleBooleanProperty; +import javafx.beans.property.SimpleListProperty; import javafx.beans.property.SimpleObjectProperty; import javafx.beans.property.SimpleStringProperty; import javafx.beans.property.StringProperty; +import javafx.collections.FXCollections; import org.jabref.gui.AbstractViewModel; +import org.jabref.gui.preferences.GuiPreferences; +import org.jabref.logic.ai.AiService; +import org.jabref.logic.ai.customimplementations.llms.ChatModel; +import org.jabref.logic.ai.pipeline.logic.parsing.UniversalContentParser; import org.jabref.logic.ai.preferences.AiPreferences; +import org.jabref.logic.ai.summarization.logic.SummarizatorFactory; +import org.jabref.logic.ai.summarization.logic.summarizationalgorithms.Summarizator; +import org.jabref.logic.ai.summarization.tasks.generatesummary.GenerateSummaryTask; +import org.jabref.logic.ai.summarization.tasks.generatesummary.GenerateSummaryTaskRequest; +import org.jabref.logic.ai.util.TrackedBackgroundTask; +import org.jabref.logic.util.CitationKeyCheck; import org.jabref.logic.util.OptionalObjectProperty; +import org.jabref.model.ai.identifiers.BibEntryAiIdentifier; +import org.jabref.model.ai.identifiers.FullBibEntryAiIdentifier; import org.jabref.model.ai.llm.AiProvider; +import org.jabref.model.ai.summarization.BibEntrySummary; +import org.jabref.model.ai.summarization.SummarizatorKind; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class AiSummaryViewModel extends AbstractViewModel { public enum State { + PENDING, PROCESSING, DONE, - ERROR + ERROR_WHILE_GENERATING, + NO_DATABASE_PATH, + NO_CITATION_KEY, + WRONG_CITATION_KEY, + NO_FILES, + NO_SUPPORTED_FILE_TYPES } + private static final Logger LOGGER = LoggerFactory.getLogger(AiSummaryViewModel.class); + + private final GuiPreferences preferences; + private final AiService aiService; + // Global properties. private final BooleanProperty showAiPrivacyPolicyGuard = new SimpleBooleanProperty(true); - private final ObjectProperty state = new SimpleObjectProperty<>(State.PROCESSING); + private final ObjectProperty state = new SimpleObjectProperty<>(State.PENDING); + + // Pending state properties. + private final ListProperty summarizatorKinds = new SimpleListProperty<>( + FXCollections.observableArrayList(SummarizatorKind.values()) + ); + private final ObjectProperty selectedSummarizatorKind = new SimpleObjectProperty<>(); // Error state properties. private final OptionalObjectProperty error = new OptionalObjectProperty<>(Optional.empty()); @@ -34,29 +74,135 @@ public enum State { private final StringProperty processingLlmName = new SimpleStringProperty(""); // Done state properties. - private final StringProperty summary = new SimpleStringProperty(""); + private final StringProperty summaryContent = new SimpleStringProperty(""); private final BooleanProperty summaryRenderMarkdown = new SimpleBooleanProperty(false); private final ObjectProperty summaryTimestamp = new SimpleObjectProperty<>(); private final ObjectProperty summaryAiProvider = new SimpleObjectProperty<>(); - private final StringProperty summaryLlmName = new SimpleStringProperty(""); + private final StringProperty summaryModel = new SimpleStringProperty(""); + + private final ObjectProperty summarizator = new SimpleObjectProperty<>(); + // Future proofing: in case it would be possible to change the chat model in the View, this property will be useful. + private final ObjectProperty chatModel = new SimpleObjectProperty<>(); + + private final ObjectProperty entry = new SimpleObjectProperty<>(); public AiSummaryViewModel( - AiPreferences aiPreferences + GuiPreferences guiPreferences, + AiService aiService ) { + this.preferences = guiPreferences; + this.aiService = aiService; + + AiPreferences aiPreferences = preferences.getAiPreferences(); + + selectedSummarizatorKind.set(aiPreferences.getSummarizatorKind()); + showAiPrivacyPolicyGuard.bind(aiPreferences.enableAiProperty()); aiPreferences.aiProviderProperty().bind(processingAiProvider); aiPreferences.addListenerToChatModels(() -> processingLlmName.set(aiPreferences.getSelectedChatModel())); - startSummarization(); + selectedSummarizatorKind.addListener((_, _, newValue) -> + summarizator.set(SummarizatorFactory.createSummarizator(aiService.getCurrentAiTemplates(), newValue))); + + chatModel.set(aiService.getChatLanguageModel()); + + entry.addListener((_, _, newEntry) -> { + if (aiPreferences.getEnableAi()) { + generate(newEntry); + } + } + ); + } + + public void bindEntry(FullBibEntryAiIdentifier entry) { + this.entry.set(entry); } public void regenerate() { - clearSummary(); - startSummarization(); + if (entry.get() != null) { + regenerate(entry.get()); + } } - private void clearSummary() { + public void generate() { + if (entry.get() != null) { + generate(entry.get()); + } + } + + private void regenerate(FullBibEntryAiIdentifier identifier) { + clearSummary(identifier); + startSummarization(identifier, true); + } + + private void generate(FullBibEntryAiIdentifier identifier) { + startSummarization(identifier, false); + } + + public void clearSummary(FullBibEntryAiIdentifier identifier) { + Optional path = identifier.databaseContext().getDatabasePath(); + Optional citationKey = identifier.entry().getCitationKey(); + + if (path.isEmpty() || citationKey.isEmpty()) { + LOGGER.warn("Could not clear stored summary for regeneration"); + return; + } + aiService.getSummariesRepository().clear(new BibEntryAiIdentifier(path.get(), citationKey.get())); + } + + private void startSummarization(FullBibEntryAiIdentifier identifier, boolean regenerate) { + BibDatabaseContext bibDatabaseContext = identifier.databaseContext(); + BibEntry entry = identifier.entry(); + + if (bibDatabaseContext.getDatabasePath().isEmpty()) { + state.set(State.NO_DATABASE_PATH); + } else if (entry.getCitationKey().isEmpty() || CitationKeyCheck.hasEmptyCitationKey(entry)) { + state.set(State.NO_CITATION_KEY); + } else if (!CitationKeyCheck.citationKeyIsUnique(bibDatabaseContext, entry.getCitationKey().get())) { + state.set(State.WRONG_CITATION_KEY); + } else if (entry.getFiles().isEmpty()) { + state.set(State.NO_FILES); + } else if (entry.getFiles().stream().map(f -> Path.of(f.getLink())).noneMatch(UniversalContentParser::isSupportedFileType)) { + state.set(State.NO_SUPPORTED_FILE_TYPES); + } + + GenerateSummaryTask task = aiService.getSummarizationTaskAggregator().start( + new GenerateSummaryTaskRequest( + preferences.getFilePreferences(), + chatModel.get(), + aiService.getSummariesRepository(), + summarizator.get(), + bibDatabaseContext, + entry, + regenerate, + aiService.getShutdownSignal() + ) + ); + + task.statusProperty().addListener((_, _, value) -> { + switch (value) { + case TrackedBackgroundTask.Status.CANCELLED -> + state.set(State.PENDING); + + case TrackedBackgroundTask.Status.PENDING -> + state.set(State.PROCESSING); + + case TrackedBackgroundTask.Status.ERROR -> { + state.set(State.ERROR_WHILE_GENERATING); + error.set(Optional.of(task.getException())); + } + + case TrackedBackgroundTask.Status.SUCCESS -> { + state.set(State.DONE); + BibEntrySummary summary = task.getResult(); + summaryContent.set(summary.content()); + summaryAiProvider.set(summary.aiProvider()); + summaryModel.set(summary.model()); + summaryTimestamp.set(summary.timestamp()); + } + } + }); } public BooleanProperty showAiPrivacyPolicyGuardProperty() { @@ -79,8 +225,8 @@ public StringProperty processingLlmNameProperty() { return processingLlmName; } - public StringProperty summaryProperty() { - return summary; + public StringProperty summaryContentProperty() { + return summaryContent; } public BooleanProperty summaryRenderMarkdownProperty() { @@ -95,7 +241,15 @@ public ObjectProperty summaryAiProviderProperty() { return summaryAiProvider; } - public StringProperty summaryLlmNameProperty() { - return summaryLlmName; + public StringProperty summaryModelProperty() { + return summaryModel; + } + + public ListProperty summarizatorKindsProperty() { + return summarizatorKinds; + } + + public ObjectProperty selectedSummarizatorKindProperty() { + return selectedSummarizatorKind; } } diff --git a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java index f082ad1d7da6..608b1c880fdb 100644 --- a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java +++ b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTab.java @@ -25,7 +25,7 @@ import org.jabref.logic.l10n.Localization; import org.jabref.model.ai.embeddings.EmbeddingModelEnumeration; import org.jabref.model.ai.llm.AiProvider; -import org.jabref.model.ai.templating.AiTemplate; +import org.jabref.model.ai.templating.AiTemplateKind; import com.airhacks.afterburner.views.ViewLoader; import com.dlsc.unitfx.IntegerInputField; @@ -109,14 +109,14 @@ private void initializeHelp() { } private void initializeTemplates() { - systemMessageTextArea.textProperty().bindBidirectional(viewModel.getTemplateSources().get(AiTemplate.CHATTING_SYSTEM_MESSAGE)); - userMessageTextArea.textProperty().bindBidirectional(viewModel.getTemplateSources().get(AiTemplate.CHATTING_USER_MESSAGE)); - summarizationChunkSystemMessageTextArea.textProperty().bindBidirectional(viewModel.getTemplateSources().get(AiTemplate.SUMMARIZATION_CHUNK_SYSTEM_MESSAGE)); - summarizationChunkUserMessageTextArea.textProperty().bindBidirectional(viewModel.getTemplateSources().get(AiTemplate.SUMMARIZATION_CHUNK_USER_MESSAGE)); - summarizationCombineSystemMessageTextArea.textProperty().bindBidirectional(viewModel.getTemplateSources().get(AiTemplate.SUMMARIZATION_COMBINE_SYSTEM_MESSAGE)); - summarizationCombineUserMessageTextArea.textProperty().bindBidirectional(viewModel.getTemplateSources().get(AiTemplate.SUMMARIZATION_COMBINE_USER_MESSAGE)); - citationParsingSystemMessageTextArea.textProperty().bindBidirectional(viewModel.getTemplateSources().get(AiTemplate.CITATION_PARSING_SYSTEM_MESSAGE)); - citationParsingUserMessageTextArea.textProperty().bindBidirectional(viewModel.getTemplateSources().get(AiTemplate.CITATION_PARSING_USER_MESSAGE)); + systemMessageTextArea.textProperty().bindBidirectional(viewModel.getTemplateSources().get(AiTemplateKind.CHATTING_SYSTEM_MESSAGE)); + userMessageTextArea.textProperty().bindBidirectional(viewModel.getTemplateSources().get(AiTemplateKind.CHATTING_USER_MESSAGE)); + summarizationChunkSystemMessageTextArea.textProperty().bindBidirectional(viewModel.getTemplateSources().get(AiTemplateKind.SUMMARIZATION_CHUNK_SYSTEM_MESSAGE)); + summarizationChunkUserMessageTextArea.textProperty().bindBidirectional(viewModel.getTemplateSources().get(AiTemplateKind.SUMMARIZATION_CHUNK_USER_MESSAGE)); + summarizationCombineSystemMessageTextArea.textProperty().bindBidirectional(viewModel.getTemplateSources().get(AiTemplateKind.SUMMARIZATION_COMBINE_SYSTEM_MESSAGE)); + summarizationCombineUserMessageTextArea.textProperty().bindBidirectional(viewModel.getTemplateSources().get(AiTemplateKind.SUMMARIZATION_COMBINE_USER_MESSAGE)); + citationParsingSystemMessageTextArea.textProperty().bindBidirectional(viewModel.getTemplateSources().get(AiTemplateKind.CITATION_PARSING_SYSTEM_MESSAGE)); + citationParsingUserMessageTextArea.textProperty().bindBidirectional(viewModel.getTemplateSources().get(AiTemplateKind.CITATION_PARSING_USER_MESSAGE)); templatesTabPane.getSelectionModel().selectedItemProperty().addListener(_ -> viewModel.selectedTemplateProperty().set(getAiTemplate())); } @@ -274,24 +274,24 @@ public ReadOnlyBooleanProperty aiEnabledProperty() { return enableAi.selectedProperty(); } - public Optional getAiTemplate() { + public Optional getAiTemplate() { Tab selectedTab = templatesTabPane.getSelectionModel().getSelectedItem(); if (selectedTab == systemMessageForChattingTab) { - return Optional.of(AiTemplate.CHATTING_SYSTEM_MESSAGE); + return Optional.of(AiTemplateKind.CHATTING_SYSTEM_MESSAGE); } else if (selectedTab == userMessageForChattingTab) { - return Optional.of(AiTemplate.CHATTING_USER_MESSAGE); + return Optional.of(AiTemplateKind.CHATTING_USER_MESSAGE); } else if (selectedTab == summarizationChunkSystemMessageTab) { - return Optional.of(AiTemplate.SUMMARIZATION_CHUNK_SYSTEM_MESSAGE); + return Optional.of(AiTemplateKind.SUMMARIZATION_CHUNK_SYSTEM_MESSAGE); } else if (selectedTab == summarizationChunkUserMessageTab) { - return Optional.of(AiTemplate.SUMMARIZATION_CHUNK_USER_MESSAGE); + return Optional.of(AiTemplateKind.SUMMARIZATION_CHUNK_USER_MESSAGE); } else if (selectedTab == summarizationCombineSystemMessageTab) { - return Optional.of(AiTemplate.SUMMARIZATION_COMBINE_SYSTEM_MESSAGE); + return Optional.of(AiTemplateKind.SUMMARIZATION_COMBINE_SYSTEM_MESSAGE); } else if (selectedTab == summarizationCombineUserMessageTab) { - return Optional.of(AiTemplate.SUMMARIZATION_COMBINE_USER_MESSAGE); + return Optional.of(AiTemplateKind.SUMMARIZATION_COMBINE_USER_MESSAGE); } else if (selectedTab == citationParsingSystemMessageTab) { - return Optional.of(AiTemplate.CITATION_PARSING_SYSTEM_MESSAGE); + return Optional.of(AiTemplateKind.CITATION_PARSING_SYSTEM_MESSAGE); } else if (selectedTab == citationParsingUserMessageTab) { - return Optional.of(AiTemplate.CITATION_PARSING_USER_MESSAGE); + return Optional.of(AiTemplateKind.CITATION_PARSING_USER_MESSAGE); } return Optional.empty(); diff --git a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java index 064c55d3357b..7d7aae760a58 100644 --- a/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java +++ b/jabgui/src/main/java/org/jabref/gui/preferences/ai/AiTabViewModel.java @@ -30,7 +30,7 @@ import org.jabref.logic.util.strings.StringUtil; import org.jabref.model.ai.embeddings.EmbeddingModelEnumeration; import org.jabref.model.ai.llm.AiProvider; -import org.jabref.model.ai.templating.AiTemplate; +import org.jabref.model.ai.templating.AiTemplateKind; import de.saxsys.mvvmfx.utils.validation.FunctionBasedValidator; import de.saxsys.mvvmfx.utils.validation.ValidationMessage; @@ -84,18 +84,18 @@ public class AiTabViewModel implements PreferenceTabViewModel { private final StringProperty huggingFaceApiBaseUrl = new SimpleStringProperty(); private final StringProperty gpt4AllApiBaseUrl = new SimpleStringProperty(); - private final Map templateSources = Map.of( - AiTemplate.CHATTING_SYSTEM_MESSAGE, new SimpleStringProperty(), - AiTemplate.CHATTING_USER_MESSAGE, new SimpleStringProperty(), - AiTemplate.SUMMARIZATION_CHUNK_SYSTEM_MESSAGE, new SimpleStringProperty(), - AiTemplate.SUMMARIZATION_CHUNK_USER_MESSAGE, new SimpleStringProperty(), - AiTemplate.SUMMARIZATION_COMBINE_SYSTEM_MESSAGE, new SimpleStringProperty(), - AiTemplate.SUMMARIZATION_COMBINE_USER_MESSAGE, new SimpleStringProperty(), - AiTemplate.CITATION_PARSING_SYSTEM_MESSAGE, new SimpleStringProperty(), - AiTemplate.CITATION_PARSING_USER_MESSAGE, new SimpleStringProperty() + private final Map templateSources = Map.of( + AiTemplateKind.CHATTING_SYSTEM_MESSAGE, new SimpleStringProperty(), + AiTemplateKind.CHATTING_USER_MESSAGE, new SimpleStringProperty(), + AiTemplateKind.SUMMARIZATION_CHUNK_SYSTEM_MESSAGE, new SimpleStringProperty(), + AiTemplateKind.SUMMARIZATION_CHUNK_USER_MESSAGE, new SimpleStringProperty(), + AiTemplateKind.SUMMARIZATION_COMBINE_SYSTEM_MESSAGE, new SimpleStringProperty(), + AiTemplateKind.SUMMARIZATION_COMBINE_USER_MESSAGE, new SimpleStringProperty(), + AiTemplateKind.CITATION_PARSING_SYSTEM_MESSAGE, new SimpleStringProperty(), + AiTemplateKind.CITATION_PARSING_USER_MESSAGE, new SimpleStringProperty() ); - private final OptionalObjectProperty selectedTemplate = OptionalObjectProperty.empty(); + private final OptionalObjectProperty selectedTemplate = OptionalObjectProperty.empty(); private final StringProperty temperature = new SimpleStringProperty(); private final IntegerProperty contextWindowSize = new SimpleIntegerProperty(); @@ -349,7 +349,7 @@ public void setValues() { selectedEmbeddingModel.setValue(aiPreferences.getEmbeddingModel()); - Arrays.stream(AiTemplate.values()).forEach(template -> + Arrays.stream(AiTemplateKind.values()).forEach(template -> templateSources.get(template).set(aiPreferences.getTemplate(template))); temperature.setValue(LocalizedNumbers.doubleToString(aiPreferences.getTemperature())); @@ -392,7 +392,7 @@ public void storeSettings() { aiPreferences.setHuggingFaceApiBaseUrl(huggingFaceApiBaseUrl.get() == null ? "" : huggingFaceApiBaseUrl.get()); aiPreferences.setGpt4AllApiBaseUrl(gpt4AllApiBaseUrl.get() == null ? "" : gpt4AllApiBaseUrl.get()); - Arrays.stream(AiTemplate.values()).forEach(template -> + Arrays.stream(AiTemplateKind.values()).forEach(template -> aiPreferences.setTemplate(template, templateSources.get(template).get())); // We already check the correctness of temperature and RAG minimum score in validators, so we don't need to check it here. @@ -419,7 +419,7 @@ public void resetExpertSettings() { } public void resetTemplates() { - Arrays.stream(AiTemplate.values()).forEach(template -> + Arrays.stream(AiTemplateKind.values()).forEach(template -> templateSources.get(template).set(AiDefaultTemplates.getTemplate(template))); } @@ -529,11 +529,11 @@ public BooleanProperty disableApiBaseUrlProperty() { return disableApiBaseUrl; } - public Map getTemplateSources() { + public Map getTemplateSources() { return templateSources; } - public OptionalObjectProperty selectedTemplateProperty() { + public OptionalObjectProperty selectedTemplateProperty() { return selectedTemplate; } diff --git a/jabgui/src/main/resources/org/jabref/gui/ai/AiPrivacyNotice.fxml b/jabgui/src/main/resources/org/jabref/gui/ai/AiPrivacyNotice.fxml new file mode 100644 index 000000000000..f2be7cf8b51d --- /dev/null +++ b/jabgui/src/main/resources/org/jabref/gui/ai/AiPrivacyNotice.fxml @@ -0,0 +1,43 @@ + + + + + + + + + + + + +