refactor: embedding 모델을 hugging face -> openai로 수정한다

Author: devxb

src/main/kotlin/org/gitanimals/quiz/infra/similarity/EsKnnTextSimilarityChecker.kt

@@ -2,6 +2,7 @@ package org.gitanimals.quiz.infra.similarity
 
 import org.gitanimals.quiz.app.SimilarityResponse
 import org.gitanimals.quiz.app.TextSimilarityChecker
+import org.slf4j.LoggerFactory
 import org.springframework.data.elasticsearch.client.elc.NativeQuery
 import org.springframework.data.elasticsearch.core.ElasticsearchOperations
 import org.springframework.stereotype.Component
@@ -12,13 +13,17 @@ class EsKnnTextSimilarityChecker(
     private val tokenizer: Tokenizer,
 ) : TextSimilarityChecker {
 
+    private val logger = LoggerFactory.getLogger(this::class.simpleName)
+
     override fun getSimilarity(text: String): SimilarityResponse {
-        val tokenizedText = tokenizer.tokenize(Tokenizer.Request(text))
+        val embeddingResponse = tokenizer.embed(Tokenizer.Request.from(text))
+
+        logger.info("[EsKnnTextSimilarityChecker] Embedding Success total token: ${embeddingResponse.usage.totalToken}, prompt token: ${embeddingResponse.usage.promptToken}")
 
         val knnQuery = NativeQuery.builder()
             .withKnnSearches {
                 it.field(QuizSimilarity::vector.name)
-                it.queryVector(tokenizedText)
+                it.queryVector(embeddingResponse.data.embedding)
                 it.similarity(0.75F)
                 it.k(MAX_RETURN_KNN_SIZE)
                 it.numCandidates(MAX_RETURN_KNN_SIZE * 5)

src/main/kotlin/org/gitanimals/quiz/infra/similarity/NewQuizCreatedEventListener.kt

@@ -18,20 +18,25 @@ class NewQuizCreatedEventListener(
     fun addQuizVectorToEs(newQuizCreated: NewQuizCreated) {
         gracefulLaunch {
             val tokenizedQuizText = runCatching {
-                tokenizer.tokenize(Tokenizer.Request(newQuizCreated.problem))
+                tokenizer.embed(Tokenizer.Request.from(newQuizCreated.problem))
+            }.onSuccess {
+                logger.info("[NewQuizCreatedEventListener] Embedding Success total token: ${it.usage.totalToken}, prompt token: ${it.usage.promptToken}")
             }.getOrElse {
-                logger.error("Tokenize fail. id: ${newQuizCreated.id}, cause: ${it.message}", it)
+                logger.error(
+                    "[NewQuizCreatedEventListener] Tokenize fail. id: ${newQuizCreated.id}, cause: ${it.message}",
+                    it
+                )
                 throw it
-            }
+            }.data.embedding
 
             runCatching {
                 val quizSimilarity = QuizSimilarity.from(newQuizCreated.id, tokenizedQuizText)
                 quizSimilarityRepository.save(quizSimilarity)
             }.onSuccess {
-                logger.info("Tokenize and save success. quizId: ${it.quizId}")
+                logger.info("[NewQuizCreatedEventListener] Tokenize and save success. quizId: ${it.quizId}")
             }.onFailure {
                 logger.error(
-                    "Tokenize success but, Fail to save es. quizId: ${newQuizCreated.id}, cause: ${it.message}",
+                    "[NewQuizCreatedEventListener] Tokenize success but, Fail to save es. quizId: ${newQuizCreated.id}, cause: ${it.message}",
                     it
                 )
             }

src/main/kotlin/org/gitanimals/quiz/infra/similarity/QuizSimilarity.kt

@@ -6,7 +6,7 @@ import org.springframework.data.elasticsearch.annotations.Document
 import org.springframework.data.elasticsearch.annotations.Field
 import org.springframework.data.elasticsearch.annotations.FieldType
 
-private const val BERT_LARGE_DIMS = 768
+private const val OPEN_AI_SMALL_DIMS = 1536
 
 @Document(indexName = "quiz_similarity", createIndex = true)
 class QuizSimilarity(
@@ -16,7 +16,7 @@ class QuizSimilarity(
     @Field(name = "quiz_id")
     val quizId: Long,
 
-    @Field(name = "vector", type = FieldType.Dense_Vector, dims = BERT_LARGE_DIMS)
+    @Field(name = "vector", type = FieldType.Dense_Vector, dims = OPEN_AI_SMALL_DIMS)
     val vector: List<Float>,
 ) {
 

src/main/kotlin/org/gitanimals/quiz/infra/similarity/Tokenizer.kt

@@ -1,14 +1,43 @@
 package org.gitanimals.quiz.infra.similarity
 
+import com.fasterxml.jackson.annotation.JsonProperty
 import org.springframework.web.bind.annotation.RequestBody
 import org.springframework.web.service.annotation.PostExchange
 
 fun interface Tokenizer {
 
-    @PostExchange("/pipeline/feature-extraction/sentence-transformers/all-mpnet-base-v2")
-    fun tokenize(@RequestBody request: Request): List<Float>
+    @PostExchange("/v1/embeddings")
+    fun embed(@RequestBody request: Request): Response
 
     data class Request(
-        val inputs: String,
-    )
+        val input: String,
+        val model: String = "text-embedding-3-small",
+        @JsonProperty("encoding_format")
+        val format: String = "float",
+    ) {
+
+        companion object {
+            fun from(input: String): Request {
+                return Request(input = input)
+            }
+        }
+    }
+
+    data class Response(
+        val usage: Usage,
+        val model: String,
+        val data: Data,
+    ) {
+        data class Usage(
+            @JsonProperty("prompt_token")
+            val promptToken: Int,
+            @JsonProperty("total_token")
+            val totalToken: Int,
+        )
+
+        data class Data(
+            val `object`: String,
+            val embedding: List<Float>,
+        )
+    }
 }

src/main/kotlin/org/gitanimals/quiz/infra/similarity/TokenizerHttpClientConfigurer.kt

@@ -12,7 +12,7 @@ import org.springframework.web.service.invoker.HttpServiceProxyFactory
 
 @Configuration
 class TokenizerHttpClientConfigurer(
-    @Value("\${tokenizer.api.key}") private val apiKey: String,
+    @Value("\${openai.key}") private val apiKey: String,
 ) {
 
     @Bean
@@ -23,7 +23,7 @@ class TokenizerHttpClientConfigurer(
                 request.headers.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
                 execution.execute(request, body)
             }
-            .baseUrl("https://api-inference.huggingface.co")
+            .baseUrl("https://api.openai.com")
             .defaultStatusHandler(tokenizerHttpClientErrorHandler())
             .build()