Merge pull request #54 from javecs/feature/simple-tokens

数字をまとめる機能を追加しました。

Author: javecs

src/main/kotlin/xyz/javecs/tools/text2expr/rules/RuleBuilder.kt

@@ -1,6 +1,5 @@
 package xyz.javecs.tools.text2expr.rules
 
-import com.atilika.kuromoji.ipadic.Token
 import com.atilika.kuromoji.ipadic.Tokenizer
 import xyz.javecs.tools.expr.Calculator
 import xyz.javecs.tools.text2expr.utils.normalize
@@ -25,7 +24,7 @@ class RuleBuilder(source: String, val name: String = "") {
         parser.visit(parser(source).text2expr())
     }
 
-    private fun indexOf(word: Word, tokens: List<Token>, start: Int, end: Int): Pair<Int, Boolean> {
+    private fun indexOf(word: Word, tokens: List<SimpleToken>, start: Int, end: Int): Pair<Int, Boolean> {
         for (i in start..end) {
             val token = tokens[i]
             var matched = 0
@@ -54,7 +53,7 @@ class RuleBuilder(source: String, val name: String = "") {
             .toTypedArray()
 
     fun matches(text: String, recognizedId: (id: Pair<String, String>) -> Unit = {}): Pair<Boolean, Double> {
-        val tokens = tokenizer.tokenize(text)
+        val tokens = tokenizer.tokenize(text).toSimpleTokens()
         var offset = BaseOffset
         var remained = rule().size
         rule().forEach {

src/main/kotlin/xyz/javecs/tools/text2expr/rules/SimpleToken.kt

@@ -0,0 +1,46 @@
+package xyz.javecs.tools.text2expr.rules
+
+import com.atilika.kuromoji.ipadic.Token
+
+private val Number = "数"
+
+fun List<Token>.toSimpleTokens(): List<SimpleToken> {
+    val merged = ArrayList<SimpleToken>()
+    for (token in this) {
+        if (merged.isEmpty()) {
+            merged.add(SimpleToken(token))
+            continue
+        }
+        if (token.partOfSpeechLevel2 == Number || token.surface == ".") {
+            val last = merged.last()
+            if (last.partOfSpeechLevel2 == Number) {
+                last.surface = last.surface + token.surface
+                continue
+            }
+        }
+        merged.add(SimpleToken(token))
+    }
+    return merged
+}
+
+class SimpleToken(token: Token) {
+    var surface: String = ""
+    var partOfSpeechLevel1: String = ""
+    var partOfSpeechLevel2: String = ""
+    var partOfSpeechLevel3: String = ""
+    var partOfSpeechLevel4: String = ""
+    var baseForm: String = ""
+    var reading: String = ""
+    var pronunciation: String = ""
+
+    init {
+        surface = token.surface
+        partOfSpeechLevel1 = token.partOfSpeechLevel1
+        partOfSpeechLevel2 = token.partOfSpeechLevel2
+        partOfSpeechLevel3 = token.partOfSpeechLevel3
+        partOfSpeechLevel4 = token.partOfSpeechLevel4
+        baseForm = token.baseForm
+        reading = token.reading
+        pronunciation = token.pronunciation
+    }
+}

src/test/kotlin/xyz/javecs/tools/text2expr/test/kotlin/SimpleTokenTest.kt

@@ -0,0 +1,32 @@
+package xyz.javecs.tools.text2expr.test.kotlin
+
+import com.atilika.kuromoji.ipadic.Tokenizer
+import org.junit.Test
+import xyz.javecs.tools.text2expr.rules.toSimpleTokens
+import kotlin.test.assertEquals
+
+class SimpleTokenTest {
+    val tokenizer = Tokenizer()
+
+    @Test fun toSimpleTokens1() {
+        val tokens = tokenizer.tokenize("3.5").toSimpleTokens()
+        assertEquals(1, tokens.size)
+        assertEquals("3.5", tokens[0].surface)
+    }
+
+    @Test fun toSimpleTokens2() {
+        val tokens = tokenizer.tokenize("3.5マイルは？").toSimpleTokens()
+        assertEquals(4, tokens.size)
+        assertEquals("3.5", tokens[0].surface)
+        assertEquals("マイル", tokens[1].surface)
+        assertEquals("は", tokens[2].surface)
+        assertEquals("？", tokens[3].surface)
+    }
+
+    @Test fun toSimpleTokens3() {
+        val tokens = tokenizer.tokenize("1 2 3.1 4 5").toSimpleTokens()
+        assertEquals(9, tokens.size)
+        assertEquals("3.1", tokens[4].surface)
+    }
+
+}

src/test/kotlin/xyz/javecs/tools/text2expr/test/kotlin/rules/arithmetic/ArithmeticTest.kt

@@ -13,6 +13,8 @@ class ArithmeticTest {
         assertEquals("16", text2Expr.eval("4 かける 4"))
         assertEquals("60", text2Expr.eval("30 × 2"))
         assertEquals("120", text2Expr.eval("24 x 5"))
+        assertEquals("7", text2Expr.eval("3.5x2"))
+        assertEquals("6", text2Expr.eval("3. x 2"))
     }
 
     @Test fun arithmetic2() {

src/test/kotlin/xyz/javecs/tools/text2expr/test/kotlin/rules/length/LengthTest.kt

@@ -8,14 +8,15 @@ class LengthTest {
 
     val text2Expr = Text2Expr()
 
-    @Test fun mileToKilometer() {
+    @Test fun milesToKilometers1() {
         assertEquals("151.27796", text2Expr.eval("９４マイルは何キロメートルですか？"))
         assertEquals("151.27796", text2Expr.eval("９４マイルは何キロですか？"))
         assertEquals("151.27796", text2Expr.eval("94mileは何kmですか？"))
         assertEquals("1.60934", text2Expr.eval("マイルは、なんキロメートル？"))
+        assertEquals("2.41401", text2Expr.eval("1.5マイルは、何キロ？"))
     }
 
-    @Test fun kilometerToMile() {
+    @Test fun kilometersToMiles1() {
         assertEquals("1.864113576", text2Expr.eval("３キロメートルは、何マイルですか？"))
         assertEquals("1.864113576", text2Expr.eval("３キロは、何マイルですか？"))
         assertEquals("1.864113576", text2Expr.eval("3kmは、何mileですか？"))