Merge remote-tracking branch 'origin/master' into fcitx

Author: Fcitx Bot

src/prediction/realtime_decoder.cc

@@ -141,6 +141,7 @@ bool RealtimeDecoder::PushBackTopConversionResult(
   // This method emulates usual converter's behavior so here disable
   // partial candidates.
   options.create_partial_candidates = false;
+  options.used_in_predictor_realtime_conversion = true;
   options.request_type = ConversionRequest::CONVERSION;
   const ConversionRequest tmp_request = ConversionRequestBuilder()
                                             .SetConversionRequestView(request)
@@ -286,6 +287,7 @@ std::optional<Result> RealtimeDecoder::DecodeSuffix(
   options.create_partial_candidates = false;
   options.kana_modifier_insensitive_conversion = false;
   options.use_actual_converter_for_realtime_conversion = false;
+  options.used_in_predictor_realtime_conversion = true;
 
   const bool has_prefix = prefix_rid != 0;
 

src/prediction/user_history_predictor.cc

@@ -1110,7 +1110,7 @@ bool UserHistoryPredictor::LookupEntry(
     const ConversionRequest& request, absl::string_view request_key,
     absl::string_view key_base,
     const Trie<std::string>* absl_nullable key_expanded, const Entry& entry,
-    const Entry* absl_nullable prev_entry,
+    const Entry* absl_nullable prev_entry, bool exact_match_only,
     EntryPriorityQueue& entry_queue) const {
   Entry* result = nullptr;
 
@@ -1139,15 +1139,25 @@ bool UserHistoryPredictor::LookupEntry(
     return false;
   }
 
+  // `exact_match_only` is called on the conversion that only requires exact
+  // match result.
+  // RIGHT_PREFIX_MATCH automatically fills the remaining suffix with
+  // the logic of partial match.
+  if (exact_match_only && mtype != MatchType::RIGHT_PREFIX_MATCH &&
+      mtype != MatchType::EXACT_MATCH) {
+    return false;
+  }
+
   // Full sentence with low frequency is suppressed unless
   // AllowLowFreqFullSentenceEntryMatch() returns true.
-  if (IsLowFreqFullSentenceEntry(request, entry) &&
+  if (!exact_match_only && IsLowFreqFullSentenceEntry(request, entry) &&
       !AllowLowFreqFullSentenceEntryMatch(request, request_key, mtype, entry)) {
     return false;
   }
 
   // For mobile, prefer exact match.
-  const bool prefer_exact_match = IsMixedConversionEnabled(request);
+  const bool prefer_exact_match =
+      IsMixedConversionEnabled(request) || exact_match_only;
 
   left_last_access_time = entry.last_access_time();
   left_most_last_access_time =
@@ -1182,13 +1192,18 @@ bool UserHistoryPredictor::LookupEntry(
       uint32_t result_attribute = 0;
       converter::InnerSegmentBoundary inner_segment_boundary;
 
-      if (GetKeyValueForExactAndRightPrefixMatch(
+      if (!exact_match_only &&
+          GetKeyValueForExactAndRightPrefixMatch(
               request, request_key, prefer_exact_match, entry, last_entry,
               left_last_access_time, left_most_last_access_time, key, value,
-              inner_segment_boundary) ||
-          GetKeyValueForPartialMatch(request, request_key, entry, key, value,
-                                     result_attribute, inner_segment_boundary,
-                                     entry_queue)) {
+              inner_segment_boundary)) {
+        result =
+            AddEntryWithNewKeyValue(request, std::move(key), std::move(value),
+                                    inner_segment_boundary, entry, entry_queue);
+        SetAttribute(*result, result_attribute);
+      } else if (GetKeyValueForPartialMatch(
+                     request, request_key, entry, key, value, result_attribute,
+                     inner_segment_boundary, entry_queue)) {
         result =
             AddEntryWithNewKeyValue(request, std::move(key), std::move(value),
                                     inner_segment_boundary, entry, entry_queue);
@@ -1268,8 +1283,8 @@ bool UserHistoryPredictor::LookupEntry(
     entry_queue.Push(result);
   }
 
-  if (IsMixedConversionEnabled(request)) {
-    // For mobile, we don't generate joined result.
+  // For mobile or conversion mode, we don't generate joined result.
+  if (IsMixedConversionEnabled(request) || exact_match_only) {
     return true;
   }
 
@@ -1369,7 +1384,7 @@ std::vector<Result> UserHistoryPredictor::Predict(
     max_prediction_size = 3;
   }
 
-  EntryPriorityQueue entry_queue = CreateEntryQueueFromHistory(
+  EntryPriorityQueue entry_queue = CreateEntryQueueFromHistoryForPrediction(
       request, prev_entry.get(), max_prediction_size * 5);
 
   if (entry_queue.size() == 0) {
@@ -1381,48 +1396,75 @@ std::vector<Result> UserHistoryPredictor::Predict(
                      entry_queue);
 }
 
-bool UserHistoryPredictor::ShouldPredict(
+std::vector<Result> UserHistoryPredictor::Convert(
     const ConversionRequest& request) const {
-  if (storage_.IsSyncerInCriticalSection()) {
-    MOZC_VLOG(2) << "Syncer is running";
-    return false;
+  if (!ShouldPredict(request)) {
+    return {};
   }
 
-  if (request.incognito_mode()) {
-    MOZC_VLOG(2) << "incognito mode";
-    return false;
+  ConstEntrySnapshot prev_entry = LookupPrevEntry(request);
+
+  constexpr int kMaxCandidatesSize = 5;
+
+  EntryPriorityQueue entry_queue = CreateEntryQueueFromHistoryForConversion(
+      request, prev_entry.get(), kMaxCandidatesSize);
+
+  if (entry_queue.size() == 0) {
+    MOZC_VLOG(2) << "no prefix match candidate is found.";
+    return {};
   }
 
-  if (request.config().history_learning_level() == config::Config::NO_HISTORY) {
-    MOZC_VLOG(2) << "history learning level is NO_HISTORY";
+  return MakeResults(request, kMaxCandidatesSize,
+                     0 /* max_prediction_char_coverage */, entry_queue);
+}
+
+bool UserHistoryPredictor::ShouldPredict(
+    const ConversionRequest& request) const {
+  if (storage_.IsSyncerInCriticalSection()) {
+    MOZC_VLOG(2) << "Syncer is running";
     return false;
   }
 
-  if (request.request_type() == ConversionRequest::CONVERSION) {
-    MOZC_VLOG(2) << "request type is CONVERSION";
+  if (storage_.IsEmpty()) {
+    MOZC_VLOG(2) << "dic is empty";
     return false;
   }
 
-  if (!request.config().use_history_suggest()) {
-    MOZC_VLOG(2) << "no history suggest";
+  if (request.incognito_mode()) {
+    MOZC_VLOG(2) << "incognito mode";
     return false;
   }
 
-  if (storage_.IsEmpty()) {
-    MOZC_VLOG(2) << "dic is empty";
+  if (request.config().history_learning_level() == config::Config::NO_HISTORY) {
+    MOZC_VLOG(2) << "history learning level is NO_HISTORY";
     return false;
   }
 
   absl::string_view request_key = request.key();
 
-  if (request_key.empty() && !IsZeroQuerySuggestionEnabled(request)) {
-    MOZC_VLOG(2) << "key length is 0";
-    return false;
-  }
+  // Prediction/suggestion specific rules.
+  if (request.request_type() == ConversionRequest::SUGGESTION ||
+      request.request_type() == ConversionRequest::PREDICTION ||
+      request.request_type() == ConversionRequest::PARTIAL_SUGGESTION ||
+      request.request_type() == ConversionRequest::PARTIAL_PREDICTION) {
+    if (request_key.empty() && !IsZeroQuerySuggestionEnabled(request)) {
+      MOZC_VLOG(2) << "key length is 0";
+      return false;
+    }
 
-  if (StartsWithPunctuation(request_key)) {
-    MOZC_VLOG(2) << "request_key starts with punctuations";
-    return false;
+    if (!request.config().use_history_suggest()) {
+      MOZC_VLOG(2) << "no history suggest";
+      return false;
+    }
+
+    if (StartsWithPunctuation(request_key)) {
+      MOZC_VLOG(2) << "request_key starts with punctuations";
+      return false;
+    }
+  } else if (request.request_type() == ConversionRequest::CONVERSION) {
+    if (request_key.empty()) {
+      return false;
+    }
   }
 
   return true;
@@ -1481,8 +1523,8 @@ UserHistoryPredictor::ConstEntrySnapshot UserHistoryPredictor::LookupPrevEntry(
 }
 
 UserHistoryPredictor::EntryPriorityQueue
-UserHistoryPredictor::CreateEntryQueueFromHistory(
-    const ConversionRequest& request, const Entry* prev_entry,
+UserHistoryPredictor::CreateEntryQueueFromHistoryForPrediction(
+    const ConversionRequest& request, const Entry* absl_nullable prev_entry,
     size_t max_entry_queue_size) const {
   // Gets romanized input key if the given preedit looks misspelled.
   const std::string roman_request_key = GetRomanMisspelledKey(request);
@@ -1521,10 +1563,12 @@ UserHistoryPredictor::CreateEntryQueueFromHistory(
       return true;
     }
 
+    static constexpr bool kDisableExactMatchOnly = false;
+
     // Lookup key from elm_value and prev_entry.
     // If a new entry is found, the entry is pushed to the entry_queue.
     if (LookupEntry(request, request_key, base_key, expanded.get(), entry,
-                    prev_entry, entry_queue)) {
+                    prev_entry, kDisableExactMatchOnly, entry_queue)) {
       return true;
     }
 
@@ -1549,7 +1593,7 @@ UserHistoryPredictor::CreateEntryQueueFromHistory(
       // in dictionary predictor.
       if (c.score > 0.0 &&
           LookupEntry(request, c.correction, c.correction, nullptr, entry,
-                      prev_entry, entry_queue)) {
+                      prev_entry, kDisableExactMatchOnly, entry_queue)) {
         break;
       }
     }
@@ -1560,6 +1604,36 @@ UserHistoryPredictor::CreateEntryQueueFromHistory(
   return entry_queue;
 }
 
+UserHistoryPredictor::EntryPriorityQueue
+UserHistoryPredictor::CreateEntryQueueFromHistoryForConversion(
+    const ConversionRequest& request, const Entry* absl_nullable prev_entry,
+    size_t max_entry_queue_size) const {
+  const std::string request_key = request.composer().GetQueryForConversion();
+  EntryPriorityQueue entry_queue;
+
+  storage_.ForEach([&](uint64_t fp, const Entry& entry) {
+    // already found enough entry_queue.
+    if (entry_queue.size() >= max_entry_queue_size) {
+      return false;
+    }
+
+    if (!IsValidEntryIgnoringRemovedField(entry)) {
+      return true;
+    }
+
+    static constexpr bool kEnableExactMatchOnly = true;
+
+    if (LookupEntry(request, request_key, request_key, nullptr, entry,
+                    prev_entry, kEnableExactMatchOnly, entry_queue)) {
+      return true;
+    }
+
+    return true;
+  });
+
+  return entry_queue;
+}
+
 // static
 std::tuple<std::string, std::string, std::unique_ptr<Trie<std::string>>>
 UserHistoryPredictor::GetInputKeyFromRequest(const ConversionRequest& request) {

src/prediction/user_history_predictor.h

@@ -70,6 +70,9 @@ class UserHistoryPredictor : public PredictorInterface {
 
   std::vector<Result> Predict(const ConversionRequest& request) const override;
 
+  // TODO(taku): Make it a virtual method of PredictorInterface.
+  std::vector<Result> Convert(const ConversionRequest& request) const;
+
   // Hook(s) for all mutable operations.
   void Finish(const ConversionRequest& request,
               absl::Span<const Result> results, uint32_t revert_id) override;
@@ -180,6 +183,8 @@ class UserHistoryPredictor : public PredictorInterface {
   };
 
   // Returns true if this predictor should return results for the input.
+  // TODO(taku): better to rename this function as it is used both for
+  // prediction and conversion.
   bool ShouldPredict(const ConversionRequest& request) const;
 
   // Gets match type from two strings
@@ -275,6 +280,7 @@ class UserHistoryPredictor : public PredictorInterface {
                    absl::string_view request_key, absl::string_view key_base,
                    const Trie<std::string>* absl_nullable key_expanded,
                    const Entry& entry, const Entry* absl_nullable prev_entry,
+                   bool exact_match_only,
                    EntryPriorityQueue& entry_queue) const;
 
   // For the EXACT and RIGHT_PREFIX match, we will generate joined
@@ -323,8 +329,13 @@ class UserHistoryPredictor : public PredictorInterface {
       converter::InnerSegmentBoundarySpan inner_segment_boundary, Entry entry,
       EntryPriorityQueue& entry_queue) const;
 
-  // Creates entry queue from request and history storage.
-  EntryPriorityQueue CreateEntryQueueFromHistory(
+  // Creates entry queue from request and history storage on prediction mode.
+  EntryPriorityQueue CreateEntryQueueFromHistoryForPrediction(
+      const ConversionRequest& request, const Entry* absl_nullable prev_entry,
+      size_t max_entry_queue_size) const;
+
+  // Creates entry queue from request and history storage on conversion mode.
+  EntryPriorityQueue CreateEntryQueueFromHistoryForConversion(
       const ConversionRequest& request, const Entry* absl_nullable prev_entry,
       size_t max_entry_queue_size) const;