|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one or more |
| 3 | + * contributor license agreements. See the NOTICE file distributed with |
| 4 | + * this work for additional information regarding copyright ownership. |
| 5 | + * The ASF licenses this file to You under the Apache License, Version 2.0 |
| 6 | + * (the "License"); you may not use this file except in compliance with |
| 7 | + * the License. You may obtain a copy of the License at |
| 8 | + * |
| 9 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | + * |
| 11 | + * Unless required by applicable law or agreed to in writing, software |
| 12 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | + * See the License for the specific language governing permissions and |
| 15 | + * limitations under the License. |
| 16 | + */ |
| 17 | + |
| 18 | +package org.apache.hop.core.search; |
| 19 | + |
| 20 | +import java.util.Locale; |
| 21 | +import java.util.regex.Pattern; |
| 22 | +import org.apache.commons.lang3.StringUtils; |
| 23 | +import org.apache.hop.core.util.HopJaroWinklerDistance; |
| 24 | +import org.apache.hop.core.util.StringUtil; |
| 25 | + |
| 26 | +/** |
| 27 | + * A single, reusable string matcher shared by every search box in Hop (global search, file explorer |
| 28 | + * filter, settings filter, the transform/action picker, ...). It is built once from a query and |
| 29 | + * then scores any number of candidate strings. |
| 30 | + * |
| 31 | + * <p>Matching is, in order of decreasing score: |
| 32 | + * |
| 33 | + * <ol> |
| 34 | + * <li>exact (the whole normalized target equals the term), |
| 35 | + * <li>word-prefix (a word in the target starts with the term), |
| 36 | + * <li>substring (the term appears somewhere in the target), |
| 37 | + * <li>fuzzy (Jaro-Winkler similarity above a threshold) - only when fuzzy matching is enabled. |
| 38 | + * </ol> |
| 39 | + * |
| 40 | + * <p>The query is normalized (diacritics removed, optionally lower-cased, whitespace collapsed) and |
| 41 | + * split into terms on whitespace and commas; <b>all</b> terms must match (AND). Alternatively a |
| 42 | + * regular-expression mode does a partial regex match. |
| 43 | + * |
| 44 | + * <p>{@link #score(String)} returns a value in {@code [0,1]} ({@code 0} means "no match") which |
| 45 | + * callers can use to rank results; {@link #matches(String)} is the boolean shortcut. |
| 46 | + */ |
| 47 | +public class SearchMatcher { |
| 48 | + |
| 49 | + /** Default Jaro-Winkler similarity above which a fuzzy match is accepted. */ |
| 50 | + public static final double DEFAULT_FUZZY_THRESHOLD = 0.86; |
| 51 | + |
| 52 | + /** Fuzzy matching is only attempted for terms at least this long (avoids noise on tiny terms). */ |
| 53 | + private static final int MIN_FUZZY_TERM_LENGTH = 3; |
| 54 | + |
| 55 | + /** |
| 56 | + * A fuzzy match is only considered when the shorter of the two strings is at least this fraction |
| 57 | + * of the longer one. Without this a long query (e.g. "tableinput") would fuzzily latch onto a |
| 58 | + * much shorter candidate (e.g. the keyword "table") thanks to the Jaro-Winkler common-prefix |
| 59 | + * bonus, which is too lenient: a genuine typo is roughly the same length as its target. |
| 60 | + */ |
| 61 | + private static final double MIN_FUZZY_LENGTH_RATIO = 0.6; |
| 62 | + |
| 63 | + private static final double SCORE_EXACT = 1.0; |
| 64 | + private static final double SCORE_WORD = 0.97; |
| 65 | + private static final double SCORE_PREFIX = 0.92; |
| 66 | + private static final double SCORE_SUBSTRING = 0.8; |
| 67 | + private static final double FUZZY_WEIGHT = 0.7; |
| 68 | + |
| 69 | + private final boolean caseSensitive; |
| 70 | + private final boolean regex; |
| 71 | + private final boolean fuzzy; |
| 72 | + private final double fuzzyThreshold; |
| 73 | + private final String[] terms; |
| 74 | + private final Pattern pattern; |
| 75 | + |
| 76 | + public SearchMatcher(String query, boolean caseSensitive, boolean regEx, boolean fuzzy) { |
| 77 | + this(query, caseSensitive, regEx, fuzzy, DEFAULT_FUZZY_THRESHOLD); |
| 78 | + } |
| 79 | + |
| 80 | + public SearchMatcher( |
| 81 | + String query, boolean caseSensitive, boolean regEx, boolean fuzzy, double fuzzyThreshold) { |
| 82 | + this.caseSensitive = caseSensitive; |
| 83 | + this.regex = regEx; |
| 84 | + this.fuzzy = fuzzy && !regEx; |
| 85 | + this.fuzzyThreshold = fuzzyThreshold; |
| 86 | + |
| 87 | + if (regEx) { |
| 88 | + Pattern compiled = null; |
| 89 | + if (StringUtils.isNotEmpty(query)) { |
| 90 | + try { |
| 91 | + compiled = Pattern.compile(query, caseSensitive ? 0 : Pattern.CASE_INSENSITIVE); |
| 92 | + } catch (Exception e) { |
| 93 | + // An incomplete/invalid expression simply matches nothing instead of throwing. |
| 94 | + compiled = null; |
| 95 | + } |
| 96 | + } |
| 97 | + this.pattern = compiled; |
| 98 | + this.terms = new String[0]; |
| 99 | + } else { |
| 100 | + this.pattern = null; |
| 101 | + String normalized = normalize(query, caseSensitive); |
| 102 | + this.terms = normalized.isEmpty() ? new String[0] : normalized.split(" "); |
| 103 | + } |
| 104 | + } |
| 105 | + |
| 106 | + /** |
| 107 | + * Normalize a string for matching: remove diacritical marks, optionally lower-case, turn commas |
| 108 | + * into spaces and collapse runs of whitespace. |
| 109 | + */ |
| 110 | + public static String normalize(String string, boolean caseSensitive) { |
| 111 | + if (StringUtils.isEmpty(string)) { |
| 112 | + return ""; |
| 113 | + } |
| 114 | + String normalized = StringUtil.removeDiacriticalMarks(string); |
| 115 | + if (!caseSensitive) { |
| 116 | + normalized = normalized.toLowerCase(Locale.ROOT); |
| 117 | + } |
| 118 | + return normalized.replace(',', ' ').replaceAll("\\s+", " ").trim(); |
| 119 | + } |
| 120 | + |
| 121 | + /** Whether the given target matches at all. */ |
| 122 | + public boolean matches(String target) { |
| 123 | + return score(target) > 0.0; |
| 124 | + } |
| 125 | + |
| 126 | + /** |
| 127 | + * Score a candidate string against the query. |
| 128 | + * |
| 129 | + * @param target the candidate |
| 130 | + * @return a relevance score in {@code [0,1]}; {@code 0} means no match |
| 131 | + */ |
| 132 | + public double score(String target) { |
| 133 | + // Regular expression mode: a partial match counts. An invalid/empty pattern matches nothing. |
| 134 | + if (regex) { |
| 135 | + return pattern != null && target != null && pattern.matcher(target).find() ? 1.0 : 0.0; |
| 136 | + } |
| 137 | + |
| 138 | + // An empty query matches everything non-empty (keeps the old "match all" behavior). |
| 139 | + if (terms.length == 0) { |
| 140 | + return StringUtils.isNotEmpty(target) ? 1.0 : 0.0; |
| 141 | + } |
| 142 | + if (StringUtils.isEmpty(target)) { |
| 143 | + return 0.0; |
| 144 | + } |
| 145 | + |
| 146 | + String normalizedTarget = normalize(target, caseSensitive); |
| 147 | + String[] words = normalizedTarget.split(" "); |
| 148 | + |
| 149 | + double total = 0.0; |
| 150 | + for (String term : terms) { |
| 151 | + double termScore = scoreTerm(term, normalizedTarget, words); |
| 152 | + if (termScore <= 0.0) { |
| 153 | + // AND semantics: every term must match. |
| 154 | + return 0.0; |
| 155 | + } |
| 156 | + total += termScore; |
| 157 | + } |
| 158 | + return total / terms.length; |
| 159 | + } |
| 160 | + |
| 161 | + private double scoreTerm(String term, String target, String[] words) { |
| 162 | + if (target.equals(term)) { |
| 163 | + return SCORE_EXACT; |
| 164 | + } |
| 165 | + if (target.contains(term)) { |
| 166 | + for (String word : words) { |
| 167 | + if (word.equals(term)) { |
| 168 | + return SCORE_WORD; |
| 169 | + } |
| 170 | + } |
| 171 | + for (String word : words) { |
| 172 | + if (word.startsWith(term)) { |
| 173 | + return SCORE_PREFIX; |
| 174 | + } |
| 175 | + } |
| 176 | + return SCORE_SUBSTRING; |
| 177 | + } |
| 178 | + if (fuzzy && term.length() >= MIN_FUZZY_TERM_LENGTH) { |
| 179 | + double best = similarity(term, target); |
| 180 | + for (String word : words) { |
| 181 | + best = Math.max(best, similarity(term, word)); |
| 182 | + } |
| 183 | + if (best >= fuzzyThreshold) { |
| 184 | + return best * FUZZY_WEIGHT; |
| 185 | + } |
| 186 | + } |
| 187 | + return 0.0; |
| 188 | + } |
| 189 | + |
| 190 | + /** |
| 191 | + * Jaro-Winkler similarity, but {@code 0} when the two strings differ too much in length to be a |
| 192 | + * plausible typo of one another (see {@link #MIN_FUZZY_LENGTH_RATIO}). |
| 193 | + */ |
| 194 | + private static double similarity(String a, String b) { |
| 195 | + if (StringUtils.isEmpty(a) || StringUtils.isEmpty(b)) { |
| 196 | + return 0.0; |
| 197 | + } |
| 198 | + int shorter = Math.min(a.length(), b.length()); |
| 199 | + int longer = Math.max(a.length(), b.length()); |
| 200 | + if ((double) shorter / longer < MIN_FUZZY_LENGTH_RATIO) { |
| 201 | + return 0.0; |
| 202 | + } |
| 203 | + HopJaroWinklerDistance distance = new HopJaroWinklerDistance(); |
| 204 | + distance.apply(a, b); |
| 205 | + Double jw = distance.getJaroWinklerDistance(); |
| 206 | + return jw == null ? 0.0 : jw; |
| 207 | + } |
| 208 | +} |
0 commit comments