Name index update: ngram-based posting lists (dart-lang#9465)

Author: isoos

app/lib/search/mem_index.dart

@@ -659,9 +659,14 @@ class PackageNameIndex {
   /// Maps the collapsed name to all the original names (e.g. `asyncmap`=> [`async_map`, `as_y_n_cmaP`]).
   late final Map<String, List<String>> _collapsedNameResolvesToMap;
 
-  /// Inverted trigram index: maps each trigram to the package indexes (ascending),
-  /// whose collapsed name contains that trigram.
-  late final Map<String, List<int>> _trigramPostings;
+  /// Inverted n-gram index: maps each n-gram (1-3 characters) to the token postings
+  /// of the packages whose collapsed name contains that n-gram.
+  ///
+  /// The weight byte is 99 or 100, depending whether the n-gram also occurs
+  /// in the original - e.g. underscored - name, as opposed to appearing only
+  /// after underscores are collapsed.
+  /// The scoring can use it without re-checking substrings.
+  late final Map<String, TokenPostings> _ngramPostings;
 
   late final _counterPool = IndexedCounterPool(_packageNames.length);
 
@@ -672,19 +677,52 @@ class PackageNameIndex {
       return _PkgNameData(lowercased, collapsed);
     }).toList();
     _collapsedNameResolvesToMap = {};
-    _trigramPostings = {};
+    final builders = <String, PostingsBuilder>{};
     for (var i = 0; i < _data.length; i++) {
+      final collapsed = _data[i].collapsed;
       _collapsedNameResolvesToMap
-          .putIfAbsent(_data[i].collapsed, () => [])
+          .putIfAbsent(collapsed, () => [])
           .add(_packageNames[i]);
-      // Register only a single trigram -> document index posting.
-      // TODO(https://github.com/dart-lang/pub-dev/issues/9462): consider posting weights
-      for (final trigram in trigrams(_data[i].collapsed).toSet()) {
-        _trigramPostings.putIfAbsent(trigram, () => <int>[]).add(i);
+      final lowercased = _data[i].lowercased;
+      // Register one posting per distinct 1-, 2- and 3-gram of the collapsed name.
+      for (final ngram in _ngrams(collapsed)) {
+        // The weight byte indicates whether the n-gram also occurs in the
+        // original (e.g. underscored) name. Postings are added in ascending
+        // document index order, as required by the delta encoding.
+        final weight = lowercased.contains(ngram)
+            ? _originalNameWeight
+            : _collapsedNameWeight;
+        builders.putIfAbsent(ngram, PostingsBuilder.new).add(i, weight);
+      }
+    }
+    _ngramPostings = builders.map((k, b) => MapEntry(k, b.build()));
+  }
+
+  Iterable<String> _ngrams(String collapsed) {
+    final result = <String>{};
+    final length = collapsed.length;
+    for (var i = 0; i < length; i++) {
+      for (var j = 1; j <= 3 && i + j <= length; j++) {
+        result.add(collapsed.substring(i, i + j));
       }
     }
+    return result;
   }
 
+  /// Score assigned when the query n-gram(s) occur in the original (underscored)
+  /// package name.
+  static const _originalNameScore = 1.0;
+
+  /// Score assigned when the query n-gram(s) occur only in the collapsed name,
+  /// i.e. the match spans a removed underscore.
+  static const _collapsedNameScore = _collapsedNameWeight / _originalNameWeight;
+
+  /// Posting weight bytes: the match score scaled by 100. Storing them as small
+  /// integers lets the trigram counter sum the weights and recognize a perfect
+  /// all-original match as `parts.length * _originalNameWeight`.
+  static const _originalNameWeight = 100;
+  static const _collapsedNameWeight = 99;
+
   String _removeUnderscores(String text) => text.replaceAll('_', '');
 
   /// Search [text] and return the matching packages with scores.
@@ -723,57 +761,50 @@ class PackageNameIndex {
     final lowercasedWord = word.toLowerCase();
     final collapsedWord = _removeUnderscores(lowercasedWord);
 
-    bool tryScoreWithSubstringMatch(int index) {
-      final entry = _data[index];
-      if (entry.collapsed.length >= collapsedWord.length &&
-          entry.collapsed.contains(collapsedWord)) {
-        // also check for non-collapsed match
-        if (entry.lowercased.length >= lowercasedWord.length &&
-            entry.lowercased.contains(lowercasedWord)) {
-          score.setValue(index, 1.0);
-        } else {
-          score.setValue(index, 0.99);
-        }
-        return true;
-      }
-      return false;
-    }
-
-    // Note: short strings (less than a trigram) are not indexed separately,
-    //       we need to do full substring check, evaluating every package.
-    // TODO(https://github.com/dart-lang/pub-dev/issues/9462): consider updating the index + the evaluation algorithm
-    if (collapsedWord.length < 3) {
-      for (var i = 0; i < _data.length; i++) {
-        if (filterOnNonZeros?.isNotPositive(i) ?? false) continue;
-        tryScoreWithSubstringMatch(i);
-      }
+    // Short queries (1-3 chars) are themselves a single indexed n-gram: look the
+    // whole query up in the postings and score each candidate directly from its
+    // stored original/collapsed weight, without any substring re-check.
+    if (collapsedWord.length <= 3) {
+      final postings = _ngramPostings[collapsedWord];
+      if (postings == null) return;
+      postings.forEach((i, weight) {
+        if (filterOnNonZeros?.isNotPositive(i) ?? false) return;
+        score.setValue(
+          i,
+          weight == _originalNameWeight
+              ? _originalNameScore
+              : _collapsedNameScore,
+        );
+      });
       return;
     }
 
     _counterPool.withPoolItemSync(
       fn: (counts) {
-        // count the trigrams for each package
+        // Sum the posting weights of the matching trigrams for each package.
         final parts = trigrams(collapsedWord);
         for (final part in parts) {
-          final postings = _trigramPostings[part];
+          final postings = _ngramPostings[part];
           if (postings == null) continue;
-          for (final i in postings) {
-            counts.increment(i);
-          }
+          postings.forEach((i, weight) => counts.add(i, weight));
         }
 
-        final acceptThreshold = parts.length ~/ 2;
+        final n = parts.length;
+        final fullMatchSum = n * _originalNameWeight;
+        final fullMatchCollapsedSum = n * _collapsedNameWeight;
+        final minSum = fullMatchCollapsedSum ~/ 2;
+
         for (var i = 0; i < _packageNames.length; i++) {
-          final matched = counts.getValue(i);
-          if (matched == 0) continue;
+          final sum = counts.getValue(i);
+          // Partial matches must still cover at least half of the trigrams.
+          if (sum < minSum) continue;
           if (filterOnNonZeros?.isNotPositive(i) ?? false) continue;
-          if (tryScoreWithSubstringMatch(i)) continue;
-          if (matched >= acceptThreshold) {
-            // making sure that match score is minimum 0.5
-            final v = matched / parts.length;
-            if (v >= 0.5) {
-              score.setValue(i, v);
-            }
+
+          if (sum >= fullMatchSum) {
+            score.setValue(i, _originalNameScore);
+          } else {
+            // Partial match: fractional relevance score in [0.5, 1.0).
+            score.setValue(i, sum / fullMatchSum);
           }
         }
       },

app/lib/search/token_index.dart

@@ -49,8 +49,9 @@ class TokenMatch {
 ///   index (or 0 for the first entry). Each byte stores 7 data bits in
 ///   bits 0-6; bit 7 is the continuation flag (1 = more bytes follow).
 ///   Typical deltas (1-127) encode in a single byte.
-/// - **Weight**: quantized token weight, 0-255, where the original float
-///   weight is recovered as `(weight + 1) / 256`.
+/// - **Weight**: an application-defined weight byte, 0-255. [TokenIndex] stores
+///   a quantized float weight recovered as `(weight + 1) / 256`; other indexes
+///   (e.g. the package name index) may use it to store a binary indicator instead.
 ///
 /// Example: doc indices [3, 5, 200] with weights [255, 128, 64] encodes as:
 ///
@@ -90,7 +91,7 @@ class TokenPostings {
 
 /// Builds a [TokenPostings] incrementally using 128-byte buffer chunks,
 /// concatenating into a single [Uint8List] at the end.
-class _PostingsBuilder {
+class PostingsBuilder {
   static const _bufferSize = 128;
 
   List<Uint8List>? _chunks;
@@ -166,7 +167,7 @@ class TokenIndex {
   TokenIndex._(this._length, this._postings);
 
   factory TokenIndex(List<String?> values, {bool skipDocumentWeight = false}) {
-    final builders = <String, _PostingsBuilder>{};
+    final builders = <String, PostingsBuilder>{};
     for (var i = 0; i < values.length; i++) {
       final text = values[i];
       if (text == null) continue;
@@ -185,7 +186,7 @@ class TokenIndex {
     List<T> objects,
     List<TokenIndexField> Function(T object) extractFields,
   ) {
-    final builders = <String, _PostingsBuilder>{};
+    final builders = <String, PostingsBuilder>{};
     for (var i = 0; i < objects.length; i++) {
       final merged = <String, double>{};
       for (final field in extractFields(objects[i])) {
@@ -212,7 +213,7 @@ class TokenIndex {
     List<List<String>?> values, {
     bool skipDocumentWeight = false,
   }) {
-    final builders = <String, _PostingsBuilder>{};
+    final builders = <String, PostingsBuilder>{};
     for (var i = 0; i < values.length; i++) {
       final parts = values[i];
       if (parts == null || parts.isEmpty) continue;
@@ -234,7 +235,7 @@ class TokenIndex {
   }
 
   static Map<String, TokenPostings> _finalize(
-    Map<String, _PostingsBuilder> builders,
+    Map<String, PostingsBuilder> builders,
   ) {
     return builders.map((token, builder) => MapEntry(token, builder.build()));
   }
@@ -244,17 +245,15 @@ class TokenIndex {
     int docIndex,
     Map<String, double>? tokens,
     bool skipDocumentWeight,
-    Map<String, _PostingsBuilder> builders,
+    Map<String, PostingsBuilder> builders,
   ) {
     if (tokens == null || tokens.isEmpty) return;
     // Document weight is a highly scaled-down proxy of the length.
     final dw = skipDocumentWeight ? 1.0 : _documentWeight(tokens.length);
     for (final e in tokens.entries) {
       final weight = e.value / dw;
       final quantized = (weight * 256 - 1).round().clamp(0, 255);
-      builders
-          .putIfAbsent(e.key, _PostingsBuilder.new)
-          .add(docIndex, quantized);
+      builders.putIfAbsent(e.key, PostingsBuilder.new).add(docIndex, quantized);
     }
   }
 
@@ -519,7 +518,7 @@ extension type IndexedCounter(List<int> values) {
 
   int getValue(int index) => values[index];
 
-  void increment(int index) {
-    values[index]++;
+  void add(int index, int amount) {
+    values[index] += amount;
   }
 }

app/test/search/package_name_index_test.dart

@@ -83,7 +83,7 @@ void main() {
     test('substring: entu', () {
       expect(index.search('entu'), {
         'fluent': 0.5,
-        'fluent_ui': 0.99, // not 1.0
+        'fluent_ui': 0.995, // not 1.0
       });
     });
   });

app/test/search/stack_trace_test.dart

@@ -38,7 +38,7 @@ void main() {
         'totalCount': 1,
         'sdkLibraryHits': [],
         'packageHits': [
-          {'package': 'stack_trace', 'score': 0.99},
+          {'package': 'stack_trace', 'score': 0.9975},
         ],
       });
     });