Merge description + readme token index. (dart-lang#9446)

Author: isoos

app/lib/search/mem_index.dart

@@ -26,8 +26,7 @@ class InMemoryPackageIndex {
   final List<PackageDocument> _documents;
   final _nameToIndex = <String, int>{};
   late final PackageNameIndex _packageNameIndex;
-  late final TokenIndex _descrIndex;
-  late final TokenIndex _readmeIndex;
+  late final TokenIndex _textIndex;
   late final List<IndexedApiDocPage> _apiDocPageKeys;
   late final TokenIndex _apiSymbolIndex;
   late final _bitArrayPool = BitArrayPool(_documents.length);
@@ -78,11 +77,13 @@ class InMemoryPackageIndex {
 
     final packageKeys = _documents.map((d) => d.package).toList();
     _packageNameIndex = PackageNameIndex(packageKeys);
-    _descrIndex = TokenIndex(
-      _documents.map((d) => d.description).toList(),
-      skipDocumentWeight: true,
-    );
-    _readmeIndex = TokenIndex(_documents.map((d) => d.readme).toList());
+    _textIndex = TokenIndex.fromFields(_documents, (doc) {
+      return [
+        if (doc.description != null)
+          TokenIndexField(doc.description!, skipDocumentWeight: true),
+        if (doc.readme != null) TokenIndexField(doc.readme!, weight: 0.75),
+      ];
+    });
     _apiDocPageKeys = apiDocPageKeys;
     _apiSymbolIndex = TokenIndex.fromValues(apiDocPageValues);
 
@@ -458,8 +459,7 @@ class InMemoryPackageIndex {
       return aborted;
     }
 
-    final matchDescription = textMatchExtent.shouldMatchDescription();
-    final matchReadme = textMatchExtent.shouldMatchReadme();
+    final matchText = textMatchExtent.shouldMatchText();
     final matchApi = textMatchExtent.shouldMatchApi();
 
     for (final word in words) {
@@ -471,15 +471,8 @@ class InMemoryPackageIndex {
             filterOnNonZeros: packageScores,
           );
 
-          if (matchDescription) {
-            await _descrIndex.searchAndAccumulate(word, score: wordScore);
-          }
-          if (matchReadme) {
-            await _readmeIndex.searchAndAccumulate(
-              word,
-              weight: 0.75,
-              score: wordScore,
-            );
+          if (matchText) {
+            await _textIndex.searchAndAccumulate(word, score: wordScore);
           }
           packageScores.multiplyAllFrom(wordScore);
         },
@@ -538,8 +531,8 @@ class InMemoryPackageIndex {
         final matchedAllPhrases = phrases.every(
           (phrase) =>
               (matchName && doc.package.contains(phrase)) ||
-              (matchDescription && doc.description!.contains(phrase)) ||
-              (matchReadme && doc.readme!.contains(phrase)),
+              (matchText && doc.description!.contains(phrase)) ||
+              (matchText && doc.readme!.contains(phrase)),
         );
         if (!matchedAllPhrases) {
           packageScores.setValue(i, 0);

app/lib/search/token_index.dart

@@ -136,6 +136,24 @@ class _PostingsBuilder {
   }
 }
 
+/// Describes a weighted field of a document.
+class TokenIndexField {
+  /// The text value of the field.
+  final String text;
+
+  /// Whether to skip the normalization for this field.
+  final bool skipDocumentWeight;
+
+  /// Token weight multiplier.
+  final double weight;
+
+  TokenIndexField(
+    this.text, {
+    this.skipDocumentWeight = false,
+    this.weight = 1.0,
+  });
+}
+
 /// Stores a token -> documentId inverted index with weights.
 class TokenIndex {
   final int _length;
@@ -157,6 +175,39 @@ class TokenIndex {
     return TokenIndex._(values.length, _finalize(builders));
   }
 
+  /// Builds a single index from a list of [objects], extracting one or more
+  /// weighted text fields from each via [extractFields] (e.g. description +
+  /// readme of the same package document).
+  ///
+  /// For each object, the tokens of all its fields are merged, keeping the
+  /// maximum weight per token.
+  static TokenIndex fromFields<T>(
+    List<T> objects,
+    List<TokenIndexField> Function(T object) extractFields,
+  ) {
+    final builders = <String, _PostingsBuilder>{};
+    for (var i = 0; i < objects.length; i++) {
+      final merged = <String, double>{};
+      for (final field in extractFields(objects[i])) {
+        final tokens = tokenize(field.text);
+        if (tokens == null || tokens.isEmpty) continue;
+        final dw = field.skipDocumentWeight
+            ? 1.0
+            : _documentWeight(tokens.length);
+        for (final e in tokens.entries) {
+          final w = e.value / dw * field.weight;
+          final old = merged[e.key];
+          if (old == null || w > old) {
+            merged[e.key] = w;
+          }
+        }
+      }
+      // Weights are already normalized above, so skip the document weight here.
+      _addTokens(i, merged, true, builders);
+    }
+    return TokenIndex._(objects.length, _finalize(builders));
+  }
+
   factory TokenIndex.fromValues(
     List<List<String>?> values, {
     bool skipDocumentWeight = false,
@@ -197,7 +248,7 @@ class TokenIndex {
   ) {
     if (tokens == null || tokens.isEmpty) return;
     // Document weight is a highly scaled-down proxy of the length.
-    final dw = skipDocumentWeight ? 1.0 : 1 + math.log(1 + tokens.length) / 100;
+    final dw = skipDocumentWeight ? 1.0 : _documentWeight(tokens.length);
     for (final e in tokens.entries) {
       final weight = e.value / dw;
       final quantized = (weight * 256 - 1).round().clamp(0, 255);
@@ -207,6 +258,8 @@ class TokenIndex {
     }
   }
 
+  static double _documentWeight(int length) => 1 + math.log(1 + length) / 100;
+
   /// Match the text against the corpus and return the tokens or
   /// their partial segments that have match.
   @visibleForTesting

app/lib/service/entrypoint/search_index.dart

@@ -203,20 +203,16 @@ class LatencyAwareSearchIndex implements SearchIndex {
       _logger.info('[text-match-normal]');
       return TextMatchExtent.api;
     }
-    if (latency < const Duration(seconds: 2)) {
-      _logger.info('[text-match-readme]');
-      return TextMatchExtent.readme;
-    }
     if (latency < const Duration(seconds: 4)) {
-      _logger.info('[text-match-description]');
-      return TextMatchExtent.description;
+      _logger.info('[text-match-text]');
+      return TextMatchExtent.text;
     }
     if (latency < const Duration(seconds: 10)) {
       _logger.info('[text-match-name]');
       return TextMatchExtent.name;
     }
     // TODO: use `TextMatchExtent.none` after we are confident about this change.
     _logger.info('[text-match-none]');
-    return TextMatchExtent.readme;
+    return TextMatchExtent.text;
   }
 }

pkg/_pub_shared/lib/search/search_request_data.dart

@@ -118,10 +118,15 @@ enum TextMatchExtent {
   /// Text search is on package names.
   name,
 
+  /// Text search is on names, descriptions, topic tags and readme content.
+  text,
+
   /// Text search is on package names, descriptions and topic tags.
+  @Deprecated('Use `text`')
   description,
 
   /// Text search is on names, descriptions, topic tags and readme content.
+  @Deprecated('Use `text`')
   readme,
 
   /// Text search is on names, descriptions, topic tags, readme content and API symbols.
@@ -130,11 +135,8 @@ enum TextMatchExtent {
   /// Text search is on package names.
   bool shouldMatchName() => index >= name.index;
 
-  /// Text search is on package names, descriptions and topic tags.
-  bool shouldMatchDescription() => index >= description.index;
-
   /// Text search is on names, descriptions, topic tags and readme content.
-  bool shouldMatchReadme() => index >= readme.index;
+  bool shouldMatchText() => index >= text.index;
 
   /// Text search is on names, descriptions, topic tags, readme content and API symbols.
   bool shouldMatchApi() => index >= api.index;