Make Selective Field Query during Reindexing (#27723)

* Make Selective Field Query
* Minor nit
* Fix Failing Tests

(cherry picked from commit 385d589)

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: mohityadav766

openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/EntityReader.java

@@ -326,7 +326,15 @@ static List<String> getSearchIndexFields(String entityType) {
     if (TIME_SERIES_ENTITIES.contains(entityType)) {
       return List.of();
     }
-    return List.of("*");
+    org.openmetadata.service.search.SearchRepository repo =
+        org.openmetadata.service.Entity.getSearchRepository();
+    if (repo == null || repo.getSearchIndexFactory() == null) {
+      // Fallback for environments where the search subsystem isn't bootstrapped (e.g. unit
+      // tests that exercise the reader without the full Entity registry). Behaves the same
+      // as the pre-selective-fields code path.
+      return List.of("*");
+    }
+    return new ArrayList<>(repo.getSearchIndexFactory().getReindexFieldsFor(entityType));
   }
 
   static int calculateNumberOfReaders(int totalEntityRecords, int batchSize) {

openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexFactory.java

@@ -104,6 +104,28 @@
 @Slf4j
 public class SearchIndexFactory {
 
+  /**
+   * Returns the minimal set of fields the reindex path must request from
+   * {@code EntityRepository.setFields} for the given entity type. Probes the corresponding
+   * index class via {@link #buildIndex(String, Object)} with a {@code null} entity and calls
+   * {@link SearchIndex#getRequiredReindexFields()}. Index constructors must be safe with a null
+   * entity for this probe to work — they are today because field declarations are static.
+   */
+  public java.util.Set<String> getReindexFieldsFor(String entityType) {
+    try {
+      SearchIndex probe = buildIndex(entityType, null);
+      if (probe != null) {
+        return probe.getRequiredReindexFields();
+      }
+    } catch (Exception e) {
+      LOG.warn(
+          "Failed to probe reindex fields for entity type {}; falling back to common set: {}",
+          entityType,
+          e.getMessage());
+    }
+    return SearchIndex.COMMON_REINDEX_FIELDS;
+  }
+
   public SearchIndex buildIndex(String entityType, Object entity) {
     return switch (entityType) {
       case Entity.TABLE -> new TableIndex((Table) entity);
@@ -171,7 +193,9 @@ public SearchIndex buildIndex(String entityType, Object entity) {
       case Entity.PIPELINE_EXECUTION -> {
         PipelineExecutionIndex.PipelineExecutionData data =
             (PipelineExecutionIndex.PipelineExecutionData) entity;
-        yield new PipelineExecutionIndex(data.getPipeline(), data.getPipelineStatus());
+        yield data == null
+            ? new PipelineExecutionIndex(null, null)
+            : new PipelineExecutionIndex(data.getPipeline(), data.getPipelineStatus());
       }
       default -> buildExternalIndexes(entityType, entity);
     };

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java

@@ -3,6 +3,7 @@
 import static org.openmetadata.service.search.EntityBuilderConstant.DATA_MODEL_COLUMNS_NAME_KEYWORD;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -26,6 +27,13 @@ public Set<String> getExcludedFields() {
     return Set.of("children");
   }
 
+  @Override
+  public Set<String> getRequiredReindexFields() {
+    Set<String> fields = new HashSet<>(ColumnIndex.super.getRequiredReindexFields());
+    fields.add("dataModel");
+    return Collections.unmodifiableSet(fields);
+  }
+
   public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     Set<List<TagLabel>> tagsWithChildren = new HashSet<>();
     List<String> columnsWithChildrenName = new ArrayList<>();

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java

@@ -23,6 +23,13 @@ public Set<String> getExcludedFields() {
     return Set.of("dataModels");
   }
 
+  @Override
+  public Set<String> getRequiredReindexFields() {
+    Set<String> fields = new java.util.HashSet<>(SearchIndex.super.getRequiredReindexFields());
+    fields.add("charts");
+    return java.util.Collections.unmodifiableSet(fields);
+  }
+
   public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.DASHBOARD, dashboard));
     Map<String, Object> commonAttributes = getCommonAttributesMap(dashboard, Entity.DASHBOARD);

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseIndex.java

@@ -17,6 +17,13 @@ public Set<String> getExcludedFields() {
     return Set.of("databaseSchemas");
   }
 
+  @Override
+  public Set<String> getRequiredReindexFields() {
+    Set<String> fields = new java.util.HashSet<>(SearchIndex.super.getRequiredReindexFields());
+    fields.add("usageSummary");
+    return java.util.Collections.unmodifiableSet(fields);
+  }
+
   public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     Map<String, Object> commonAttributes = getCommonAttributesMap(database, Entity.DATABASE);
     doc.putAll(commonAttributes);

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryTermIndex.java

@@ -22,6 +22,13 @@ public Set<String> getExcludedFields() {
     return Set.of("children");
   }
 
+  @Override
+  public Set<String> getRequiredReindexFields() {
+    Set<String> fields = new java.util.HashSet<>(SearchIndex.super.getRequiredReindexFields());
+    fields.add("relatedTerms");
+    return java.util.Collections.unmodifiableSet(fields);
+  }
+
   public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     Map<String, Object> commonAttributes =
         getCommonAttributesMap(glossaryTerm, Entity.GLOSSARY_TERM);

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/IngestionPipelineIndex.java

@@ -29,6 +29,13 @@ public Set<String> getExcludedFields() {
     return excludeFields;
   }
 
+  @Override
+  public Set<String> getRequiredReindexFields() {
+    Set<String> fields = new java.util.HashSet<>(SearchIndex.super.getRequiredReindexFields());
+    fields.add("pipelineStatuses");
+    return java.util.Collections.unmodifiableSet(fields);
+  }
+
   public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     ParseTags parseTags =
         new ParseTags(Entity.getEntityTags(Entity.INGESTION_PIPELINE, ingestionPipeline));

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java

@@ -1,6 +1,7 @@
 package org.openmetadata.service.search.indexes;
 
 import java.util.Map;
+import java.util.Set;
 import org.openmetadata.schema.entity.data.Pipeline;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.ParseTags;
@@ -17,6 +18,14 @@ public Object getEntity() {
     return pipeline;
   }
 
+  @Override
+  public Set<String> getRequiredReindexFields() {
+    Set<String> fields = new java.util.HashSet<>(SearchIndex.super.getRequiredReindexFields());
+    fields.add("tasks");
+    return java.util.Collections.unmodifiableSet(fields);
+  }
+
+
   public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.PIPELINE, pipeline));
     Map<String, Object> commonAttributes = getCommonAttributesMap(pipeline, Entity.PIPELINE);

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java

@@ -56,6 +56,22 @@ public interface SearchIndex {
           "connection",
           "changeSummary");
 
+  /**
+   * Relationship/enrichment fields fetched by {@code EntityRepository.setFields} that every search
+   * document populates. Stored-JSON fields (name, displayName, description, service, entity-native
+   * counts) are NOT in this set — they live on the entity row and need no extra fetch.
+   */
+  Set<String> COMMON_REINDEX_FIELDS =
+      Set.of(
+          "owners",
+          "domains",
+          "reviewers",
+          "followers",
+          "votes",
+          "extension",
+          "certification",
+          "dataProducts");
+
   public static final SearchClient searchClient = Entity.getSearchRepository().getSearchClient();
   static final Logger LOG = LoggerFactory.getLogger(SearchIndex.class);
 
@@ -92,6 +108,21 @@ default Set<String> getExcludedFields() {
 
   Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> esDoc);
 
+  /**
+   * Returns the minimal set of fields the {@code SearchIndexApp} reindex path must ask
+   * {@code EntityRepository.setFields} to populate for this index to build a correct document.
+   *
+   * <p>Default is {@link #COMMON_REINDEX_FIELDS} plus {@code "tags"}. Individual index classes
+   * override to add entity-specific relationships. Keep this method side-effect-free and safe to
+   * call on a probe instance whose entity is {@code null} — it is invoked without an entity to
+   * discover fields statically.
+   */
+  default Set<String> getRequiredReindexFields() {
+    Set<String> fields = new java.util.HashSet<>(COMMON_REINDEX_FIELDS);
+    fields.add("tags");
+    return java.util.Collections.unmodifiableSet(fields);
+  }
+
   default Map<String, Object> getCommonAttributesMap(EntityInterface entity, String entityType) {
     Map<String, Object> map = new HashMap<>();
     map.put(

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SpreadsheetIndex.java

@@ -28,6 +28,14 @@ public Set<String> getExcludedFields() {
     return excludeSpreadsheetFields;
   }
 
+  @Override
+  public Set<String> getRequiredReindexFields() {
+    Set<String> fields = new java.util.HashSet<>(SearchIndex.super.getRequiredReindexFields());
+    fields.add("worksheets");
+    return java.util.Collections.unmodifiableSet(fields);
+  }
+
+
   public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     ParseTags parseTags = new ParseTags(Entity.getEntityTags(Entity.SPREADSHEET, spreadsheet));
     List<TagLabel> tags = new ArrayList<>();