Make Selective Field Query during Reindexing (#27723)

* Make Selective Field Query

* Minor nit

* Fix Failing Tests

Author: mohityadav766

openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/EntityReader.java

@@ -326,7 +326,15 @@ static List<String> getSearchIndexFields(String entityType) {
     if (TIME_SERIES_ENTITIES.contains(entityType)) {
       return List.of();
     }
-    return List.of("*");
+    org.openmetadata.service.search.SearchRepository repo =
+        org.openmetadata.service.Entity.getSearchRepository();
+    if (repo == null || repo.getSearchIndexFactory() == null) {
+      // Fallback for environments where the search subsystem isn't bootstrapped (e.g. unit
+      // tests that exercise the reader without the full Entity registry). Behaves the same
+      // as the pre-selective-fields code path.
+      return List.of("*");
+    }
+    return new ArrayList<>(repo.getSearchIndexFactory().getReindexFieldsFor(entityType));
   }
 
   static int calculateNumberOfReaders(int totalEntityRecords, int batchSize) {

openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexFactory.java

@@ -107,6 +107,28 @@
 @Slf4j
 public class SearchIndexFactory {
 
+  /**
+   * Returns the minimal set of fields the reindex path must request from
+   * {@code EntityRepository.setFields} for the given entity type. Probes the corresponding
+   * index class via {@link #buildIndex(String, Object)} with a {@code null} entity and calls
+   * {@link SearchIndex#getRequiredReindexFields()}. Index constructors must be safe with a null
+   * entity for this probe to work — they are today because field declarations are static.
+   */
+  public java.util.Set<String> getReindexFieldsFor(String entityType) {
+    try {
+      SearchIndex probe = buildIndex(entityType, null);
+      if (probe != null) {
+        return probe.getRequiredReindexFields();
+      }
+    } catch (Exception e) {
+      LOG.warn(
+          "Failed to probe reindex fields for entity type {}; falling back to common set: {}",
+          entityType,
+          e.getMessage());
+    }
+    return SearchIndex.COMMON_REINDEX_FIELDS;
+  }
+
   public SearchIndex buildIndex(String entityType, Object entity) {
     return switch (entityType) {
       case Entity.TABLE -> new TableIndex((Table) entity);
@@ -177,7 +199,9 @@ public SearchIndex buildIndex(String entityType, Object entity) {
       case Entity.PIPELINE_EXECUTION -> {
         PipelineExecutionIndex.PipelineExecutionData data =
             (PipelineExecutionIndex.PipelineExecutionData) entity;
-        yield new PipelineExecutionIndex(data.getPipeline(), data.getPipelineStatus());
+        yield data == null
+            ? new PipelineExecutionIndex(null, null)
+            : new PipelineExecutionIndex(data.getPipeline(), data.getPipelineStatus());
       }
       default -> buildExternalIndexes(entityType, entity);
     };

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/ContainerIndex.java

@@ -3,6 +3,7 @@
 import static org.openmetadata.service.search.EntityBuilderConstant.DATA_MODEL_COLUMNS_NAME_KEYWORD;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -35,6 +36,13 @@ public Set<String> getExcludedFields() {
     return Set.of("children");
   }
 
+  @Override
+  public Set<String> getRequiredReindexFields() {
+    Set<String> fields = new HashSet<>(DataAssetIndex.super.getRequiredReindexFields());
+    fields.add("dataModel");
+    return Collections.unmodifiableSet(fields);
+  }
+
   public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     if (container.getDataModel() != null && container.getDataModel().getColumns() != null) {
       List<FlattenColumn> cols = new ArrayList<>();

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DashboardIndex.java

@@ -32,6 +32,13 @@ public Set<String> getExcludedFields() {
     return Set.of("dataModels");
   }
 
+  @Override
+  public Set<String> getRequiredReindexFields() {
+    Set<String> fields = new java.util.HashSet<>(DataAssetIndex.super.getRequiredReindexFields());
+    fields.add("charts");
+    return java.util.Collections.unmodifiableSet(fields);
+  }
+
   public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     return doc;
   }

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/DatabaseIndex.java

@@ -22,6 +22,13 @@ public Set<String> getExcludedFields() {
     return Set.of("databaseSchemas");
   }
 
+  @Override
+  public Set<String> getRequiredReindexFields() {
+    Set<String> fields = new java.util.HashSet<>(TaggableIndex.super.getRequiredReindexFields());
+    fields.add("usageSummary");
+    return java.util.Collections.unmodifiableSet(fields);
+  }
+
   public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     return doc;
   }

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/GlossaryTermIndex.java

@@ -29,6 +29,13 @@ public Set<String> getExcludedFields() {
     return Set.of("children");
   }
 
+  @Override
+  public Set<String> getRequiredReindexFields() {
+    Set<String> fields = new java.util.HashSet<>(TaggableIndex.super.getRequiredReindexFields());
+    fields.add("relatedTerms");
+    return java.util.Collections.unmodifiableSet(fields);
+  }
+
   public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     if (doc.containsKey("glossary") && glossaryTerm.getGlossary() != null) {
       @SuppressWarnings("unchecked")

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/IngestionPipelineIndex.java

@@ -33,6 +33,13 @@ public Set<String> getExcludedFields() {
     return excludeFields;
   }
 
+  @Override
+  public Set<String> getRequiredReindexFields() {
+    Set<String> fields = new java.util.HashSet<>(TaggableIndex.super.getRequiredReindexFields());
+    fields.add("pipelineStatuses");
+    return java.util.Collections.unmodifiableSet(fields);
+  }
+
   public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     doc.put(
         "name",

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/PipelineIndex.java

@@ -1,6 +1,7 @@
 package org.openmetadata.service.search.indexes;
 
 import java.util.Map;
+import java.util.Set;
 import org.openmetadata.schema.entity.data.Pipeline;
 import org.openmetadata.service.Entity;
 
@@ -21,6 +22,13 @@ public String getEntityTypeName() {
     return Entity.PIPELINE;
   }
 
+  @Override
+  public Set<String> getRequiredReindexFields() {
+    Set<String> fields = new java.util.HashSet<>(DataAssetIndex.super.getRequiredReindexFields());
+    fields.add("tasks");
+    return java.util.Collections.unmodifiableSet(fields);
+  }
+
   @Override
   public Object getIndexServiceType() {
     return pipeline.getServiceType();

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java

@@ -53,6 +53,23 @@ public interface SearchIndex {
           "connection",
           "changeSummary");
 
+  /**
+   * Relationship/enrichment fields fetched by {@code EntityRepository.setFields} that every search
+   * document populates via {@link #populateCommonFields(Map, EntityInterface, String)}. Stored-JSON
+   * fields (name, displayName, description, service, entity-native counts) are NOT in this set —
+   * they live on the entity row and need no extra fetch.
+   */
+  Set<String> COMMON_REINDEX_FIELDS =
+      Set.of(
+          "owners",
+          "domains",
+          "reviewers",
+          "followers",
+          "votes",
+          "extension",
+          "certification",
+          "dataProducts");
+
   SearchClient searchClient = Entity.getSearchRepository().getSearchClient();
   Logger LOG = LoggerFactory.getLogger(SearchIndex.class);
 
@@ -114,6 +131,23 @@ default Set<String> getExcludedFields() {
 
   Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> esDoc);
 
+  /**
+   * Returns the minimal set of fields the {@code SearchIndexApp} reindex path must ask
+   * {@code EntityRepository.setFields} to populate for this index to build a correct document.
+   *
+   * <p>Default is {@link #COMMON_REINDEX_FIELDS}, augmented with {@code "tags"} when the index
+   * implements {@link TaggableIndex}. Individual index classes override to add entity-specific
+   * relationships. Keep this method side-effect-free and safe to call on a probe instance whose
+   * entity is {@code null} — it is invoked without an entity to discover fields statically.
+   */
+  default Set<String> getRequiredReindexFields() {
+    Set<String> fields = new java.util.HashSet<>(COMMON_REINDEX_FIELDS);
+    if (this instanceof TaggableIndex) {
+      fields.add("tags");
+    }
+    return java.util.Collections.unmodifiableSet(fields);
+  }
+
   /**
    * Populates common entity fields into the search index document. Called automatically by {@link
    * #buildSearchIndexDoc()} for all EntityInterface-based entities. Individual index classes should

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SpreadsheetIndex.java

@@ -36,6 +36,13 @@ public Object getIndexServiceType() {
     return spreadsheet.getServiceType();
   }
 
+  @Override
+  public Set<String> getRequiredReindexFields() {
+    Set<String> fields = new java.util.HashSet<>(DataAssetIndex.super.getRequiredReindexFields());
+    fields.add("worksheets");
+    return java.util.Collections.unmodifiableSet(fields);
+  }
+
   public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc) {
     doc.put("directory", getEntityWithDisplayName(spreadsheet.getDirectory()));
     doc.put("mimeType", spreadsheet.getMimeType());