Preserve highlighting for returnable: false fields

Author: jwils

config/schema/artifacts/datastore_config.yaml

@@ -1469,10 +1469,6 @@ index_templates:
           required: true
         _size:
           enabled: true
-        _source:
-          excludes:
-          - internal_name
-          - internal_details.*
       settings:
         index.mapping.ignore_malformed: false
         index.mapping.coerce: false

config/schema/artifacts_with_apollo/datastore_config.yaml

@@ -1469,10 +1469,6 @@ index_templates:
           required: true
         _size:
           enabled: true
-        _source:
-          excludes:
-          - internal_name
-          - internal_details.*
       settings:
         index.mapping.ignore_malformed: false
         index.mapping.coerce: false

elasticgraph-graphql/spec/acceptance/returnable_fields_spec.rb

@@ -0,0 +1,222 @@
+# Copyright 2024 - 2026 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require_relative "elasticgraph_graphql_acceptance_support"
+
+module ElasticGraph
+  RSpec.describe "ElasticGraph::GraphQL--returnable fields" do
+    include_context "ElasticGraph GraphQL acceptance support"
+
+    with_both_casing_forms do
+      let(:aggregated_values) { case_correctly("aggregated_values") }
+      let(:approximate_distinct_value_count) { case_correctly("approximate_distinct_value_count") }
+      let(:grouped_by) { case_correctly("grouped_by") }
+      let(:internal_details) { case_correctly("internal_details") }
+      let(:internal_name) { case_correctly("internal_name") }
+      let(:widget_aggregations) { case_correctly("widget_aggregations") }
+      let(:widgets) { case_correctly("widgets") }
+
+      it "supports filtering, sorting, grouping, aggregating, and highlighting a hidden leaf field" do
+        index_records(
+          build(:widget, id: "w1", internal_name: "alpha secret"),
+          build(:widget, id: "w2", internal_name: "beta secret"),
+          build(:widget, id: "w3", internal_name: "gamma hidden")
+        )
+
+        response = call_graphql_query(<<~QUERY)
+          query {
+            widgets(
+              filter: {
+                internal_name: {
+                  contains: {
+                    any_substring_of: ["secret"]
+                  }
+                }
+              }
+              order_by: [#{internal_name}_ASC]
+            ) {
+              edges {
+                node {
+                  id
+                }
+
+                highlights {
+                  internal_name
+                }
+              }
+            }
+
+            widget_aggregations {
+              nodes {
+                grouped_by {
+                  internal_name
+                }
+
+                count
+
+                aggregated_values {
+                  internal_name {
+                    approximate_distinct_value_count
+                  }
+                }
+              }
+            }
+          }
+        QUERY
+
+        expect(response.dig("data", widgets, "edges").map { |edge| edge.dig("node", "id") }).to eq(%w[w1 w2])
+        expect(response.dig("data", widgets, "edges").to_h { |edge| [edge.dig("node", "id"), edge.dig("highlights", internal_name)] }).to eq({
+          "w1" => ["<em>alpha secret</em>"],
+          "w2" => ["<em>beta secret</em>"]
+        })
+
+        aggregation_nodes = response
+          .dig("data", widget_aggregations, "nodes")
+          .sort_by { |node| node.dig(grouped_by, internal_name) }
+
+        expect(aggregation_nodes).to eq([
+          {
+            grouped_by => {internal_name => "alpha secret"},
+            "count" => 1,
+            aggregated_values => {internal_name => {approximate_distinct_value_count => 1}}
+          },
+          {
+            grouped_by => {internal_name => "beta secret"},
+            "count" => 1,
+            aggregated_values => {internal_name => {approximate_distinct_value_count => 1}}
+          },
+          {
+            grouped_by => {internal_name => "gamma hidden"},
+            "count" => 1,
+            aggregated_values => {internal_name => {approximate_distinct_value_count => 1}}
+          }
+        ])
+
+        expect {
+          hidden_field_response = call_graphql_query(<<~QUERY, allow_errors: true)
+            query {
+              widgets {
+                edges {
+                  node {
+                    id
+                    internal_name
+                  }
+                }
+              }
+            }
+          QUERY
+
+          expect_error_related_to(hidden_field_response, "Widget", internal_name, "doesn't exist on type")
+        }.to log(a_string_including("Widget", internal_name, "doesn't exist on type"))
+      end
+
+      it "supports filtering, grouping, aggregating, and highlighting a hidden object field" do
+        index_records(
+          build(:widget, id: "w1", internal_details: build(:widget_internal_details, name: "alpha vault")),
+          build(:widget, id: "w2", internal_details: build(:widget_internal_details, name: "beta vault")),
+          build(:widget, id: "w3", internal_details: build(:widget_internal_details, name: "gamma archive"))
+        )
+
+        response = call_graphql_query(<<~QUERY)
+          query {
+            widgets(
+              filter: {
+                internal_details: {
+                  name: {
+                    contains: {
+                      any_substring_of: ["vault"]
+                    }
+                  }
+                }
+              }
+            ) {
+              edges {
+                node {
+                  id
+                }
+
+                highlights {
+                  internal_details {
+                    name
+                  }
+                }
+              }
+            }
+
+            widget_aggregations {
+              nodes {
+                grouped_by {
+                  internal_details {
+                    name
+                  }
+                }
+
+                count
+
+                aggregated_values {
+                  internal_details {
+                    name {
+                      approximate_distinct_value_count
+                    }
+                  }
+                }
+              }
+            }
+          }
+        QUERY
+
+        expect(response.dig("data", widgets, "edges").map { |edge| edge.dig("node", "id") }).to match_array(%w[w1 w2])
+        expect(response.dig("data", widgets, "edges").to_h { |edge| [edge.dig("node", "id"), edge.dig("highlights", internal_details, "name")] }).to eq({
+          "w1" => ["<em>alpha vault</em>"],
+          "w2" => ["<em>beta vault</em>"]
+        })
+
+        aggregation_nodes = response
+          .dig("data", widget_aggregations, "nodes")
+          .sort_by { |node| node.dig(grouped_by, internal_details, "name") }
+
+        expect(aggregation_nodes).to eq([
+          {
+            grouped_by => {internal_details => {"name" => "alpha vault"}},
+            "count" => 1,
+            aggregated_values => {internal_details => {"name" => {approximate_distinct_value_count => 1}}}
+          },
+          {
+            grouped_by => {internal_details => {"name" => "beta vault"}},
+            "count" => 1,
+            aggregated_values => {internal_details => {"name" => {approximate_distinct_value_count => 1}}}
+          },
+          {
+            grouped_by => {internal_details => {"name" => "gamma archive"}},
+            "count" => 1,
+            aggregated_values => {internal_details => {"name" => {approximate_distinct_value_count => 1}}}
+          }
+        ])
+
+        expect {
+          hidden_field_response = call_graphql_query(<<~QUERY, allow_errors: true)
+            query {
+              widgets {
+                edges {
+                  node {
+                    id
+                    internal_details {
+                      name
+                    }
+                  }
+                }
+              }
+            }
+          QUERY
+
+          expect_error_related_to(hidden_field_response, "Widget", internal_details, "doesn't exist on type")
+        }.to log(a_string_including("Widget", internal_details, "doesn't exist on type"))
+      end
+    end
+  end
+end

elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_indices.rb

@@ -292,22 +292,33 @@ def fields_with_sources
           indexing_fields_by_name_in_index.values.reject { |f| f.source.nil? }
         end
 
-        # Returns the list of `_source.excludes` paths for non-returnable fields.
+        # Returns the list of `_source.excludes` paths for non-returnable, non-highlightable fields.
+        #
+        # Hidden highlightable fields must remain in `_source` so the datastore can still
+        # produce search highlight snippets for them.
         #
         # Uses `indexing_fields_by_name_in_index` for traversal (same as
         # `index_field_runtime_metadata_tuples`) to avoid infinite recursion
         # through interface/union subtype cycles.
         #
         # @private
-        def source_excludes_paths(path_prefix: "")
+        def source_excludes_paths(path_prefix: "", under_non_returnable_parent: false)
           indexing_fields_by_name_in_index.flat_map do |name, field|
             path = path_prefix + name
             object_type = field.type.fully_unwrapped.as_object_type
+            non_returnable = under_non_returnable_parent || !field.returnable?
 
-            if !field.returnable?
-              [object_type ? "#{path}.*" : path]
-            elsif object_type
-              object_type.source_excludes_paths(path_prefix: "#{path}.")
+            if object_type
+              if non_returnable && !field.highlightable?
+                ["#{path}.*"]
+              else
+                object_type.source_excludes_paths(
+                  path_prefix: "#{path}.",
+                  under_non_returnable_parent: non_returnable
+                )
+              end
+            elsif non_returnable && !field.highlightable?
+              [path]
             else
               []
             end

elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/miscellaneous_spec.rb

@@ -117,25 +117,38 @@ module SchemaDefinition
         })
       end
 
-      it "uses `name_in_index` in `_source.excludes` for `returnable: false` fields" do
+      it "uses `name_in_index` in `_source.excludes` for non-highlightable `returnable: false` fields" do
         mapping = index_mapping_for "my_type" do |s|
           s.object_type "MyType" do |t|
             t.field "id", "ID"
-            t.field "name", "String"
-            t.field "internal_code_gql", "String", name_in_index: "internal_code", returnable: false
+            t.field "internal_count_gql", "Int", name_in_index: "internal_count", returnable: false
             t.index "my_type"
           end
         end
 
-        expect(mapping.dig("_source", "excludes")).to contain_exactly("internal_code")
+        expect(mapping.dig("_source", "excludes")).to contain_exactly("internal_count")
         # The field should still appear in properties (it's indexed, just not in _source)
+        expect(mapping.dig("properties", "internal_count")).to eq({"type" => "integer"})
+      end
+
+      it "keeps highlightable `returnable: false` fields in `_source` so they can still be highlighted" do
+        mapping = index_mapping_for "my_type" do |s|
+          s.object_type "MyType" do |t|
+            t.field "id", "ID"
+            t.field "internal_code", "String", returnable: false
+            t.index "my_type"
+          end
+        end
+
+        expect(mapping).not_to have_key("_source")
         expect(mapping.dig("properties", "internal_code")).to eq({"type" => "keyword"})
       end
 
-      it "includes `.*` in `_source.excludes` for `returnable: false` object fields" do
+      it "only excludes non-highlightable descendants for highlightable `returnable: false` object fields" do
         mapping = index_mapping_for "my_type" do |s|
           s.object_type "InternalMetadata" do |t|
             t.field "internal_code", "String"
+            t.field "internal_count", "Int"
           end
 
           s.object_type "MyType" do |t|
@@ -145,33 +158,35 @@ module SchemaDefinition
           end
         end
 
-        expect(mapping).to include("_source" => {"excludes" => ["internal_metadata.*"]})
+        expect(mapping).to include("_source" => {"excludes" => ["internal_metadata.internal_count"]})
         expect(mapping.dig("properties", "internal_metadata", "properties", "internal_code")).to eq({"type" => "keyword"})
+        expect(mapping.dig("properties", "internal_metadata", "properties", "internal_count")).to eq({"type" => "integer"})
       end
 
-      it "excludes `returnable: false` indexing-only fields but not `graphql_only` fields" do
+      it "excludes non-highlightable `returnable: false` indexing-only fields but not `graphql_only` fields" do
         mapping = index_mapping_for "my_type" do |s|
           s.object_type "MyType" do |t|
             t.field "id", "ID"
             t.field "name", "String"
-            t.field "legacy_name", "String", graphql_only: true, name_in_index: "name", returnable: false
-            t.field "internal_code", "String", indexing_only: true, returnable: false
+            t.field "legacy_count", "Int", graphql_only: true, name_in_index: "legacy_count_index", returnable: false
+            t.field "internal_count", "Int", indexing_only: true, returnable: false
             t.index "my_type"
           end
         end
 
-        expect(mapping.dig("_source", "excludes")).to contain_exactly("internal_code")
+        expect(mapping.dig("_source", "excludes")).to contain_exactly("internal_count")
         expect(mapping.fetch("properties")).to include(
           "name" => {"type" => "keyword"},
-          "internal_code" => {"type" => "keyword"}
+          "internal_count" => {"type" => "integer"}
         )
-        expect(mapping.fetch("properties")).not_to include("legacy_name")
+        expect(mapping.fetch("properties")).not_to include("legacy_count")
+        expect(mapping.fetch("properties")).not_to include("legacy_count_index")
       end
 
       it "uses full indexed paths in `_source.excludes` for `returnable: false` fields under nested mappings" do
         mapping = index_mapping_for "my_type" do |s|
           s.object_type "Parent" do |t|
-            t.field "child", "String", name_in_index: "child_in_index", returnable: false
+            t.field "child_count_gql", "Int", name_in_index: "child_count", returnable: false
           end
 
           s.object_type "Grandparent" do |t|
@@ -187,13 +202,13 @@ module SchemaDefinition
           end
         end
 
-        expect(mapping.dig("_source", "excludes")).to contain_exactly("grandparents_in_index.parent_in_index.child_in_index")
+        expect(mapping.dig("_source", "excludes")).to contain_exactly("grandparents_in_index.parent_in_index.child_count")
         expect(mapping.dig("properties", "grandparents_in_index")).to include(
           "type" => "nested",
           "properties" => {
             "parent_in_index" => {
               "properties" => {
-                "child_in_index" => {"type" => "keyword"}
+                "child_count" => {"type" => "integer"}
               }
             }
           }