Prototype: fetchable: false field option to exclude fields from _source

Adds a new `fetchable: false` option for field definitions that allows fields
to remain filterable, sortable, groupable, and aggregatable while being excluded
from the GraphQL output type and from `_source` in the datastore mapping.

Changes:
- Field struct: new `:fetchable` member with `fetchable?` predicate (defaults true)
- fields_sdl: filters out non-fetchable fields from output type SDL
- Index#mappings: emits `_source.excludes` for non-fetchable field paths
- TypeWithSubfields#non_fetchable_field_paths: collects paths using
  indexing_fields_by_name_in_index to avoid interface/union recursion cycles
- to_filter_field: resets fetchable to nil so filter fields still appear in
  filter input types

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: jwils

elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb

@@ -326,6 +326,14 @@ def mappings
             # made against the wrong shard.
             hash["_routing"] = {"required" => true} if uses_custom_routing?
             hash["_size"] = {"enabled" => true} if schema_def_state.index_document_sizes?
+
+            # Exclude non-fetchable fields from `_source` to save storage. These fields are still
+            # indexed (in the inverted index and/or doc_values) for filtering, sorting, and aggregation,
+            # but their values are not stored in the compressed `_source` blob.
+            if indexed_type.respond_to?(:non_fetchable_field_paths)
+              source_excludes = indexed_type.non_fetchable_field_paths
+              hash["_source"] = {"excludes" => source_excludes} if source_excludes.any?
+            end
           end
         end
 

elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/field.rb

@@ -91,7 +91,7 @@ class Field < Struct.new(
         :name, :original_type, :parent_type, :original_type_for_derived_types, :schema_def_state, :accuracy_confidence,
         :filter_customizations, :grouped_by_customizations, :highlights_customizations, :sub_aggregations_customizations,
         :aggregated_values_customizations, :sort_order_enum_value_customizations, :args,
-        :sortable, :filterable, :aggregatable, :groupable, :highlightable,
+        :sortable, :filterable, :aggregatable, :groupable, :highlightable, :fetchable,
         :graphql_only, :source, :runtime_field_script, :relationship, :singular_name,
         :computation_detail, :non_nullable_in_json_schema, :as_input,
         :name_in_index, :resolver
@@ -106,7 +106,7 @@ def initialize(
           name:, type:, parent_type:, schema_def_state:,
           accuracy_confidence: :high, name_in_index: name,
           type_for_derived_types: nil, graphql_only: nil, singular: nil,
-          sortable: nil, filterable: nil, aggregatable: nil, groupable: nil, highlightable: nil,
+          sortable: nil, filterable: nil, aggregatable: nil, groupable: nil, highlightable: nil, fetchable: nil,
           as_input: false, resolver: nil
         )
           type_ref = schema_def_state.type_ref(type)
@@ -129,6 +129,7 @@ def initialize(
             aggregatable: aggregatable,
             groupable: groupable,
             highlightable: highlightable,
+            fetchable: fetchable,
             graphql_only: graphql_only,
             source: nil,
             runtime_field_script: nil,
@@ -743,6 +744,16 @@ def highlightable?
           type_for_derived_types.fully_unwrapped.as_object_type&.supports?(&:highlightable?)
         end
 
+        # Indicates if this field is fetchable in GraphQL query responses. When `false`, the field will
+        # still be available for filtering, sorting, grouping, and aggregation, but will not appear in the
+        # GraphQL output type and its data will be excluded from `_source` in the datastore for storage savings.
+        #
+        # @return [Boolean] true if this field's data can be fetched (default: true)
+        def fetchable?
+          return true if fetchable.nil?
+          fetchable
+        end
+
         # Defines an argument on the field.
         #
         # @note ElasticGraph takes care of defining arguments for all the query features it supports, so there is generally no need to use
@@ -892,7 +903,10 @@ def to_filter_field(parent_type:, for_single_value: !type_for_derived_types.list
             parent_type: parent_type,
             name_in_index: name_in_index,
             type_for_derived_types: nil,
-            resolver: nil
+            resolver: nil,
+            # Filter fields should always appear in their parent input type's SDL regardless
+            # of the source field's fetchability.
+            fetchable: nil
           )
 
           schema_def_state.factory.new_field(**params).tap do |f|

elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_with_subfields.rb

@@ -137,6 +137,9 @@ def name
         #   ElasticGraph will infer field sortability based on the field's GraphQL type and mapping type.
         # @option options [Boolean] highlightable force-enables or disables the ability to request search highlights for this field. When
         #   not provided, ElasticGraph will infer field highlightable based on the field's mapping type.
+        # @option options [Boolean] fetchable when set to `false`, the field will not appear in the GraphQL output type and its data
+        #   will be excluded from `_source` in the datastore for storage savings. The field will still be available for filtering,
+        #   sorting, grouping, and aggregation. Defaults to `true`.
         # @yield [Field] the field for further customization
         # @return [void]
         #
@@ -482,6 +485,27 @@ def to_indexing_field_type
           )
         end
 
+        # Returns the list of field paths (in dotted notation) for fields that have `fetchable: false`.
+        # These paths are used to populate `_source.excludes` in the datastore mapping so that
+        # non-fetchable field data is not stored in `_source`, saving storage space.
+        #
+        # Uses `indexing_fields_by_name_in_index` for traversal (same as `index_field_runtime_metadata_tuples`)
+        # to avoid infinite recursion through interface/union subtype cycles.
+        #
+        # @private
+        def non_fetchable_field_paths(path_prefix: "")
+          indexing_fields_by_name_in_index.flat_map do |name, field|
+            path = path_prefix + name
+            if !field.fetchable?
+              [path]
+            elsif (object_type = field.type.fully_unwrapped.as_object_type) && object_type.respond_to?(:non_fetchable_field_paths)
+              object_type.non_fetchable_field_paths(path_prefix: "#{path}.")
+            else
+              []
+            end
+          end
+        end
+
         # @private
         def current_sources
           indexing_fields_by_name_in_index.values.flat_map do |field|
@@ -531,6 +555,7 @@ def index_field_runtime_metadata_tuples(
 
         def fields_sdl(&arg_selector)
           graphql_fields_by_name.values
+            .select(&:fetchable?)
             .map { |f| f.to_sdl(&arg_selector) }
             .flat_map { |sdl| sdl.split("\n") }
             .join("\n  ")

elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/index_mappings/miscellaneous_spec.rb

@@ -117,6 +117,33 @@ module SchemaDefinition
         })
       end
 
+      it "includes `_source.excludes` for `fetchable: false` fields" do
+        mapping = index_mapping_for "my_type" do |s|
+          s.object_type "MyType" do |t|
+            t.field "id", "ID"
+            t.field "name", "String"
+            t.field "internal_code", "String", fetchable: false
+            t.index "my_type"
+          end
+        end
+
+        expect(mapping).to include("_source" => {"excludes" => ["internal_code"]})
+        # The field should still appear in properties (it's indexed, just not in _source)
+        expect(mapping.dig("properties", "internal_code")).to eq({"type" => "keyword"})
+      end
+
+      it "does not include `_source` config when all fields are fetchable" do
+        mapping = index_mapping_for "my_type" do |s|
+          s.object_type "MyType" do |t|
+            t.field "id", "ID"
+            t.field "name", "String"
+            t.index "my_type"
+          end
+        end
+
+        expect(mapping).not_to have_key("_source")
+      end
+
       it "keeps `source_from` fields in the mapping so that indexed documents support the field even though it comes from an alternate source" do
         mapping = index_mapping_for "components" do |s|
           s.object_type "Widget" do |t|

elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/object_type_spec.rb

@@ -643,6 +643,39 @@ module SchemaDefinition
           end
         end
 
+        it "excludes `fetchable: false` fields from the output type but keeps them in filter, sort, grouped_by, aggregated_values, and highlights types" do
+          result = define_schema do |api|
+            api.object_type "Widget" do |t|
+              t.field "id", "ID"
+              t.field "name", "String"
+              t.field "internal_code", "String", fetchable: false
+              t.index "widgets"
+            end
+          end
+
+          expect(type_def_from(result, "Widget")).to eq(<<~EOS.strip)
+            type Widget {
+              id: ID
+              name: String
+            }
+          EOS
+
+          # fetchable: false field should still appear in filter input
+          expect(filter_type_from(result, "Widget")).to include("internal_code: StringFilterInput")
+
+          # fetchable: false field should still appear in sort order
+          expect(sort_order_type_from(result, "Widget")).to include("internal_code_ASC")
+
+          # fetchable: false field should still appear in grouped_by
+          expect(grouped_by_type_from(result, "Widget")).to include("internal_code: String")
+
+          # fetchable: false field should still appear in aggregated_values
+          expect(aggregated_values_type_from(result, "Widget")).to include("internal_code: NonNumericAggregatedValues")
+
+          # fetchable: false field should still appear in highlights
+          expect(highlights_type_from(result, "Widget")).to include("internal_code: [String!]!")
+        end
+
         def object_type(name, *args, pre_def: nil, include_docs: true, &block)
           result = define_schema do |api|
             pre_def&.call(api)