Support retrieved_from: :doc_values for direct leaf fields (#1110)

Add a narrow secondary retrieval path for fields that should stay
returnable in GraphQL while being excluded from stored `_source`.

When a field is marked `retrieved_from: :doc_values`, ElasticGraph:

- keeps the field in GraphQL output types
- excludes the field from datastore `_source` via `_source.excludes`
- requests the field through datastore `docvalue_fields`
- resolves the field from `fields` in datastore hits when `_source` omits it

The change is intentionally narrow: only direct, non-list, non-text
GraphQL leaf fields on indexed root document types are supported.
Query planning only uses `docvalue_fields` when every participating
index definition agrees on the retrieval method.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: jwils

config/schema/artifacts/datastore_config.yaml

@@ -1463,6 +1463,9 @@ index_templates:
           required: true
         _size:
           enabled: true
+        _source:
+          excludes:
+          - workspace_id2
       settings:
         index.mapping.ignore_malformed: false
         index.mapping.coerce: false
@@ -1505,6 +1508,9 @@ indices:
           dynamic: 'false'
       _size:
         enabled: true
+      _source:
+        excludes:
+        - full_address
     settings:
       index.mapping.ignore_malformed: false
       index.mapping.coerce: false

config/schema/artifacts/runtime_metadata.yaml

@@ -1446,6 +1446,7 @@ index_definitions_by_name:
       __counts.shapes|type:
         source: __self
       full_address:
+        retrieved_from: doc_values
         source: __self
       geo_location.lat:
         source: __self
@@ -2613,6 +2614,7 @@ index_definitions_by_name:
       weight_in_ng_str:
         source: __self
       workspace_id2:
+        retrieved_from: doc_values
         source: __self
       workspace_name:
         source: workspace

config/schema/artifacts_with_apollo/datastore_config.yaml

@@ -1463,6 +1463,9 @@ index_templates:
           required: true
         _size:
           enabled: true
+        _source:
+          excludes:
+          - workspace_id2
       settings:
         index.mapping.ignore_malformed: false
         index.mapping.coerce: false
@@ -1505,6 +1508,9 @@ indices:
           dynamic: 'false'
       _size:
         enabled: true
+      _source:
+        excludes:
+        - full_address
     settings:
       index.mapping.ignore_malformed: false
       index.mapping.coerce: false

config/schema/artifacts_with_apollo/runtime_metadata.yaml

@@ -1475,6 +1475,7 @@ index_definitions_by_name:
       __counts.shapes|type:
         source: __self
       full_address:
+        retrieved_from: doc_values
         source: __self
       geo_location.lat:
         source: __self
@@ -2642,6 +2643,7 @@ index_definitions_by_name:
       weight_in_ng_str:
         source: __self
       workspace_id2:
+        retrieved_from: doc_values
         source: __self
       workspace_name:
         source: workspace

config/schema/widgets.rb

@@ -75,8 +75,8 @@
     t.field "id", "ID!"
 
     # Here we use an alternate name for this field since it's the routing field and want to verify
-    # that `name_in_index` works correctly on routing fields.
-    t.field "workspace_id", "ID", name_in_index: "workspace_id2"
+    # that `name_in_index` works correctly on routing fields, including when fetched from doc values.
+    t.field "workspace_id", "ID", name_in_index: "workspace_id2", retrieved_from: :doc_values
 
     # It's a bit funny we have both `amount_cents` and `cost` but it's nice to be able to test
     # aggregations on both a root numeric field and on a nested one, so we are keeping both here.
@@ -367,7 +367,7 @@
     # We use `indexing_only: true` here to verify that `id` can be an indexing-only field.
     t.field "id", "ID!", indexing_only: true
 
-    t.field "full_address", "String!"
+    t.field "full_address", "String!", retrieved_from: :doc_values
     t.field "timestamps", "AddressTimestamps"
     t.field "geo_location", "GeoLocation"
 

elasticgraph-graphql/lib/elastic_graph/graphql/datastore_query.rb

@@ -303,7 +303,7 @@ def ignored_values_for_routing
       def to_datastore_body
         @to_datastore_body ||= aggregations_datastore_body
           .merge(document_paginator.to_datastore_body)
-          .merge({highlight: highlight, query: filter_interpreter.build_query(all_filters), _source: source}.compact)
+          .merge({docvalue_fields: docvalue_fields, highlight: highlight, query: filter_interpreter.build_query(all_filters), _source: source}.compact)
       end
 
       def aggregations_datastore_body
@@ -323,13 +323,33 @@ def aggregations_datastore_body
       # we only ask for the fields we need to return.
       def source
         return true if request_all_fields
-        requested_source_fields = requested_fields - ["id"]
+        requested_source_fields = requested_fields_for_source - ["id"]
         return false if requested_source_fields.empty?
         # Merging in requested_fields as _source:{includes:} based on Elasticsearch documentation:
         # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html#include-exclude
         {includes: requested_source_fields.to_a}
       end
 
+      def docvalue_fields
+        requested_docvalue_fields =
+          if request_all_fields
+            # When requesting all fields we send `_source: true`, but fields excluded from stored
+            # `_source` (because they use `retrieved_from: :doc_values`) still need an alternative
+            # retrieval path. We therefore request docvalue_fields for any field that ANY index
+            # definition stores in doc values, unlike the selective path below which requires
+            # unanimity across all index definitions.
+            all_docvalue_fields
+          else
+            requested_fields.select do |field_path|
+              requested_via_doc_values?(field_path)
+            end
+          end
+
+        return nil if requested_docvalue_fields.empty?
+
+        requested_docvalue_fields.to_a
+      end
+
       def highlight
         return nil if !request_all_highlights && requested_highlights.empty?
 
@@ -343,6 +363,35 @@ def highlight
         {fields:, highlight_query:}.compact
       end
 
+      def requested_fields_for_source
+        @requested_fields_for_source ||= requested_fields.reject do |field_path|
+          requested_via_doc_values?(field_path)
+        end
+      end
+
+      def all_docvalue_fields
+        @all_docvalue_fields ||= search_index_definitions.flat_map do |index_def|
+          index_def.fields_by_path.filter_map do |field_path, field|
+            field_path if field.retrieved_from_doc_values?
+          end
+        end.to_set
+      end
+
+      # Returns true only when every participating index definition agrees the field should be
+      # retrieved via doc values. When they disagree we fall back to `_source` so that source-backed
+      # indices can return the field normally; the doc-values-backed index will also have the value
+      # available in `_source` in that case (a disagreement like this should not happen in practice,
+      # since `retrieved_from` is set once per field definition and propagates to all index definitions).
+      def requested_via_doc_values?(field_path)
+        return false if field_path == "id"
+
+        field_definitions = search_index_definitions.filter_map do |index_def|
+          index_def.fields_by_path[field_path]
+        end
+
+        field_definitions.any? && field_definitions.all?(&:retrieved_from_doc_values?)
+      end
+
       # Encapsulates dependencies of `Query`, giving us something we can expose off of `application`
       # to build queries when desired.
       class Builder < Support::MemoizableData.define(:runtime_metadata, :logger, :filter_interpreter, :filter_node_interpreter, :default_page_size, :max_page_size)

elasticgraph-graphql/lib/elastic_graph/graphql/datastore_response/document.rb

@@ -8,20 +8,23 @@
 
 require "elastic_graph/graphql/decoded_cursor"
 require "elastic_graph/support/memoizable_data"
+require "elastic_graph/support/hash_util"
 require "forwardable"
 
 module ElasticGraph
   class GraphQL
     module DatastoreResponse
+      # @private
+      # Sentinel value to distinguish "no default given" from an explicit `nil` default in {Document#fetch_value_at}.
+      UNSET = ::Object.new.freeze
+
       # Represents a document fetched from the datastore. Exposes both the raw metadata
       # provided by the datastore and the doc payload itself. In addition, you can treat
       # it just like a document hash using `#[]` or `#fetch`.
       Document = Support::MemoizableData.define(:raw_data, :payload, :decoded_cursor_factory) do
         # @implements Document
         extend Forwardable
 
-        def_delegators :payload, :[], :fetch
-
         def self.build(raw_data, decoded_cursor_factory: DecodedCursor::Factory::Null)
           source = raw_data.fetch("_source") do
             {} # : ::Hash[::String, untyped]
@@ -51,6 +54,41 @@ def id
           raw_data["_id"]
         end
 
+        def [](key)
+          return payload[key] if payload.key?(key)
+          docvalue_field(key)&.first
+        end
+
+        def fetch(key, *default)
+          if !default.empty? && block_given?
+            raise ArgumentError, "wrong number of arguments (given #{default.size + 1}, expected 1)"
+          end
+          return payload[key] if payload.key?(key)
+          if (field_values = docvalue_field(key))
+            return field_values.first
+          end
+          return yield(key) if block_given?
+          return default.first unless default.empty?
+          raise KeyError, "key not found: #{key}"
+        end
+
+        def fetch_value_at(path, default_value: UNSET)
+          Support::HashUtil.fetch_value_at_path(payload, path) do
+            if (field_values = docvalue_field(path.join(".")))
+              next field_values.first
+            end
+            next yield(path) if block_given?
+            next default_value unless default_value.equal?(UNSET)
+            raise KeyError, "path not found: #{path.join(".")}"
+          end
+        end
+
+        def value_at(path)
+          Support::HashUtil.fetch_value_at_path(payload, path) do
+            docvalue_field(path.join("."))&.first
+          end
+        end
+
         def sort
           raw_data["sort"]
         end
@@ -77,6 +115,14 @@ def to_s
           "#<#{self.class.name} #{datastore_path}>"
         end
         alias_method :inspect, :to_s
+
+        private
+
+        # Returns the doc_values field array for the given key, or nil if not present.
+        # Datastore doc_values are always returned as arrays (e.g. `{"name" => ["Bob"]}`).
+        def docvalue_field(key)
+          raw_data.dig("fields", key)
+        end
       end
     end
   end

elasticgraph-graphql/lib/elastic_graph/graphql/datastore_response/search_response.rb

@@ -114,7 +114,7 @@ def filter_results(field_path, values, size)
               # `id` within `_source`, given it's available as `_id`.
               ->(hit) { values.include?(hit.fetch("_id")) }
             else
-              ->(hit) { values.intersect?(Support::HashUtil.fetch_leaf_values_at_path(hit.fetch("_source"), field_path).to_set) }
+              ->(hit) { values.intersect?(hit_values_at_path(hit, field_path).to_set) }
             end
 
           hits = raw_data.fetch("hits").fetch("hits").select(&filter).first(size)
@@ -131,6 +131,28 @@ def docs_description
           (documents.size < 3) ? documents.inspect : "[#{documents.first}, ..., #{documents.last}]"
         end
 
+        # Extracts leaf values from a hit, checking `_source` first and falling back to `fields`
+        # (populated by `docvalue_fields` in the query). When a field is excluded from `_source`
+        # (e.g. `retrieved_from: :doc_values`), the datastore still returns it under the `fields`
+        # key because the query explicitly requested it via `docvalue_fields`.
+        def hit_values_at_path(hit, field_path)
+          source = hit["_source"]
+          if source
+            Support::HashUtil.fetch_leaf_values_at_path(source, field_path) do
+              docvalue_fields_for(hit, field_path)
+            end
+          else
+            docvalue_fields_for(hit, field_path)
+          end
+        end
+
+        def docvalue_fields_for(hit, field_path)
+          fields = hit["fields"]
+          joined_path = field_path.join(".")
+          raise KeyError, "key not found: #{joined_path}" unless fields
+          fields.fetch(joined_path)
+        end
+
         def total_document_count(default: nil)
           super() || default || raise(Errors::CountUnavailableError, "#{__method__} is unavailable; set `query.total_document_count_needed = true` to make it available")
         end

elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/get_record_field_value.rb

@@ -19,15 +19,14 @@ def initialize(elasticgraph_graphql:, config:)
         end
 
         def resolve(field:, object:, args:, context:)
-          data =
+          value =
             case object
             when DatastoreResponse::Document
-              object.payload
+              object.value_at(field.path_in_index)
             else
-              object
+              Support::HashUtil.fetch_value_at_path(object, field.path_in_index) { nil }
             end
 
-          value = Support::HashUtil.fetch_value_at_path(data, field.path_in_index) { nil }
           value = [] if value.nil? && field.type.list?
 
           if field.type.relay_connection?

elasticgraph-graphql/sig/elastic_graph/graphql/datastore_response/document.rbs

@@ -1,6 +1,8 @@
 module ElasticGraph
   class GraphQL
     module DatastoreResponse
+      UNSET: ::Object
+
       class Document
         extend Forwardable
 
@@ -29,6 +31,12 @@ module ElasticGraph
 
         def []: (::String) -> untyped
         def fetch: (::String) -> untyped
+                 | (::String, untyped) -> untyped
+                 | (::String) { (::String) -> untyped } -> untyped
+        def fetch_value_at: (::Array[::String]) -> untyped
+                          | (::Array[::String], default_value: untyped) -> untyped
+                          | (::Array[::String]) { (::Array[::String]) -> untyped } -> untyped
+        def value_at: (::Array[::String]) -> untyped
         def index_name: () -> ::String
         def index_definition_name: () -> ::String
         def id: () -> ::String