Support retrieved_from: :doc_values for direct leaf fields

Author: jwils

config/site/src/guides/ai-tools.md

@@ -43,7 +43,7 @@ The [elasticgraph-mcp-server](https://pypi.org/project/elasticgraph-mcp-server/)
 
 ### Installation
 
-Install and run the MCP server, for example as a [Goose extension](https://block.github.io/goose/docs/getting-started/using-extensions), using:
+Install and run the MCP server, for example as a [Goose extension](https://goose-docs.ai/docs/getting-started/using-extensions/), using:
 
 {% include copyable_code_snippet.html language="shell" code="uvx elasticgraph-mcp-server" %}
 

elasticgraph-graphql/lib/elastic_graph/graphql/datastore_query.rb

@@ -303,7 +303,7 @@ def ignored_values_for_routing
       def to_datastore_body
         @to_datastore_body ||= aggregations_datastore_body
           .merge(document_paginator.to_datastore_body)
-          .merge({highlight: highlight, query: filter_interpreter.build_query(all_filters), _source: source}.compact)
+          .merge({docvalue_fields: docvalue_fields, highlight: highlight, query: filter_interpreter.build_query(all_filters), _source: source}.compact)
       end
 
       def aggregations_datastore_body
@@ -323,13 +323,28 @@ def aggregations_datastore_body
       # we only ask for the fields we need to return.
       def source
         return true if request_all_fields
-        requested_source_fields = requested_fields - ["id"]
+        requested_source_fields = requested_fields_for_source - ["id"]
         return false if requested_source_fields.empty?
         # Merging in requested_fields as _source:{includes:} based on Elasticsearch documentation:
         # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html#include-exclude
         {includes: requested_source_fields.to_a}
       end
 
+      def docvalue_fields
+        requested_docvalue_fields =
+          if request_all_fields
+            all_docvalue_fields
+          else
+            requested_fields.select do |field_path|
+              requested_via_doc_values?(field_path)
+            end
+          end
+
+        return nil if requested_docvalue_fields.empty?
+
+        requested_docvalue_fields.to_a
+      end
+
       def highlight
         return nil if !request_all_highlights && requested_highlights.empty?
 
@@ -343,6 +358,30 @@ def highlight
         {fields:, highlight_query:}.compact
       end
 
+      def requested_fields_for_source
+        @requested_fields_for_source ||= requested_fields.reject do |field_path|
+          requested_via_doc_values?(field_path)
+        end
+      end
+
+      def all_docvalue_fields
+        @all_docvalue_fields ||= search_index_definitions.flat_map do |index_def|
+          index_def.fields_by_path.filter_map do |field_path, field|
+            field_path if field.retrieved_from_doc_values?
+          end
+        end.to_set
+      end
+
+      def requested_via_doc_values?(field_path)
+        return false if field_path == "id"
+
+        field_definitions = search_index_definitions.filter_map do |index_def|
+          index_def.fields_by_path[field_path]
+        end
+
+        field_definitions.any? && field_definitions.all?(&:retrieved_from_doc_values?)
+      end
+
       # Encapsulates dependencies of `Query`, giving us something we can expose off of `application`
       # to build queries when desired.
       class Builder < Support::MemoizableData.define(:runtime_metadata, :logger, :filter_interpreter, :filter_node_interpreter, :default_page_size, :max_page_size)

elasticgraph-graphql/lib/elastic_graph/graphql/datastore_response/document.rb

@@ -8,20 +8,21 @@
 
 require "elastic_graph/graphql/decoded_cursor"
 require "elastic_graph/support/memoizable_data"
+require "elastic_graph/support/hash_util"
 require "forwardable"
 
 module ElasticGraph
   class GraphQL
     module DatastoreResponse
+      DOCUMENT_MISSING_VALUE = ::Object.new.freeze
+
       # Represents a document fetched from the datastore. Exposes both the raw metadata
       # provided by the datastore and the doc payload itself. In addition, you can treat
       # it just like a document hash using `#[]` or `#fetch`.
       Document = Support::MemoizableData.define(:raw_data, :payload, :decoded_cursor_factory) do
         # @implements Document
         extend Forwardable
 
-        def_delegators :payload, :[], :fetch
-
         def self.build(raw_data, decoded_cursor_factory: DecodedCursor::Factory::Null)
           source = raw_data.fetch("_source") do
             {} # : ::Hash[::String, untyped]
@@ -51,6 +52,33 @@ def id
           raw_data["_id"]
         end
 
+        def [](key)
+          fetch(key) { nil }
+        end
+
+        def fetch(key, default_value = DOCUMENT_MISSING_VALUE)
+          value = lookup_value_at([key], list: false, missing_value: DOCUMENT_MISSING_VALUE)
+          return value unless value.equal?(DOCUMENT_MISSING_VALUE)
+          return yield(key) if block_given?
+          return default_value unless default_value.equal?(DOCUMENT_MISSING_VALUE)
+
+          raise KeyError, "key not found: #{key}"
+        end
+
+        def fetch_value_at(path, list:, default_value: DOCUMENT_MISSING_VALUE)
+          value = lookup_value_at(path, list: list, missing_value: DOCUMENT_MISSING_VALUE)
+          return value unless value.equal?(DOCUMENT_MISSING_VALUE)
+          return yield(path) if block_given?
+          return default_value unless default_value.equal?(DOCUMENT_MISSING_VALUE)
+
+          raise KeyError, "path not found: #{path.join(".")}"
+        end
+
+        def value_at(path, list:)
+          value = lookup_value_at(path, list: list, missing_value: DOCUMENT_MISSING_VALUE)
+          value.equal?(DOCUMENT_MISSING_VALUE) ? nil : value
+        end
+
         def sort
           raw_data["sort"]
         end
@@ -77,6 +105,18 @@ def to_s
           "#<#{self.class.name} #{datastore_path}>"
         end
         alias_method :inspect, :to_s
+
+        private
+
+        def lookup_value_at(path, list:, missing_value:)
+          value = Support::HashUtil.fetch_value_at_path(payload, path) { missing_value }
+          return value unless value.equal?(missing_value)
+
+          field_values = raw_data.fetch("fields", {}).fetch(path.join("."), missing_value)
+          return missing_value if field_values.equal?(missing_value)
+
+          list ? field_values : field_values.first
+        end
       end
     end
   end

elasticgraph-graphql/lib/elastic_graph/graphql/datastore_response/search_response.rb

@@ -114,7 +114,7 @@ def filter_results(field_path, values, size)
               # `id` within `_source`, given it's available as `_id`.
               ->(hit) { values.include?(hit.fetch("_id")) }
             else
-              ->(hit) { values.intersect?(Support::HashUtil.fetch_leaf_values_at_path(hit.fetch("_source"), field_path).to_set) }
+              ->(hit) { values.intersect?(hit_values_at_path(hit, field_path).to_set) }
             end
 
           hits = raw_data.fetch("hits").fetch("hits").select(&filter).first(size)
@@ -131,6 +131,15 @@ def docs_description
           (documents.size < 3) ? documents.inspect : "[#{documents.first}, ..., #{documents.last}]"
         end
 
+        def hit_values_at_path(hit, field_path)
+          Support::HashUtil.fetch_leaf_values_at_path(hit.fetch("_source"), field_path)
+        rescue KeyError => error
+          fields = hit.fetch("fields", {})
+          return fields.fetch(field_path.join(".")) if fields.key?(field_path.join("."))
+
+          raise error
+        end
+
         def total_document_count(default: nil)
           super() || default || raise(Errors::CountUnavailableError, "#{__method__} is unavailable; set `query.total_document_count_needed = true` to make it available")
         end

elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/get_record_field_value.rb

@@ -19,15 +19,15 @@ def initialize(elasticgraph_graphql:, config:)
         end
 
         def resolve(field:, object:, args:, context:)
-          data =
+          value =
             case object
             when DatastoreResponse::Document
-              object.payload
+              object.value_at(field.path_in_index, list: field.type.list?)
             else
-              object
+              data = object
+              Support::HashUtil.fetch_value_at_path(data, field.path_in_index) { nil }
             end
 
-          value = Support::HashUtil.fetch_value_at_path(data, field.path_in_index) { nil }
           value = [] if value.nil? && field.type.list?
 
           if field.type.relay_connection?

elasticgraph-graphql/lib/elastic_graph/graphql/schema/relation_join.rb

@@ -7,6 +7,7 @@
 # frozen_string_literal: true
 
 require "elastic_graph/graphql/datastore_response/search_response"
+require "elastic_graph/graphql/datastore_response/document"
 
 module ElasticGraph
   class GraphQL
@@ -51,10 +52,19 @@ def blank_value
 
         # Extracts a single id or a list of ids from the given document, as required by the relation.
         def extract_id_or_ids_from(document, log_warning)
-          id_or_ids = document.fetch(document_id_field_name) do
-            log_warning.call(document: document, problem: "#{document_id_field_name} is missing from the document")
-            blank_value
-          end
+          id_or_ids =
+            case document
+            when DatastoreResponse::Document
+              document.fetch_value_at([document_id_field_name], list: id_cardinality == Cardinality::Many) do
+                log_warning.call(document: document, problem: "#{document_id_field_name} is missing from the document")
+                blank_value
+              end
+            else
+              document.fetch(document_id_field_name) do
+                log_warning.call(document: document, problem: "#{document_id_field_name} is missing from the document")
+                blank_value
+              end
+            end
 
           normalize_ids(id_or_ids) do |problem|
             log_warning.call(document: document, problem: "#{document_id_field_name}: #{problem}")

elasticgraph-graphql/sig/elastic_graph/graphql/datastore_response/document.rbs

@@ -29,6 +29,12 @@ module ElasticGraph
 
         def []: (::String) -> untyped
         def fetch: (::String) -> untyped
+                 | (::String, untyped) -> untyped
+                 | (::String) { (::String) -> untyped } -> untyped
+        def fetch_value_at: (::Array[::String], list: bool) -> untyped
+                          | (::Array[::String], list: bool, default_value: untyped) -> untyped
+                          | (::Array[::String], list: bool) { (::Array[::String]) -> untyped } -> untyped
+        def value_at: (::Array[::String], list: bool) -> untyped
         def index_name: () -> ::String
         def index_definition_name: () -> ::String
         def id: () -> ::String

elasticgraph-graphql/sig/elastic_graph/graphql/schema/relation_join.rbs

@@ -55,8 +55,8 @@ module ElasticGraph
         def blank_value: () -> (nil | DatastoreResponse::SearchResponse)
 
         def extract_id_or_ids_from: (
-          ::Hash[::String, untyped],
-          ^(document: ::Hash[::String, untyped], problem: ::String) -> void
+          DatastoreResponse::Document | ::Hash[::String, untyped],
+          ^(document: DatastoreResponse::Document | ::Hash[::String, untyped], problem: ::String) -> void
         ) -> (nil | ::String | ::Enumerable[::String])
 
         def normalize_documents: (

elasticgraph-graphql/spec/integration/elastic_graph/graphql/datastore_query/requested_fields_spec.rb

@@ -50,6 +50,30 @@ class GraphQL
         expect(results.first.payload.keys).to include("name", "id", "options", "the_opts", "tags")
         expect(results.first["options"].keys).to include("size", "the_sighs")
       end
+
+      specify "returns docvalue-backed fields if passed `request_all_fields: true`" do
+        graphql = build_graphql(schema_definition: lambda do |schema|
+          schema.object_type "Widget" do |t|
+            t.field "id", "ID!"
+            t.field "name", "String"
+            t.field "internal_code", "String", retrieved_from: :doc_values
+            t.index "widgets"
+          end
+        end)
+
+        index_into(graphql, {
+          "__typename" => "Widget",
+          "__version" => 1,
+          "__json_schema_version" => 1,
+          "id" => "widget-1",
+          "name" => "Widget 1",
+          "internal_code" => "INT-001"
+        })
+
+        results = search_datastore(graphql: graphql, request_all_fields: true)
+
+        expect(results.first["internal_code"]).to eq "INT-001"
+      end
     end
   end
 end

elasticgraph-graphql/spec/unit/elastic_graph/graphql/datastore_query/requested_fields_spec.rb

@@ -42,6 +42,88 @@ class GraphQL
 
         expect(datastore_body_of(query)[:_source]).to eq(true)
       end
+
+      it "still requests doc values when requesting all fields" do
+        graphql = build_graphql(schema_definition: lambda do |schema|
+          schema.object_type "Widget" do |t|
+            t.field "id", "ID!"
+            t.field "name", "String"
+            t.field "internal_code", "String", retrieved_from: :doc_values
+            t.index "widgets"
+          end
+        end)
+        query = graphql.datastore_query_builder.new_query(
+          search_index_definitions: graphql.datastore_core.index_definitions_by_graphql_type.fetch("Widget"),
+          request_all_fields: true
+        )
+
+        expect(datastore_body_of(query)[:_source]).to eq true
+        expect(datastore_body_of(query)[:docvalue_fields]).to contain_exactly("internal_code")
+      end
+
+      it "still requests doc values when requesting all fields from mixed index definitions" do
+        graphql = build_graphql(schema_definition: lambda do |schema|
+          schema.object_type "Widget" do |t|
+            t.field "id", "ID!"
+            t.field "internal_code", "String", retrieved_from: :doc_values
+            t.index "widgets"
+          end
+        end)
+        doc_values_index_def = graphql.datastore_core.index_definitions_by_name.fetch("widgets")
+        source_backed_index_def = doc_values_index_def.with(
+          fields_by_path: doc_values_index_def.fields_by_path.merge(
+            "internal_code" => doc_values_index_def.fields_by_path.fetch("internal_code").with(retrieved_from: nil)
+          )
+        )
+        query = graphql.datastore_query_builder.new_query(
+          search_index_definitions: [doc_values_index_def, source_backed_index_def],
+          request_all_fields: true
+        )
+
+        expect(datastore_body_of(query)[:_source]).to eq true
+        expect(datastore_body_of(query)[:docvalue_fields]).to contain_exactly("internal_code")
+      end
+
+      it "requests doc values for fields marked `retrieved_from: :doc_values`" do
+        graphql = build_graphql(schema_definition: lambda do |schema|
+          schema.object_type "Widget" do |t|
+            t.field "id", "ID!"
+            t.field "name", "String"
+            t.field "internal_code", "String", retrieved_from: :doc_values
+            t.index "widgets"
+          end
+        end)
+        query = graphql.datastore_query_builder.new_query(
+          search_index_definitions: graphql.datastore_core.index_definitions_by_graphql_type.fetch("Widget"),
+          requested_fields: ["internal_code", "name"]
+        )
+
+        expect(datastore_body_of(query)[:docvalue_fields]).to contain_exactly("internal_code")
+        expect(datastore_body_of(query)[:_source][:includes]).to contain_exactly("name")
+      end
+
+      it "keeps requesting `_source` when index definitions disagree on how to retrieve a field" do
+        graphql = build_graphql(schema_definition: lambda do |schema|
+          schema.object_type "Widget" do |t|
+            t.field "id", "ID!"
+            t.field "internal_code", "String", retrieved_from: :doc_values
+            t.index "widgets"
+          end
+        end)
+        doc_values_index_def = graphql.datastore_core.index_definitions_by_name.fetch("widgets")
+        source_backed_index_def = doc_values_index_def.with(
+          fields_by_path: doc_values_index_def.fields_by_path.merge(
+            "internal_code" => doc_values_index_def.fields_by_path.fetch("internal_code").with(retrieved_from: nil)
+          )
+        )
+        query = graphql.datastore_query_builder.new_query(
+          search_index_definitions: [doc_values_index_def, source_backed_index_def],
+          requested_fields: ["internal_code"]
+        )
+
+        expect(datastore_body_of(query)[:docvalue_fields]).to eq nil
+        expect(datastore_body_of(query)[:_source][:includes]).to contain_exactly("internal_code")
+      end
     end
   end
 end