Support retrieved_from: :doc_values for direct leaf fields

Author: jwils

config/schema/artifacts/datastore_config.yaml

@@ -1245,6 +1245,9 @@ index_templates:
           required: true
         _size:
           enabled: true
+        _source:
+          excludes:
+          - workspace_id2
       settings:
         index.mapping.ignore_malformed: false
         index.mapping.coerce: false
@@ -1505,6 +1508,9 @@ indices:
           dynamic: 'false'
       _size:
         enabled: true
+      _source:
+        excludes:
+        - full_address
     settings:
       index.mapping.ignore_malformed: false
       index.mapping.coerce: false

config/schema/artifacts/runtime_metadata.yaml

@@ -1446,6 +1446,7 @@ index_definitions_by_name:
       __counts.shapes|type:
         source: __self
       full_address:
+        retrieved_from: doc_values
         source: __self
       geo_location.lat:
         source: __self
@@ -2613,6 +2614,7 @@ index_definitions_by_name:
       weight_in_ng_str:
         source: __self
       workspace_id2:
+        retrieved_from: doc_values
         source: __self
       workspace_name:
         source: workspace

config/schema/artifacts_with_apollo/datastore_config.yaml

@@ -1245,6 +1245,9 @@ index_templates:
           required: true
         _size:
           enabled: true
+        _source:
+          excludes:
+          - workspace_id2
       settings:
         index.mapping.ignore_malformed: false
         index.mapping.coerce: false
@@ -1505,6 +1508,9 @@ indices:
           dynamic: 'false'
       _size:
         enabled: true
+      _source:
+        excludes:
+        - full_address
     settings:
       index.mapping.ignore_malformed: false
       index.mapping.coerce: false

config/schema/artifacts_with_apollo/runtime_metadata.yaml

@@ -1475,6 +1475,7 @@ index_definitions_by_name:
       __counts.shapes|type:
         source: __self
       full_address:
+        retrieved_from: doc_values
         source: __self
       geo_location.lat:
         source: __self
@@ -2642,6 +2643,7 @@ index_definitions_by_name:
       weight_in_ng_str:
         source: __self
       workspace_id2:
+        retrieved_from: doc_values
         source: __self
       workspace_name:
         source: workspace

config/schema/widgets.rb

@@ -75,8 +75,8 @@
     t.field "id", "ID!"
 
     # Here we use an alternate name for this field since it's the routing field and want to verify
-    # that `name_in_index` works correctly on routing fields.
-    t.field "workspace_id", "ID", name_in_index: "workspace_id2"
+    # that `name_in_index` works correctly on routing fields, including when fetched from doc values.
+    t.field "workspace_id", "ID", name_in_index: "workspace_id2", retrieved_from: :doc_values
 
     # It's a bit funny we have both `amount_cents` and `cost` but it's nice to be able to test
     # aggregations on both a root numeric field and on a nested one, so we are keeping both here.
@@ -367,7 +367,7 @@
     # We use `indexing_only: true` here to verify that `id` can be an indexing-only field.
     t.field "id", "ID!", indexing_only: true
 
-    t.field "full_address", "String!"
+    t.field "full_address", "String!", retrieved_from: :doc_values
     t.field "timestamps", "AddressTimestamps"
     t.field "geo_location", "GeoLocation"
 

config/site/src/guides/ai-tools.md

@@ -43,7 +43,7 @@ The [elasticgraph-mcp-server](https://pypi.org/project/elasticgraph-mcp-server/)
 
 ### Installation
 
-Install and run the MCP server, for example as a [Goose extension](https://block.github.io/goose/docs/getting-started/using-extensions), using:
+Install and run the MCP server, for example as a [Goose extension](https://goose-docs.ai/docs/getting-started/using-extensions/), using:
 
 {% include copyable_code_snippet.html language="shell" code="uvx elasticgraph-mcp-server" %}
 

elasticgraph-graphql/lib/elastic_graph/graphql/datastore_query.rb

@@ -303,7 +303,7 @@ def ignored_values_for_routing
       def to_datastore_body
         @to_datastore_body ||= aggregations_datastore_body
           .merge(document_paginator.to_datastore_body)
-          .merge({highlight: highlight, query: filter_interpreter.build_query(all_filters), _source: source}.compact)
+          .merge({docvalue_fields: docvalue_fields, highlight: highlight, query: filter_interpreter.build_query(all_filters), _source: source}.compact)
       end
 
       def aggregations_datastore_body
@@ -323,13 +323,28 @@ def aggregations_datastore_body
       # we only ask for the fields we need to return.
       def source
         return true if request_all_fields
-        requested_source_fields = requested_fields - ["id"]
+        requested_source_fields = requested_fields_for_source - ["id"]
         return false if requested_source_fields.empty?
         # Merging in requested_fields as _source:{includes:} based on Elasticsearch documentation:
         # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html#include-exclude
         {includes: requested_source_fields.to_a}
       end
 
+      def docvalue_fields
+        requested_docvalue_fields =
+          if request_all_fields
+            all_docvalue_fields
+          else
+            requested_fields.select do |field_path|
+              requested_via_doc_values?(field_path)
+            end
+          end
+
+        return nil if requested_docvalue_fields.empty?
+
+        requested_docvalue_fields.to_a
+      end
+
       def highlight
         return nil if !request_all_highlights && requested_highlights.empty?
 
@@ -343,6 +358,30 @@ def highlight
         {fields:, highlight_query:}.compact
       end
 
+      def requested_fields_for_source
+        @requested_fields_for_source ||= requested_fields.reject do |field_path|
+          requested_via_doc_values?(field_path)
+        end
+      end
+
+      def all_docvalue_fields
+        @all_docvalue_fields ||= search_index_definitions.flat_map do |index_def|
+          index_def.fields_by_path.filter_map do |field_path, field|
+            field_path if field.retrieved_from_doc_values?
+          end
+        end.to_set
+      end
+
+      def requested_via_doc_values?(field_path)
+        return false if field_path == "id"
+
+        field_definitions = search_index_definitions.filter_map do |index_def|
+          index_def.fields_by_path[field_path]
+        end
+
+        field_definitions.any? && field_definitions.all?(&:retrieved_from_doc_values?)
+      end
+
       # Encapsulates dependencies of `Query`, giving us something we can expose off of `application`
       # to build queries when desired.
       class Builder < Support::MemoizableData.define(:runtime_metadata, :logger, :filter_interpreter, :filter_node_interpreter, :default_page_size, :max_page_size)

elasticgraph-graphql/lib/elastic_graph/graphql/datastore_response/document.rb

@@ -8,20 +8,21 @@
 
 require "elastic_graph/graphql/decoded_cursor"
 require "elastic_graph/support/memoizable_data"
+require "elastic_graph/support/hash_util"
 require "forwardable"
 
 module ElasticGraph
   class GraphQL
     module DatastoreResponse
+      DOCUMENT_MISSING_VALUE = ::Object.new.freeze
+
       # Represents a document fetched from the datastore. Exposes both the raw metadata
       # provided by the datastore and the doc payload itself. In addition, you can treat
       # it just like a document hash using `#[]` or `#fetch`.
       Document = Support::MemoizableData.define(:raw_data, :payload, :decoded_cursor_factory) do
         # @implements Document
         extend Forwardable
 
-        def_delegators :payload, :[], :fetch
-
         def self.build(raw_data, decoded_cursor_factory: DecodedCursor::Factory::Null)
           source = raw_data.fetch("_source") do
             {} # : ::Hash[::String, untyped]
@@ -51,6 +52,33 @@ def id
           raw_data["_id"]
         end
 
+        def [](key)
+          fetch(key) { nil }
+        end
+
+        def fetch(key, default_value = DOCUMENT_MISSING_VALUE)
+          value = lookup_value_at([key], list: false, missing_value: DOCUMENT_MISSING_VALUE)
+          return value unless value.equal?(DOCUMENT_MISSING_VALUE)
+          return yield(key) if block_given?
+          return default_value unless default_value.equal?(DOCUMENT_MISSING_VALUE)
+
+          raise KeyError, "key not found: #{key}"
+        end
+
+        def fetch_value_at(path, list:, default_value: DOCUMENT_MISSING_VALUE)
+          value = lookup_value_at(path, list: list, missing_value: DOCUMENT_MISSING_VALUE)
+          return value unless value.equal?(DOCUMENT_MISSING_VALUE)
+          return yield(path) if block_given?
+          return default_value unless default_value.equal?(DOCUMENT_MISSING_VALUE)
+
+          raise KeyError, "path not found: #{path.join(".")}"
+        end
+
+        def value_at(path, list:)
+          value = lookup_value_at(path, list: list, missing_value: DOCUMENT_MISSING_VALUE)
+          value.equal?(DOCUMENT_MISSING_VALUE) ? nil : value
+        end
+
         def sort
           raw_data["sort"]
         end
@@ -77,6 +105,18 @@ def to_s
           "#<#{self.class.name} #{datastore_path}>"
         end
         alias_method :inspect, :to_s
+
+        private
+
+        def lookup_value_at(path, list:, missing_value:)
+          value = Support::HashUtil.fetch_value_at_path(payload, path) { missing_value }
+          return value unless value.equal?(missing_value)
+
+          field_values = raw_data.fetch("fields", {}).fetch(path.join("."), missing_value)
+          return missing_value if field_values.equal?(missing_value)
+
+          list ? field_values : field_values.first
+        end
       end
     end
   end

elasticgraph-graphql/lib/elastic_graph/graphql/datastore_response/search_response.rb

@@ -114,7 +114,7 @@ def filter_results(field_path, values, size)
               # `id` within `_source`, given it's available as `_id`.
               ->(hit) { values.include?(hit.fetch("_id")) }
             else
-              ->(hit) { values.intersect?(Support::HashUtil.fetch_leaf_values_at_path(hit.fetch("_source"), field_path).to_set) }
+              ->(hit) { values.intersect?(hit_values_at_path(hit, field_path).to_set) }
             end
 
           hits = raw_data.fetch("hits").fetch("hits").select(&filter).first(size)
@@ -131,6 +131,15 @@ def docs_description
           (documents.size < 3) ? documents.inspect : "[#{documents.first}, ..., #{documents.last}]"
         end
 
+        def hit_values_at_path(hit, field_path)
+          Support::HashUtil.fetch_leaf_values_at_path(hit.fetch("_source"), field_path)
+        rescue KeyError => error
+          fields = hit.fetch("fields", {})
+          return fields.fetch(field_path.join(".")) if fields.key?(field_path.join("."))
+
+          raise error
+        end
+
         def total_document_count(default: nil)
           super() || default || raise(Errors::CountUnavailableError, "#{__method__} is unavailable; set `query.total_document_count_needed = true` to make it available")
         end

elasticgraph-graphql/lib/elastic_graph/graphql/resolvers/get_record_field_value.rb

@@ -19,15 +19,15 @@ def initialize(elasticgraph_graphql:, config:)
         end
 
         def resolve(field:, object:, args:, context:)
-          data =
+          value =
             case object
             when DatastoreResponse::Document
-              object.payload
+              object.value_at(field.path_in_index, list: field.type.list?)
             else
-              object
+              data = object
+              Support::HashUtil.fetch_value_at_path(data, field.path_in_index) { nil }
             end
 
-          value = Support::HashUtil.fetch_value_at_path(data, field.path_in_index) { nil }
           value = [] if value.nil? && field.type.list?
 
           if field.type.relay_connection?