Add indexer_extension_modules to runtime metadata (#1298)

## Why

To support ingestion in any data format while keeping format-specific
logic in its own gem (`elasticgraph-json_ingestion`, the upcoming
`elasticgraph-proto_ingestion`), the indexer needs an extension
mechanism analogous to the GraphQL one: schema definition extensions
register runtime extensions in the schema artifacts, and the runtime
component applies them at boot. This PR adds the runtime metadata
storage for that.

## What

- Add `indexer_extension_modules` to `RuntimeMetadata::Schema`,
mirroring `graphql_extension_modules`
- Add a `RuntimeMetadata::IndexerExtension` wrapper (twin of
`GraphQLExtension`)
- Nothing registers or consumes these yet, and empty lists are pruned
from dumped YAML, so no schema artifacts change

## Verification

- `script/lint`, `script/type_check`
- `script/run_gem_specs elasticgraph-schema_artifacts` (100% coverage)
- `script/run_gem_specs elasticgraph-schema_definition`
- `bundle exec rake schema_artifacts:check` (no artifact changes)
## Stack
Current PR is marked with `->`.

- -> [#1298 Add indexer_extension_modules to runtime
metadata](#1298)
- [#1299 Add register_indexer_extension schema definition
API](#1299)
- [#1300 Apply indexer extension modules when the Indexer
boots](#1300)
- [#1301 Extract an ingestion adapter seam inside
elasticgraph-indexer](#1301)
- [#1302 Move JSON ingestion into elasticgraph-json_ingestion via an
indexer extension](#1302)

Author: jwils

…ts/runtime_metadata/graphql_extension.rb …/runtime_metadata/component_extension.rbelasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/graphql_extension.rb renamed to elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/component_extension.rb elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/graphql_extension.rb renamed to elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/component_extension.rb

@@ -12,15 +12,15 @@ module ElasticGraph
   module SchemaArtifacts
     module RuntimeMetadata
       # @private
-      class GraphQLExtension < ::Data.define(:extension_ref)
+      class ComponentExtension < ::Data.define(:extension_ref)
         def self.loader
           @loader ||= ExtensionLoader.new(Module.new)
         end
 
         EXTENSION_REF = "extension_ref"
 
         def load_extension
-          Extension.load_from_hash(extension_ref, via: GraphQLExtension.loader)
+          Extension.load_from_hash(extension_ref, via: ComponentExtension.loader)
         end
 
         def self.from_hash(hash)

elasticgraph-schema_artifacts/lib/elastic_graph/schema_artifacts/runtime_metadata/schema.rb

@@ -7,10 +7,10 @@
 # frozen_string_literal: true
 
 require "elastic_graph/constants"
+require "elastic_graph/schema_artifacts/runtime_metadata/component_extension"
 require "elastic_graph/schema_artifacts/runtime_metadata/enum"
 require "elastic_graph/schema_artifacts/runtime_metadata/extension"
 require "elastic_graph/schema_artifacts/runtime_metadata/extension_loader"
-require "elastic_graph/schema_artifacts/runtime_metadata/graphql_extension"
 require "elastic_graph/schema_artifacts/runtime_metadata/graphql_resolver"
 require "elastic_graph/schema_artifacts/runtime_metadata/hash_dumper"
 require "elastic_graph/schema_artifacts/runtime_metadata/index_definition"
@@ -34,6 +34,7 @@ class Schema < ::Data.define(
         :schema_element_names,
         :graphql_extension_modules,
         :graphql_resolvers_by_name,
+        :indexer_extension_modules,
         :static_script_ids_by_scoped_name
       )
         # @private
@@ -53,6 +54,8 @@ class Schema < ::Data.define(
         # @private
         GRAPHQL_RESOLVERS_BY_NAME = "graphql_resolvers_by_name"
         # @private
+        INDEXER_EXTENSION_MODULES = "indexer_extension_modules"
+        # @private
         STATIC_SCRIPT_IDS_BY_NAME = "static_script_ids_by_scoped_name"
 
         # Loads a {RuntimeMetadata::Schema} from the given hash.
@@ -90,13 +93,18 @@ def self.from_hash(hash)
 
           graphql_extension_modules =
             hash[GRAPHQL_EXTENSION_MODULES]&.map do |ext_mod_hash|
-              GraphQLExtension.from_hash(ext_mod_hash)
+              ComponentExtension.from_hash(ext_mod_hash)
             end || []
 
           graphql_resolvers_by_name = hash[GRAPHQL_RESOLVERS_BY_NAME]&.to_h do |name, resolver_hash|
             [name.to_sym, GraphQLResolver.from_hash(resolver_hash)]
           end || {}
 
+          indexer_extension_modules =
+            hash[INDEXER_EXTENSION_MODULES]&.map do |ext_mod_hash|
+              ComponentExtension.from_hash(ext_mod_hash)
+            end || []
+
           static_script_ids_by_scoped_name = hash[STATIC_SCRIPT_IDS_BY_NAME] || {}
 
           new(
@@ -108,6 +116,7 @@ def self.from_hash(hash)
             schema_element_names: schema_element_names,
             graphql_extension_modules: graphql_extension_modules,
             graphql_resolvers_by_name: graphql_resolvers_by_name,
+            indexer_extension_modules: indexer_extension_modules,
             static_script_ids_by_scoped_name: static_script_ids_by_scoped_name
           )
         end
@@ -123,6 +132,7 @@ def to_dumpable_hash
             GRAPHQL_EXTENSION_MODULES => graphql_extension_modules.map(&:to_dumpable_hash),
             GRAPHQL_RESOLVERS_BY_NAME => HashDumper.dump_hash(graphql_resolvers_by_name.transform_keys(&:to_s), &:to_dumpable_hash),
             INDEX_DEFINITIONS_BY_NAME => HashDumper.dump_hash(index_definitions_by_name, &:to_dumpable_hash),
+            INDEXER_EXTENSION_MODULES => indexer_extension_modules.map(&:to_dumpable_hash),
             OBJECT_TYPES_BY_NAME => HashDumper.dump_hash(object_types_by_name, &:to_dumpable_hash),
             SCALAR_TYPES_BY_NAME => HashDumper.dump_hash(scalar_types_by_name, &:to_dumpable_hash),
             SCHEMA_ELEMENT_NAMES => schema_element_names.to_dumpable_hash,

…s/runtime_metadata/graphql_extension.rbs …runtime_metadata/component_extension.rbselasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/graphql_extension.rbs renamed to elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/component_extension.rbs elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/graphql_extension.rbs renamed to elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/component_extension.rbs

@@ -1,7 +1,7 @@
 module ElasticGraph
   module SchemaArtifacts
     module RuntimeMetadata
-      class GraphQLExtensionSupertype
+      class ComponentExtensionSupertype
         attr_reader extension_ref: ::Hash[::String, ::String]
 
         def initialize: (
@@ -17,14 +17,14 @@ module ElasticGraph
           | (::Hash[::String, ::String]) -> instance
       end
 
-      class GraphQLExtension < GraphQLExtensionSupertype
+      class ComponentExtension < ComponentExtensionSupertype
         self.@loader: ExtensionLoader?
         def self.loader: () -> ExtensionLoader
 
         EXTENSION_REF: "extension_ref"
 
         def load_extension: () -> Extension
-        def self.from_hash: (::Hash[::String, untyped]) -> GraphQLExtension
+        def self.from_hash: (::Hash[::String, untyped]) -> ComponentExtension
         def to_dumpable_hash: () -> ::Hash[::String, untyped]
       end
     end

elasticgraph-schema_artifacts/sig/elastic_graph/schema_artifacts/runtime_metadata/schema.rbs

@@ -8,8 +8,9 @@ module ElasticGraph
         attr_reader enum_types_by_name: ::Hash[::String, Enum::Type]
         attr_reader index_definitions_by_name: ::Hash[::String, IndexDefinition]
         attr_reader schema_element_names: SchemaElementNames
-        attr_reader graphql_extension_modules: ::Array[GraphQLExtension]
+        attr_reader graphql_extension_modules: ::Array[ComponentExtension]
         attr_reader graphql_resolvers_by_name: ::Hash[::Symbol, GraphQLResolver]
+        attr_reader indexer_extension_modules: ::Array[ComponentExtension]
         attr_reader static_script_ids_by_scoped_name: ::Hash[::String, ::String]
 
         def initialize: (
@@ -19,8 +20,9 @@ module ElasticGraph
           enum_types_by_name: ::Hash[::String, Enum::Type],
           index_definitions_by_name: ::Hash[::String, IndexDefinition],
           schema_element_names: SchemaElementNames,
-          graphql_extension_modules: ::Array[GraphQLExtension],
+          graphql_extension_modules: ::Array[ComponentExtension],
           graphql_resolvers_by_name: ::Hash[::Symbol, GraphQLResolver],
+          indexer_extension_modules: ::Array[ComponentExtension],
           static_script_ids_by_scoped_name: ::Hash[::String, ::String]) -> void
 
         def with: (
@@ -30,8 +32,9 @@ module ElasticGraph
           ?enum_types_by_name: ::Hash[::String, Enum::Type],
           ?index_definitions_by_name: ::Hash[::String, IndexDefinition],
           ?schema_element_names: SchemaElementNames,
-          ?graphql_extension_modules: ::Array[GraphQLExtension],
+          ?graphql_extension_modules: ::Array[ComponentExtension],
           ?graphql_resolvers_by_name: ::Hash[::Symbol, GraphQLResolver],
+          ?indexer_extension_modules: ::Array[ComponentExtension],
           ?static_script_ids_by_scoped_name: ::Hash[::String, ::String]) -> Schema
       end
 
@@ -44,6 +47,7 @@ module ElasticGraph
         SCHEMA_ELEMENT_NAMES: "schema_element_names"
         GRAPHQL_EXTENSION_MODULES: "graphql_extension_modules"
         GRAPHQL_RESOLVERS_BY_NAME: "graphql_resolvers_by_name"
+        INDEXER_EXTENSION_MODULES: "indexer_extension_modules"
         STATIC_SCRIPT_IDS_BY_NAME: "static_script_ids_by_scoped_name"
 
         def self.from_hash: (::Hash[::String, untyped]) -> Schema

…_extensions/graphql_extension_modules.rb …xtensions/component_extension_modules.rbelasticgraph-schema_artifacts/spec/support/example_extensions/graphql_extension_modules.rb renamed to elasticgraph-schema_artifacts/spec/support/example_extensions/component_extension_modules.rb elasticgraph-schema_artifacts/spec/support/example_extensions/graphql_extension_modules.rb renamed to elasticgraph-schema_artifacts/spec/support/example_extensions/component_extension_modules.rb

@@ -8,7 +8,7 @@
 
 module ElasticGraph
   module SchemaArtifacts
-    module GraphQLExtensionModule1
+    module ComponentExtensionModule1
     end
   end
 end

…ntime_metadata/graphql_extension_spec.rb …ime_metadata/component_extension_spec.rbelasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/graphql_extension_spec.rb renamed to elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/component_extension_spec.rb elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/graphql_extension_spec.rb renamed to elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/component_extension_spec.rb

@@ -6,21 +6,21 @@
 #
 # frozen_string_literal: true
 
-require "elastic_graph/schema_artifacts/runtime_metadata/graphql_extension"
+require "elastic_graph/schema_artifacts/runtime_metadata/component_extension"
 
 module ElasticGraph
   module SchemaArtifacts
     module RuntimeMetadata
-      RSpec.describe GraphQLExtension do
+      RSpec.describe ComponentExtension do
         it "loads extension lazily" do
-          graphql_extension = GraphQLExtension.new(
+          component_extension = ComponentExtension.new(
             extension_ref: {
               "name" => "ElasticGraph::Extensions::Valid",
               "require_path" => "support/example_extensions/valid"
             }
           )
 
-          extension = graphql_extension.load_extension
+          extension = component_extension.load_extension
 
           expect(extension).to be_a(RuntimeMetadata::Extension)
           expect(extension.extension_class).to be_a(::Module).and have_attributes(name: "ElasticGraph::Extensions::Valid")

elasticgraph-schema_artifacts/spec/unit/elastic_graph/schema_artifacts/runtime_metadata/schema_spec.rb

@@ -15,7 +15,7 @@
 require "elastic_graph/schema_artifacts/runtime_metadata/schema"
 require "elastic_graph/spec_support/runtime_metadata_support"
 require "elastic_graph/spec_support/example_extensions/graphql_resolvers"
-require "support/example_extensions/graphql_extension_modules"
+require "support/example_extensions/component_extension_modules"
 require "support/example_extensions/indexing_preparers"
 require "support/example_extensions/scalar_coercion_adapters"
 require "yaml"
@@ -158,13 +158,14 @@ module RuntimeMetadata
               form: :snake_case,
               overrides: {"any_of" => "or"}
             ),
-            graphql_extension_modules: [graphql_extension_module1],
+            graphql_extension_modules: [component_extension_module1],
             graphql_resolvers_by_name: {
               resolver1: graphql_resolver_with(
                 needs_lookahead: true,
                 resolver_ref: graphql_resolver_with_lookahead(limit: 10).to_dumpable_hash
               )
             },
+            indexer_extension_modules: [component_extension_module1],
             static_script_ids_by_scoped_name: {
               "filter/time_of_day" => "time_of_day_4474b200b6a00f385ed49f7c9669cbf3"
             }
@@ -302,8 +303,8 @@ module RuntimeMetadata
             },
             "graphql_extension_modules" => [{
               "extension_ref" => {
-                "name" => "ElasticGraph::SchemaArtifacts::GraphQLExtensionModule1",
-                "require_path" => "support/example_extensions/graphql_extension_modules"
+                "name" => "ElasticGraph::SchemaArtifacts::ComponentExtensionModule1",
+                "require_path" => "support/example_extensions/component_extension_modules"
               }
             }],
             "graphql_resolvers_by_name" => {
@@ -316,6 +317,12 @@ module RuntimeMetadata
                 }
               }
             },
+            "indexer_extension_modules" => [{
+              "extension_ref" => {
+                "name" => "ElasticGraph::SchemaArtifacts::ComponentExtensionModule1",
+                "require_path" => "support/example_extensions/component_extension_modules"
+              }
+            }],
             "static_script_ids_by_scoped_name" => {
               "filter/time_of_day" => "time_of_day_4474b200b6a00f385ed49f7c9669cbf3"
             }
@@ -387,6 +394,7 @@ module RuntimeMetadata
             schema_element_names: SchemaElementNames.from_hash({"form" => "camelCase"}),
             graphql_extension_modules: [],
             graphql_resolvers_by_name: {},
+            indexer_extension_modules: [],
             static_script_ids_by_scoped_name: {}
           )
         end

elasticgraph-schema_definition/lib/elastic_graph/schema_definition/api.rb

@@ -344,7 +344,7 @@ def deleted_type(name)
       #   end
       def register_graphql_extension(extension_module, defined_at:, **config)
         extension = SchemaArtifacts::RuntimeMetadata::Extension.new(extension_module, defined_at, config)
-        @state.graphql_extension_modules << SchemaArtifacts::RuntimeMetadata::GraphQLExtension.new(extension.to_dumpable_hash)
+        @state.graphql_extension_modules << SchemaArtifacts::RuntimeMetadata::ComponentExtension.new(extension.to_dumpable_hash)
         nil
       end
 

elasticgraph-schema_definition/lib/elastic_graph/schema_definition/results.rb

@@ -125,6 +125,7 @@ def build_runtime_metadata
           schema_element_names: state.schema_elements,
           graphql_extension_modules: state.graphql_extension_modules,
           graphql_resolvers_by_name: state.graphql_resolvers_by_name,
+          indexer_extension_modules: [],
           static_script_ids_by_scoped_name: STATIC_SCRIPT_REPO.script_ids_by_scoped_name
         ).tap { |rm| verify_runtime_metadata(rm) }
       end

elasticgraph-schema_definition/sig/elastic_graph/schema_definition/state.rbs

@@ -15,7 +15,7 @@ module ElasticGraph
       attr_reader paginated_collection_element_types: ::Set[::String]
       attr_reader user_defined_fields: ::Set[SchemaElements::Field]
       attr_reader reserved_type_names: ::Set[::String]
-      attr_reader graphql_extension_modules: ::Array[SchemaArtifacts::RuntimeMetadata::GraphQLExtension]
+      attr_reader graphql_extension_modules: ::Array[SchemaArtifacts::RuntimeMetadata::ComponentExtension]
       attr_reader graphql_resolvers_by_name: ::Hash[::Symbol, SchemaArtifacts::RuntimeMetadata::GraphQLResolver]
       attr_reader built_in_graphql_resolvers: ::Set[::Symbol]
       attr_accessor initially_registered_built_in_types: ::Set[::String]
@@ -48,7 +48,7 @@ module ElasticGraph
         paginated_collection_element_types: ::Set[::String],
         user_defined_fields: ::Set[SchemaElements::Field],
         reserved_type_names: ::Set[::String],
-        graphql_extension_modules: ::Array[SchemaArtifacts::RuntimeMetadata::GraphQLExtension],
+        graphql_extension_modules: ::Array[SchemaArtifacts::RuntimeMetadata::ComponentExtension],
         graphql_resolvers_by_name: ::Hash[::Symbol, SchemaArtifacts::RuntimeMetadata::GraphQLResolver],
         built_in_graphql_resolvers: ::Set[::Symbol],
         initially_registered_built_in_types: ::Set[::String],