Use JSON ingestion as schema definition extension

Author: jwils

config/site/support/doctest_helper.rb

@@ -9,6 +9,7 @@
 require "elastic_graph/apollo/schema_definition/api_extension"
 require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names"
 require "elastic_graph/schema_definition/api"
+require "elastic_graph/schema_definition/extension_module_support"
 require "elastic_graph/schema_definition/schema_artifact_manager"
 require "elastic_graph/warehouse/schema_definition/api_extension"
 require "rspec/mocks"
@@ -60,7 +61,7 @@ module ElasticGraph
         @api = SchemaDefinition::API.new(
           SchemaArtifacts::RuntimeMetadata::SchemaElementNames.new(form: :camelCase, overrides: {}),
           true,
-          extension_modules: extension_modules
+          extension_modules: SchemaDefinition::ExtensionModuleSupport.default_extension_modules + extension_modules
         )
 
         # This is required in all schemas, but we don't want to have to put in all our examples,
@@ -96,6 +97,7 @@ module ElasticGraph
         # `schema.json_schema_version` raises an error when the version is set more than once.
         # By default we set it above. Here we clear it to allow our example to set it.
         schema.state.json_schema_version = nil
+        schema.state.json_schema_version_setter_location = nil
       end
     end
 

elasticgraph-apollo/spec/unit/elastic_graph/apollo/apollo_directives_spec.rb

@@ -552,7 +552,7 @@ def self.with_both_casing_forms(&block)
         end
 
         def define_schema(&block)
-          extension_modules = [SchemaDefinition::APIExtension]
+          extension_modules = ::ElasticGraph::SchemaDefinition::ExtensionModuleSupport.default_extension_modules + [SchemaDefinition::APIExtension]
           super(schema_element_name_form: schema_element_name_form, extension_modules: extension_modules, &block)
         end
       end

elasticgraph-apollo/spec/unit/elastic_graph/apollo/schema_definition_spec.rb

@@ -1431,7 +1431,9 @@ def expect_identifiable_type_tagging_of_token(&type_def_for)
         end
 
         def define_schema(with_apollo: true, &block)
-          extension_modules = with_apollo ? [SchemaDefinition::APIExtension] : []
+          # Always include the JSON ingestion default so `Results#json_schemas_for` is available in both modes.
+          extension_modules = ::ElasticGraph::SchemaDefinition::ExtensionModuleSupport.default_extension_modules
+          extension_modules += [SchemaDefinition::APIExtension] if with_apollo
           super(schema_element_name_form: schema_element_name_form, extension_modules: extension_modules, &block)
         end
       end

elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/results_extension.rb

@@ -121,7 +121,7 @@ def json_schema_indexing_field_types_by_name
             end
             .sort_by(&:name)
             .to_h do |type|
-              # @type var indexing_field_type: ::ElasticGraph::SchemaDefinition::Indexing::_FieldType
+              # @type var indexing_field_type: Indexing::_JSONFieldType
               indexing_field_type = _ = type.to_indexing_field_type
               [type.name, indexing_field_type]
             end

elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion.rbs

@@ -39,7 +39,7 @@ module ElasticGraph::JSONIngestion
 
           attr_reader unused_deprecated_elements: ::Set[::ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement]
 
-          def initialize: (::ElasticGraph::SchemaDefinition::Results) -> void
+          def initialize: ((::ElasticGraph::SchemaDefinition::Results & ResultsExtension)) -> void
           def merge_metadata_into: (::Hash[::String, untyped]) -> JSONSchemaWithMetadata
 
           private

elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata.rbs

@@ -18,12 +18,12 @@ module ElasticGraph
         @json_schema_field_metadata_by_type_and_field_name: ::Hash[::String, ::Hash[::String, Indexing::JSONSchemaFieldMetadata]]?
         @current_public_json_schema: ::Hash[::String, untyped]?
         @json_schema_with_metadata_merger: Indexing::JSONSchemaWithMetadata::Merger?
-        @json_schema_indexing_field_types_by_name: ::Hash[::String, ::ElasticGraph::SchemaDefinition::Indexing::_FieldType]?
+        @json_schema_indexing_field_types_by_name: ::Hash[::String, Indexing::_JSONFieldType]?
 
         def json_ingestion_state: () -> (::ElasticGraph::SchemaDefinition::State & StateExtension)
         def json_schema_with_metadata_merger: () -> Indexing::JSONSchemaWithMetadata::Merger
         def build_public_json_schema: () -> ::Hash[::String, untyped]
-        def json_schema_indexing_field_types_by_name: () -> ::Hash[::String, ::ElasticGraph::SchemaDefinition::Indexing::_FieldType]
+        def json_schema_indexing_field_types_by_name: () -> ::Hash[::String, Indexing::_JSONFieldType]
       end
     end
   end

elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/results_extension.rbs

@@ -6,7 +6,7 @@ module ElasticGraph
 
         private
 
-        @json_schemas_artifact: ::ElasticGraph::SchemaDefinition::SchemaArtifact[untyped]
+        @json_schemas_artifact: ::ElasticGraph::SchemaDefinition::SchemaArtifact[untyped]?
 
         def json_ingestion_schema_definition_results: () -> (::ElasticGraph::SchemaDefinition::Results & ResultsExtension)
         def artifacts_from_schema_def: () -> ::Array[::ElasticGraph::SchemaDefinition::SchemaArtifact[untyped]]

elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_artifact_manager_extension.rbs

@@ -43,6 +43,9 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "elasticgraph-graphql", ElasticGraph::VERSION # needed since we validate that scalar `coerce_with` options are valid (which loads scalar coercion adapters)
   spec.add_dependency "elasticgraph-indexer", ElasticGraph::VERSION # needed since we validate that scalar `prepare_for_indexing_with` options are valid (which loads indexing preparer adapters)
+  # `elasticgraph-json_ingestion` is the default ingestion serializer extension. It's a soft dep here
+  # so `default_extension_modules` can fall back to `[]` when the gem isn't installed (e.g. for apps
+  # that supply a different serializer via `Gemfile-custom`), but most apps want it.
   spec.add_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION
   spec.add_dependency "elasticgraph-schema_artifacts", ElasticGraph::VERSION
   spec.add_dependency "elasticgraph-support", ElasticGraph::VERSION

elasticgraph-schema_definition/elasticgraph-schema_definition.gemspec

@@ -295,7 +295,7 @@ def union_type(name, &block)
       #   ElasticGraph.define_schema do |schema|
       #     schema.scalar_type "URL" do |t|
       #       t.mapping type: "keyword"
-      #       t.json_schema type: "string", format: "uri"
+      #       t.json_schema type: "string"
       #     end
       #   end
       def scalar_type(name, &block)
@@ -454,67 +454,23 @@ def results
         @results ||= @factory.new_results
       end
 
-      # Defines the version number of the current JSON schema. Importantly, every time a change is made that impacts the JSON schema
-      # artifact, the version number must be incremented to ensure that each different version of the JSON schema is identified by a unique
-      # version number. The publisher will then include this version number in published events to identify the version of the schema it
-      # was using. This avoids the need to deploy the publisher and ElasticGraph indexer at the same time to keep them in sync.
+      # Records the JSON schema version of this schema definition.
       #
-      # @note While this is an important part of how ElasticGraph is designed to support schema evolution, it can be annoying constantly
-      #   have to increment this while rapidly changing the schema during prototyping. You can disable the requirement to increment this
-      #   on every JSON schema change by setting `enforce_json_schema_version` to `false` in your `Rakefile`.
+      # The default implementation is a no-op so that callers (and the test helpers) can invoke this
+      # uniformly regardless of whether an ingestion-serializer extension is loaded. When
+      # `elasticgraph-json_ingestion` is loaded, its `APIExtension` overrides this with real behavior.
       #
-      # @param version [Integer] current version number of the JSON schema artifact
+      # @param version [Integer] the JSON schema version
       # @return [void]
-      # @see Local::RakeTasks#enforce_json_schema_version
-      #
-      # @example Set the JSON schema version to 1
-      #   ElasticGraph.define_schema do |schema|
-      #     schema.json_schema_version 1
-      #   end
       def json_schema_version(version)
-        if !version.is_a?(Integer) || version < 1
-          raise Errors::SchemaError, "`json_schema_version` must be a positive integer. Specified version: #{version}"
-        end
-
-        if @state.json_schema_version
-          raise Errors::SchemaError, "`json_schema_version` can only be set once on a schema. Previously-set version: #{@state.json_schema_version}"
-        end
-
-        @state.json_schema_version = version
-        @state.json_schema_version_setter_location = caller_locations(1, 1).to_a.first
-        nil
       end
 
-      # Defines strictness of the JSON schema validation. By default, the JSON schema will require all fields to be provided by the
-      # publisher (but they can be nullable) and will ignore extra fields that are not defined in the schema. Use this method to
-      # configure this behavior.
-      #
-      # @param allow_omitted_fields [bool] Whether nullable fields can be omitted from indexing events.
-      # @param allow_extra_fields [bool] Whether extra fields (e.g. beyond fields defined in the schema) can be included in indexing events.
-      # @return [void]
+      # Records JSON schema strictness configuration.
       #
-      # @note If you allow both omitted fields and extra fields, ElasticGraph's JSON schema validation will allow (and ignore) misspelled
-      #   field names in indexing events. For example, if the ElasticGraph schema has a nullable field named `parentId` but the publisher
-      #   accidentally provides it as `parent_id`, ElasticGraph would happily ignore the `parent_id` field entirely, because `parentId`
-      #   is allowed to be omitted and `parent_id` would be treated as an extra field. Therefore, we recommend that you only set one of
-      #   these to `true` (or none).
+      # The default implementation is a no-op; overridden by the JSON ingestion extension.
       #
-      # @example Allow omitted fields and disallow extra fields
-      #   ElasticGraph.define_schema do |schema|
-      #     schema.json_schema_strictness allow_omitted_fields: true, allow_extra_fields: false
-      #   end
+      # @return [void]
       def json_schema_strictness(allow_omitted_fields: false, allow_extra_fields: true)
-        unless [true, false].include?(allow_omitted_fields)
-          raise Errors::SchemaError, "`allow_omitted_fields` must be true or false"
-        end
-
-        unless [true, false].include?(allow_extra_fields)
-          raise Errors::SchemaError, "`allow_extra_fields` must be true or false"
-        end
-
-        @state.allow_omitted_json_schema_fields = allow_omitted_fields
-        @state.allow_extra_json_schema_fields = allow_extra_fields
-        nil
       end
 
       # Registers a customization callback that will be applied to every built-in type automatically provided by ElasticGraph. Provides