Use JSON ingestion as schema definition extension

Author: jwils

CODEBASE_OVERVIEW.md

@@ -111,7 +111,6 @@ graph LR;
     rackup["rackup"];
     rake["rake"];
     webrick["webrick"];
-    elasticgraph-json_ingestion["eg-json_ingestion"];
     elasticgraph-schema_artifacts["eg-schema_artifacts"];
     graphql["graphql"];
     elasticgraph --> elasticgraph-support;
@@ -126,7 +125,6 @@ graph LR;
     elasticgraph-local --> webrick;
     elasticgraph-schema_definition --> elasticgraph-graphql;
     elasticgraph-schema_definition --> elasticgraph-indexer;
-    elasticgraph-schema_definition --> elasticgraph-json_ingestion;
     elasticgraph-schema_definition --> elasticgraph-schema_artifacts;
     elasticgraph-schema_definition --> elasticgraph-support;
     elasticgraph-schema_definition --> graphql;
@@ -143,7 +141,6 @@ graph LR;
     class rackup externalGemCatStyle;
     class rake externalGemCatStyle;
     class webrick externalGemCatStyle;
-    class elasticgraph-json_ingestion otherEgGemStyle;
     class elasticgraph-schema_artifacts otherEgGemStyle;
     class graphql externalGemCatStyle;
     click thor href "https://rubygems.org/gems/thor" "Open on RubyGems.org" _blank;

Gemfile.lock

@@ -198,7 +198,6 @@ PATH
     elasticgraph-schema_definition (1.2.1.pre)
       elasticgraph-graphql (= 1.2.1.pre)
       elasticgraph-indexer (= 1.2.1.pre)
-      elasticgraph-json_ingestion (= 1.2.1.pre)
       elasticgraph-schema_artifacts (= 1.2.1.pre)
       elasticgraph-support (= 1.2.1.pre)
       graphql (~> 2.6.2)

Rakefile

@@ -8,6 +8,7 @@
 
 require "delegate"
 require "elastic_graph/apollo/schema_definition/api_extension"
+require "elastic_graph/json_ingestion/schema_definition/api_extension"
 require "elastic_graph/warehouse/schema_definition/api_extension"
 require "elastic_graph/local/rake_tasks"
 require "elastic_graph/schema_definition/rake_tasks"
@@ -49,6 +50,7 @@ configure_local_rake_tasks = ->(tasks) do
   tasks.index_document_sizes = true
   tasks.env_port_mapping = {test: test_port}
   tasks.output = schema_def_output
+  tasks.schema_definition_extension_modules = [ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension]
 
   tasks.define_fake_data_batch_for(:widgets) do
     require "rspec/core" # the factories file expects RSpec to be loaded, so load it.

config/site/support/doctest_helper.rb

@@ -7,6 +7,7 @@
 # frozen_string_literal: true
 
 require "elastic_graph/apollo/schema_definition/api_extension"
+require "elastic_graph/json_ingestion/schema_definition/api_extension"
 require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names"
 require "elastic_graph/schema_definition/api"
 require "elastic_graph/schema_definition/schema_artifact_manager"
@@ -60,7 +61,7 @@ module ElasticGraph
         @api = SchemaDefinition::API.new(
           SchemaArtifacts::RuntimeMetadata::SchemaElementNames.new(form: :camelCase, overrides: {}),
           true,
-          extension_modules: extension_modules
+          extension_modules: [JSONIngestion::SchemaDefinition::APIExtension] + extension_modules
         )
 
         # This is required in all schemas, but we don't want to have to put in all our examples,
@@ -95,6 +96,7 @@ module ElasticGraph
         # `schema.json_schema_version` raises an error when the version is set more than once.
         # By default we set it above. Here we clear it to allow our example to set it.
         schema.state.json_schema_version = nil
+        schema.state.json_schema_version_setter_location = nil
       end
     end
 

elasticgraph-apollo/spec/unit/elastic_graph/apollo/apollo_directives_spec.rb

@@ -7,6 +7,7 @@
 # frozen_string_literal: true
 
 require "elastic_graph/apollo/schema_definition/api_extension"
+require "elastic_graph/json_ingestion/schema_definition/api_extension"
 require "elastic_graph/spec_support/schema_definition_helpers"
 
 module ElasticGraph
@@ -552,7 +553,7 @@ def self.with_both_casing_forms(&block)
         end
 
         def define_schema(&block)
-          extension_modules = [SchemaDefinition::APIExtension]
+          extension_modules = [::ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension, SchemaDefinition::APIExtension]
           super(schema_element_name_form: schema_element_name_form, extension_modules: extension_modules, &block)
         end
       end

elasticgraph-apollo/spec/unit/elastic_graph/apollo/schema_definition_spec.rb

@@ -7,6 +7,7 @@
 # frozen_string_literal: true
 
 require "elastic_graph/apollo/schema_definition/api_extension"
+require "elastic_graph/json_ingestion/schema_definition/api_extension"
 require "elastic_graph/spec_support/runtime_metadata_support"
 require "elastic_graph/spec_support/schema_definition_helpers"
 require "graphql"
@@ -1431,7 +1432,8 @@ def expect_identifiable_type_tagging_of_token(&type_def_for)
         end
 
         def define_schema(with_apollo: true, &block)
-          extension_modules = with_apollo ? [SchemaDefinition::APIExtension] : []
+          extension_modules = [::ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension]
+          extension_modules += [SchemaDefinition::APIExtension] if with_apollo
           super(schema_element_name_form: schema_element_name_form, extension_modules: extension_modules, &block)
         end
       end

elasticgraph-json_ingestion/README.md

@@ -17,7 +17,4 @@ graph LR;
     elasticgraph-support["elasticgraph-support"];
     elasticgraph-json_ingestion --> elasticgraph-support;
     class elasticgraph-support otherEgGemStyle;
-    elasticgraph-schema_definition["elasticgraph-schema_definition"];
-    elasticgraph-schema_definition --> elasticgraph-json_ingestion;
-    class elasticgraph-schema_definition otherEgGemStyle;
 ```

elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/api_extension.rb

@@ -99,11 +99,11 @@ def self.extended(api)
         #
         # @note While this is an important part of how ElasticGraph is designed to support schema evolution, it can be annoying constantly
         #   have to increment this while rapidly changing the schema during prototyping. You can disable the requirement to increment this
-        #   on every JSON schema change by setting `enforce_json_schema_version` to `false` in your `Rakefile`.
+        #   on every JSON schema change with {#enforce_json_schema_version}.
         #
         # @param version [Integer] current version number of the JSON schema artifact
         # @return [void]
-        # @see Local::RakeTasks#enforce_json_schema_version
+        # @see #enforce_json_schema_version
         def json_schema_version(version)
           state = json_ingestion_state
 
@@ -120,6 +120,26 @@ def json_schema_version(version)
           nil
         end
 
+        # Configures whether JSON schema artifact dumping enforces that {#json_schema_version} gets bumped every time the JSON schemas
+        # artifact changes. This should generally remain enabled for production applications, but disabling it can be useful during early
+        # prototyping.
+        #
+        # @param value [Boolean] whether JSON schema version bumps should be enforced
+        # @return [void]
+        #
+        # @example Disable JSON schema version enforcement while prototyping
+        #   ElasticGraph.define_schema do |schema|
+        #     schema.enforce_json_schema_version false
+        #   end
+        def enforce_json_schema_version(value)
+          unless value == true || value == false
+            raise Errors::SchemaError, "`enforce_json_schema_version` must be a boolean. Specified value: #{value.inspect}"
+          end
+
+          json_ingestion_state.enforce_json_schema_version = value
+          nil
+        end
+
         # Defines strictness of the JSON schema validation. By default, the JSON schema will require all fields to be provided by the
         # publisher (but they can be nullable) and will ignore extra fields that are not defined in the schema. Use this method to
         # configure this behavior.

elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/results_extension.rb

@@ -121,7 +121,7 @@ def json_schema_indexing_field_types_by_name
             end
             .sort_by(&:name)
             .to_h do |type|
-              # @type var indexing_field_type: ElasticGraph::SchemaDefinition::Indexing::_FieldType
+              # @type var indexing_field_type: Indexing::_JSONFieldType
               indexing_field_type = _ = type.to_indexing_field_type
               [type.name, indexing_field_type]
             end

elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_artifact_manager_extension.rb

@@ -18,17 +18,12 @@ module SchemaDefinition
       #
       # @private
       module SchemaArtifactManagerExtension
-        # Configuration key consumed from {ElasticGraph::SchemaDefinition::SchemaArtifactManager}'s
-        # `extension_artifact_options:` hash. When set to `false`, JSON schema artifact dumps don't
-        # require an accompanying `json_schema_version` bump.
-        ENFORCE_JSON_SCHEMA_VERSION_KEY = :enforce_json_schema_version
-
         # Overrides `dump_artifacts` to add JSON schema version bump checking before dumping.
         def dump_artifacts
           schema_results = json_ingestion_schema_definition_results
 
           check_if_needs_json_schema_version_bump do |recommended_json_schema_version|
-            if @extension_artifact_options.fetch(ENFORCE_JSON_SCHEMA_VERSION_KEY, true)
+            if schema_results.state.enforce_json_schema_version
               # @type var setter_location: ::Thread::Backtrace::Location
               # We use `_ =` because while `json_schema_version_setter_location` can be nil,
               # it'll never be nil if we get here and we want the type to be non-nilable.
@@ -39,13 +34,12 @@ def dump_artifacts
                 "increase the schema's version, and then run the `bundle exec rake schema_artifacts:dump` command again.\n\n" \
                 "To update the schema version to the expected version, change line #{setter_location.lineno} at `#{setter_location_path}` to:\n" \
                 "  `schema.json_schema_version #{recommended_json_schema_version}`\n\n" \
-                "Alternately, pass `extension_artifact_options: {enforce_json_schema_version: false}` to `ElasticGraph::SchemaDefinition::RakeTasks.new` " \
-                "(or set `tasks.enforce_json_schema_version = false` on `ElasticGraph::Local::RakeTasks`) to allow the JSON schemas file to " \
-                "change without requiring a version bump, but that is only recommended for non-production applications during initial schema prototyping."
+                "Alternately, call `schema.enforce_json_schema_version false` in your schema definition to allow the JSON schemas file " \
+                "to change without requiring a version bump, but that is only recommended for non-production applications during initial schema prototyping."
             else
               @output.puts <<~EOS
                 WARNING: the `json_schemas.yaml` artifact is being updated without the `json_schema_version` being correspondingly incremented.
-                This is not recommended for production applications, but is currently allowed because you have set `enforce_json_schema_version: false`.
+                This is not recommended for production applications, but is currently allowed because you have called `schema.enforce_json_schema_version false`.
               EOS
             end
           end