Use JSON ingestion as schema definition extension

CODEBASE_OVERVIEW.md

@@ -111,7 +111,6 @@ graph LR;
     rackup["rackup"];
     rake["rake"];
     webrick["webrick"];
-    elasticgraph-json_ingestion["eg-json_ingestion"];
     elasticgraph-schema_artifacts["eg-schema_artifacts"];
     graphql["graphql"];
     elasticgraph --> elasticgraph-support;
@@ -126,7 +125,6 @@ graph LR;
     elasticgraph-local --> webrick;
     elasticgraph-schema_definition --> elasticgraph-graphql;
     elasticgraph-schema_definition --> elasticgraph-indexer;
-    elasticgraph-schema_definition --> elasticgraph-json_ingestion;
     elasticgraph-schema_definition --> elasticgraph-schema_artifacts;
     elasticgraph-schema_definition --> elasticgraph-support;
     elasticgraph-schema_definition --> graphql;
@@ -143,7 +141,6 @@ graph LR;
     class rackup externalGemCatStyle;
     class rake externalGemCatStyle;
     class webrick externalGemCatStyle;
-    class elasticgraph-json_ingestion otherEgGemStyle;
     class elasticgraph-schema_artifacts otherEgGemStyle;
     class graphql externalGemCatStyle;
     click thor href "https://rubygems.org/gems/thor" "Open on RubyGems.org" _blank;

Gemfile.lock

@@ -175,7 +175,6 @@ PATH
     elasticgraph-schema_definition (1.2.1.pre)
       elasticgraph-graphql (= 1.2.1.pre)
       elasticgraph-indexer (= 1.2.1.pre)
-      elasticgraph-json_ingestion (= 1.2.1.pre)
       elasticgraph-schema_artifacts (= 1.2.1.pre)
       elasticgraph-support (= 1.2.1.pre)
       graphql (~> 2.6.2)

Rakefile

@@ -8,9 +8,10 @@
 
 require "delegate"
 require "elastic_graph/apollo/schema_definition/api_extension"
-require "elastic_graph/warehouse/schema_definition/api_extension"
+require "elastic_graph/json_ingestion/schema_definition/api_extension"
 require "elastic_graph/local/rake_tasks"
 require "elastic_graph/schema_definition/rake_tasks"
+require "elastic_graph/warehouse/schema_definition/api_extension"
 require "yaml"
 
 project_root = File.expand_path(__dir__)
@@ -91,6 +92,7 @@ configure_local_rake_tasks = ->(tasks) do
   tasks.opensearch_versions = tested_datastore_versions.fetch("opensearch")
 
   ENV["DEMONSTRATE_WAREHOUSE_APIS"] = "true"
+  tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension
   tasks.schema_definition_extension_modules << ElasticGraph::Warehouse::SchemaDefinition::APIExtension
 end
 

config/site/Rakefile

@@ -433,6 +433,7 @@ module ElasticGraph
 
         # When we are releasing gems, these dependencies aren't available.
         unless ENV["BUNDLE_GEMFILE"].to_s.end_with?("config/release/Gemfile")
+          require "elastic_graph/json_ingestion/schema_definition/api_extension"
           require "elastic_graph/local/rake_tasks"
           require "elastic_graph/query_registry/rake_tasks"
 
@@ -460,6 +461,7 @@ module ElasticGraph
                   # Here we provide a "default Rakefile" for examples that don't have any specialized needs and that aren't trying
                   # to demonstrate any `Rakefile` APIs.
                   ::ElasticGraph::Local::RakeTasks.new(local_config_yaml: settings_file, path_to_schema: schema_file) do |tasks|
+                    tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension
                     tasks.opensearch_versions = []
                   end
                 end

config/site/examples/music/Rakefile

@@ -6,13 +6,15 @@
 #
 # frozen_string_literal: true
 
+require "elastic_graph/json_ingestion/schema_definition/api_extension"
 require "elastic_graph/local/rake_tasks"
 
 ElasticGraph::Local::RakeTasks.new(
   local_config_yaml: File.expand_path("local_settings.yaml", __dir__),
   path_to_schema: File.expand_path("schema.rb", __dir__)
 ) do |tasks|
   tasks.opensearch_versions = []
+  tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension
 end
 
 # :nocov: -- only used for manual live-server query validation, not during the build

config/site/examples/schema_customization_rake_tasks/Rakefile

@@ -6,13 +6,15 @@
 #
 # frozen_string_literal: true
 
+require "elastic_graph/json_ingestion/schema_definition/api_extension"
 require "elastic_graph/local/rake_tasks"
 
 ElasticGraph::Local::RakeTasks.new(
   local_config_yaml: File.expand_path("local_settings.yaml", __dir__),
   path_to_schema: File.expand_path("schema.rb", __dir__)
 ) do |tasks|
   tasks.opensearch_versions = []
+  tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension
 
   # :snippet-start: schema_element_name_form
   # Within `ElasticGraph::Local::RakeTasks.new { ... }` in your `Rakefile`:

config/site/support/doctest_helper.rb

@@ -51,10 +51,10 @@ module ElasticGraph
     # API instance being active.
     descriptions_needing_schema_def_api_and_extension_modules = {
       "ElasticGraph.define_schema" => [],
-      "ElasticGraph::Apollo::SchemaDefinition" => [ElasticGraph::Apollo::SchemaDefinition::APIExtension],
-      "ElasticGraph::JSONIngestion::SchemaDefinition" => [ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension],
+      "ElasticGraph::Apollo::SchemaDefinition" => [Apollo::SchemaDefinition::APIExtension],
+      "ElasticGraph::JSONIngestion::SchemaDefinition" => [JSONIngestion::SchemaDefinition::APIExtension],
       "ElasticGraph::SchemaDefinition" => [],
-      "ElasticGraph::Warehouse::SchemaDefinition" => [ElasticGraph::Warehouse::SchemaDefinition::APIExtension]
+      "ElasticGraph::Warehouse::SchemaDefinition" => [Warehouse::SchemaDefinition::APIExtension]
     }
 
     descriptions_needing_schema_def_api_and_extension_modules.each do |description, extension_modules|
@@ -65,9 +65,10 @@ module ElasticGraph
           extension_modules: extension_modules
         )
 
-        # This is required in all schemas, but we don't want to have to put in all our examples,
-        # so we set it here.
-        @api.json_schema_version 1
+        # This is required in all JSON ingestion schemas, but we don't want to have to put it in all
+        # our examples, so we set it here. (Without a JSON ingestion extension, the
+        # `json_schema_version` API does not exist and there is no version to set.)
+        @api.json_schema_version 1 if @api.respond_to?(:json_schema_version)
 
         @api.object_type "SomeIndexedTypeToEnsureQueryTypeHasFields" do |t|
           t.field "id", "ID"
@@ -92,16 +93,11 @@ module ElasticGraph
       end
     end
 
-    [
-      "ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension#json_schema_version",
-      "ElasticGraph::SchemaDefinition::API#json_schema_version"
-    ].each do |description|
-      doctest.before description do
-        ElasticGraph.define_schema do |schema|
-          # `schema.json_schema_version` raises an error when the version is set more than once.
-          # By default we set it above. Here we clear it to allow our example to set it.
-          schema.state.json_schema_version = nil
-        end
+    doctest.before "ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension#json_schema_version" do
+      ElasticGraph.define_schema do |schema|
+        # `schema.json_schema_version` raises an error when the version is set more than once.
+        # By default we set it above. Here we clear it to allow our example to set it.
+        schema.state.json_schema_version = nil
       end
     end
 

elasticgraph-admin/elasticgraph-admin.gemspec

@@ -48,6 +48,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "rake", "~> 13.4", ">= 13.4.2"
 
   spec.add_development_dependency "elasticgraph-elasticsearch", ElasticGraph::VERSION
+  spec.add_development_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-opensearch", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION
 end

elasticgraph-apollo/README.md

@@ -49,29 +49,28 @@ index 4a5ef1e..5c16c2b 100644
 
 ```
 
-Finally, update `Rakefile` so that `ElasticGraph::GraphQL::Apollo::SchemaDefinition::APIExtension` is
+Finally, update `Rakefile` so that `ElasticGraph::Apollo::SchemaDefinition::APIExtension` is
 used as one of the `extension_modules`:
 
 ```diff
 diff --git a/Rakefile b/Rakefile
-index 2943335..26633c3 100644
+index b4dfe2e..731c0d0 100644
 --- a/Rakefile
 +++ b/Rakefile
-@@ -1,5 +1,6 @@
+@@ -1,4 +1,5 @@
  project_root = File.expand_path(__dir__)
 
 +require "elastic_graph/apollo/schema_definition/api_extension"
+ require "elastic_graph/json_ingestion/schema_definition/api_extension"
  require "elastic_graph/local/rake_tasks"
- require "elastic_graph/query_registry/rake_tasks"
- require "rspec/core/rake_task"
-@@ -12,5 +13,7 @@ ElasticGraph::Local::RakeTasks.new(
-   local_config_yaml: settings_file,
-   path_to_schema: "#{project_root}/config/schema.rb"
+@@ -15,6 +16,7 @@ ElasticGraph::Local::RakeTasks.new(
  ) do |tasks|
-+  tasks.schema_definition_extension_modules << ElasticGraph::Apollo::SchemaDefinition::APIExtension
-+
    # Determines casing of field names. Can be either `:camelCase` or `:snake_case`.
    tasks.schema_element_name_form = :camelCase
++  tasks.schema_definition_extension_modules << ElasticGraph::Apollo::SchemaDefinition::APIExtension
+   tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension
+
+   # Customizes the names of fields generated by ElasticGraph.
 ```
 
 That's it!

elasticgraph-apollo/apollo_tests_implementation/Rakefile

@@ -6,8 +6,9 @@
 #
 # frozen_string_literal: true
 
-require "elastic_graph/schema_definition/rake_tasks"
 require "elastic_graph/apollo/schema_definition/api_extension"
+require "elastic_graph/json_ingestion/schema_definition/api_extension"
+require "elastic_graph/schema_definition/rake_tasks"
 require "pathname"
 
 project_root = Pathname.new(__dir__)
@@ -17,5 +18,8 @@ ElasticGraph::SchemaDefinition::RakeTasks.new(
   index_document_sizes: false,
   path_to_schema: project_root / "config/products_schema.rb",
   schema_artifacts_directory: project_root / "config/schema/artifacts",
-  extension_modules: [ElasticGraph::Apollo::SchemaDefinition::APIExtension]
+  extension_modules: [
+    ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension,
+    ElasticGraph::Apollo::SchemaDefinition::APIExtension
+  ]
 )

elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/api_extension.rb

@@ -224,23 +224,20 @@ def define_apollo_schema_elements
 
           apollo_scalar_type "link__Import" do |t|
             t.documentation "Scalar type used by the `@link` directive required for Apollo Federation V2."
-            # `scalar_type` requires we set these but this scalar type is only in GraphQL.
+            # `scalar_type` requires a mapping but this scalar type is only in GraphQL.
             t.mapping type: nil
-            t.json_schema type: "null"
           end
 
           apollo_scalar_type "federation__Scope" do |t|
             t.documentation "Scalar type used by the `@requiresScopes` directive required for Apollo Federation V2.5+."
-            # `scalar_type` requires we set these but this scalar type is only in GraphQL.
+            # `scalar_type` requires a mapping but this scalar type is only in GraphQL.
             t.mapping type: nil
-            t.json_schema type: "null"
           end
 
           apollo_scalar_type "federation__Policy" do |t|
             t.documentation "Scalar type used by the `@policy` directive required for Apollo Federation V2.6+."
-            # `scalar_type` requires we set these but this scalar type is only in GraphQL.
+            # `scalar_type` requires a mapping but this scalar type is only in GraphQL.
             t.mapping type: nil
-            t.json_schema type: "null"
           end
 
           # Copied from https://github.com/apollographql/federation/blob/b3a3cb84d8d67d1d6e817dc85b9ae0ecdd9908d1/docs/source/subgraph-spec.mdx#subgraph-schema-additions
@@ -271,9 +268,8 @@ def define_apollo_schema_elements
               Not intended for use by clients other than Apollo.
             EOS
 
-            # `scalar_type` requires we set these but this scalar type is only in GraphQL.
+            # `scalar_type` requires a mapping but this scalar type is only in GraphQL.
             t.mapping type: nil
-            t.json_schema type: "null"
           end
 
           apollo_scalar_type "_Any" do |t|
@@ -297,9 +293,8 @@ def define_apollo_schema_elements
               Not intended for use by clients other than Apollo.
             EOS
 
-            # `scalar_type` requires we set these but this scalar type is only in GraphQL.
+            # `scalar_type` requires a mapping but this scalar type is only in GraphQL.
             t.mapping type: nil
-            t.json_schema type: "null"
           end
 
           apollo_object_type "_Service" do |t|

elasticgraph-apollo/spec/unit/elastic_graph/apollo/apollo_directives_spec.rb

@@ -56,7 +56,6 @@ def self.with_both_casing_forms(&block)
             schema.scalar_type "Url" do |f|
               f.apollo_authenticated
               f.mapping type: "keyword"
-              f.json_schema type: "string"
             end
           end
 
@@ -166,7 +165,6 @@ def self.with_both_casing_forms(&block)
             schema.scalar_type "Url" do |f|
               f.apollo_inaccessible
               f.mapping type: "keyword"
-              f.json_schema type: "string"
 
               f.customize_derived_types "UrlFilterInput" do |dt|
                 dt.apollo_inaccessible
@@ -319,7 +317,6 @@ def self.with_both_casing_forms(&block)
             schema.scalar_type "Url" do |f|
               f.apollo_policy(policies: [["Policy1", "Policy2"], ["Policy3"]])
               f.mapping type: "keyword"
-              f.json_schema type: "string"
             end
           end
 
@@ -407,7 +404,6 @@ def self.with_both_casing_forms(&block)
             schema.scalar_type "Url" do |f|
               f.apollo_requires_scopes(scopes: [["Scope1", "Scope2"], ["Scope3"]])
               f.mapping type: "keyword"
-              f.json_schema type: "string"
             end
           end
 
@@ -493,7 +489,6 @@ def self.with_both_casing_forms(&block)
             schema.scalar_type "Url" do |f|
               f.apollo_tag name: "test"
               f.mapping type: "keyword"
-              f.json_schema type: "string"
 
               f.customize_derived_types "UrlFilterInput" do |dt|
                 dt.apollo_tag name: "test"

elasticgraph-apollo/spec/unit/elastic_graph/apollo/schema_definition_spec.rb

@@ -453,14 +453,14 @@ def self.with_both_casing_forms(&block)
 
           # Ensure `Query` doesn't have a typical ElasticGraph query field that it includes for all indexed types (including union indexed types)
           expect(type_def_from(schema_string, "Query")).to exclude("__entitys", "_EntityConnection")
+          expect(schema_state.object_types_by_name.fetch("_Entity")).not_to be_root_document_type
         end
 
         it "has minimal impact on schema artifacts that are not used by the ElasticGraph GraphQL engine" do
           with_apollo_results = define_schema(with_apollo: true) { |s| define_some_types_on(s) }
           without_apollo_results = define_schema(with_apollo: false) { |s| define_some_types_on(s) }
 
           expect(with_apollo_results.datastore_scripts).to eq(without_apollo_results.datastore_scripts)
-          expect(with_apollo_results.json_schemas_for(1)).to eq(without_apollo_results.json_schemas_for(1))
           expect(with_apollo_results.indices).to eq(without_apollo_results.indices)
           expect(with_apollo_results.index_templates).to eq(without_apollo_results.index_templates)
 

elasticgraph-datastore_core/elasticgraph-datastore_core.gemspec

@@ -46,6 +46,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "elasticgraph-admin", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-elasticsearch", ElasticGraph::VERSION
+  spec.add_development_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-opensearch", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION
 end

elasticgraph-graphql/elasticgraph-graphql.gemspec

@@ -48,6 +48,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "elasticgraph-admin", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-elasticsearch", ElasticGraph::VERSION
+  spec.add_development_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-opensearch", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-indexer", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION

elasticgraph-graphql/spec/acceptance/schema_evolution_spec.rb

@@ -8,6 +8,7 @@
 
 require "elastic_graph/graphql"
 require "elastic_graph/indexer"
+require "elastic_graph/json_ingestion/schema_definition/api_extension"
 require "elastic_graph/schema_definition/rake_tasks"
 require "support/graphql"
 
@@ -65,6 +66,7 @@ def dump_schema_artifacts(json_schema_version:, team_extras: "")
       run_rake "schema_artifacts:dump" do |output|
         SchemaDefinition::RakeTasks.new(
           schema_element_name_form: :snake_case,
+          extension_modules: [JSONIngestion::SchemaDefinition::APIExtension],
           index_document_sizes: true,
           path_to_schema: path_to_schema,
           schema_artifacts_directory: "config/schema/artifacts",

elasticgraph-indexer/elasticgraph-indexer.gemspec

@@ -48,6 +48,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "elasticgraph-admin", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-elasticsearch", ElasticGraph::VERSION
+  spec.add_development_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-opensearch", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION
 end

elasticgraph-indexer/spec/acceptance/schema_evolution_spec.rb

@@ -7,6 +7,7 @@
 # frozen_string_literal: true
 
 require "elastic_graph/indexer"
+require "elastic_graph/json_ingestion/schema_definition/api_extension"
 require "elastic_graph/schema_definition/rake_tasks"
 
 module ElasticGraph
@@ -401,6 +402,7 @@ def dump_artifacts
       run_rake "schema_artifacts:dump" do |output|
         SchemaDefinition::RakeTasks.new(
           schema_element_name_form: :snake_case,
+          extension_modules: [JSONIngestion::SchemaDefinition::APIExtension],
           index_document_sizes: true,
           path_to_schema: path_to_schema,
           schema_artifacts_directory: "config/schema/artifacts",

elasticgraph-indexer/spec/unit/elastic_graph/indexer/record_preparer_spec.rb

@@ -576,7 +576,11 @@ def build_preparer_for_old_json_schema_version(v1_def:, v2_def:)
         end
 
         def define_schema(&schema_definition)
-          super(schema_element_name_form: "snake_case", &schema_definition)
+          super(
+            schema_element_name_form: "snake_case",
+            extension_modules: [JSONIngestion::SchemaDefinition::APIExtension],
+            &schema_definition
+          )
         end
       end
 

elasticgraph-json_ingestion/README.md

@@ -17,7 +17,4 @@ graph LR;
     elasticgraph-support["elasticgraph-support"];
     elasticgraph-json_ingestion --> elasticgraph-support;
     class elasticgraph-support otherEgGemStyle;
-    elasticgraph-schema_definition["elasticgraph-schema_definition"];
-    elasticgraph-schema_definition --> elasticgraph-json_ingestion;
-    class elasticgraph-schema_definition otherEgGemStyle;
 ```