Use JSON ingestion as schema definition extension

Author: jwils

CODEBASE_OVERVIEW.md

@@ -111,7 +111,6 @@ graph LR;
     rackup["rackup"];
     rake["rake"];
     webrick["webrick"];
-    elasticgraph-json_ingestion["eg-json_ingestion"];
     elasticgraph-schema_artifacts["eg-schema_artifacts"];
     graphql["graphql"];
     elasticgraph --> elasticgraph-support;
@@ -126,7 +125,6 @@ graph LR;
     elasticgraph-local --> webrick;
     elasticgraph-schema_definition --> elasticgraph-graphql;
     elasticgraph-schema_definition --> elasticgraph-indexer;
-    elasticgraph-schema_definition --> elasticgraph-json_ingestion;
     elasticgraph-schema_definition --> elasticgraph-schema_artifacts;
     elasticgraph-schema_definition --> elasticgraph-support;
     elasticgraph-schema_definition --> graphql;
@@ -143,7 +141,6 @@ graph LR;
     class rackup externalGemCatStyle;
     class rake externalGemCatStyle;
     class webrick externalGemCatStyle;
-    class elasticgraph-json_ingestion otherEgGemStyle;
     class elasticgraph-schema_artifacts otherEgGemStyle;
     class graphql externalGemCatStyle;
     click thor href "https://rubygems.org/gems/thor" "Open on RubyGems.org" _blank;

Gemfile.lock

@@ -198,7 +198,6 @@ PATH
     elasticgraph-schema_definition (1.2.1.pre)
       elasticgraph-graphql (= 1.2.1.pre)
       elasticgraph-indexer (= 1.2.1.pre)
-      elasticgraph-json_ingestion (= 1.2.1.pre)
       elasticgraph-schema_artifacts (= 1.2.1.pre)
       elasticgraph-support (= 1.2.1.pre)
       graphql (~> 2.6.2)

Rakefile

@@ -8,9 +8,10 @@
 
 require "delegate"
 require "elastic_graph/apollo/schema_definition/api_extension"
-require "elastic_graph/warehouse/schema_definition/api_extension"
+require "elastic_graph/json_ingestion/schema_definition/api_extension"
 require "elastic_graph/local/rake_tasks"
 require "elastic_graph/schema_definition/rake_tasks"
+require "elastic_graph/warehouse/schema_definition/api_extension"
 require "yaml"
 
 project_root = File.expand_path(__dir__)
@@ -49,6 +50,7 @@ configure_local_rake_tasks = ->(tasks) do
   tasks.index_document_sizes = true
   tasks.env_port_mapping = {test: test_port}
   tasks.output = schema_def_output
+  tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension
 
   tasks.define_fake_data_batch_for(:widgets) do
     require "rspec/core" # the factories file expects RSpec to be loaded, so load it.

config/site/Rakefile

@@ -433,6 +433,7 @@ module ElasticGraph
 
         # When we are releasing gems, these dependencies aren't available.
         unless ENV["BUNDLE_GEMFILE"].to_s.end_with?("config/release/Gemfile")
+          require "elastic_graph/json_ingestion/schema_definition/api_extension"
           require "elastic_graph/local/rake_tasks"
           require "elastic_graph/query_registry/rake_tasks"
 
@@ -460,6 +461,7 @@ module ElasticGraph
                   # Here we provide a "default Rakefile" for examples that don't have any specialized needs and that aren't trying
                   # to demonstrate any `Rakefile` APIs.
                   ::ElasticGraph::Local::RakeTasks.new(local_config_yaml: settings_file, path_to_schema: schema_file) do |tasks|
+                    tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension
                     tasks.opensearch_versions = []
                   end
                 end

config/site/support/doctest_helper.rb

@@ -52,7 +52,7 @@ module ElasticGraph
     descriptions_needing_schema_def_api_and_extension_modules = {
       "ElasticGraph.define_schema" => [],
       "ElasticGraph::Apollo::SchemaDefinition" => [ElasticGraph::Apollo::SchemaDefinition::APIExtension],
-      "ElasticGraph::JSONIngestion::SchemaDefinition" => [ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension],
+      "ElasticGraph::JSONIngestion::SchemaDefinition" => [JSONIngestion::SchemaDefinition::APIExtension],
       "ElasticGraph::SchemaDefinition" => [],
       "ElasticGraph::Warehouse::SchemaDefinition" => [ElasticGraph::Warehouse::SchemaDefinition::APIExtension]
     }
@@ -92,16 +92,11 @@ module ElasticGraph
       end
     end
 
-    [
-      "ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension#json_schema_version",
-      "ElasticGraph::SchemaDefinition::API#json_schema_version"
-    ].each do |description|
-      doctest.before description do
-        ElasticGraph.define_schema do |schema|
-          # `schema.json_schema_version` raises an error when the version is set more than once.
-          # By default we set it above. Here we clear it to allow our example to set it.
-          schema.state.json_schema_version = nil
-        end
+    doctest.before "ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension#json_schema_version" do
+      ElasticGraph.define_schema do |schema|
+        # `schema.json_schema_version` raises an error when the version is set more than once.
+        # By default we set it above. Here we clear it to allow our example to set it.
+        schema.state.json_schema_version = nil
       end
     end
 

elasticgraph-admin/elasticgraph-admin.gemspec

@@ -48,6 +48,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "rake", "~> 13.4", ">= 13.4.2"
 
   spec.add_development_dependency "elasticgraph-elasticsearch", ElasticGraph::VERSION
+  spec.add_development_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-opensearch", ElasticGraph::VERSION
   spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION
 end

elasticgraph-apollo/apollo_tests_implementation/Rakefile

@@ -6,8 +6,8 @@
 #
 # frozen_string_literal: true
 
-require "elastic_graph/schema_definition/rake_tasks"
 require "elastic_graph/apollo/schema_definition/api_extension"
+require "elastic_graph/schema_definition/rake_tasks"
 require "pathname"
 
 project_root = Pathname.new(__dir__)

elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/api_extension.rb

@@ -224,23 +224,20 @@ def define_apollo_schema_elements
 
           apollo_scalar_type "link__Import" do |t|
             t.documentation "Scalar type used by the `@link` directive required for Apollo Federation V2."
-            # `scalar_type` requires we set these but this scalar type is only in GraphQL.
+            # `scalar_type` requires a mapping but this scalar type is only in GraphQL.
             t.mapping type: nil
-            t.json_schema type: "null"
           end
 
           apollo_scalar_type "federation__Scope" do |t|
             t.documentation "Scalar type used by the `@requiresScopes` directive required for Apollo Federation V2.5+."
-            # `scalar_type` requires we set these but this scalar type is only in GraphQL.
+            # `scalar_type` requires a mapping but this scalar type is only in GraphQL.
             t.mapping type: nil
-            t.json_schema type: "null"
           end
 
           apollo_scalar_type "federation__Policy" do |t|
             t.documentation "Scalar type used by the `@policy` directive required for Apollo Federation V2.6+."
-            # `scalar_type` requires we set these but this scalar type is only in GraphQL.
+            # `scalar_type` requires a mapping but this scalar type is only in GraphQL.
             t.mapping type: nil
-            t.json_schema type: "null"
           end
 
           # Copied from https://github.com/apollographql/federation/blob/b3a3cb84d8d67d1d6e817dc85b9ae0ecdd9908d1/docs/source/subgraph-spec.mdx#subgraph-schema-additions
@@ -271,9 +268,8 @@ def define_apollo_schema_elements
               Not intended for use by clients other than Apollo.
             EOS
 
-            # `scalar_type` requires we set these but this scalar type is only in GraphQL.
+            # `scalar_type` requires a mapping but this scalar type is only in GraphQL.
             t.mapping type: nil
-            t.json_schema type: "null"
           end
 
           apollo_scalar_type "_Any" do |t|
@@ -297,9 +293,8 @@ def define_apollo_schema_elements
               Not intended for use by clients other than Apollo.
             EOS
 
-            # `scalar_type` requires we set these but this scalar type is only in GraphQL.
+            # `scalar_type` requires a mapping but this scalar type is only in GraphQL.
             t.mapping type: nil
-            t.json_schema type: "null"
           end
 
           apollo_object_type "_Service" do |t|

elasticgraph-apollo/spec/unit/elastic_graph/apollo/apollo_directives_spec.rb

@@ -56,7 +56,6 @@ def self.with_both_casing_forms(&block)
             schema.scalar_type "Url" do |f|
               f.apollo_authenticated
               f.mapping type: "keyword"
-              f.json_schema type: "string"
             end
           end
 
@@ -166,7 +165,6 @@ def self.with_both_casing_forms(&block)
             schema.scalar_type "Url" do |f|
               f.apollo_inaccessible
               f.mapping type: "keyword"
-              f.json_schema type: "string"
 
               f.customize_derived_types "UrlFilterInput" do |dt|
                 dt.apollo_inaccessible
@@ -319,7 +317,6 @@ def self.with_both_casing_forms(&block)
             schema.scalar_type "Url" do |f|
               f.apollo_policy(policies: [["Policy1", "Policy2"], ["Policy3"]])
               f.mapping type: "keyword"
-              f.json_schema type: "string"
             end
           end
 
@@ -407,7 +404,6 @@ def self.with_both_casing_forms(&block)
             schema.scalar_type "Url" do |f|
               f.apollo_requires_scopes(scopes: [["Scope1", "Scope2"], ["Scope3"]])
               f.mapping type: "keyword"
-              f.json_schema type: "string"
             end
           end
 
@@ -493,7 +489,6 @@ def self.with_both_casing_forms(&block)
             schema.scalar_type "Url" do |f|
               f.apollo_tag name: "test"
               f.mapping type: "keyword"
-              f.json_schema type: "string"
 
               f.customize_derived_types "UrlFilterInput" do |dt|
                 dt.apollo_tag name: "test"

elasticgraph-apollo/spec/unit/elastic_graph/apollo/schema_definition_spec.rb

@@ -460,7 +460,6 @@ def self.with_both_casing_forms(&block)
           without_apollo_results = define_schema(with_apollo: false) { |s| define_some_types_on(s) }
 
           expect(with_apollo_results.datastore_scripts).to eq(without_apollo_results.datastore_scripts)
-          expect(with_apollo_results.json_schemas_for(1)).to eq(without_apollo_results.json_schemas_for(1))
           expect(with_apollo_results.indices).to eq(without_apollo_results.indices)
           expect(with_apollo_results.index_templates).to eq(without_apollo_results.index_templates)