diff --git a/CODEBASE_OVERVIEW.md b/CODEBASE_OVERVIEW.md index 8cc77e3b8..a7f4b58ea 100644 --- a/CODEBASE_OVERVIEW.md +++ b/CODEBASE_OVERVIEW.md @@ -111,7 +111,6 @@ graph LR; rackup["rackup"]; rake["rake"]; webrick["webrick"]; - elasticgraph-json_ingestion["eg-json_ingestion"]; elasticgraph-schema_artifacts["eg-schema_artifacts"]; graphql["graphql"]; elasticgraph --> elasticgraph-support; @@ -126,7 +125,6 @@ graph LR; elasticgraph-local --> webrick; elasticgraph-schema_definition --> elasticgraph-graphql; elasticgraph-schema_definition --> elasticgraph-indexer; - elasticgraph-schema_definition --> elasticgraph-json_ingestion; elasticgraph-schema_definition --> elasticgraph-schema_artifacts; elasticgraph-schema_definition --> elasticgraph-support; elasticgraph-schema_definition --> graphql; @@ -143,7 +141,6 @@ graph LR; class rackup externalGemCatStyle; class rake externalGemCatStyle; class webrick externalGemCatStyle; - class elasticgraph-json_ingestion otherEgGemStyle; class elasticgraph-schema_artifacts otherEgGemStyle; class graphql externalGemCatStyle; click thor href "https://rubygems.org/gems/thor" "Open on RubyGems.org" _blank; diff --git a/Gemfile.lock b/Gemfile.lock index 1f3734453..841dc4527 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -175,7 +175,6 @@ PATH elasticgraph-schema_definition (1.2.1.pre) elasticgraph-graphql (= 1.2.1.pre) elasticgraph-indexer (= 1.2.1.pre) - elasticgraph-json_ingestion (= 1.2.1.pre) elasticgraph-schema_artifacts (= 1.2.1.pre) elasticgraph-support (= 1.2.1.pre) graphql (~> 2.6.2) diff --git a/Rakefile b/Rakefile index 831dc68b3..d1bc451ae 100644 --- a/Rakefile +++ b/Rakefile @@ -8,9 +8,10 @@ require "delegate" require "elastic_graph/apollo/schema_definition/api_extension" -require "elastic_graph/warehouse/schema_definition/api_extension" +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/local/rake_tasks" require "elastic_graph/schema_definition/rake_tasks" +require "elastic_graph/warehouse/schema_definition/api_extension" require "yaml" project_root = File.expand_path(__dir__) @@ -91,6 +92,7 @@ configure_local_rake_tasks = ->(tasks) do tasks.opensearch_versions = tested_datastore_versions.fetch("opensearch") ENV["DEMONSTRATE_WAREHOUSE_APIS"] = "true" + tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension tasks.schema_definition_extension_modules << ElasticGraph::Warehouse::SchemaDefinition::APIExtension end diff --git a/config/site/Rakefile b/config/site/Rakefile index c3eec0216..b76f8eb7b 100644 --- a/config/site/Rakefile +++ b/config/site/Rakefile @@ -433,6 +433,7 @@ module ElasticGraph # When we are releasing gems, these dependencies aren't available. unless ENV["BUNDLE_GEMFILE"].to_s.end_with?("config/release/Gemfile") + require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/local/rake_tasks" require "elastic_graph/query_registry/rake_tasks" @@ -460,6 +461,7 @@ module ElasticGraph # Here we provide a "default Rakefile" for examples that don't have any specialized needs and that aren't trying # to demonstrate any `Rakefile` APIs. ::ElasticGraph::Local::RakeTasks.new(local_config_yaml: settings_file, path_to_schema: schema_file) do |tasks| + tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension tasks.opensearch_versions = [] end end diff --git a/config/site/examples/music/Rakefile b/config/site/examples/music/Rakefile index c54dd64f6..028d899ae 100644 --- a/config/site/examples/music/Rakefile +++ b/config/site/examples/music/Rakefile @@ -6,6 +6,7 @@ # # frozen_string_literal: true +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/local/rake_tasks" ElasticGraph::Local::RakeTasks.new( @@ -13,6 +14,7 @@ ElasticGraph::Local::RakeTasks.new( path_to_schema: File.expand_path("schema.rb", __dir__) ) do |tasks| tasks.opensearch_versions = [] + tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension end # :nocov: -- only used for manual live-server query validation, not during the build diff --git a/config/site/examples/schema_customization_rake_tasks/Rakefile b/config/site/examples/schema_customization_rake_tasks/Rakefile index c5b091a54..0ca1c66a9 100644 --- a/config/site/examples/schema_customization_rake_tasks/Rakefile +++ b/config/site/examples/schema_customization_rake_tasks/Rakefile @@ -6,6 +6,7 @@ # # frozen_string_literal: true +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/local/rake_tasks" ElasticGraph::Local::RakeTasks.new( @@ -13,6 +14,7 @@ ElasticGraph::Local::RakeTasks.new( path_to_schema: File.expand_path("schema.rb", __dir__) ) do |tasks| tasks.opensearch_versions = [] + tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension # :snippet-start: schema_element_name_form # Within `ElasticGraph::Local::RakeTasks.new { ... }` in your `Rakefile`: diff --git a/config/site/support/doctest_helper.rb b/config/site/support/doctest_helper.rb index 3be66a000..ee2ec66e7 100644 --- a/config/site/support/doctest_helper.rb +++ b/config/site/support/doctest_helper.rb @@ -51,10 +51,10 @@ module ElasticGraph # API instance being active. descriptions_needing_schema_def_api_and_extension_modules = { "ElasticGraph.define_schema" => [], - "ElasticGraph::Apollo::SchemaDefinition" => [ElasticGraph::Apollo::SchemaDefinition::APIExtension], - "ElasticGraph::JSONIngestion::SchemaDefinition" => [ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension], + "ElasticGraph::Apollo::SchemaDefinition" => [Apollo::SchemaDefinition::APIExtension], + "ElasticGraph::JSONIngestion::SchemaDefinition" => [JSONIngestion::SchemaDefinition::APIExtension], "ElasticGraph::SchemaDefinition" => [], - "ElasticGraph::Warehouse::SchemaDefinition" => [ElasticGraph::Warehouse::SchemaDefinition::APIExtension] + "ElasticGraph::Warehouse::SchemaDefinition" => [Warehouse::SchemaDefinition::APIExtension] } descriptions_needing_schema_def_api_and_extension_modules.each do |description, extension_modules| @@ -65,9 +65,10 @@ module ElasticGraph extension_modules: extension_modules ) - # This is required in all schemas, but we don't want to have to put in all our examples, - # so we set it here. - @api.json_schema_version 1 + # This is required in all JSON ingestion schemas, but we don't want to have to put it in all + # our examples, so we set it here. (Without a JSON ingestion extension, the + # `json_schema_version` API does not exist and there is no version to set.) + @api.json_schema_version 1 if @api.respond_to?(:json_schema_version) @api.object_type "SomeIndexedTypeToEnsureQueryTypeHasFields" do |t| t.field "id", "ID" @@ -92,16 +93,11 @@ module ElasticGraph end end - [ - "ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension#json_schema_version", - "ElasticGraph::SchemaDefinition::API#json_schema_version" - ].each do |description| - doctest.before description do - ElasticGraph.define_schema do |schema| - # `schema.json_schema_version` raises an error when the version is set more than once. - # By default we set it above. Here we clear it to allow our example to set it. - schema.state.json_schema_version = nil - end + doctest.before "ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension#json_schema_version" do + ElasticGraph.define_schema do |schema| + # `schema.json_schema_version` raises an error when the version is set more than once. + # By default we set it above. Here we clear it to allow our example to set it. + schema.state.json_schema_version = nil end end diff --git a/elasticgraph-admin/elasticgraph-admin.gemspec b/elasticgraph-admin/elasticgraph-admin.gemspec index 722116506..cc90e68a4 100644 --- a/elasticgraph-admin/elasticgraph-admin.gemspec +++ b/elasticgraph-admin/elasticgraph-admin.gemspec @@ -48,6 +48,7 @@ Gem::Specification.new do |spec| spec.add_dependency "rake", "~> 13.4", ">= 13.4.2" spec.add_development_dependency "elasticgraph-elasticsearch", ElasticGraph::VERSION + spec.add_development_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-opensearch", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION end diff --git a/elasticgraph-apollo/README.md b/elasticgraph-apollo/README.md index 2503b0377..3bf3a1103 100644 --- a/elasticgraph-apollo/README.md +++ b/elasticgraph-apollo/README.md @@ -49,29 +49,28 @@ index 4a5ef1e..5c16c2b 100644 ``` -Finally, update `Rakefile` so that `ElasticGraph::GraphQL::Apollo::SchemaDefinition::APIExtension` is +Finally, update `Rakefile` so that `ElasticGraph::Apollo::SchemaDefinition::APIExtension` is used as one of the `extension_modules`: ```diff diff --git a/Rakefile b/Rakefile -index 2943335..26633c3 100644 +index b4dfe2e..731c0d0 100644 --- a/Rakefile +++ b/Rakefile -@@ -1,5 +1,6 @@ +@@ -1,4 +1,5 @@ project_root = File.expand_path(__dir__) +require "elastic_graph/apollo/schema_definition/api_extension" + require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/local/rake_tasks" - require "elastic_graph/query_registry/rake_tasks" - require "rspec/core/rake_task" -@@ -12,5 +13,7 @@ ElasticGraph::Local::RakeTasks.new( - local_config_yaml: settings_file, - path_to_schema: "#{project_root}/config/schema.rb" +@@ -15,6 +16,7 @@ ElasticGraph::Local::RakeTasks.new( ) do |tasks| -+ tasks.schema_definition_extension_modules << ElasticGraph::Apollo::SchemaDefinition::APIExtension -+ # Determines casing of field names. Can be either `:camelCase` or `:snake_case`. tasks.schema_element_name_form = :camelCase ++ tasks.schema_definition_extension_modules << ElasticGraph::Apollo::SchemaDefinition::APIExtension + tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension + + # Customizes the names of fields generated by ElasticGraph. ``` That's it! diff --git a/elasticgraph-apollo/apollo_tests_implementation/Rakefile b/elasticgraph-apollo/apollo_tests_implementation/Rakefile index befade088..88a9ae811 100644 --- a/elasticgraph-apollo/apollo_tests_implementation/Rakefile +++ b/elasticgraph-apollo/apollo_tests_implementation/Rakefile @@ -6,8 +6,9 @@ # # frozen_string_literal: true -require "elastic_graph/schema_definition/rake_tasks" require "elastic_graph/apollo/schema_definition/api_extension" +require "elastic_graph/json_ingestion/schema_definition/api_extension" +require "elastic_graph/schema_definition/rake_tasks" require "pathname" project_root = Pathname.new(__dir__) @@ -17,5 +18,8 @@ ElasticGraph::SchemaDefinition::RakeTasks.new( index_document_sizes: false, path_to_schema: project_root / "config/products_schema.rb", schema_artifacts_directory: project_root / "config/schema/artifacts", - extension_modules: [ElasticGraph::Apollo::SchemaDefinition::APIExtension] + extension_modules: [ + ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension, + ElasticGraph::Apollo::SchemaDefinition::APIExtension + ] ) diff --git a/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/api_extension.rb b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/api_extension.rb index fe834e38f..63c59039f 100644 --- a/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/api_extension.rb +++ b/elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/api_extension.rb @@ -224,23 +224,20 @@ def define_apollo_schema_elements apollo_scalar_type "link__Import" do |t| t.documentation "Scalar type used by the `@link` directive required for Apollo Federation V2." - # `scalar_type` requires we set these but this scalar type is only in GraphQL. + # `scalar_type` requires a mapping but this scalar type is only in GraphQL. t.mapping type: nil - t.json_schema type: "null" end apollo_scalar_type "federation__Scope" do |t| t.documentation "Scalar type used by the `@requiresScopes` directive required for Apollo Federation V2.5+." - # `scalar_type` requires we set these but this scalar type is only in GraphQL. + # `scalar_type` requires a mapping but this scalar type is only in GraphQL. t.mapping type: nil - t.json_schema type: "null" end apollo_scalar_type "federation__Policy" do |t| t.documentation "Scalar type used by the `@policy` directive required for Apollo Federation V2.6+." - # `scalar_type` requires we set these but this scalar type is only in GraphQL. + # `scalar_type` requires a mapping but this scalar type is only in GraphQL. t.mapping type: nil - t.json_schema type: "null" end # Copied from https://github.com/apollographql/federation/blob/b3a3cb84d8d67d1d6e817dc85b9ae0ecdd9908d1/docs/source/subgraph-spec.mdx#subgraph-schema-additions @@ -271,9 +268,8 @@ def define_apollo_schema_elements Not intended for use by clients other than Apollo. EOS - # `scalar_type` requires we set these but this scalar type is only in GraphQL. + # `scalar_type` requires a mapping but this scalar type is only in GraphQL. t.mapping type: nil - t.json_schema type: "null" end apollo_scalar_type "_Any" do |t| @@ -297,9 +293,8 @@ def define_apollo_schema_elements Not intended for use by clients other than Apollo. EOS - # `scalar_type` requires we set these but this scalar type is only in GraphQL. + # `scalar_type` requires a mapping but this scalar type is only in GraphQL. t.mapping type: nil - t.json_schema type: "null" end apollo_object_type "_Service" do |t| diff --git a/elasticgraph-apollo/spec/unit/elastic_graph/apollo/apollo_directives_spec.rb b/elasticgraph-apollo/spec/unit/elastic_graph/apollo/apollo_directives_spec.rb index 915c79c34..f03f0760c 100644 --- a/elasticgraph-apollo/spec/unit/elastic_graph/apollo/apollo_directives_spec.rb +++ b/elasticgraph-apollo/spec/unit/elastic_graph/apollo/apollo_directives_spec.rb @@ -56,7 +56,6 @@ def self.with_both_casing_forms(&block) schema.scalar_type "Url" do |f| f.apollo_authenticated f.mapping type: "keyword" - f.json_schema type: "string" end end @@ -166,7 +165,6 @@ def self.with_both_casing_forms(&block) schema.scalar_type "Url" do |f| f.apollo_inaccessible f.mapping type: "keyword" - f.json_schema type: "string" f.customize_derived_types "UrlFilterInput" do |dt| dt.apollo_inaccessible @@ -319,7 +317,6 @@ def self.with_both_casing_forms(&block) schema.scalar_type "Url" do |f| f.apollo_policy(policies: [["Policy1", "Policy2"], ["Policy3"]]) f.mapping type: "keyword" - f.json_schema type: "string" end end @@ -407,7 +404,6 @@ def self.with_both_casing_forms(&block) schema.scalar_type "Url" do |f| f.apollo_requires_scopes(scopes: [["Scope1", "Scope2"], ["Scope3"]]) f.mapping type: "keyword" - f.json_schema type: "string" end end @@ -493,7 +489,6 @@ def self.with_both_casing_forms(&block) schema.scalar_type "Url" do |f| f.apollo_tag name: "test" f.mapping type: "keyword" - f.json_schema type: "string" f.customize_derived_types "UrlFilterInput" do |dt| dt.apollo_tag name: "test" diff --git a/elasticgraph-apollo/spec/unit/elastic_graph/apollo/schema_definition_spec.rb b/elasticgraph-apollo/spec/unit/elastic_graph/apollo/schema_definition_spec.rb index 740bc003e..338b5af62 100644 --- a/elasticgraph-apollo/spec/unit/elastic_graph/apollo/schema_definition_spec.rb +++ b/elasticgraph-apollo/spec/unit/elastic_graph/apollo/schema_definition_spec.rb @@ -453,6 +453,7 @@ def self.with_both_casing_forms(&block) # Ensure `Query` doesn't have a typical ElasticGraph query field that it includes for all indexed types (including union indexed types) expect(type_def_from(schema_string, "Query")).to exclude("__entitys", "_EntityConnection") + expect(schema_state.object_types_by_name.fetch("_Entity")).not_to be_root_document_type end it "has minimal impact on schema artifacts that are not used by the ElasticGraph GraphQL engine" do @@ -460,7 +461,6 @@ def self.with_both_casing_forms(&block) without_apollo_results = define_schema(with_apollo: false) { |s| define_some_types_on(s) } expect(with_apollo_results.datastore_scripts).to eq(without_apollo_results.datastore_scripts) - expect(with_apollo_results.json_schemas_for(1)).to eq(without_apollo_results.json_schemas_for(1)) expect(with_apollo_results.indices).to eq(without_apollo_results.indices) expect(with_apollo_results.index_templates).to eq(without_apollo_results.index_templates) diff --git a/elasticgraph-datastore_core/elasticgraph-datastore_core.gemspec b/elasticgraph-datastore_core/elasticgraph-datastore_core.gemspec index 05ee33e5f..89fd32983 100644 --- a/elasticgraph-datastore_core/elasticgraph-datastore_core.gemspec +++ b/elasticgraph-datastore_core/elasticgraph-datastore_core.gemspec @@ -46,6 +46,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency "elasticgraph-admin", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-elasticsearch", ElasticGraph::VERSION + spec.add_development_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-opensearch", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION end diff --git a/elasticgraph-graphql/elasticgraph-graphql.gemspec b/elasticgraph-graphql/elasticgraph-graphql.gemspec index 8bc5ab51b..eba63698a 100644 --- a/elasticgraph-graphql/elasticgraph-graphql.gemspec +++ b/elasticgraph-graphql/elasticgraph-graphql.gemspec @@ -48,6 +48,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency "elasticgraph-admin", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-elasticsearch", ElasticGraph::VERSION + spec.add_development_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-opensearch", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-indexer", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION diff --git a/elasticgraph-graphql/spec/acceptance/schema_evolution_spec.rb b/elasticgraph-graphql/spec/acceptance/schema_evolution_spec.rb index 494152ebf..05d18f5ab 100644 --- a/elasticgraph-graphql/spec/acceptance/schema_evolution_spec.rb +++ b/elasticgraph-graphql/spec/acceptance/schema_evolution_spec.rb @@ -8,6 +8,7 @@ require "elastic_graph/graphql" require "elastic_graph/indexer" +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/schema_definition/rake_tasks" require "support/graphql" @@ -65,6 +66,7 @@ def dump_schema_artifacts(json_schema_version:, team_extras: "") run_rake "schema_artifacts:dump" do |output| SchemaDefinition::RakeTasks.new( schema_element_name_form: :snake_case, + extension_modules: [JSONIngestion::SchemaDefinition::APIExtension], index_document_sizes: true, path_to_schema: path_to_schema, schema_artifacts_directory: "config/schema/artifacts", diff --git a/elasticgraph-indexer/elasticgraph-indexer.gemspec b/elasticgraph-indexer/elasticgraph-indexer.gemspec index 9cd116162..9760addb5 100644 --- a/elasticgraph-indexer/elasticgraph-indexer.gemspec +++ b/elasticgraph-indexer/elasticgraph-indexer.gemspec @@ -48,6 +48,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency "elasticgraph-admin", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-elasticsearch", ElasticGraph::VERSION + spec.add_development_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-opensearch", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION end diff --git a/elasticgraph-indexer/spec/acceptance/schema_evolution_spec.rb b/elasticgraph-indexer/spec/acceptance/schema_evolution_spec.rb index a7f241102..baa04ec86 100644 --- a/elasticgraph-indexer/spec/acceptance/schema_evolution_spec.rb +++ b/elasticgraph-indexer/spec/acceptance/schema_evolution_spec.rb @@ -7,6 +7,7 @@ # frozen_string_literal: true require "elastic_graph/indexer" +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/schema_definition/rake_tasks" module ElasticGraph @@ -401,6 +402,7 @@ def dump_artifacts run_rake "schema_artifacts:dump" do |output| SchemaDefinition::RakeTasks.new( schema_element_name_form: :snake_case, + extension_modules: [JSONIngestion::SchemaDefinition::APIExtension], index_document_sizes: true, path_to_schema: path_to_schema, schema_artifacts_directory: "config/schema/artifacts", diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/record_preparer_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/record_preparer_spec.rb index 66eb66525..1ecc94fc3 100644 --- a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/record_preparer_spec.rb +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/record_preparer_spec.rb @@ -576,7 +576,11 @@ def build_preparer_for_old_json_schema_version(v1_def:, v2_def:) end def define_schema(&schema_definition) - super(schema_element_name_form: "snake_case", &schema_definition) + super( + schema_element_name_form: "snake_case", + extension_modules: [JSONIngestion::SchemaDefinition::APIExtension], + &schema_definition + ) end end diff --git a/elasticgraph-json_ingestion/README.md b/elasticgraph-json_ingestion/README.md index 38d29c1d1..db1eb3988 100644 --- a/elasticgraph-json_ingestion/README.md +++ b/elasticgraph-json_ingestion/README.md @@ -17,7 +17,4 @@ graph LR; elasticgraph-support["elasticgraph-support"]; elasticgraph-json_ingestion --> elasticgraph-support; class elasticgraph-support otherEgGemStyle; - elasticgraph-schema_definition["elasticgraph-schema_definition"]; - elasticgraph-schema_definition --> elasticgraph-json_ingestion; - class elasticgraph-schema_definition otherEgGemStyle; ``` diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/api_extension.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/api_extension.rb index 4070d8eab..a8b8ef645 100644 --- a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/api_extension.rb +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/api_extension.rb @@ -25,7 +25,8 @@ module APIExtension # @return [void] # @api private def self.extended(api) - api.state.extend(StateExtension) + state = api.state.extend(StateExtension) # : ElasticGraph::SchemaDefinition::State & StateExtension + state.reserved_type_names << EVENT_ENVELOPE_JSON_SCHEMA_NAME api.factory.extend(FactoryExtension) api.on_built_in_types do |type| diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field.rb index 850bbd0f4..4b175fdeb 100644 --- a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field.rb +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field.rb @@ -21,13 +21,16 @@ module Indexing # # @api private class Field < DelegateClass(ElasticGraph::SchemaDefinition::Indexing::Field) - # @dynamic __getobj__, json_schema_layers, json_schema_customizations + # @dynamic __getobj__, json_schema_layers, json_schema_customizations, doc_comment # @return [Array] JSON schema wrapper layers from the field type reference attr_reader :json_schema_layers # @return [Hash] user-defined JSON schema customizations attr_reader :json_schema_customizations + # @return [String, nil] documentation for the field + attr_reader :doc_comment + # JSON schema overrides that automatically apply to specific mapping types so that the JSON schema # validation will reject values which cannot be indexed into fields of a specific mapping type. # @@ -44,10 +47,12 @@ class Field < DelegateClass(ElasticGraph::SchemaDefinition::Indexing::Field) # @param field [ElasticGraph::SchemaDefinition::Indexing::Field] the indexing field to wrap # @param json_schema_layers [Array] JSON schema wrapper layers from the field type reference # @param json_schema_customizations [Hash] user-defined JSON schema customizations - def initialize(field, json_schema_layers:, json_schema_customizations:) + # @param doc_comment [String, nil] documentation for the field + def initialize(field, json_schema_layers:, json_schema_customizations:, doc_comment:) super(field) @json_schema_layers = json_schema_layers @json_schema_customizations = json_schema_customizations + @doc_comment = doc_comment end # Returns the JSON schema definition for this field. @@ -81,7 +86,8 @@ def ==(other) when Field __getobj__ == other.__getobj__ && json_schema_layers == other.json_schema_layers && - json_schema_customizations == other.json_schema_customizations + json_schema_customizations == other.json_schema_customizations && + doc_comment == other.doc_comment else super end @@ -95,7 +101,7 @@ def eql?(other) # # @return [Integer] the hash code def hash - [__getobj__, json_schema_layers, json_schema_customizations].hash + [__getobj__, json_schema_layers, json_schema_customizations, doc_comment].hash end private diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_reference.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_reference.rb index c25e73ead..2fa5e42d8 100644 --- a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_reference.rb +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_reference.rb @@ -18,20 +18,25 @@ module Indexing # # @api private class FieldReference < DelegateClass(ElasticGraph::SchemaDefinition::Indexing::FieldReference) - # @dynamic __getobj__, json_schema_layers, json_schema_customizations + # @dynamic __getobj__, json_schema_layers, json_schema_customizations, doc_comment # @return [Array] JSON schema wrapper layers from the field type reference attr_reader :json_schema_layers # @return [Hash] user-defined JSON schema customizations attr_reader :json_schema_customizations + # @return [String, nil] documentation for the referenced field + attr_reader :doc_comment + # @param field_reference [ElasticGraph::SchemaDefinition::Indexing::FieldReference] the field reference to wrap # @param json_schema_layers [Array] JSON schema wrapper layers from the field type reference # @param json_schema_customizations [Hash] user-defined JSON schema customizations - def initialize(field_reference, json_schema_layers:, json_schema_customizations:) + # @param doc_comment [String, nil] documentation for the referenced field + def initialize(field_reference, json_schema_layers:, json_schema_customizations:, doc_comment:) super(field_reference) @json_schema_layers = json_schema_layers @json_schema_customizations = json_schema_customizations + @doc_comment = doc_comment end # Resolves this reference to a JSON-schema-aware indexing field. @@ -43,7 +48,8 @@ def resolve Field.new( resolved_field, json_schema_layers: json_schema_layers, - json_schema_customizations: json_schema_customizations + json_schema_customizations: json_schema_customizations, + doc_comment: doc_comment ) end @@ -56,7 +62,8 @@ def ==(other) when FieldReference __getobj__ == other.__getobj__ && json_schema_layers == other.json_schema_layers && - json_schema_customizations == other.json_schema_customizations + json_schema_customizations == other.json_schema_customizations && + doc_comment == other.doc_comment else super end @@ -70,7 +77,7 @@ def eql?(other) # # @return [Integer] the hash code def hash - [__getobj__, json_schema_layers, json_schema_customizations].hash + [__getobj__, json_schema_layers, json_schema_customizations, doc_comment].hash end end end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/object.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/object.rb index ba8733732..38ef9ff12 100644 --- a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/object.rb +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/object.rb @@ -19,14 +19,18 @@ module FieldType # # @private class Object < DelegateClass(ElasticGraph::SchemaDefinition::Indexing::FieldType::Object) - # @dynamic __getobj__, json_schema_options, json_schema_options= + # @dynamic __getobj__, json_schema_options, json_schema_options=, doc_comment, doc_comment= # @return [Hash] JSON schema options for this object type attr_accessor :json_schema_options + # @return [String, nil] documentation for the object type + attr_accessor :doc_comment + # @param field_type [ElasticGraph::SchemaDefinition::Indexing::FieldType::Object] the object field type to wrap def initialize(field_type) super(field_type) @json_schema_options = {} + @doc_comment = nil end # @return [Hash] field metadata keyed by field name @@ -74,7 +78,8 @@ def ==(other) case other when Object __getobj__ == other.__getobj__ && - json_schema_options == other.json_schema_options + json_schema_options == other.json_schema_options && + doc_comment == other.doc_comment else super end @@ -85,7 +90,7 @@ def eql?(other) end def hash - [__getobj__, json_schema_options].hash + [__getobj__, json_schema_options, doc_comment].hash end private diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/scalar.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/scalar.rb index a7b637a7d..e30a1969c 100644 --- a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/scalar.rb +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/scalar.rb @@ -32,7 +32,8 @@ def format_field_json_schema_customizations(customizations) # @return [Hash] the JSON schema definition for this scalar type def to_json_schema - Support::HashUtil.stringify_keys(scalar_type.json_schema_options) + json_scalar_type = scalar_type # : ElasticGraph::SchemaDefinition::SchemaElements::ScalarType & SchemaElements::ScalarTypeExtension + Support::HashUtil.stringify_keys(json_scalar_type.json_schema_options) end end end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/field_extension.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/field_extension.rb index 9e4a558ef..918bd13d0 100644 --- a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/field_extension.rb +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/field_extension.rb @@ -100,7 +100,8 @@ def to_indexing_field_reference Indexing::FieldReference.new( reference.with(type: type_for_json_schema), json_schema_layers: JSONSchemaLayers.for(type_for_json_schema), - json_schema_customizations: json_schema_options + json_schema_customizations: json_schema_options, + doc_comment: doc_comment ) end end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/scalar_type_extension.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/scalar_type_extension.rb index 9f0307d46..0189fc50c 100644 --- a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/scalar_type_extension.rb +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/scalar_type_extension.rb @@ -18,27 +18,22 @@ module ScalarTypeExtension # Validates that json_schema has been configured on this scalar type, and applies # post-yield runtime metadata derived from the final JSON schema configuration. + # GraphQL-only scalar types are skipped because they are not part of ingestion. # - # @raise [Errors::SchemaError] if json_schema has not been configured + # @raise [Errors::SchemaError] if json_schema has not been configured on an ingested scalar type # @return [void] def finalize_json_schema_configuration! - validate_json_schema_configuration! + return if graphql_only? + + if json_schema_options.empty? + raise Errors::SchemaError, "Scalar types require `json_schema` to be configured, but `#{name}` lacks `json_schema`." + end if !grouping_missing_value_placeholder_overridden && (placeholder = inferred_grouping_missing_value_placeholder) self.runtime_metadata = runtime_metadata.with(grouping_missing_value_placeholder: placeholder) end end - # Validates that json_schema has been configured on this scalar type. - # - # @raise [Errors::SchemaError] if json_schema has not been configured - # @return [void] - def validate_json_schema_configuration! - return unless json_schema_options.empty? - - raise Errors::SchemaError, "Scalar types require `json_schema` to be configured, but `#{name}` lacks `json_schema`." - end - private def inferred_grouping_missing_value_placeholder diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/type_with_subfields_extension.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/type_with_subfields_extension.rb index 67d21caa2..78085fb66 100644 --- a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/type_with_subfields_extension.rb +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/schema_elements/type_with_subfields_extension.rb @@ -20,6 +20,7 @@ module TypeWithSubfieldsExtension def to_indexing_field_type field_type = super # : Indexing::FieldType::Object field_type.json_schema_options = json_schema_options + field_type.doc_comment = doc_comment field_type end end diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/factory_extension.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/factory_extension.rbs index 95fd5e1d0..da1871370 100644 --- a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/factory_extension.rbs +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/factory_extension.rbs @@ -11,9 +11,7 @@ module ElasticGraph def new_object_indexing_field_type: ( type_name: ::String, subfields: ::Array[Indexing::Field], - mapping_options: ::ElasticGraph::SchemaDefinition::Mixins::HasTypeInfo::optionsHash, - doc_comment: ::String?, - json_schema_options: ::Hash[::Symbol, untyped] + mapping_options: ::ElasticGraph::SchemaDefinition::Mixins::HasTypeInfo::optionsHash ) -> Indexing::FieldType::Object def new_scalar_type: (::String) ?{ (::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType & SchemaElements::ScalarTypeExtension) -> void } -> (::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType & SchemaElements::ScalarTypeExtension) def new_scalar_indexing_field_type: (scalar_type: ::ElasticGraph::SchemaDefinition::SchemaElements::ScalarType) -> Indexing::FieldType::Scalar diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field.rbs index d3ff6d0e0..585584b08 100644 --- a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field.rbs +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field.rbs @@ -9,6 +9,7 @@ module ElasticGraph class Field < FieldSupertype attr_reader json_schema_layers: ::ElasticGraph::SchemaDefinition::jsonSchemaLayersArray attr_reader json_schema_customizations: ::Hash[::Symbol, untyped] + attr_reader doc_comment: ::String? @json_schema: ::Hash[::String, untyped]? @@ -17,7 +18,8 @@ module ElasticGraph def initialize: ( ::ElasticGraph::SchemaDefinition::Indexing::Field field, json_schema_layers: ::ElasticGraph::SchemaDefinition::jsonSchemaLayersArray, - json_schema_customizations: ::Hash[::Symbol, untyped] + json_schema_customizations: ::Hash[::Symbol, untyped], + doc_comment: ::String? ) -> void def json_schema: () -> ::Hash[::String, untyped] diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_reference.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_reference.rbs index 84c7faed8..0e0318da2 100644 --- a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_reference.rbs +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_reference.rbs @@ -9,11 +9,13 @@ module ElasticGraph class FieldReference < FieldReferenceSupertype attr_reader json_schema_layers: ::ElasticGraph::SchemaDefinition::jsonSchemaLayersArray attr_reader json_schema_customizations: ::Hash[::Symbol, untyped] + attr_reader doc_comment: ::String? def initialize: ( ::ElasticGraph::SchemaDefinition::Indexing::FieldReference field_reference, json_schema_layers: ::ElasticGraph::SchemaDefinition::jsonSchemaLayersArray, - json_schema_customizations: ::Hash[::Symbol, untyped] + json_schema_customizations: ::Hash[::Symbol, untyped], + doc_comment: ::String? ) -> void def resolve: () -> Field? diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_type/object.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_type/object.rbs index 370698dde..03d779555 100644 --- a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_type/object.rbs +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/field_type/object.rbs @@ -9,6 +9,7 @@ module ElasticGraph class Object < ObjectSupertype attr_accessor json_schema_options: ::Hash[::Symbol, untyped] + attr_accessor doc_comment: ::String? @to_json_schema: ::Hash[::String, untyped]? diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata.rbs index f78578800..4ddde82f6 100644 --- a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata.rbs +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata.rbs @@ -40,7 +40,7 @@ module ElasticGraph attr_reader unused_deprecated_elements: ::Set[ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement] - def initialize: (ElasticGraph::SchemaDefinition::Results) -> void + def initialize: ((ElasticGraph::SchemaDefinition::Results & ResultsExtension)) -> void def merge_metadata_into: (::Hash[::String, untyped]) -> JSONSchemaWithMetadata private diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/json_schema_builder.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/json_schema_builder.rbs index ae129f100..4d6c8131d 100644 --- a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/json_schema_builder.rbs +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/json_schema_builder.rbs @@ -3,7 +3,7 @@ module ElasticGraph module SchemaDefinition class JSONSchemaBuilder def initialize: ( - state: ::ElasticGraph::SchemaDefinition::State, + state: ::ElasticGraph::SchemaDefinition::State & StateExtension, all_types: ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType], derived_indexing_type_names: ::Set[::String] ) -> void @@ -13,7 +13,7 @@ module ElasticGraph private - @state: ::ElasticGraph::SchemaDefinition::State + @state: ::ElasticGraph::SchemaDefinition::State & StateExtension @all_types: ::Array[::ElasticGraph::SchemaDefinition::SchemaElements::graphQLType] @derived_indexing_type_names: ::Set[::String] @indexing_field_types_by_name: ::Hash[::String, Indexing::_JSONFieldType]? diff --git a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_elements/scalar_type_extension.rbs b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_elements/scalar_type_extension.rbs index 8a914177c..3ed0e80b6 100644 --- a/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_elements/scalar_type_extension.rbs +++ b/elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/schema_elements/scalar_type_extension.rbs @@ -6,7 +6,6 @@ module ElasticGraph include HasJSONSchema def finalize_json_schema_configuration!: () -> void - def validate_json_schema_configuration!: () -> void private diff --git a/elasticgraph-local/elasticgraph-local.gemspec b/elasticgraph-local/elasticgraph-local.gemspec index ebdd0a74a..43e9165c0 100644 --- a/elasticgraph-local/elasticgraph-local.gemspec +++ b/elasticgraph-local/elasticgraph-local.gemspec @@ -52,5 +52,6 @@ Gem::Specification.new do |spec| spec.add_development_dependency "elasticgraph-apollo", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-elasticsearch", ElasticGraph::VERSION + spec.add_development_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-opensearch", ElasticGraph::VERSION end diff --git a/elasticgraph-local/spec/acceptance/rake_tasks_spec.rb b/elasticgraph-local/spec/acceptance/rake_tasks_spec.rb index 7a3b0d783..3c8d60519 100644 --- a/elasticgraph-local/spec/acceptance/rake_tasks_spec.rb +++ b/elasticgraph-local/spec/acceptance/rake_tasks_spec.rb @@ -6,6 +6,7 @@ # # frozen_string_literal: true +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/local/rake_tasks" require "elastic_graph/schema_definition/rake_tasks" require "json" @@ -134,6 +135,7 @@ def run_rake(*cli_args, daemon_timeout: nil, batch_size: 1) ) do |t| t.index_document_sizes = true t.schema_element_name_form = :snake_case + t.schema_definition_extension_modules << JSONIngestion::SchemaDefinition::APIExtension t.env_port_mapping = {"example" => 9615} t.elasticsearch_versions = ["8.18.0", "9.0.0"] t.opensearch_versions = ["2.19.0"] diff --git a/elasticgraph-schema_definition/README.md b/elasticgraph-schema_definition/README.md index 5c06af483..b85f79b9e 100644 --- a/elasticgraph-schema_definition/README.md +++ b/elasticgraph-schema_definition/README.md @@ -21,9 +21,6 @@ graph LR; elasticgraph-indexer["elasticgraph-indexer"]; elasticgraph-schema_definition --> elasticgraph-indexer; class elasticgraph-indexer otherEgGemStyle; - elasticgraph-json_ingestion["elasticgraph-json_ingestion"]; - elasticgraph-schema_definition --> elasticgraph-json_ingestion; - class elasticgraph-json_ingestion otherEgGemStyle; elasticgraph-schema_artifacts["elasticgraph-schema_artifacts"]; elasticgraph-schema_definition --> elasticgraph-schema_artifacts; class elasticgraph-schema_artifacts otherEgGemStyle; diff --git a/elasticgraph-schema_definition/elasticgraph-schema_definition.gemspec b/elasticgraph-schema_definition/elasticgraph-schema_definition.gemspec index da50e4335..69b268ad5 100644 --- a/elasticgraph-schema_definition/elasticgraph-schema_definition.gemspec +++ b/elasticgraph-schema_definition/elasticgraph-schema_definition.gemspec @@ -43,7 +43,6 @@ Gem::Specification.new do |spec| spec.add_dependency "elasticgraph-graphql", ElasticGraph::VERSION # needed since we validate that scalar `coerce_with` options are valid (which loads scalar coercion adapters) spec.add_dependency "elasticgraph-indexer", ElasticGraph::VERSION # needed since we validate that scalar `prepare_for_indexing_with` options are valid (which loads indexing preparer adapters) - spec.add_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION spec.add_dependency "elasticgraph-schema_artifacts", ElasticGraph::VERSION spec.add_dependency "elasticgraph-support", ElasticGraph::VERSION spec.add_dependency "graphql", "~> 2.6.2" @@ -52,5 +51,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency "elasticgraph-admin", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-datastore_core", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-elasticsearch", ElasticGraph::VERSION + # The test suite exercises the default JSON ingestion extension, but the gem remains optional at runtime. + spec.add_development_dependency "elasticgraph-json_ingestion", ElasticGraph::VERSION spec.add_development_dependency "elasticgraph-opensearch", ElasticGraph::VERSION end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/api.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/api.rb index 80306e237..7b9ed412e 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/api.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/api.rb @@ -294,7 +294,6 @@ def union_type(name, &block) # ElasticGraph.define_schema do |schema| # schema.scalar_type "URL" do |t| # t.mapping type: "keyword" - # t.json_schema type: "string", format: "uri" # end # end def scalar_type(name, &block) @@ -453,99 +452,6 @@ def results @results ||= @factory.new_results end - # Defines the version number of the current JSON schema. Importantly, every time a change is made that impacts the JSON schema - # artifact, the version number must be incremented to ensure that each different version of the JSON schema is identified by a unique - # version number. The publisher will then include this version number in published events to identify the version of the schema it - # was using. This avoids the need to deploy the publisher and ElasticGraph indexer at the same time to keep them in sync. - # - # @note While this is an important part of how ElasticGraph is designed to support schema evolution, it can be annoying constantly - # have to increment this while rapidly changing the schema during prototyping. You can disable the requirement to increment this - # on every JSON schema change with {#enforce_json_schema_version}. - # - # @param version [Integer] current version number of the JSON schema artifact - # @return [void] - # @see #enforce_json_schema_version - # - # @example Set the JSON schema version to 1 - # ElasticGraph.define_schema do |schema| - # schema.json_schema_version 1 - # end - def json_schema_version(version) - if !version.is_a?(Integer) || version < 1 - raise Errors::SchemaError, "`json_schema_version` must be a positive integer. Specified version: #{version}" - end - - if @state.json_schema_version - raise Errors::SchemaError, "`json_schema_version` can only be set once on a schema. Previously-set version: #{@state.json_schema_version}" - end - - @state.json_schema_version = version - @state.json_schema_version_setter_location = caller_locations(1, 1).to_a.first - nil - end - - # Configures whether {SchemaArtifactManager} enforces the requirement that the JSON schema version is incremented every time - # dumping the JSON schemas results in a changed artifact. Defaults to `true`. - # - # @note Generally speaking, you will want this to be `true` for any ElasticGraph application that is in - # production as the versioning of JSON schemas is what supports safe schema evolution as it allows - # ElasticGraph to identify which version of the JSON schema the publishing system was operating on - # when it published an event. - # - # It can be useful to set it to `false` before your application is in production, as you do not want - # to be forced to bump the version after every single schema change while you are building an initial - # prototype. - # - # @param value [Boolean] whether to require `json_schema_version` to be incremented on changes that impact `json_schemas.yaml` - # @return [void] - # @see #json_schema_version - # - # @example Disable enforcement during initial prototyping - # ElasticGraph.define_schema do |schema| - # # TODO: remove this once we're past the prototyping stage - # schema.enforce_json_schema_version false - # end - def enforce_json_schema_version(value) - unless value == true || value == false - raise Errors::SchemaError, "`enforce_json_schema_version` must be a boolean. Specified value: #{value.inspect}" - end - - @state.enforce_json_schema_version = value - nil - end - - # Defines strictness of the JSON schema validation. By default, the JSON schema will require all fields to be provided by the - # publisher (but they can be nullable) and will ignore extra fields that are not defined in the schema. Use this method to - # configure this behavior. - # - # @param allow_omitted_fields [bool] Whether nullable fields can be omitted from indexing events. - # @param allow_extra_fields [bool] Whether extra fields (e.g. beyond fields defined in the schema) can be included in indexing events. - # @return [void] - # - # @note If you allow both omitted fields and extra fields, ElasticGraph's JSON schema validation will allow (and ignore) misspelled - # field names in indexing events. For example, if the ElasticGraph schema has a nullable field named `parentId` but the publisher - # accidentally provides it as `parent_id`, ElasticGraph would happily ignore the `parent_id` field entirely, because `parentId` - # is allowed to be omitted and `parent_id` would be treated as an extra field. Therefore, we recommend that you only set one of - # these to `true` (or none). - # - # @example Allow omitted fields and disallow extra fields - # ElasticGraph.define_schema do |schema| - # schema.json_schema_strictness allow_omitted_fields: true, allow_extra_fields: false - # end - def json_schema_strictness(allow_omitted_fields: false, allow_extra_fields: true) - unless [true, false].include?(allow_omitted_fields) - raise Errors::SchemaError, "`allow_omitted_fields` must be true or false" - end - - unless [true, false].include?(allow_extra_fields) - raise Errors::SchemaError, "`allow_extra_fields` must be true or false" - end - - @state.allow_omitted_json_schema_fields = allow_omitted_fields - @state.allow_extra_json_schema_fields = allow_extra_fields - nil - end - # Registers a customization callback that will be applied to every built-in type automatically provided by ElasticGraph. Provides # an opportunity to customize the built-in types (e.g. to add directives to them or whatever). # diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/factory.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/factory.rb index a1efea5b3..a560794ea 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/factory.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/factory.rb @@ -250,14 +250,12 @@ def new_object_type(name) end @@object_type_new = prevent_non_factory_instantiation_of(SchemaElements::ObjectType) - def new_object_indexing_field_type(type_name:, subfields:, mapping_options:, json_schema_options:, doc_comment:) + def new_object_indexing_field_type(type_name:, subfields:, mapping_options:) @@object_indexing_field_type_new.call( schema_def_state: @state, type_name: type_name, subfields: subfields, - mapping_options: mapping_options, - json_schema_options: json_schema_options, - doc_comment: doc_comment + mapping_options: mapping_options ) end @@object_indexing_field_type_new = prevent_non_factory_instantiation_of(Indexing::FieldType::Object) diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field.rb index 1401aa949..810f2fcea 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field.rb @@ -7,7 +7,6 @@ # frozen_string_literal: true require "elastic_graph/constants" -require "elastic_graph/json_ingestion/schema_definition/indexing/json_schema_field_metadata" require "elastic_graph/schema_definition/indexing/list_counts_mapping" require "elastic_graph/support/hash_util" require "elastic_graph/support/memoizable_data" @@ -22,28 +21,12 @@ class Field < Support::MemoizableData.define( :name, :name_in_index, :type, - :json_schema_layers, :indexing_field_type, :accuracy_confidence, - :json_schema_customizations, :mapping_customizations, :source, - :runtime_field_script, - :doc_comment + :runtime_field_script ) - # JSON schema overrides that automatically apply to specific mapping types so that the JSON schema - # validation will reject values which cannot be indexed into fields of a specific mapping type. - # - # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/number.html Elasticsearch numeric field type documentation - # @note We don't handle `integer` here because it's the default numeric type (handled by our definition of the `Int` scalar type). - # @note Likewise, we don't handle `long` here because a custom scalar type must be used for that since GraphQL's `Int` type can't handle long values. - JSON_SCHEMA_OVERRIDES_BY_MAPPING_TYPE = { - "byte" => {"minimum" => -(2**7), "maximum" => (2**7) - 1}, - "short" => {"minimum" => -(2**15), "maximum" => (2**15) - 1}, - "keyword" => {"maxLength" => DEFAULT_MAX_KEYWORD_LENGTH}, - "text" => {"maxLength" => DEFAULT_MAX_TEXT_LENGTH} - } - # @return [Hash] the mapping for this field. The returned hash should be composed entirely # of Ruby primitives that, when converted to a JSON string, match the structure required by # [Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html). @@ -63,23 +46,6 @@ def mapping end end - # @return [Hash] the JSON schema definition for this field. The returned object should - # be composed entirely of Ruby primitives that, when converted to a JSON string, match the - # requirements of [the JSON schema spec](https://json-schema.org/). - def json_schema - json_schema_layers - .reverse # resolve layers from innermost to outermost wrappings - .reduce(inner_json_schema) { |acc, layer| process_layer(layer, acc) } - .merge(outer_json_schema_customizations) - .merge({"description" => doc_comment}.compact) - .then { |h| Support::HashUtil.stringify_keys(h) } - end - - # @return [JSONSchemaFieldMetadata] additional ElasticGraph metadata to be stored in the JSON schema for this field. - def json_schema_metadata - JSONIngestion::SchemaDefinition::Indexing::JSONSchemaFieldMetadata.new(type: type.name, name_in_index: name_in_index) - end - # Builds a hash containing the mapping for the provided fields, normalizing it in the same way that the # datastore does so that consistency checks between our index configuration and what's in the datastore # work properly. @@ -107,80 +73,6 @@ def self.normalized_mapping_hash_for(fields) mapping_hash end - - def nullable? - json_schema_layers.include?(:nullable) - end - - private - - def inner_json_schema - user_specified_customizations = - if user_specified_json_schema_customizations_go_on_outside? - {} # : ::Hash[::String, untyped] - else - Support::HashUtil.stringify_keys(json_schema_customizations) - end - - customizations_from_mapping = JSON_SCHEMA_OVERRIDES_BY_MAPPING_TYPE[mapping["type"]] || {} - customizations = customizations_from_mapping.merge(user_specified_customizations) - customizations = indexing_field_type.format_field_json_schema_customizations(customizations) - - ref = {"$ref" => "#/$defs/#{type.unwrapped_name}"} - return ref if customizations.empty? - - # Combine any customizations with type ref under an "allOf" subschema: - # All of these properties must hold true for the type to be valid. - # - # Note that if we simply combine the customizations with the `$ref` - # at the same level, it will not work, because other subschema - # properties are ignored when they are in the same object as a `$ref`: - # https://github.com/json-schema-org/JSON-Schema-Test-Suite/blob/2.0.0/tests/draft7/ref.json#L165-L168 - {"allOf" => [ref, customizations]} - end - - def outer_json_schema_customizations - return {} unless user_specified_json_schema_customizations_go_on_outside? - Support::HashUtil.stringify_keys(json_schema_customizations) - end - - # Indicates if the user-specified JSON schema customizations should go on the inside - # (where they normally go) or on the outside. They only go on the outside when it's - # an array field, because then they apply to the array itself instead of the items in the - # array. - def user_specified_json_schema_customizations_go_on_outside? - json_schema_layers.include?(:array) - end - - def process_layer(layer, schema) - case layer - when :nullable - make_nullable(schema) - when :array - make_array(schema) - else - # :nocov: - layer is only ever `:nullable` or `:array` so we never get here - schema - # :nocov: - end - end - - def make_nullable(schema) - # Here we use "anyOf" to ensure that JSON can either match the schema OR null. - # - # (Using "oneOf" would mean that if we had a schema that also allowed null, - # null would never be allowed, since "oneOf" must match exactly one subschema). - { - "anyOf" => [ - schema, - {"type" => "null"} - ] - } - end - - def make_array(schema) - {"type" => "array", "items" => schema} - end end end end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_reference.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_reference.rb index 070f70db3..4c91b3446 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_reference.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_reference.rb @@ -9,24 +9,19 @@ module ElasticGraph module SchemaDefinition module Indexing - # @!parse class FieldReference < ::Data; end - FieldReference = ::Data.define( + # A lazy reference to a {Field}. It contains all attributes needed to build a {Field}, but the referenced `type` may not be + # resolvable yet (which is why this exists). + # + # @api private + class FieldReference < ::Data.define( :name, :name_in_index, :type, :mapping_options, - :json_schema_options, :accuracy_confidence, :source, - :runtime_field_script, - :doc_comment + :runtime_field_script ) - - # A lazy reference to a {Field}. It contains all attributes needed to build a {Field}, but the referenced `type` may not be - # resolvable yet (which is why this exists). - # - # @api private - class FieldReference < ::Data # @return [Field, nil] the {Field} this reference resolves to (if it can be resolved) def resolve return nil unless (resolved_type = type.fully_unwrapped.resolved) @@ -35,18 +30,15 @@ def resolve name: name, name_in_index: name_in_index, type: type, - json_schema_layers: type.json_schema_layers, indexing_field_type: resolved_type.to_indexing_field_type, accuracy_confidence: accuracy_confidence, - json_schema_customizations: json_schema_options, mapping_customizations: mapping_options, source: source, - runtime_field_script: runtime_field_script, - doc_comment: doc_comment + runtime_field_script: runtime_field_script ) end - # @dynamic initialize, with, name, name_in_index, type, mapping_options, json_schema_options, accuracy_confidence, source, runtime_field_script, doc_comment + # @dynamic initialize, with, name, name_in_index, type, mapping_options, accuracy_confidence, source, runtime_field_script end end end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/enum.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/enum.rb index d2f72fb6f..dc096ba25 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/enum.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/enum.rb @@ -13,50 +13,18 @@ module Indexing # # @api private module FieldType - # @!parse class Enum < ::Data; end - Enum = ::Data.define(:enum_value_names) - - # Responsible for the JSON schema and mapping of a {SchemaElements::EnumType}. + # Responsible for the mapping of a {SchemaElements::EnumType}. # # @!attribute [r] enum_value_names # @return [Array] list of names of values in this enum type. # # @api private - class Enum < ::Data - # @return [Hash] the JSON schema for this enum type. - def to_json_schema - {"type" => "string", "enum" => enum_value_names} - end - + class Enum < ::Data.define(:enum_value_names) # @return [Hash] the datastore mapping for this enum type. def to_mapping {"type" => "keyword"} end - # @return [Hash] additional ElasticGraph metadata to put in the JSON schema for this enum type. - def json_schema_field_metadata_by_field_name - {} - end - - # @param customizations [Hash] JSON schema customizations - # @return [Hash] formatted customizations. - def format_field_json_schema_customizations(customizations) - # Since an enum type already restricts the values to a small set of allowed values, we do not need to keep - # other customizations (such as the `maxLength` field customization EG automatically applies to fields - # indexed as a `keyword`--we don't allow enum values to exceed that length, anyway). - # - # It's desirable to restrict what customizations are applied because when a publisher uses the JSON schema - # to generate code using a library such as https://github.com/pwall567/json-kotlin-schema-codegen, we found - # that the presence of extra field customizations inhibits the library's ability to generate code in the way - # we want (it causes the type of the enum to change since the JSON schema changes from a direct `$ref` to - # being wrapped in an `allOf`). - # - # However, we still want to apply `enum` customizations--this allows a user to "narrow" the set of allowed - # values for a field. For example, a `Currency` enum could contain every currency, and a user may want to - # restrict a specific `currency` field to a subset of currencies (e.g. to just USD, CAD, and EUR). - customizations.slice("enum") - end - # @dynamic initialize, enum_value_names end end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/object.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/object.rb index c01fcbd4e..f52c811c6 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/object.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/object.rb @@ -14,7 +14,7 @@ module ElasticGraph module SchemaDefinition module Indexing module FieldType - # Responsible for the JSON schema and mapping of a {SchemaElements::ObjectType}. + # Responsible for the mapping of a {SchemaElements::ObjectType}. # # @!attribute [r] type_name # @return [String] name of the object type @@ -22,13 +22,9 @@ module FieldType # @return [Array] the subfields of this object type # @!attribute [r] mapping_options # @return [Hash] options to be included in the mapping - # @!attribute [r] json_schema_options - # @return [Hash] options to be included in the JSON schema - # @!attribute [r] doc_comment - # @return [String, nil] documentation for the type # # @api private - class Object < Support::MemoizableData.define(:schema_def_state, :type_name, :subfields, :mapping_options, :json_schema_options, :doc_comment) + class Object < Support::MemoizableData.define(:schema_def_state, :type_name, :subfields, :mapping_options) # @return [Hash] the datastore mapping for this object type. def to_mapping @to_mapping ||= begin @@ -41,77 +37,10 @@ def to_mapping end end - # @return [Hash] the JSON schema for this object type. - def to_json_schema - @to_json_schema ||= - if json_schema_options.empty? - # Fields that are `sourced_from` an alternate type must not be included in this types JSON schema, - # since events of this type won't include them. - other_source_subfields, json_schema_candidate_subfields = subfields.partition(&:source) - validate_sourced_fields_have_no_json_schema_overrides(other_source_subfields) - json_schema_subfields = json_schema_candidate_subfields.reject(&:runtime_field_script) - required_fields = json_schema_subfields - required_fields = required_fields.reject(&:nullable?) if schema_def_state.allow_omitted_json_schema_fields - - { - "type" => "object", - "properties" => json_schema_subfields.to_h { |f| [f.name, f.json_schema] }.merge(json_schema_typename_field), - # Note: `__typename` is intentionally not included in the `required` list. If `__typename` is present - # we want it validated (as we do by merging in `json_schema_typename_field`) but we only want - # to require it in the context of a union type. The union's json schema requires the field. - "required" => required_fields.map(&:name).freeze, - "additionalProperties" => (false unless schema_def_state.allow_extra_json_schema_fields), - "description" => doc_comment - }.compact.freeze - else - Support::HashUtil.stringify_keys(json_schema_options) - end - end - - # @return [Hash] additional ElasticGraph metadata to put in the JSON schema for this object type. - def json_schema_field_metadata_by_field_name - subfields.to_h { |f| [f.name, f.json_schema_metadata] } - end - - # @param customizations [Hash] JSON schema customizations - # @return [Hash] formatted customizations. - def format_field_json_schema_customizations(customizations) - customizations - end - - private - + # @private def after_initialize subfields.freeze end - - # Returns a __typename property which we use for union types. - # - # This must always be set to the name of the type (thus the const value). - # - # We also add a "default" value. This does not impact validation, but rather - # aids tools like our kotlin codegen to save publishers from having to set the - # property explicitly when creating events. - def json_schema_typename_field - { - "__typename" => { - "type" => "string", - "const" => type_name, - "default" => type_name - } - } - end - - def validate_sourced_fields_have_no_json_schema_overrides(other_source_subfields) - problem_fields = other_source_subfields.reject { |f| f.json_schema_customizations.empty? } - return if problem_fields.empty? - - field_descriptions = problem_fields.map(&:name).sort.map { |f| "`#{f}`" }.join(", ") - raise Errors::SchemaError, - "`#{type_name}` has #{problem_fields.size} field(s) (#{field_descriptions}) that are `sourced_from` " \ - "another type and also have JSON schema customizations. Instead, put the JSON schema " \ - "customizations on the source type's field definitions." - end end end end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/scalar.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/scalar.rb index cb9c3132e..4c5994dbb 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/scalar.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/scalar.rb @@ -12,37 +12,18 @@ module ElasticGraph module SchemaDefinition module Indexing module FieldType - # @!parse class Scalar < ::Data; end - Scalar = ::Data.define(:scalar_type) - - # Responsible for the JSON schema and mapping of a {SchemaElements::ScalarType}. + # Responsible for the mapping of a {SchemaElements::ScalarType}. # # @!attribute [r] scalar_type # @return [SchemaElements::ScalarType] the scalar type # # @api private - class Scalar < ::Data + class Scalar < ::Data.define(:scalar_type) # @return [Hash] the datastore mapping for this scalar type. def to_mapping Support::HashUtil.stringify_keys(scalar_type.mapping_options) end - # @return [Hash] the JSON schema for this scalar type. - def to_json_schema - Support::HashUtil.stringify_keys(scalar_type.json_schema_options) - end - - # @return [Hash] additional ElasticGraph metadata to put in the JSON schema for this scalar type. - def json_schema_field_metadata_by_field_name - {} - end - - # @param customizations [Hash] JSON schema customizations - # @return [Hash] formatted customizations. - def format_field_json_schema_customizations(customizations) - customizations - end - # @dynamic initialize, scalar_type end end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/union.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/union.rb index e4d6e634f..8f8358528 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/union.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field_type/union.rb @@ -13,9 +13,8 @@ module ElasticGraph module SchemaDefinition module Indexing module FieldType - # Responsible for the JSON schema and mapping of a {SchemaElements::UnionType}. + # Responsible for the mapping of a {SchemaElements::UnionType}. # - # @note In JSON schema, we model this with a `oneOf`, and a `__typename` field on each subtype. # @note Within the mapping, we have a single object type that has a set union of the properties # of the subtypes (and also a `__typename` keyword field). # @@ -24,25 +23,6 @@ module FieldType # # @api private class Union < ::Data.define(:subtypes_by_name) - # @return [Hash] the JSON schema for this union type. - def to_json_schema - subtype_json_schemas = subtypes_by_name.keys.map { |name| {"$ref" => "#/$defs/#{name}"} } - - # A union type can represent multiple subtypes, referenced by the "anyOf" clause below. - # We also add a requirement for the presence of __typename to indicate which type - # is being referenced (this property is pre-defined on the type itself as a constant). - # - # Note: Although both "oneOf" and "anyOf" keywords are valid for combining schemas - # to form a union, and validate equivalently when no object can satisfy multiple of the - # subschemas (which is the case here given the __typename requirements are mutually - # exclusive), we chose to use "oneOf" here because it works better with this library: - # https://github.com/pwall567/json-kotlin-schema-codegen - { - "required" => %w[__typename], - "oneOf" => subtype_json_schemas - } - end - # @return [Hash] the datastore mapping for this union type. def to_mapping mapping_subfields = subtypes_by_name.values.map(&:subfields).reduce([], :union) @@ -53,16 +33,7 @@ def to_mapping ) end - # @return [Hash] additional ElasticGraph metadata to put in the JSON schema for this union type. - def json_schema_field_metadata_by_field_name - {} - end - - # @param customizations [Hash] JSON schema customizations - # @return [Hash] formatted customizations. - def format_field_json_schema_customizations(customizations) - customizations - end + # @dynamic initialize, subtypes_by_name end end end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb index b084d7998..631255cb7 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb @@ -79,7 +79,6 @@ def initialize(name, settings, schema_def_state, indexed_type) # us a nice efficiency boost. id_field_path = public_field_path("id", explanation: "indexed types must have an `id` field") self.routing_field_path = id_field_path - id_field_path.last_part.json_schema nullable: false end yield self if block_given? @@ -152,8 +151,6 @@ def rollover(frequency, timestamp_field_path_name) raise Errors::SchemaError, "rollover field `#{timestamp_field_path.full_description}` cannot be used for rollover since it is a list field." end - timestamp_field_path.path_parts.each { |f| f.json_schema nullable: false } - self.rollover_config = RolloverConfig.new( frequency: frequency, timestamp_field_path: timestamp_field_path @@ -200,8 +197,6 @@ def route_with(routing_field_path_name) self.routing_field_path = routing_field_path - routing_field_path.path_parts[0..-2].each { |f| f.json_schema nullable: false } - routing_field_path.last_part.json_schema nullable: false, pattern: HAS_NON_WHITE_SPACE_REGEX indexed_type.append_to_documentation "For more performant queries on this type, please filter on `#{routing_field_path_name}` if possible." end end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_type_info.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_type_info.rb index 454093ff6..c9fbed0e2 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_type_info.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/mixins/has_type_info.rb @@ -6,29 +6,22 @@ # # frozen_string_literal: true -require "elastic_graph/support/json_schema/meta_schema_validator" - module ElasticGraph module SchemaDefinition module Mixins - # Mixin used to specify non-GraphQL type info (datastore index and JSON schema type info). + # Mixin used to specify non-GraphQL type info on schema elements. # Exists as a mixin so we can apply the same consistent API to every place we need to use this. # Currently it's used in 3 places: # - # - {SchemaElements::ScalarType}: allows specification of how scalars are represented in JSON schema and the index. - # - {SchemaElements::TypeWithSubfields}: allows customization of how an object type is represented in JSON schema and the index. - # - {SchemaElements::Field}: allows customization of a specific field over the field type's standard JSON schema and the index mapping. + # - {SchemaElements::ScalarType}: allows specification of how scalars are represented in the datastore index. + # - {SchemaElements::TypeWithSubfields}: allows customization of how an object type is represented in the datastore index. + # - {SchemaElements::Field}: allows customization of a specific field over the field type's standard index mapping. module HasTypeInfo # @return [Hash] datastore mapping options def mapping_options @mapping_options ||= {} end - # @return [Hash] JSON schema options - def json_schema_options - @json_schema_options ||= {} - end - # Set of mapping parameters that it makes sense to allow customization of, based on # [the Elasticsearch docs](https://www.elastic.co/guide/en/elasticsearch/reference/8.15/mapping-params.html). CUSTOMIZABLE_DATASTORE_PARAMS = Set[ @@ -70,7 +63,6 @@ def json_schema_options # ElasticGraph.define_schema do |schema| # schema.scalar_type "URL" do |t| # t.mapping type: "keyword" - # t.json_schema type: "string", format: "uri" # end # end # @@ -87,13 +79,6 @@ def json_schema_options # t.field "expYear", "Int" do |f| # # Use a smaller numeric type to save space in the datastore # f.mapping type: "short" - # f.json_schema minimum: 2000, maximum: 2099 - # end - # - # t.field "expMonth", "Int" do |f| - # # Use a smaller numeric type to save space in the datastore - # f.mapping type: "byte" - # f.json_schema minimum: 1, maximum: 12 # end # # t.index "cards" @@ -108,73 +93,6 @@ def mapping(**options) mapping_options.update(options) end - - # Defines the [JSON schema](https://json-schema.org/understanding-json-schema/) validations for this field or type. Validations - # defined here will be included in the generated `json_schemas.yaml` artifact, which is used by the ElasticGraph indexer to - # validate events before indexing their data in the datastore. In addition, the publisher may use `json_schemas.yaml` for code - # generation and to apply validation before publishing an event to ElasticGraph. - # - # Can be called multiple times; each time, the options will be merged into the existing options. - # - # This is _required_ on a {SchemaElements::ScalarType} (since we don’t know how a custom scalar type should be represented in - # JSON!). On a {SchemaElements::Field}, this is optional, but can be used to make the JSON schema validation stricter then it - # would otherwise be. For example, you could use `json_schema maxLength: 30` on a `String` field to limit the length. - # - # You can use any of the JSON schema validation keywords here. In addition, `nullable: false` is supported to configure the - # generated JSON schema to disallow `null` values for the field. Note that if you define a field with a non-nullable GraphQL type - # (e.g. `Int!`), the JSON schema will automatically disallow nulls. However, as explained in the - # {SchemaElements::TypeWithSubfields#field} documentation, we generally recommend against defining non-nullable GraphQL fields. - # `json_schema nullable: false` will disallow `null` values from being indexed, while still keeping the field nullable in the - # GraphQL schema. If you think you might want to make a field non-nullable in the GraphQL schema some day, it’s a good idea to use - # `json_schema nullable: false` now to ensure every indexed record has a non-null value for the field. - # - # @note We recommend using JSON schema validations in a limited fashion. Validations that are appropriate to apply when data is - # entering the system-of-record are often not appropriate on a secondary index like ElasticGraph. Events that violate a JSON - # schema validation will fail to index (typically they will be sent to the dead letter queue and page an oncall engineer). If an - # ElasticGraph instance is meant to contain all the data of some source system, you probably don’t want it applying stricter - # validations than the source system itself has. We recommend limiting your JSON schema validations to situations where - # violations would prevent ElasticGraph from operating correctly. - # - # @param options [Hash] JSON schema options - # @return [void] - # - # @example Define the JSON schema validations of a custom scalar type - # ElasticGraph.define_schema do |schema| - # schema.scalar_type "URL" do |t| - # t.mapping type: "keyword" - # - # # JSON schema has a built-in URI format validator: - # # https://json-schema.org/understanding-json-schema/reference/string.html#resource-identifiers - # t.json_schema type: "string", format: "uri" - # end - # end - # - # @example Define additional validations on a field - # ElasticGraph.define_schema do |schema| - # schema.object_type "Card" do |t| - # t.field "id", "ID!" - # - # t.field "expYear", "Int" do |f| - # # Use JSON schema to ensure the publisher is sending us 4 digit years, not 2 digit years. - # f.json_schema minimum: 2000, maximum: 2099 - # end - # - # t.field "expMonth", "Int" do |f| - # f.json_schema minimum: 1, maximum: 12 - # end - # - # t.index "cards" - # end - # end - def json_schema(**options) - validatable_json_schema = Support::HashUtil.stringify_keys(options) - - if (error_msg = Support::JSONSchema.strict_meta_schema_validator.validate_with_error_message(validatable_json_schema)) - raise Errors::SchemaError, "Invalid JSON schema options set on #{self}:\n\n#{error_msg}" - end - - json_schema_options.update(options) - end end end end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/rake_tasks.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/rake_tasks.rb index 8ab08390e..4ae693147 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/rake_tasks.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/rake_tasks.rb @@ -6,8 +6,8 @@ # # frozen_string_literal: true -require "rake/tasklib" require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names" +require "rake/tasklib" module ElasticGraph module SchemaDefinition @@ -40,7 +40,7 @@ class RakeTasks < ::Rake::TaskLib # @param enum_value_overrides_by_type [Hash>] overrides for the names of specific enum values for # specific enum types. For example, to rename the `DayOfWeek.MONDAY` enum to `DayOfWeek.MON`, pass `{DayOfWeek: {MONDAY: "MON"}}`. # @param extension_modules [Array] List of Ruby modules to extend onto the `SchemaDefinition::API` instance. Designed to - # support ElasticGraph extension gems (such as `elasticgraph-apollo`). + # support ElasticGraph extension gems (such as `elasticgraph-apollo` and `elasticgraph-json_ingestion`). # @param output [IO] used for printing task output # # @example Minimal setup with defaults diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/results.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/results.rb index d6a428785..93f71fa66 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/results.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/results.rb @@ -8,8 +8,6 @@ require "elastic_graph/constants" require "elastic_graph/errors" -require "elastic_graph/json_ingestion/schema_definition/indexing/event_envelope" -require "elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata" require "elastic_graph/schema_artifacts/artifacts_helper_methods" require "elastic_graph/schema_artifacts/runtime_metadata/schema" require "elastic_graph/schema_definition/indexing/sourced_from_update_targets_resolver" @@ -45,53 +43,6 @@ def runtime_metadata @runtime_metadata ||= build_runtime_metadata end - # @param version [Integer] desired JSON schema version - # @return [Hash] the JSON schema for the requested version, if available - # @raise [Errors::NotFoundError] if the requested JSON schema version is not available - def json_schemas_for(version) - unless available_json_schema_versions.include?(version) - raise Errors::NotFoundError, "The requested json schema version (#{version}) is not available. Available versions: #{available_json_schema_versions.to_a.join(", ")}." - end - - @latest_versioned_json_schema ||= merge_field_metadata_into_json_schema(current_public_json_schema).json_schema - end - - # @return [Set] set of available JSON schema versions - def available_json_schema_versions - @available_json_schema_versions ||= Set[latest_json_schema_version] - end - - # @return [Hash] the newly generated JSON schema - def latest_json_schema_version - current_public_json_schema[JSON_SCHEMA_VERSION_KEY] - end - - # @private - def json_schema_version_setter_location - state.json_schema_version_setter_location - end - - # @private - def json_schema_field_metadata_by_type_and_field_name - @json_schema_field_metadata_by_type_and_field_name ||= json_schema_indexing_field_types_by_name - .transform_values(&:json_schema_field_metadata_by_field_name) - end - - # @private - def current_public_json_schema - @current_public_json_schema ||= build_public_json_schema - end - - # @private - def merge_field_metadata_into_json_schema(json_schema) - json_schema_with_metadata_merger.merge_metadata_into(json_schema) - end - - # @private - def unused_deprecated_elements - json_schema_with_metadata_merger.unused_deprecated_elements - end - # @private STATIC_SCRIPT_REPO = Scripting::FileSystemRepository.new(::File.join(__dir__.to_s, "scripting", "scripts")) @@ -112,10 +63,6 @@ def after_initialize state.user_definition_complete_callbacks.each(&:call) end - def json_schema_with_metadata_merger - @json_schema_with_metadata_merger ||= JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata::Merger.new(self) - end - def generate_datastore_config # We need to check this before generating our datastore configuration. # We can't generate a mapping from a recursively defined schema type. @@ -196,46 +143,6 @@ def generate_sdl [type_defs + state.sdl_parts].join("\n\n") end - def build_public_json_schema - json_schema_version = state.json_schema_version - if json_schema_version.nil? - raise Errors::SchemaError, "`json_schema_version` must be specified in the schema. To resolve, add `schema.json_schema_version 1` in a schema definition block." - end - - root_document_type_names = state.object_types_by_name.values - .select { |type| type.root_document_type? && !type.abstract? } - .reject { |type| derived_indexing_type_names.include?(type.name) } - .map(&:name) - - definitions_by_name = json_schema_indexing_field_types_by_name - .transform_values(&:to_json_schema) - .compact - - event_envelope = JSONIngestion::SchemaDefinition::Indexing::EventEnvelope - - { - "$schema" => JSON_META_SCHEMA, - JSON_SCHEMA_VERSION_KEY => json_schema_version, - "$defs" => { - "ElasticGraphEventEnvelope" => event_envelope.json_schema(root_document_type_names, json_schema_version) - }.merge(definitions_by_name) - } - end - - def json_schema_indexing_field_types_by_name - @json_schema_indexing_field_types_by_name ||= state - .types_by_name - .except("Query") - .values - .reject do |t| - derived_indexing_type_names.include?(t.name) || - # Skip graphql types (including namespace types, which are GraphQL-only). - t.graphql_only? - end - .sort_by(&:name) - .to_h { |type| [type.name, type.to_indexing_field_type] } - end - def verify_runtime_metadata(runtime_metadata) registered_resolvers = runtime_metadata.graphql_resolvers_by_name diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_artifact_manager.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_artifact_manager.rb index 465b03d1b..0ae2e7ff7 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_artifact_manager.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_artifact_manager.rb @@ -8,7 +8,6 @@ require "did_you_mean" require "elastic_graph/constants" -require "elastic_graph/json_ingestion/schema_definition/json_schema_pruner" require "elastic_graph/support/graphql_gem_loader" require "elastic_graph/support/memoizable_data" require "fileutils" @@ -36,41 +35,10 @@ def initialize(schema_definition_results:, schema_artifacts_directory:, output:, @schema_artifacts_directory = schema_artifacts_directory @output = output @max_diff_lines = max_diff_lines - - @json_schemas_artifact = new_yaml_artifact( - JSON_SCHEMAS_FILE, - JSONIngestion::SchemaDefinition::JSONSchemaPruner.prune(schema_definition_results.current_public_json_schema), - extra_comment_lines: [ - "This is the \"public\" JSON schema file and is intended to be provided to publishers so that", - "they can perform code generation and event validation." - ] - ) end # Dumps all the schema artifacts to disk. def dump_artifacts - check_if_needs_json_schema_version_bump do |recommended_json_schema_version| - if schema_definition_results.state.enforce_json_schema_version - # @type var setter_location: ::Thread::Backtrace::Location - # We use `_ =` because while `json_schema_version_setter_location` can be nil, - # it'll never be nil if we get here and we want the type to be non-nilable. - setter_location = _ = schema_definition_results.json_schema_version_setter_location - setter_location_path = ::Pathname.new(setter_location.absolute_path.to_s).relative_path_from(::Dir.pwd) - - abort "A change has been attempted to `json_schemas.yaml`, but the `json_schema_version` has not been correspondingly incremented. Please " \ - "increase the schema's version, and then run the `bundle exec rake schema_artifacts:dump` command again.\n\n" \ - "To update the schema version to the expected version, change line #{setter_location.lineno} at `#{setter_location_path}` to:\n" \ - " `schema.json_schema_version #{recommended_json_schema_version}`\n\n" \ - "Alternately, call `schema.enforce_json_schema_version false` in your schema definition to allow the JSON schemas file " \ - "to change without requiring a version bump, but that is only recommended for non-production applications during initial schema prototyping." - else - @output.puts <<~EOS - WARNING: the `json_schemas.yaml` artifact is being updated without the `json_schema_version` being correspondingly incremented. - This is not recommended for production applications, but is currently allowed because you have called `schema.enforce_json_schema_version false`. - EOS - end - end - ::FileUtils.mkdir_p(@schema_artifacts_directory) artifacts.each { |artifact| artifact.dump(@output) } end @@ -111,18 +79,11 @@ def artifacts_from_schema_def # schema elements. graphql_schema = ::GraphQL::Schema.from_definition(schema_definition_results.graphql_schema_string).to_definition.chomp - unversioned_artifacts = [ + [ new_yaml_artifact(DATASTORE_CONFIG_FILE, schema_definition_results.datastore_config), new_yaml_artifact(RUNTIME_METADATA_FILE, pruned_runtime_metadata(graphql_schema).to_dumpable_hash), - @json_schemas_artifact, new_raw_artifact(GRAPHQL_SCHEMA_FILE, "\n" + graphql_schema) ] - - versioned_artifacts = build_desired_versioned_json_schemas(@json_schemas_artifact.desired_contents).values.map do |versioned_schema| - new_versioned_json_schema_artifact(versioned_schema) - end - - unversioned_artifacts + versioned_artifacts end def notify_about_unused_type_name_overrides @@ -172,157 +133,6 @@ def notify_about_unused_enum_value_overrides EOS end - def build_desired_versioned_json_schemas(current_public_json_schema) - versioned_parsed_yamls = ::Dir.glob(::File.join(@schema_artifacts_directory, JSON_SCHEMAS_BY_VERSION_DIRECTORY, "v*.yaml")).map do |file| - ::YAML.safe_load_file(file) - end + [current_public_json_schema] - - results_by_json_schema_version = versioned_parsed_yamls.to_h do |parsed_yaml| - merged_schema = @schema_definition_results.merge_field_metadata_into_json_schema(parsed_yaml) - [merged_schema.json_schema_version, merged_schema] - end - - report_json_schema_merge_errors(results_by_json_schema_version.values) - report_json_schema_merge_warnings - - results_by_json_schema_version.transform_values(&:json_schema) - end - - def report_json_schema_merge_errors(merged_results) - json_schema_versions_by_missing_field = ::Hash.new { |h, k| h[k] = [] } # : ::Hash[::String, ::Array[::Integer]] - json_schema_versions_by_missing_type = ::Hash.new { |h, k| h[k] = [] } # : ::Hash[::String, ::Array[::Integer]] - json_schema_versions_by_missing_necessary_field = ::Hash.new { |h, k| h[k] = [] } # : ::Hash[JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata::MissingNecessaryField, ::Array[::Integer]] - - merged_results.each do |result| - result.missing_fields.each do |field| - json_schema_versions_by_missing_field[field] << result.json_schema_version - end - - result.missing_types.each do |type| - json_schema_versions_by_missing_type[type] << result.json_schema_version - end - - result.missing_necessary_fields.each do |missing_necessary_field| - json_schema_versions_by_missing_necessary_field[missing_necessary_field] << result.json_schema_version - end - end - - missing_field_errors = json_schema_versions_by_missing_field.map do |field, json_schema_versions| - missing_field_error_for(field, json_schema_versions) - end - - missing_type_errors = json_schema_versions_by_missing_type.map do |type, json_schema_versions| - missing_type_error_for(type, json_schema_versions) - end - - missing_necessary_field_errors = json_schema_versions_by_missing_necessary_field.map do |field, json_schema_versions| - missing_necessary_field_error_for(field, json_schema_versions) - end - - definition_conflict_errors = merged_results - .flat_map { |result| result.definition_conflicts.to_a } - .group_by(&:name) - .map do |name, deprecated_elements| - <<~EOS - The schema definition of `#{name}` has conflicts. To resolve the conflict, remove the unneeded definitions from the following: - - #{format_deprecated_elements(deprecated_elements)} - EOS - end - - errors = missing_field_errors + missing_type_errors + missing_necessary_field_errors + definition_conflict_errors - return if errors.empty? - - abort errors.join("\n\n") - end - - def report_json_schema_merge_warnings - unused_elements = @schema_definition_results.unused_deprecated_elements - return if unused_elements.empty? - - @output.puts <<~EOS - The schema definition has #{unused_elements.size} unneeded reference(s) to deprecated schema elements. These can all be safely deleted: - - #{format_deprecated_elements(unused_elements)} - - EOS - end - - def format_deprecated_elements(deprecated_elements) - descriptions = deprecated_elements - .sort_by { |e| [e.defined_at.path, e.defined_at.lineno] } - .map(&:description) - .uniq - - descriptions.each.with_index(1).map { |desc, idx| "#{idx}. #{desc}" }.join("\n") - end - - def missing_field_error_for(qualified_field, json_schema_versions) - type, field = qualified_field.split(".") - - <<~EOS - The `#{qualified_field}` field (which existed in #{describe_json_schema_versions(json_schema_versions, "and")}) no longer exists in the current schema definition. - ElasticGraph cannot guess what it should do with this field's data when ingesting events at #{old_versions(json_schema_versions)}. - To continue, do one of the following: - - 1. If the `#{qualified_field}` field has been renamed, indicate this by calling `field.renamed_from "#{field}"` on the renamed field. - 2. If the `#{qualified_field}` field has been dropped, indicate this by calling `type.deleted_field "#{field}"` on the `#{type}` type. - 3. Alternately, if no publishers or in-flight events use #{describe_json_schema_versions(json_schema_versions, "or")}, delete #{files_noun_phrase(json_schema_versions)} from `#{JSON_SCHEMAS_BY_VERSION_DIRECTORY}`, and no further changes are required. - EOS - end - - def missing_type_error_for(type, json_schema_versions) - <<~EOS - The `#{type}` type (which existed in #{describe_json_schema_versions(json_schema_versions, "and")}) no longer exists in the current schema definition. - ElasticGraph cannot guess what it should do with this type's data when ingesting events at #{old_versions(json_schema_versions)}. - To continue, do one of the following: - - 1. If the `#{type}` type has been renamed, indicate this by calling `type.renamed_from "#{type}"` on the renamed type. - 2. If the `#{type}` field has been dropped, indicate this by calling `schema.deleted_type "#{type}"` on the schema. - 3. Alternately, if no publishers or in-flight events use #{describe_json_schema_versions(json_schema_versions, "or")}, delete #{files_noun_phrase(json_schema_versions)} from `#{JSON_SCHEMAS_BY_VERSION_DIRECTORY}`, and no further changes are required. - EOS - end - - def missing_necessary_field_error_for(field, json_schema_versions) - path = field.fully_qualified_path.split(".").last - # :nocov: -- we only cover one side of this ternary. - has_or_have = (json_schema_versions.size == 1) ? "has" : "have" - # :nocov: - - <<~EOS - #{describe_json_schema_versions(json_schema_versions, "and")} #{has_or_have} no field that maps to the #{field.field_type} field path of `#{field.fully_qualified_path}`. - Since the field path is required for #{field.field_type}, ElasticGraph cannot ingest events that lack it. To continue, do one of the following: - - 1. If the `#{field.fully_qualified_path}` field has been renamed, indicate this by calling `field.renamed_from "#{path}"` on the renamed field rather than using `deleted_field`. - 2. Alternately, if no publishers or in-flight events use #{describe_json_schema_versions(json_schema_versions, "or")}, delete #{files_noun_phrase(json_schema_versions)} from `#{JSON_SCHEMAS_BY_VERSION_DIRECTORY}`, and no further changes are required. - EOS - end - - def describe_json_schema_versions(json_schema_versions, conjunction) - json_schema_versions = json_schema_versions.sort - - # Steep doesn't support pattern matching yet, so have to skip type checking here. - __skip__ = case json_schema_versions - in [single_version] - "JSON schema version #{single_version}" - in [version1, version2] - "JSON schema versions #{version1} #{conjunction} #{version2}" - else - *versions, last_version = json_schema_versions - "JSON schema versions #{versions.join(", ")}, #{conjunction} #{last_version}" - end - end - - def old_versions(json_schema_versions) - return "this old version" if json_schema_versions.size == 1 - "these old versions" - end - - def files_noun_phrase(json_schema_versions) - return "its file" if json_schema_versions.size == 1 - "their files" - end - def artifacts_out_of_date_error(out_of_date_artifacts) # @type var diffs: ::Array[[SchemaArtifact[untyped], ::String]] diffs = [] @@ -377,20 +187,6 @@ def new_yaml_artifact(file_name, desired_contents, extra_comment_lines: []) ) end - def new_versioned_json_schema_artifact(desired_contents) - # File name depends on the schema_version field in the json schema. - schema_version = desired_contents[JSON_SCHEMA_VERSION_KEY] - - new_yaml_artifact( - ::File.join(JSON_SCHEMAS_BY_VERSION_DIRECTORY, "v#{schema_version}.yaml"), - desired_contents, - extra_comment_lines: [ - "This JSON schema file contains internal ElasticGraph metadata and should be considered private.", - "The unversioned JSON schema file is public and intended to be provided to publishers." - ] - ) - end - def new_raw_artifact(file_name, desired_contents) SchemaArtifact.new( ::File.join(@schema_artifacts_directory, file_name), @@ -401,17 +197,6 @@ def new_raw_artifact(file_name, desired_contents) ) end - def check_if_needs_json_schema_version_bump(&block) - if @json_schemas_artifact.out_of_date? - existing_schema_version = @json_schemas_artifact.existing_dumped_contents&.dig(JSON_SCHEMA_VERSION_KEY) || -1 - desired_schema_version = @json_schemas_artifact.desired_contents[JSON_SCHEMA_VERSION_KEY] - - if existing_schema_version >= desired_schema_version - yield existing_schema_version + 1 - end - end - end - def pruned_runtime_metadata(graphql_schema_string) schema = ::GraphQL::Schema.from_definition(graphql_schema_string) runtime_meta = schema_definition_results.runtime_metadata diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/built_in_types.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/built_in_types.rb index 7aee7630d..483b1da6e 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/built_in_types.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/built_in_types.rb @@ -422,19 +422,11 @@ def register_standard_elastic_graph_types # As per the Elasticsearch docs, the field MUST come in named `lat` in Elastisearch (but we want the full name in GraphQL). t.field names.latitude, "Float", name_in_index: "lat" do |f| f.documentation "Angular distance north or south of the Earth's equator, measured in degrees from -90 to +90." - - # Note: we use `nullable: false` because we index it as a single `geo_point` field, and therefore can't - # support a `latitude` without a `longitude` or vice-versa. - f.json_schema minimum: -90, maximum: 90, nullable: false end # As per the Elasticsearch docs, the field MUST come in named `lon` in Elastisearch (but we want the full name in GraphQL). t.field names.longitude, "Float", name_in_index: "lon" do |f| f.documentation "Angular distance east or west of the Prime Meridian at Greenwich, UK, measured in degrees from -180 to +180." - - # Note: we use `nullable: false` because we index it as a single `geo_point` field, and therefore can't - # support a `latitude` without a `longitude` or vice-versa. - f.json_schema minimum: -180, maximum: 180, nullable: false end t.mapping type: "geo_point" @@ -676,17 +668,15 @@ def register_standard_elastic_graph_types # Registers the standard GraphQL scalar types. Note that the SDL for the scalar type itself isn't # included in the dumped SDL, but registering it allows us to derive a filter for each, - # which we need. In addition, this lets us define the mapping and JSON schema for each standard - # scalar type. + # which we need. In addition, this lets us define the mapping for each standard scalar type. + # Ingestion serializers can layer their own built-in configuration on top. def register_standard_graphql_scalars schema_def_api.scalar_type "Boolean" do |t| t.mapping type: "boolean" - t.json_schema type: "boolean" end schema_def_api.scalar_type "Float" do |t| t.mapping type: "double" - t.json_schema type: "number" t.customize_aggregated_values_type do |avt| # not nullable, since sum(empty_set) == 0 @@ -726,12 +716,10 @@ def register_standard_graphql_scalars schema_def_api.scalar_type "ID" do |t| t.mapping type: "keyword" - t.json_schema type: "string" end schema_def_api.scalar_type "Int" do |t| t.mapping type: "integer" - t.json_schema type: "integer", minimum: INT_MIN, maximum: INT_MAX t.prepare_for_indexing_with "ElasticGraph::Indexer::IndexingPreparers::Integer", defined_at: "elastic_graph/indexer/indexing_preparers/integer" @@ -746,7 +734,6 @@ def register_standard_graphql_scalars schema_def_api.scalar_type "String" do |t| t.mapping type: "keyword" - t.json_schema type: "string" t.customize_filter_input_type do |fit| fit.field names.contains, schema_def_state.type_ref("StringContains").as_filter_input.name do |f| @@ -778,12 +765,11 @@ def register_custom_elastic_graph_scalars cursor_type_name = @schema_def_state.type_namer.cursor_type_name unless STOCK_GRAPHQL_SCALARS.include?(cursor_type_name) schema_def_api.scalar_type "Cursor" do |t| - # Technically, we don't use the mapping or json_schema on this type since it's a return-only + # Technically, we don't use the mapping or ingestion config on this type since it's a return-only # type and isn't indexed. However, `scalar_type` requires them to be set (since custom scalars # defined by users will need those set) so we set them here to what they would be if we actually # used them. t.mapping type: "keyword" - t.json_schema type: "string" t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::Cursor", defined_at: "elastic_graph/graphql/scalar_coercion_adapters/cursor" @@ -797,7 +783,6 @@ def register_custom_elastic_graph_scalars schema_def_api.scalar_type "Date" do |t| t.mapping type: "date", format: DATASTORE_DATE_FORMAT - t.json_schema type: "string", format: "date" t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::Date", defined_at: "elastic_graph/graphql/scalar_coercion_adapters/date" @@ -817,7 +802,6 @@ def register_custom_elastic_graph_scalars schema_def_api.scalar_type "DateTime" do |t| t.mapping type: "date", format: DATASTORE_DATE_TIME_FORMAT - t.json_schema type: "string", format: "date-time" t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::DateTime", defined_at: "elastic_graph/graphql/scalar_coercion_adapters/date_time" t.prepare_for_indexing_with "ElasticGraph::Indexer::IndexingPreparers::DateTime", @@ -908,8 +892,6 @@ def register_custom_elastic_graph_scalars t.mapping type: "date", format: "HH:mm:ss||HH:mm:ss.S||HH:mm:ss.SS||HH:mm:ss.SSS" - t.json_schema type: "string", pattern: VALID_LOCAL_TIME_JSON_SCHEMA_PATTERN - t.customize_aggregated_values_type do |avt| define_exact_min_max_and_approx_avg_on_aggregated_values(avt, "LocalTime") do |adjective:, full_name:| <<~EOS @@ -922,7 +904,6 @@ def register_custom_elastic_graph_scalars schema_def_api.scalar_type "TimeZone" do |t| t.mapping type: "keyword" - t.json_schema type: "string", enum: GraphQL::ScalarCoercionAdapters::VALID_TIME_ZONES.to_a t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::TimeZone", defined_at: "elastic_graph/graphql/scalar_coercion_adapters/time_zone" @@ -939,8 +920,6 @@ def register_custom_elastic_graph_scalars # https://github.com/json-schema-org/json-schema-spec/blob/draft-07/schema.json#L23-L29 # # ...except we are omitting `null` here; it'll be added by the nullability decorator if the field is defined as nullable. - t.json_schema type: ["array", "boolean", "integer", "number", "object", "string"] - # In the index we store this as a JSON string in a `keyword` field. t.mapping type: "keyword" @@ -965,7 +944,6 @@ def register_custom_elastic_graph_scalars schema_def_api.scalar_type "JsonSafeLong" do |t| t.mapping type: "long" - t.json_schema type: "integer", minimum: JSON_SAFE_LONG_MIN, maximum: JSON_SAFE_LONG_MAX t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::JsonSafeLong", defined_at: "elastic_graph/graphql/scalar_coercion_adapters/longs" @@ -1009,7 +987,6 @@ def register_custom_elastic_graph_scalars # to do if we ingest them as strings. (The `pattern` regex to validate the range # would be *extremely* complicated). t.mapping type: "long" - t.json_schema type: "integer", minimum: LONG_STRING_MIN, maximum: LONG_STRING_MAX t.coerce_with "ElasticGraph::GraphQL::ScalarCoercionAdapters::LongString", defined_at: "elastic_graph/graphql/scalar_coercion_adapters/longs" t.prepare_for_indexing_with "ElasticGraph::Indexer::IndexingPreparers::Integer", diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enum_type.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enum_type.rb index 02b0a9ab4..67b3576bc 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enum_type.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/enum_type.rb @@ -161,7 +161,7 @@ def derived_graphql_types # @return [void] # @api private def configure_derived_scalar_type(scalar_type) - scalar_type.json_schema type: "string" + # No-op by default; extensions (e.g. JSONIngestion) override this. end # @return [Indexing::FieldType::Enum] indexing representation of this enum type diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/field.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/field.rb index 3d66eeb62..42ad8eaa0 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/field.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/field.rb @@ -85,8 +85,6 @@ module SchemaElements # @private # @!attribute [rw] computation_detail # @private - # @!attribute [rw] non_nullable_in_json_schema - # @private # @!attribute [rw] as_input # @private class Field < Struct.new( @@ -95,7 +93,7 @@ class Field < Struct.new( :aggregated_values_customizations, :sort_order_enum_value_customizations, :args, :sortable, :filterable, :aggregatable, :groupable, :highlightable, :returnable, :graphql_only, :source, :runtime_field_script, :relationship, :singular_name, - :computation_detail, :non_nullable_in_json_schema, :as_input, + :computation_detail, :as_input, :name_in_index, :resolver ) include Mixins::HasDocumentation @@ -140,7 +138,6 @@ def initialize( # the `_name` suffix on the attribute for clarity. singular_name: singular, name_in_index: name_in_index, - non_nullable_in_json_schema: false, as_input: as_input, resolver: resolver ) @@ -468,22 +465,6 @@ def on_each_generated_schema_element(&customization_block) customize_sort_order_enum_values(&customization_block) end - # (see Mixins::HasTypeInfo#json_schema) - def json_schema(nullable: nil, **options) - if options.key?(:type) - raise Errors::SchemaError, "Cannot override JSON schema type of field `#{name}` with `#{options.fetch(:type)}`" - end - - case nullable - when true - raise Errors::SchemaError, "`nullable: true` is not allowed on a field--just declare the GraphQL field as being nullable (no `!` suffix) instead." - when false - self.non_nullable_in_json_schema = true - end - - super(**options) - end - # (see Mixins::HasTypeInfo#mapping) def mapping(**options) # ElasticGraph has special handling for the nested type (e.g. we generate sub-aggregation types in the GraphQL schema for @@ -1010,13 +991,11 @@ def to_indexing_field_reference Indexing::FieldReference.new( name: name, name_in_index: name_in_index, - type: non_nullable_in_json_schema ? type.wrap_non_null : type, + type: type, mapping_options: mapping_options, - json_schema_options: json_schema_options, accuracy_confidence: accuracy_confidence, source: source, - runtime_field_script: runtime_field_script, - doc_comment: doc_comment + runtime_field_script: runtime_field_script ) end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/scalar_type.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/scalar_type.rb index 11b961ef9..b9639b73c 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/scalar_type.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/scalar_type.rb @@ -25,7 +25,6 @@ module SchemaElements # ElasticGraph.define_schema do |schema| # schema.scalar_type "URL" do |t| # t.mapping type: "keyword" - # t.json_schema type: "string", format: "uri" # end # end # @@ -60,7 +59,7 @@ class ScalarType < Struct.new( include Mixins::HasReadableToSAndInspect.new { |t| t.name } # `HasTypeInfo` provides the following methods: - # @dynamic mapping_options, json_schema_options + # @dynamic mapping_options include Mixins::HasTypeInfo # @dynamic graphql_only? @@ -78,13 +77,8 @@ def initialize(schema_def_state, name) yield self - missing = [ - ("`mapping`" if mapping_options.empty?), - ("`json_schema`" if json_schema_options.empty?) - ].compact - - if missing.any? - raise Errors::SchemaError, "Scalar types require `mapping` and `json_schema` to be configured, but `#{name}` lacks #{missing.join(" and ")}." + if mapping_options.empty? + raise Errors::SchemaError, "Scalar types require `mapping` to be configured, but `#{name}` lacks `mapping`." end if (placeholder = inferred_grouping_missing_value_placeholder) @@ -121,7 +115,6 @@ def mapping(**options) # ElasticGraph.define_schema do |schema| # schema.scalar_type "PhoneNumber" do |t| # t.mapping type: "keyword" - # t.json_schema type: "string", pattern: "^\\+[1-9][0-9]{1,14}$" # t.coerce_with "CoercionAdapters::PhoneNumber", defined_at: "./coercion_adapters/phone_number" # end # end @@ -147,7 +140,6 @@ def coerce_with(adapter_name, defined_at:) # ElasticGraph.define_schema do |schema| # schema.scalar_type "PhoneNumber" do |t| # t.mapping type: "keyword" - # t.json_schema type: "string", pattern: "^\\+[1-9][0-9]{1,14}$" # # t.prepare_for_indexing_with "IndexingPreparers::PhoneNumber", # defined_at: "./indexing_preparers/phone_number" @@ -172,7 +164,6 @@ def prepare_for_indexing_with(preparer_name, defined_at:) # ElasticGraph.define_schema do |schema| # schema.scalar_type "BigInt" do |t| # t.mapping type: "long" - # t.json_schema type: "integer", minimum: -(2**53) + 1, maximum: (2**53) - 1 # t.grouping_missing_value_placeholder "NaN" # end # end @@ -343,23 +334,14 @@ def inferred_grouping_missing_value_placeholder MISSING_STRING_PLACEHOLDER elsif FLOAT_TYPES.include?(mapping_type) MISSING_NUMERIC_PLACEHOLDER - elsif mapping_type == "long" - # It is only safe to use NaN for a long when the long's range is safe to coerce to a float - # without loss of precision. This is because using NaN as the missing value will cause - # the datastore to coerce the other bucket keys to float. - # JSON schema min/max only constrains newly indexed values, not existing data that may fall outside the range before the constraints were added. - # This is an edge case where the long range may exceed safe float precision. - # In this case, users can set grouping_missing_value_placeholder to nil. - if (json_schema_options[:minimum] || LONG_STRING_MIN) >= JSON_SAFE_LONG_MIN && - (json_schema_options[:maximum] || LONG_STRING_MAX) <= JSON_SAFE_LONG_MAX - inferred_numeric_placeholder_for_integer_type - end - elsif mapping_type == "unsigned_long" - # Similar to the checks above for long except we only need to check the max - # (since the min is zero even if not specified) - if (json_schema_options[:maximum] || LONG_STRING_MAX) <= JSON_SAFE_LONG_MAX - inferred_numeric_placeholder_for_integer_type - end + elsif mapping_type == "long" || mapping_type == "unsigned_long" + # It is only safe to use NaN as the missing value placeholder when every value can be coerced to a + # float without loss of precision (using NaN causes the datastore to coerce the other bucket keys to + # float). Core ElasticGraph has no knowledge of the range of values of a `long` or `unsigned_long` + # field, so we cannot safely infer a placeholder here. Ingestion extensions that know the allowed + # range (e.g. from the JSON schema `minimum`/`maximum` in `elasticgraph-json_ingestion`) override + # this method to infer a placeholder when it is safe. + nil elsif INTEGER_TYPES.include?(mapping_type) # All other integer types can safely be coerced to float without loss of precision inferred_numeric_placeholder_for_integer_type diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_reference.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_reference.rb index 483d296bf..22c574977 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_reference.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_reference.rb @@ -171,20 +171,6 @@ def with_reverted_override schema_def_state.type_ref(type_namer.revert_override_for(name)) end - # Returns all the JSON schema array/nullable layers of a type, from outermost to innermost. - # For example, [[Int]] will return [:nullable, :array, :nullable, :array, :nullable] - def json_schema_layers - @json_schema_layers ||= begin - layers, inner_type = peel_json_schema_layers_once - - if layers.empty? || inner_type == self - layers - else - layers + inner_type.json_schema_layers - end - end - end - # Most of ElasticGraph's derived GraphQL types have a static suffix (e.g. the full type name # is source_type + suffix). This is a map of all of these. STATIC_FORMAT_NAME_BY_CATEGORY = TypeNamer::REQUIRED_PLACEHOLDERS.filter_map do |format_name, placeholders| @@ -304,16 +290,6 @@ def after_initialize Mixins::VerifiesGraphQLName.verify_name!(unwrapped_name) end - def peel_json_schema_layers_once - if list? - return [[:array], unwrap_list] if non_null? - return [[:nullable, :array], unwrap_list] - end - - return [[], unwrap_non_null] if non_null? - [[:nullable], self] - end - def matches_format_of?(category) format_name = STATIC_FORMAT_NAME_BY_CATEGORY.fetch(category) type_namer.matches_format?(name, format_name) diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_with_subfields.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_with_subfields.rb index 833d5f2db..44099ad46 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_with_subfields.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/schema_elements/type_with_subfields.rb @@ -157,8 +157,7 @@ def name # via an AuthZ layer (as such a layer would have no way to force a field value to `null` for a client denied field access). # Therefore, we recommend limiting your use of `!` to only a few situations such as defining a type’s primary key (e.g. # `t.field "id", "ID!"`) or defining a list field (e.g. `t.field "authors", "[String!]!"`) since empty lists already provide a - # "no data" representation. You can still configure the ElasticGraph indexer to require a non-null value for a field using - # `f.json_schema nullable: false`. + # "no data" representation. You can still configure an ingestion validation extension to require a non-null indexed value. # # @note ElasticGraph’s understanding of datastore capabilities may override your configured # `aggregatable`/`filterable`/`groupable`/`sortable` options. For example, a field indexed as `text` for full text search will @@ -494,9 +493,7 @@ def to_indexing_field_type schema_def_state.factory.new_object_indexing_field_type( type_name: name, subfields: indexing_fields_by_name_in_index.values.map(&:to_indexing_field).compact, - mapping_options: mapping_options, - json_schema_options: json_schema_options, - doc_comment: doc_comment + mapping_options: mapping_options ) end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/state.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/state.rb index ccc4e76c5..aa4431ff5 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/state.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/state.rb @@ -6,14 +6,13 @@ # # frozen_string_literal: true -require "elastic_graph/constants" require "elastic_graph/errors" require "elastic_graph/schema_definition/factory" require "elastic_graph/schema_definition/mixins/has_readable_to_s_and_inspect" require "elastic_graph/schema_definition/schema_elements/enum_value_namer" require "elastic_graph/schema_definition/schema_elements/field_path" -require "elastic_graph/schema_definition/schema_elements/type_namer" require "elastic_graph/schema_definition/schema_elements/sub_aggregation_path" +require "elastic_graph/schema_definition/schema_elements/type_namer" module ElasticGraph module SchemaDefinition @@ -41,9 +40,7 @@ class State < Struct.new( :deleted_types_by_old_name, :renamed_fields_by_type_name_and_old_field_name, :deleted_fields_by_type_name_and_old_field_name, - :json_schema_version, - :json_schema_version_setter_location, - :enforce_json_schema_version, + :reserved_type_names, :graphql_extension_modules, :graphql_resolvers_by_name, :built_in_graphql_resolvers, @@ -56,8 +53,6 @@ class State < Struct.new( :output, :type_namer, :enum_value_namer, - :allow_omitted_json_schema_fields, - :allow_extra_json_schema_fields, :indexed_types_by_index_name ) include Mixins::HasReadableToSAndInspect.new @@ -92,9 +87,7 @@ def self.with( deleted_types_by_old_name: {}, renamed_fields_by_type_name_and_old_field_name: ::Hash.new { |h, k| h[k] = {} }, deleted_fields_by_type_name_and_old_field_name: ::Hash.new { |h, k| h[k] = {} }, - json_schema_version_setter_location: nil, - json_schema_version: nil, - enforce_json_schema_version: true, + reserved_type_names: ::Set.new, graphql_extension_modules: [], graphql_resolvers_by_name: {}, built_in_graphql_resolvers: ::Set.new, @@ -110,8 +103,6 @@ def self.with( ), enum_value_namer: SchemaElements::EnumValueNamer.new(enum_value_overrides_by_type), output: output, - allow_omitted_json_schema_fields: false, - allow_extra_json_schema_fields: true, indexed_types_by_index_name: {} ) end @@ -226,12 +217,10 @@ def field_path_resolver private - RESERVED_TYPE_NAMES = [EVENT_ENVELOPE_JSON_SCHEMA_NAME].to_set - def register_type(type, additional_type_index = nil) name = (_ = type).name - if RESERVED_TYPE_NAMES.include?(name) + if reserved_type_names.include?(name) raise Errors::SchemaError, "`#{name}` cannot be used as a schema type because it is a reserved name." end diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/test_support.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/test_support.rb index c4c10fb89..9dcc2533a 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/test_support.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/test_support.rb @@ -75,9 +75,12 @@ def define_schema_with_schema_elements( yield api if block_given? - # Set the json_schema_version to the provided value, if needed. - if !json_schema_version.nil? && api.state.json_schema_version.nil? - api.json_schema_version(json_schema_version) + # Set the json_schema_version to the provided value, if needed. The `json_schema_version` API only + # exists when an extension that implements JSON schema generation (such as + # `elasticgraph-json_ingestion`) is loaded; without one there is no JSON schema version to set. + if !json_schema_version.nil? && api.respond_to?(:json_schema_version) + versioned_api = api # : untyped + versioned_api.json_schema_version(json_schema_version) if versioned_api.state.json_schema_version.nil? end # :nocov: -- the else branch and code past this aren't used by tests in elasticgraph-schema_definition. @@ -85,7 +88,10 @@ def define_schema_with_schema_elements( # Reloading the schema artifacts takes extra time that we don't usually want to spend (so it's opt-in) # but it can be useful in some cases because there is a bit of extra pruning/validation that it applies. - api.enforce_json_schema_version false + if api.respond_to?(:enforce_json_schema_version) + versioned_api = api # : untyped + versioned_api.enforce_json_schema_version false + end tmp_dir = ::Dir.mktmpdir artifacts_manager = api.factory.new_schema_artifact_manager( schema_definition_results: api.results, diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/api.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/api.rbs index 797db12cb..6eb85354a 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/api.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/api.rbs @@ -65,7 +65,6 @@ module ElasticGraph ?output: io ) -> void - def json_schema_strictness: (?allow_omitted_fields: bool, ?allow_extra_fields: bool) -> void def raw_sdl: (::String) -> void def object_type: (::String) { (SchemaElements::ObjectType) -> void } -> void def namespace_type: (::String) ?{ (SchemaElements::NamespaceType) -> void } -> void @@ -77,8 +76,6 @@ module ElasticGraph def as_active_instance: { () -> void } -> void @results: Results? def results: () -> Results - def json_schema_version: (::Integer) -> void - def enforce_json_schema_version: (bool) -> void def register_graphql_extension: (::Module, defined_at: ::String, **untyped) -> void def register_graphql_resolver: (::Symbol, ::Class, defined_at: ::String, ?built_in: bool, **untyped) -> void def on_built_in_types: () { (SchemaElements::graphQLType) -> void } -> void diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/factory.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/factory.rbs index 82f027fdd..c029adb22 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/factory.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/factory.rbs @@ -88,9 +88,7 @@ module ElasticGraph def new_object_indexing_field_type: ( type_name: ::String, subfields: ::Array[Indexing::Field], - mapping_options: Mixins::HasTypeInfo::optionsHash, - json_schema_options: Mixins::HasTypeInfo::optionsHash, - doc_comment: ::String? + mapping_options: Mixins::HasTypeInfo::optionsHash ) -> Indexing::FieldType::Object @@object_indexing_field_type_new: ::Method diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field.rbs index 80f6eab04..97270cd20 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field.rbs @@ -5,64 +5,41 @@ module ElasticGraph attr_reader name: ::String attr_reader name_in_index: ::String attr_reader type: SchemaElements::TypeReference - attr_reader json_schema_layers: jsonSchemaLayersArray attr_reader indexing_field_type: _FieldType attr_reader accuracy_confidence: Field::accuracyConfidence - attr_reader json_schema_customizations: ::Hash[::Symbol, untyped] attr_reader mapping_customizations: ::Hash[::Symbol, untyped] attr_reader source: SchemaElements::FieldSource? attr_accessor runtime_field_script: ::String? - attr_reader doc_comment: ::String? def initialize: ( name: ::String, name_in_index: ::String, type: SchemaElements::TypeReference, - json_schema_layers: jsonSchemaLayersArray, indexing_field_type: _FieldType, accuracy_confidence: Field::accuracyConfidence, - json_schema_customizations: ::Hash[::Symbol, untyped], mapping_customizations: ::Hash[::Symbol, untyped], source: SchemaElements::FieldSource?, - runtime_field_script: ::String?, - doc_comment: ::String? + runtime_field_script: ::String? ) -> Field def with: ( ?name: ::String, ?name_in_index: ::String, ?type: SchemaElements::TypeReference, - ?json_schema_layers: jsonSchemaLayersArray, ?indexing_field_type: _FieldType, ?accuracy_confidence: Field::accuracyConfidence, - ?json_schema_customizations: ::Hash[::Symbol, untyped], ?mapping_customizations: ::Hash[::Symbol, untyped], ?source: SchemaElements::FieldSource?, - ?runtime_field_script: ::String?, - ?doc_comment: ::String? + ?runtime_field_script: ::String? ) -> Field end class Field < FieldSupertype - JSON_SCHEMA_OVERRIDES_BY_MAPPING_TYPE: ::Hash[::String, untyped] - type accuracyConfidence = SchemaElements::Field::accuracyConfidence @mapping: ::Hash[::String, untyped]? def mapping: () -> ::Hash[::String, untyped] - def json_schema: () -> ::Hash[::String, untyped] - def json_schema_metadata: () -> JSONIngestion::SchemaDefinition::Indexing::JSONSchemaFieldMetadata def self.normalized_mapping_hash_for: (::Array[Field]) -> ::Hash[::String, untyped] - - def inner_json_schema: () -> ::Hash[::String, untyped] - def outer_json_schema_customizations: () -> ::Hash[::String, untyped] - - def nullable?: () -> bool - - def user_specified_json_schema_customizations_go_on_outside?: () -> bool - def process_layer: (::Symbol, ::Hash[::String, untyped]) -> ::Hash[::String, untyped] - def make_nullable: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] - def make_array: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] end end end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_reference.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_reference.rbs index 55a8f0724..bd2936cb1 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_reference.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_reference.rbs @@ -6,22 +6,18 @@ module ElasticGraph attr_reader name_in_index: ::String attr_reader type: SchemaElements::TypeReference attr_reader mapping_options: ::Hash[::Symbol, untyped] - attr_reader json_schema_options: ::Hash[::Symbol, untyped] attr_reader accuracy_confidence: Field::accuracyConfidence attr_reader source: SchemaElements::FieldSource? attr_reader runtime_field_script: ::String? - attr_reader doc_comment: ::String? def initialize: ( name: ::String, name_in_index: ::String, type: SchemaElements::TypeReference, mapping_options: ::Hash[::Symbol, untyped], - json_schema_options: ::Hash[::Symbol, untyped], accuracy_confidence: Field::accuracyConfidence, source: SchemaElements::FieldSource?, - runtime_field_script: ::String?, - doc_comment: ::String? + runtime_field_script: ::String? ) -> void def with: ( @@ -29,11 +25,9 @@ module ElasticGraph ?name_in_index: ::String, ?type: SchemaElements::TypeReference, ?mapping_options: ::Hash[::Symbol, untyped], - ?json_schema_options: ::Hash[::Symbol, untyped], ?accuracy_confidence: Field::accuracyConfidence, ?source: SchemaElements::FieldSource?, - ?runtime_field_script: ::String?, - ?doc_comment: ::String? + ?runtime_field_script: ::String? ) -> FieldReference def resolve: () -> Field? diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type.rbs index bc5e08829..d01805f99 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type.rbs @@ -3,9 +3,6 @@ module ElasticGraph module Indexing interface _FieldType def to_mapping: () -> ::Hash[::String, untyped] - def to_json_schema: () -> ::Hash[::String, untyped] - def json_schema_field_metadata_by_field_name: () -> ::Hash[::String, JSONIngestion::SchemaDefinition::Indexing::JSONSchemaFieldMetadata] - def format_field_json_schema_customizations: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] end end end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/enum.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/enum.rbs index 7f8f9afd6..c5ca26562 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/enum.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/enum.rbs @@ -4,6 +4,7 @@ module ElasticGraph module FieldType class Enum include _FieldType + attr_reader enum_value_names: ::Array[::String] def initialize: (::Array[::String]) -> void end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/object.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/object.rbs index 5dd23dbaa..0ecdae84a 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/object.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/object.rbs @@ -3,29 +3,23 @@ module ElasticGraph module Indexing module FieldType class ObjectSuperType + attr_reader schema_def_state: State attr_reader type_name: ::String attr_reader subfields: ::Array[Field] attr_reader mapping_options: Mixins::HasTypeInfo::optionsHash - attr_reader json_schema_options: Mixins::HasTypeInfo::optionsHash - attr_reader doc_comment: ::String? - attr_reader schema_def_state: State def initialize: ( + schema_def_state: State, type_name: ::String, subfields: ::Array[Field], - mapping_options: Mixins::HasTypeInfo::optionsHash, - json_schema_options: Mixins::HasTypeInfo::optionsHash, - doc_comment: ::String?, - schema_def_state: State + mapping_options: Mixins::HasTypeInfo::optionsHash ) -> void def with: ( + ?schema_def_state: State, ?type_name: ::String, ?subfields: ::Array[Field], - ?mapping_options: Mixins::HasTypeInfo::optionsHash, - ?json_schema_options: Mixins::HasTypeInfo::optionsHash, - ?doc_comment: ::String?, - ?schema_def_state: State + ?mapping_options: Mixins::HasTypeInfo::optionsHash ) -> Object end @@ -34,12 +28,6 @@ module ElasticGraph include Support::_MemoizableDataClass @to_mapping: ::Hash[::String, untyped]? - @to_json_schema: ::Hash[::String, untyped]? - - private - - def json_schema_typename_field: () -> ::Hash[::String, untyped] - def validate_sourced_fields_have_no_json_schema_overrides: (::Array[Field]) -> void end end end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/scalar.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/scalar.rbs index 32cff29a8..40a5d6e2f 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/scalar.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/scalar.rbs @@ -4,6 +4,7 @@ module ElasticGraph module FieldType class Scalar include _FieldType + attr_reader scalar_type: SchemaElements::ScalarType def initialize: (scalar_type: SchemaElements::ScalarType) -> Scalar diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/union.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/union.rbs index d100ef666..323d68590 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/union.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/field_type/union.rbs @@ -2,13 +2,11 @@ module ElasticGraph module SchemaDefinition module Indexing module FieldType - class UnionSuperType - attr_reader subtypes_by_name: ::Hash[::String, Object] - def self.new: (::Hash[::String, Object]) -> instance - end - - class Union < UnionSuperType + class Union include _FieldType + + attr_reader subtypes_by_name: ::Hash[::String, Object] + def initialize: (::Hash[::String, Object]) -> void end end end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_type_info.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_type_info.rbs index 1f3c1391b..665759c4c 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_type_info.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/mixins/has_type_info.rbs @@ -6,10 +6,8 @@ module ElasticGraph type optionsHash = ::Hash[::Symbol, untyped] attr_reader mapping_options: optionsHash - attr_reader json_schema_options: optionsHash def mapping: (**untyped) -> void - def json_schema: (**untyped) -> void end end end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/results.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/results.rbs index 8af2cd816..73fc21719 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/results.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/results.rbs @@ -6,42 +6,29 @@ module ElasticGraph end class Results < ResultsSupertype - include _SchemaArtifacts include Support::_MemoizableDataClass - def json_schema_version_setter_location: () -> ::Thread::Backtrace::Location? - def json_schema_field_metadata_by_type_and_field_name: () -> ::Hash[::String, ::Hash[::String, JSONIngestion::SchemaDefinition::Indexing::JSONSchemaFieldMetadata]] - def current_public_json_schema: () -> ::Hash[::String, untyped] - def merge_field_metadata_into_json_schema: (::Hash[::String, untyped]) -> JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata - def unused_deprecated_elements: () -> ::Set[SchemaElements::DeprecatedElement] + def graphql_schema_string: () -> ::String + def datastore_config: () -> ::Hash[::String, untyped] + def runtime_metadata: () -> SchemaArtifacts::RuntimeMetadata::Schema def derived_indexing_type_names: () -> ::Set[::String] @graphql_schema_string: ::String? @datastore_config: ::Hash[::String, untyped] @runtime_metadata: SchemaArtifacts::RuntimeMetadata::Schema? - @current_json_schemas: ::Hash[::String, untyped]? @static_script_repo: Scripting::FileSystemRepository? - @available_json_schema_versions: ::Set[::Integer]? @no_circular_dependencies: bool? @field_path_resolver: SchemaElements::FieldPath::Resolver? - @json_schema_indexing_field_types_by_name: ::Hash[::String, Indexing::_FieldType]? @derived_indexing_type_names: ::Set[::String]? - @json_schema_field_metadata_by_type_and_field_name: ::Hash[::String, ::Hash[::String, JSONIngestion::SchemaDefinition::Indexing::JSONSchemaFieldMetadata]]? - @current_public_json_schema: ::Hash[::String, untyped]? - @latest_versioned_json_schema: ::Hash[::String, untyped]? - @json_schema_with_metadata_merger: JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata::Merger? STATIC_SCRIPT_REPO: Scripting::FileSystemRepository private - def json_schema_with_metadata_merger: () -> JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata::Merger def generate_datastore_config: () -> ::Hash[::String, untyped] def build_dynamic_scripts: () -> ::Array[Scripting::Script] def build_runtime_metadata: () -> SchemaArtifacts::RuntimeMetadata::Schema def generate_sdl: () -> ::String - def build_public_json_schema: () -> ::Hash[::String, untyped] - def json_schema_indexing_field_types_by_name: () -> ::Hash[::String, Indexing::_FieldType] def verify_runtime_metadata: (SchemaArtifacts::RuntimeMetadata::Schema) -> void def strip_trailing_whitespace: (::String) -> ::String def check_for_circular_dependencies!: () -> void diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_artifact_manager.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_artifact_manager.rbs index 389dfbfdd..713693ac3 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_artifact_manager.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_artifact_manager.rbs @@ -20,22 +20,11 @@ module ElasticGraph @output: io @max_diff_lines: ::Integer @artifacts: ::Array[SchemaArtifact[untyped]]? - @json_schemas_artifact: SchemaArtifact[untyped] def artifacts: () -> ::Array[SchemaArtifact[untyped]] def artifacts_from_schema_def: () -> ::Array[SchemaArtifact[untyped]] def notify_about_unused_type_name_overrides: () -> void def notify_about_unused_enum_value_overrides: () -> void - def build_desired_versioned_json_schemas: (::Hash[::String, untyped]) -> ::Hash[::Integer, ::Hash[::String, untyped]] - def report_json_schema_merge_errors: (::Array[JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata]) -> void - def report_json_schema_merge_warnings: () -> void - def format_deprecated_elements: (::Enumerable[SchemaElements::DeprecatedElement]) -> ::String - def missing_field_error_for: (::String, ::Array[::Integer]) -> ::String - def missing_type_error_for: (::String, ::Array[::Integer]) -> ::String - def missing_necessary_field_error_for: (JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata::MissingNecessaryField, ::Array[::Integer]) -> ::String - def describe_json_schema_versions: (::Array[::Integer], ::String) -> ::String - def old_versions: (::Array[::Integer]) -> ::String - def files_noun_phrase: (::Array[::Integer]) -> ::String def artifacts_out_of_date_error: (::Array[SchemaArtifact[untyped]]) -> ::String def truncate_diff: (::String, ::Integer) -> [::String, ::String] @@ -45,9 +34,7 @@ module ElasticGraph ?extra_comment_lines: ::Array[::String] ) -> SchemaArtifact[::Hash[::String, untyped]] - def new_versioned_json_schema_artifact: (::Hash[::String, untyped]) -> SchemaArtifact[::Hash[::String, untyped]] def new_raw_artifact: (::String, ::String) -> SchemaArtifact[::String] - def check_if_needs_json_schema_version_bump: () { (::Integer) -> void } -> void def pruned_runtime_metadata: (::String) -> SchemaArtifacts::RuntimeMetadata::Schema end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/field.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/field.rbs index b32e01646..07f2dc653 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/field.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/field.rbs @@ -22,7 +22,6 @@ module ElasticGraph attr_accessor computation_detail: SchemaArtifacts::RuntimeMetadata::ComputationDetail attr_reader filter_customizations: ::Array[^(Field) -> void] attr_reader sort_order_enum_value_customizations: ::Array[^(SortOrderEnumValue) -> void] - attr_reader non_nullable_in_json_schema: bool attr_reader source: FieldSource? attr_accessor relationship: Relationship? attr_reader resolver: SchemaArtifacts::RuntimeMetadata::ConfiguredGraphQLResolver? diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/scalar_type.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/scalar_type.rbs index 181b563a6..47cb75435 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/scalar_type.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/scalar_type.rbs @@ -31,7 +31,6 @@ module ElasticGraph def runtime_metadata: () -> SchemaArtifacts::RuntimeMetadata::ScalarType def mapping_options: () -> Mixins::HasTypeInfo::optionsHash - def json_schema_options: () -> Mixins::HasTypeInfo::optionsHash private diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/type_reference.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/type_reference.rbs index 9d418a38f..388116f00 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/type_reference.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/schema_elements/type_reference.rbs @@ -30,9 +30,6 @@ module ElasticGraph def scalar_type_needing_grouped_by_object?: () -> bool def with_reverted_override: () -> TypeReference - @json_schema_layers: jsonSchemaLayersArray? - def json_schema_layers: () -> jsonSchemaLayersArray - def to_final_form: (?as_input: bool) -> TypeReference STATIC_FORMAT_NAME_BY_CATEGORY: ::Hash[::Symbol, ::Symbol] @@ -59,8 +56,6 @@ module ElasticGraph private - def peel_json_schema_layers_once: () -> [jsonSchemaLayersArray, TypeReference] - def matches_format_of?: (::Symbol) -> bool def parent_aggregation_type: (::Array[::String]) -> ::String def renamed_with_same_wrappings: (::String) -> TypeReference diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/state.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/state.rbs index bb9a73291..f095bab18 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/state.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/state.rbs @@ -18,9 +18,7 @@ module ElasticGraph attr_reader deleted_types_by_old_name: ::Hash[::String, SchemaElements::DeprecatedElement] attr_reader renamed_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, SchemaElements::DeprecatedElement]] attr_reader deleted_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, SchemaElements::DeprecatedElement]] - attr_accessor json_schema_version: ::Integer? - attr_accessor json_schema_version_setter_location: ::Thread::Backtrace::Location? - attr_accessor enforce_json_schema_version: bool + attr_reader reserved_type_names: ::Set[::String] attr_reader graphql_extension_modules: ::Array[SchemaArtifacts::RuntimeMetadata::GraphQLExtension] attr_reader graphql_resolvers_by_name: ::Hash[::Symbol, SchemaArtifacts::RuntimeMetadata::GraphQLResolver] attr_reader built_in_graphql_resolvers: ::Set[::Symbol] @@ -33,8 +31,6 @@ module ElasticGraph attr_reader type_namer: SchemaElements::TypeNamer attr_reader enum_value_namer: SchemaElements::EnumValueNamer attr_accessor output: io - attr_accessor allow_omitted_json_schema_fields: bool - attr_accessor allow_extra_json_schema_fields: bool attr_accessor indexed_types_by_index_name: ::Hash[::String, indexableType] def initialize: ( @@ -55,9 +51,7 @@ module ElasticGraph deleted_types_by_old_name: ::Hash[::String, SchemaElements::DeprecatedElement], renamed_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, SchemaElements::DeprecatedElement]], deleted_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, SchemaElements::DeprecatedElement]], - json_schema_version: Integer?, - json_schema_version_setter_location: ::Thread::Backtrace::Location?, - enforce_json_schema_version: bool, + reserved_type_names: ::Set[::String], graphql_extension_modules: ::Array[SchemaArtifacts::RuntimeMetadata::GraphQLExtension], graphql_resolvers_by_name: ::Hash[::Symbol, SchemaArtifacts::RuntimeMetadata::GraphQLResolver], built_in_graphql_resolvers: ::Set[::Symbol], @@ -70,8 +64,6 @@ module ElasticGraph type_namer: SchemaElements::TypeNamer, enum_value_namer: SchemaElements::EnumValueNamer, output: io, - allow_omitted_json_schema_fields: bool, - allow_extra_json_schema_fields: bool, indexed_types_by_index_name: ::Hash[::String, indexableType], ) -> void end @@ -119,7 +111,6 @@ module ElasticGraph private - RESERVED_TYPE_NAMES: ::Set[::String] def register_type: [T] (T & SchemaElements::graphQLType, ?::Hash[::String, T]?) -> T end end diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/test_support.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/test_support.rbs index 211039ae3..1b92524c5 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/test_support.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/test_support.rbs @@ -12,7 +12,7 @@ module ElasticGraph ?enum_value_overrides_by_type: ::Hash[::Symbol, ::Hash[::Symbol, ::String]], ?output: io?, ?reload_schema_artifacts: bool, - ) ?{ (API) -> void } -> _SchemaArtifacts + ) ?{ (API) -> void } -> (Results | SchemaArtifacts::FromDisk) def define_schema_with_schema_elements: ( SchemaArtifacts::RuntimeMetadata::SchemaElementNames, @@ -24,7 +24,7 @@ module ElasticGraph ?enum_value_overrides_by_type: ::Hash[::Symbol, ::Hash[::Symbol, ::String]], ?output: io?, ?reload_schema_artifacts: bool, - ) ?{ (API) -> void } -> _SchemaArtifacts + ) ?{ (API) -> void } -> (Results | SchemaArtifacts::FromDisk) DOC_COMMENTS: ::String diff --git a/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/rake_tasks_spec.rb b/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/rake_tasks_spec.rb index 54a235ec8..edb0a967e 100644 --- a/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/rake_tasks_spec.rb +++ b/elasticgraph-schema_definition/spec/integration/elastic_graph/schema_definition/rake_tasks_spec.rb @@ -8,6 +8,7 @@ require "bundler" require "elastic_graph/constants" +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/schema_definition/rake_tasks" require "elastic_graph/schema_definition/schema_elements/type_namer" require "graphql" @@ -216,7 +217,12 @@ module SchemaDefinition "`schema.json_schema_version 4`" ).and matching(json_schema_version_setter_location_regex) - write_elastic_graph_schema_def_code(component_suffix: "3", json_schema_version: 3, component_extras: "t.renamed_from 'Component'", enforce_json_schema_version: false) + write_elastic_graph_schema_def_code( + component_suffix: "3", + json_schema_version: 3, + component_extras: "t.renamed_from 'Component'", + enforce_json_schema_version: false + ) expect { output = run_rake("schema_artifacts:dump") @@ -494,9 +500,13 @@ module SchemaDefinition }) ) - # Here we add a different new field (`ordinal: Int!`), without bumping the version, and disable - # enforcement so we do not have to bump the version. - write_elastic_graph_schema_def_code(json_schema_version: 2, component_name_extras: "\nt.field 'ordinal', 'Int!'", enforce_json_schema_version: false) + # Here we add a different new field (`ordinal: Int!`), without bumping the version (and using `enforce_json_schema_version: false` + # to not have to bump the version)... + write_elastic_graph_schema_def_code( + json_schema_version: 2, + component_name_extras: "\nt.field 'ordinal', 'Int!'", + enforce_json_schema_version: false + ) run_rake("schema_artifacts:dump") # It should not be added to the v1 schema... @@ -585,7 +595,7 @@ module SchemaDefinition To continue, do one of the following: 1. If the `Component` type has been renamed, indicate this by calling `type.renamed_from "Component"` on the renamed type. - 2. If the `Component` field has been dropped, indicate this by calling `schema.deleted_type "Component"` on the schema. + 2. If the `Component` type has been dropped, indicate this by calling `schema.deleted_type "Component"` on the schema. 3. Alternately, if no publishers or in-flight events use JSON schema version 1, delete its file from `json_schemas_by_version`, and no further changes are required. EOS @@ -600,7 +610,7 @@ module SchemaDefinition To continue, do one of the following: 1. If the `Component` type has been renamed, indicate this by calling `type.renamed_from "Component"` on the renamed type. - 2. If the `Component` field has been dropped, indicate this by calling `schema.deleted_type "Component"` on the schema. + 2. If the `Component` type has been dropped, indicate this by calling `schema.deleted_type "Component"` on the schema. 3. Alternately, if no publishers or in-flight events use JSON schema versions 1 or 2, delete their files from `json_schemas_by_version`, and no further changes are required. EOS @@ -615,7 +625,7 @@ module SchemaDefinition To continue, do one of the following: 1. If the `Component` type has been renamed, indicate this by calling `type.renamed_from "Component"` on the renamed type. - 2. If the `Component` field has been dropped, indicate this by calling `schema.deleted_type "Component"` on the schema. + 2. If the `Component` type has been dropped, indicate this by calling `schema.deleted_type "Component"` on the schema. 3. Alternately, if no publishers or in-flight events use JSON schema versions 1, 2, or 3, delete their files from `json_schemas_by_version`, and no further changes are required. EOS @@ -822,7 +832,7 @@ module SchemaDefinition expect(runtime_meta["object_types_by_name"].keys).to exclude("DateTimeListFilterInput") end - it "successfully checks schema artifacts when the rake task is run within a bundle that only includes the `elasticgraph-schema_definition` gem" do + it "successfully checks schema artifacts when the rake task is run within a minimal schema definition bundle" do # We want to ensure that `elasticgraph-schema_definition` gem declares (in its gemspec) all the # dependencies necessary for the schema definition rake tasks. Unfortunately, it's test suite # alone can't detect this, even when run via `script/run_gem_specs`, due to transitive dependencies @@ -837,6 +847,7 @@ module SchemaDefinition source "https://rubygems.org" gem "elasticgraph-schema_definition", path: "#{CommonSpecHelpers::REPO_ROOT}/elasticgraph-schema_definition" + gem "elasticgraph-json_ingestion", path: "#{CommonSpecHelpers::REPO_ROOT}/elasticgraph-json_ingestion" register_gemspec_gems_with_path = lambda do |eg_gem_name| gemspec_contents = ::File.read("#{CommonSpecHelpers::REPO_ROOT}/\#{eg_gem_name}/\#{eg_gem_name}.gemspec") @@ -854,6 +865,7 @@ module SchemaDefinition ::File.write("Rakefile", <<~EOS) project_root = "#{CommonSpecHelpers::REPO_ROOT}" + require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/schema_definition/rake_tasks" ElasticGraph::SchemaDefinition::RakeTasks.new( @@ -861,6 +873,7 @@ module SchemaDefinition index_document_sizes: true, path_to_schema: "\#{project_root}/config/schema.rb", schema_artifacts_directory: "\#{project_root}/config/schema/artifacts", + extension_modules: [ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension] ) EOS @@ -902,7 +915,7 @@ def expect_successful_run_of(*shell_commands) /line 7 at `(\S*\/?)schema\.rb`/ end - def write_elastic_graph_schema_def_code(json_schema_version:, enforce_json_schema_version: true, component_suffix: "", extra_sdl: "", component_name_extras: "", component_extras: "", omit_component_name_field: false) + def write_elastic_graph_schema_def_code(json_schema_version:, component_suffix: "", extra_sdl: "", component_name_extras: "", component_extras: "", omit_component_name_field: false, enforce_json_schema_version: true) code = <<~EOS Thread.current[:eg_schema_load_count] = (Thread.current[:eg_schema_load_count] || 0) + 1 if Thread.current[:eg_schema_load_count] > 1 @@ -1088,7 +1101,7 @@ def as_active_instance index_document_sizes: true, path_to_schema: path_to_schema, schema_artifacts_directory: "config/schema/artifacts", - extension_modules: [extension_module].compact, + extension_modules: [JSONIngestion::SchemaDefinition::APIExtension, extension_module].compact, derived_type_name_formats: derived_type_name_formats, type_name_overrides: type_name_overrides, enum_value_overrides_by_type: enum_value_overrides_by_type, diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/factory_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/factory_spec.rb new file mode 100644 index 000000000..4e87d4a19 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/factory_spec.rb @@ -0,0 +1,42 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names" +require "elastic_graph/schema_definition/api" + +module ElasticGraph + module SchemaDefinition + RSpec.describe Factory do + it "can create an object type without a customization block" do + object_type = build_api.factory.new_object_type("Widget") + + expect(object_type.name).to eq("Widget") + end + + it "can create an index without a customization block" do + api = build_api + + api.object_type "Widget" do |t| + t.field "id", "ID!" + t.index "widgets" + end + + index = api.state.object_types_by_name.fetch("Widget").own_index_def + expect(index.name).to eq("widgets") + end + + def build_api + API.new( + SchemaArtifacts::RuntimeMetadata::SchemaElementNames.new(form: :snake_case, overrides: {}), + true, + extension_modules: [] + ) + end + end + end +end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/define_schema_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/define_schema_spec.rb index b0850973a..e6cf32482 100644 --- a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/define_schema_spec.rb +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/graphql_schema/define_schema_spec.rb @@ -21,7 +21,6 @@ module SchemaDefinition api.as_active_instance do ElasticGraph.define_schema do |schema| - schema.json_schema_version 1 schema.object_type("Person") do |t| t.field "id", "ID" end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/indexing/json_schema_with_metadata_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/indexing/json_schema_with_metadata_spec.rb index f3cd503ac..2e87e57ce 100644 --- a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/indexing/json_schema_with_metadata_spec.rb +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/indexing/json_schema_with_metadata_spec.rb @@ -1054,7 +1054,11 @@ def metadata_for(json_schema, type, field) end def define_schema(&schema_definition) - super(schema_element_name_form: "snake_case", &schema_definition) + super( + schema_element_name_form: "snake_case", + extension_modules: [JSONIngestion::SchemaDefinition::APIExtension], + &schema_definition + ) end def have_dumped_metadata(name_in_index, type) diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_field_metadata_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_field_metadata_spec.rb index 1e0b20b8c..450ae447d 100644 --- a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_field_metadata_spec.rb +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_field_metadata_spec.rb @@ -141,7 +141,11 @@ def dump_metadata(&schema_definition) end def define_schema(&schema_definition) - super(schema_element_name_form: "snake_case", &schema_definition) + super( + schema_element_name_form: "snake_case", + extension_modules: [JSONIngestion::SchemaDefinition::APIExtension], + &schema_definition + ) end def field_meta_of(type, name_in_index) diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_spec.rb index 0ca98186c..cdaa8ba7b 100644 --- a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_spec.rb +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/json_schema_spec.rb @@ -2925,6 +2925,24 @@ def link_supertype_to_subtypes(interface_type, *subtype_names) expect(envelope_type_enum_values(schemas)).to eq ["Widget"] end + it "does not require or dump a schema for a GraphQL-only scalar type because it has no backing data to index" do + schemas = all_type_definitions_for do |s| + s.object_type "Widget" do |t| + t.field "id", "ID!" + t.index "widgets" + end + + s.scalar_type "GraphQLOnly" do |t| + t.graphql_only true + t.mapping type: nil + end + end + + expect(schemas.keys).to include(EVENT_ENVELOPE_JSON_SCHEMA_NAME, "Widget") + expect(schemas.keys).to exclude("GraphQLOnly") + expect(envelope_type_enum_values(schemas)).to eq ["Widget"] + end + it "omits fields that reference a namespace type from the JSON schema of an indexed type" do # This test would fail if `s.namespace_type` were replaced with `s.object_type` -- the `olap` # property would appear under Widget's JSON schema. Namespace types have no backing data, so diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/runtime_metadata_support.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/runtime_metadata_support.rb index 3f1b1df12..b0dcbdc1d 100644 --- a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/runtime_metadata_support.rb +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/runtime_metadata/runtime_metadata_support.rb @@ -16,7 +16,12 @@ module SchemaDefinition include SchemaArtifacts::RuntimeMetadata::RuntimeMetadataSupport def define_schema(**options, &block) - super(schema_element_name_form: "snake_case", **options) + super( + schema_element_name_form: "snake_case", + extension_modules: [JSONIngestion::SchemaDefinition::APIExtension], + **options, + &block + ) end end end diff --git a/elasticgraph-warehouse/README.md b/elasticgraph-warehouse/README.md index 6890bbe63..9bf1ce88a 100644 --- a/elasticgraph-warehouse/README.md +++ b/elasticgraph-warehouse/README.md @@ -46,21 +46,21 @@ diff --git a/Rakefile b/Rakefile index 2943335..26633c3 100644 --- a/Rakefile +++ b/Rakefile -@@ -1,5 +1,6 @@ +@@ -1,6 +1,7 @@ project_root = File.expand_path(__dir__) -+require "elastic_graph/warehouse/schema_definition/api_extension" + require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/local/rake_tasks" require "elastic_graph/query_registry/rake_tasks" ++require "elastic_graph/warehouse/schema_definition/api_extension" require "rspec/core/rake_task" -@@ -12,5 +13,7 @@ ElasticGraph::Local::RakeTasks.new( - local_config_yaml: settings_file, - path_to_schema: "#{project_root}/config/schema.rb" - ) do |tasks| -+ tasks.schema_definition_extension_modules << ElasticGraph::Warehouse::SchemaDefinition::APIExtension -+ +@@ -16,5 +17,6 @@ ElasticGraph::Local::RakeTasks.new( # Determines casing of field names. Can be either `:camelCase` or `:snake_case`. tasks.schema_element_name_form = :camelCase + tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension ++ tasks.schema_definition_extension_modules << ElasticGraph::Warehouse::SchemaDefinition::APIExtension + + # Customizes the names of fields generated by ElasticGraph. ``` After running `bundle exec rake schema_artifacts:dump`, a `data_warehouse.yaml` file will be diff --git a/elasticgraph-warehouse/elasticgraph-warehouse.gemspec b/elasticgraph-warehouse/elasticgraph-warehouse.gemspec index 1467bdbcc..3d03e7e20 100644 --- a/elasticgraph-warehouse/elasticgraph-warehouse.gemspec +++ b/elasticgraph-warehouse/elasticgraph-warehouse.gemspec @@ -34,6 +34,7 @@ Gem::Specification.new do |spec| spec.required_ruby_version = [">= 3.4", "< 4.1"] - spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION spec.add_dependency "elasticgraph-support", ElasticGraph::VERSION + + spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION end diff --git a/elasticgraph-warehouse/spec/integration/elastic_graph/warehouse/schema_definition/rake_tasks_spec.rb b/elasticgraph-warehouse/spec/integration/elastic_graph/warehouse/schema_definition/rake_tasks_spec.rb index da74eb579..f07a875f1 100644 --- a/elasticgraph-warehouse/spec/integration/elastic_graph/warehouse/schema_definition/rake_tasks_spec.rb +++ b/elasticgraph-warehouse/spec/integration/elastic_graph/warehouse/schema_definition/rake_tasks_spec.rb @@ -85,7 +85,7 @@ module SchemaDefinition original_content = read_warehouse_artifact expect(original_content).not_to include("added_field") - write_warehouse_schema(enforce_json_schema_version: false, table_defs: <<~EOS) + write_warehouse_schema(table_defs: <<~EOS) s.object_type "Product" do |t| t.field "id", "ID" t.field "added_field", "String" @@ -157,12 +157,9 @@ module SchemaDefinition end end - def write_warehouse_schema(table_defs:, enforce_json_schema_version: true) + def write_warehouse_schema(table_defs:) ::File.write("schema.rb", <<~EOS) ElasticGraph.define_schema do |s| - s.json_schema_version 1 - #{"s.enforce_json_schema_version false" unless enforce_json_schema_version} - # Add a dummy indexed type to ensure the Query type has at least one field. # This prevents GraphQL-Ruby warnings about empty Query types in tests. # We exclude it from the warehouse so it doesn't interfere with test expectations. diff --git a/elasticgraph-warehouse/spec/unit/elastic_graph/warehouse/schema_definition/scalar_type_extension_spec.rb b/elasticgraph-warehouse/spec/unit/elastic_graph/warehouse/schema_definition/scalar_type_extension_spec.rb index 8839a646a..f0457b8ad 100644 --- a/elasticgraph-warehouse/spec/unit/elastic_graph/warehouse/schema_definition/scalar_type_extension_spec.rb +++ b/elasticgraph-warehouse/spec/unit/elastic_graph/warehouse/schema_definition/scalar_type_extension_spec.rb @@ -35,7 +35,6 @@ module SchemaDefinition results = define_warehouse_schema do |s| s.scalar_type "CustomTimestamp" do |t| t.mapping type: "date" - t.json_schema type: "string", format: "date-time" t.warehouse_column type: "TIMESTAMP" end @@ -53,7 +52,6 @@ module SchemaDefinition results = define_warehouse_schema do |s| s.scalar_type "UnconfiguredScalar" do |t| t.mapping type: "keyword" - t.json_schema type: "string" # Intentionally NOT calling t.warehouse_column end diff --git a/elasticgraph/lib/elastic_graph/project_template/Rakefile.tt b/elasticgraph/lib/elastic_graph/project_template/Rakefile.tt index 7507ebfbe..6a8f878e3 100644 --- a/elasticgraph/lib/elastic_graph/project_template/Rakefile.tt +++ b/elasticgraph/lib/elastic_graph/project_template/Rakefile.tt @@ -1,5 +1,6 @@ project_root = File.expand_path(__dir__) +require "elastic_graph/json_ingestion/schema_definition/api_extension" require "elastic_graph/local/rake_tasks" require "elastic_graph/query_registry/rake_tasks" require "rspec/core/rake_task" @@ -14,6 +15,7 @@ ElasticGraph::Local::RakeTasks.new( ) do |tasks| # Determines casing of field names. Can be either `:camelCase` or `:snake_case`. tasks.schema_element_name_form = :camelCase + tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension # Customizes the names of fields generated by ElasticGraph. tasks.schema_element_name_overrides = { diff --git a/spec_support/lib/elastic_graph/spec_support/schema_definition_helpers.rb b/spec_support/lib/elastic_graph/spec_support/schema_definition_helpers.rb index 0c3cf8538..d279fd460 100644 --- a/spec_support/lib/elastic_graph/spec_support/schema_definition_helpers.rb +++ b/spec_support/lib/elastic_graph/spec_support/schema_definition_helpers.rb @@ -8,32 +8,49 @@ require "elastic_graph/schema_definition/test_support" +module ElasticGraph + module SpecSupport + DEFAULT_SCHEMA_DEFINITION_EXTENSION_MODULES = begin + require "elastic_graph/json_ingestion/schema_definition/api_extension" + [::ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension] + rescue LoadError => e + # :nocov: -- per-gem spec bundles may not include the optional `elasticgraph-json_ingestion` gem. + raise unless e.path == "elastic_graph/json_ingestion/schema_definition/api_extension" + + [] + # :nocov: + end.freeze + end +end + # Combines `:capture_logs` with `ElasicGraph::SchemaDefinition::TestSupport` in order # to silence log output and fail if any tests result in logged warnings. ::RSpec.shared_context "SchemaDefinitionHelpers", :capture_logs do include ::ElasticGraph::SchemaDefinition::TestSupport - def define_schema_with_schema_elements( - schema_elements, - index_document_sizes: true, - json_schema_version: 1, - extension_modules: [], - derived_type_name_formats: {}, - type_name_overrides: {}, - enum_value_overrides_by_type: {}, - reload_schema_artifacts: false, - output: nil - ) + # Defaults `extension_modules` and `output` for tests; all other options are forwarded to + # `TestSupport` unchanged. `output` must be handled with `||` (rather than a keyword default) + # because `TestSupport#define_schema` passes `output: nil` explicitly when no output is given. + def define_schema(extension_modules: default_schema_definition_extension_modules, output: nil, **options, &block) + super( + extension_modules: extension_modules, + output: output || log_device, + **options, + &block + ) + end + + def define_schema_with_schema_elements(schema_elements, extension_modules: default_schema_definition_extension_modules, output: nil, **options, &block) super( schema_elements, - index_document_sizes: index_document_sizes, - json_schema_version: json_schema_version, extension_modules: extension_modules, - derived_type_name_formats: derived_type_name_formats, - type_name_overrides: type_name_overrides, - enum_value_overrides_by_type: enum_value_overrides_by_type, - reload_schema_artifacts: false, - output: output || log_device + output: output || log_device, + **options, + &block ) end + + def default_schema_definition_extension_modules + ::ElasticGraph::SpecSupport::DEFAULT_SCHEMA_DEFINITION_EXTENSION_MODULES.dup + end end diff --git a/spec_support/spec_helper.rb b/spec_support/spec_helper.rb index e62430bd5..655e16663 100644 --- a/spec_support/spec_helper.rb +++ b/spec_support/spec_helper.rb @@ -365,6 +365,7 @@ def generate_schema_artifacts( reload_schema_artifacts: false ) require "elastic_graph/schema_definition/test_support" + require "elastic_graph/spec_support/schema_definition_helpers" require "stringio" output = ::StringIO.new # to silence warnings. @@ -374,6 +375,7 @@ def generate_schema_artifacts( derived_type_name_formats: derived_type_name_formats, type_name_overrides: type_name_overrides, enum_value_overrides_by_type: enum_value_overrides_by_type, + extension_modules: ::ElasticGraph::SpecSupport::DEFAULT_SCHEMA_DEFINITION_EXTENSION_MODULES.dup, reload_schema_artifacts: reload_schema_artifacts, output: output ) do |schema|