Polish JSON ingestion extension APIs and docs

Author: jwils

AGENTS.md

@@ -85,9 +85,10 @@ All gems follow the pattern: `elasticgraph-[name]/` containing:
 - `elasticgraph-elasticsearch`: Elasticsearch client wrapper
 - `elasticgraph-opensearch`: OpenSearch client wrapper
 
-**Extensions** (5 gems): Optional functionality
+**Extensions** (6 gems): Optional functionality plus the default JSON Schema ingestion serializer
 - `elasticgraph-apollo`: Apollo Federation support
 - `elasticgraph-health_check`: Health checks
+- `elasticgraph-json_ingestion`: JSON Schema ingestion serializer
 - `elasticgraph-query_interceptor`: Query interception
 - `elasticgraph-query_registry`: Source-controlled query registry
 - `elasticgraph-warehouse`: Data warehouse ingestion

elasticgraph-apollo/lib/elastic_graph/apollo/schema_definition/api_extension.rb

@@ -35,7 +35,7 @@ module ElasticGraph
   #     local_config_yaml: "config/settings/local.yaml",
   #     path_to_schema: "config/schema.rb"
   #   ) do |tasks|
-  #     tasks.schema_definition_extension_modules = [ElasticGraph::Apollo::SchemaDefinition::APIExtension]
+  #     tasks.schema_definition_extension_modules += [ElasticGraph::Apollo::SchemaDefinition::APIExtension]
   #   end
   module Apollo
     # Namespace for all Apollo schema definition support.
@@ -55,7 +55,7 @@ module SchemaDefinition
       #     local_config_yaml: "config/settings/local.yaml",
       #     path_to_schema: "config/schema.rb"
       #   ) do |tasks|
-      #     tasks.schema_definition_extension_modules = [ElasticGraph::Apollo::SchemaDefinition::APIExtension]
+      #     tasks.schema_definition_extension_modules += [ElasticGraph::Apollo::SchemaDefinition::APIExtension]
       #   end
       module APIExtension
         # Applies an apollo tag to built-in types so that they are included in the Apollo contract schema.

elasticgraph-json_ingestion/README.md

@@ -1,9 +1,98 @@
 # ElasticGraph::JSONIngestion
 
-JSON Schema ingestion support for ElasticGraph.
+Default JSON Schema ingestion support for ElasticGraph.
 
 This gem provides the schema-definition extension that generates JSON Schema artifacts for indexing
-events and validates JSON-ingestion-specific schema options.
+events and validates JSON-ingestion-specific schema options. Generated ElasticGraph projects install
+and enable it by default. Applications that wire schema-definition tasks manually enable it by adding
+`ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension` to their schema-definition extension modules.
+
+## Schema Definition APIs
+
+Use `schema.json_schema_version` to identify the current JSON schema artifact. Every change that affects
+the JSON schema should increment this version so publishers and indexers can safely evolve independently.
+
+```diff
+diff --git a/config/schema.rb b/config/schema.rb
+index 015c5fa..b8eeaef 100644
+--- a/config/schema.rb
++++ b/config/schema.rb
+@@ -1,5 +1,5 @@
+ ElasticGraph.define_schema do |schema|
+   # ElasticGraph will tell you when you need to bump this.
+-  schema.json_schema_version 1
++  schema.json_schema_version 2
+-
++
+   # Set this to true once you're beyond the prototyping stage.
+```
+
+Use `schema.json_schema_strictness` to configure whether indexing events may omit nullable fields or include
+extra fields. We recommend enabling at most one of these options, because enabling both can hide misspelled
+event fields.
+
+```ruby
+# in config/schema/json_schema_strictness.rb
+
+ElasticGraph.define_schema do |schema|
+  schema.json_schema_strictness allow_omitted_fields: true, allow_extra_fields: false
+end
+```
+
+Custom scalar types must declare how they are represented in JSON Schema:
+
+```ruby
+# in config/schema/url.rb
+
+ElasticGraph.define_schema do |schema|
+  schema.scalar_type "URL" do |t|
+    t.mapping type: "keyword"
+    t.json_schema type: "string", format: "uri"
+  end
+end
+```
+
+Fields and object/interface types can add JSON Schema validations. Use field-level validations sparingly:
+they run while indexing events, so violations can send otherwise valid source-system data to the dead letter
+queue. They are best reserved for constraints that ElasticGraph needs in order to index correctly.
+
+```ruby
+# in config/schema/card.rb
+
+ElasticGraph.define_schema do |schema|
+  schema.object_type "Card" do |t|
+    t.field "id", "ID!"
+
+    t.field "expYear", "Int" do |f|
+      f.json_schema minimum: 2000, maximum: 2099
+    end
+
+    t.field "expMonth", "Int" do |f|
+      f.json_schema minimum: 1, maximum: 12
+    end
+
+    t.index "cards"
+  end
+end
+```
+
+On fields, `nullable: false` disallows `null` in indexing events while keeping the GraphQL field nullable:
+
+```ruby
+# in config/schema/widget.rb
+
+ElasticGraph.define_schema do |schema|
+  schema.object_type "Widget" do |t|
+    t.field "id", "ID!"
+
+    t.field "name", "String" do |f|
+      f.json_schema nullable: false
+    end
+
+    t.index "widgets"
+  end
+end
+```
 
 ## Dependency Diagram
 

elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/api_extension.rb

@@ -9,15 +9,7 @@
 require "elastic_graph/constants"
 require "elastic_graph/graphql/scalar_coercion_adapters/valid_time_zones"
 require "elastic_graph/json_ingestion/schema_definition/factory_extension"
-require "elastic_graph/json_ingestion/schema_definition/indexing/field_type/enum_extension"
-require "elastic_graph/json_ingestion/schema_definition/indexing/field_type/object_extension"
-require "elastic_graph/json_ingestion/schema_definition/indexing/field_type/scalar_extension"
-require "elastic_graph/json_ingestion/schema_definition/indexing/field_type/union_extension"
 require "elastic_graph/json_ingestion/schema_definition/state_extension"
-require "elastic_graph/schema_definition/indexing/field_type/enum"
-require "elastic_graph/schema_definition/indexing/field_type/object"
-require "elastic_graph/schema_definition/indexing/field_type/scalar"
-require "elastic_graph/schema_definition/indexing/field_type/union"
 
 module ElasticGraph
   module JSONIngestion
@@ -47,20 +39,12 @@ module APIExtension
           "LongString" => {type: "integer", minimum: LONG_STRING_MIN, maximum: LONG_STRING_MAX}
         }.freeze
 
-        # Wires up the factory extension when this module is extended onto an API instance.
+        # Wires up the JSON ingestion extensions when this module is extended onto an API instance.
         #
         # @param api [ElasticGraph::SchemaDefinition::API] the API instance to extend
         # @return [void]
         # @api private
         def self.extended(api)
-          # Prepend our indexing-field-type extensions onto the core classes so they participate in
-          # `to_json_schema` / `format_field_json_schema_customizations` / `json_schema_field_metadata_by_field_name`.
-          # Guarded so re-extending an already-extended API instance is a no-op.
-          ElasticGraph::SchemaDefinition::Indexing::FieldType::Enum.prepend(Indexing::FieldType::EnumExtension) unless ElasticGraph::SchemaDefinition::Indexing::FieldType::Enum < Indexing::FieldType::EnumExtension
-          ElasticGraph::SchemaDefinition::Indexing::FieldType::Object.prepend(Indexing::FieldType::ObjectExtension) unless ElasticGraph::SchemaDefinition::Indexing::FieldType::Object < Indexing::FieldType::ObjectExtension
-          ElasticGraph::SchemaDefinition::Indexing::FieldType::Scalar.prepend(Indexing::FieldType::ScalarExtension) unless ElasticGraph::SchemaDefinition::Indexing::FieldType::Scalar < Indexing::FieldType::ScalarExtension
-          ElasticGraph::SchemaDefinition::Indexing::FieldType::Union.prepend(Indexing::FieldType::UnionExtension) unless ElasticGraph::SchemaDefinition::Indexing::FieldType::Union < Indexing::FieldType::UnionExtension
-
           state = api.state.extend(StateExtension) # : ElasticGraph::SchemaDefinition::State & StateExtension
           state.reserved_type_names << EVENT_ENVELOPE_JSON_SCHEMA_NAME
           api.factory.extend FactoryExtension

elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/factory_extension.rb

@@ -14,6 +14,7 @@
 require "elastic_graph/json_ingestion/schema_definition/schema_elements/object_interface_extension"
 require "elastic_graph/json_ingestion/schema_definition/schema_elements/scalar_type_extension"
 require "elastic_graph/json_ingestion/schema_definition/schema_elements/type_reference_extension"
+require "elastic_graph/json_ingestion/schema_definition/schema_elements/union_type_extension"
 
 module ElasticGraph
   module JSONIngestion
@@ -78,6 +79,14 @@ def new_type_reference(name)
           super(name).extend(SchemaElements::TypeReferenceExtension)
         end
 
+        # @private
+        def new_union_type(name)
+          super(name) do |type|
+            type.extend SchemaElements::UnionTypeExtension
+            yield type if block_given?
+          end
+        end
+
         # Creates a new Results instance with JSON Schema extensions.
         #
         # @return [ElasticGraph::SchemaDefinition::Results] the created results instance

elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/enum.rb

@@ -0,0 +1,43 @@
+# Copyright 2024 - 2026 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/json_ingestion/schema_definition/indexing/field_type/enum_extension"
+require "elastic_graph/schema_definition/indexing/field_type/enum"
+
+module ElasticGraph
+  module JSONIngestion
+    module SchemaDefinition
+      module Indexing
+        module FieldType
+          # JSON-ingestion indexing field type wrapper for enums.
+          #
+          # @private
+          class Enum
+            include EnumExtension
+
+            def self.wrap(field_type)
+              new(field_type)
+            end
+
+            def initialize(field_type)
+              @field_type = field_type
+            end
+
+            def enum_value_names
+              @field_type.enum_value_names
+            end
+
+            def to_mapping
+              @field_type.to_mapping
+            end
+          end
+        end
+      end
+    end
+  end
+end

elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/object.rb

@@ -0,0 +1,59 @@
+# Copyright 2024 - 2026 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/json_ingestion/schema_definition/indexing/field_type/object_extension"
+require "elastic_graph/schema_definition/indexing/field_type/object"
+
+module ElasticGraph
+  module JSONIngestion
+    module SchemaDefinition
+      module Indexing
+        module FieldType
+          # JSON-ingestion indexing field type wrapper for objects.
+          #
+          # @private
+          class Object
+            include ObjectExtension
+
+            def self.wrap(field_type, json_schema_options:)
+              new(field_type).with_json_schema_options(json_schema_options)
+            end
+
+            def initialize(field_type)
+              @field_type = field_type
+            end
+
+            def schema_def_state
+              @field_type.schema_def_state
+            end
+
+            def type_name
+              @field_type.type_name
+            end
+
+            def subfields
+              @field_type.subfields
+            end
+
+            def mapping_options
+              @field_type.mapping_options
+            end
+
+            def doc_comment
+              @field_type.doc_comment
+            end
+
+            def to_mapping
+              @field_type.to_mapping
+            end
+          end
+        end
+      end
+    end
+  end
+end

elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/scalar.rb

@@ -0,0 +1,43 @@
+# Copyright 2024 - 2026 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/json_ingestion/schema_definition/indexing/field_type/scalar_extension"
+require "elastic_graph/schema_definition/indexing/field_type/scalar"
+
+module ElasticGraph
+  module JSONIngestion
+    module SchemaDefinition
+      module Indexing
+        module FieldType
+          # JSON-ingestion indexing field type wrapper for scalars.
+          #
+          # @private
+          class Scalar
+            include ScalarExtension
+
+            def self.wrap(field_type)
+              new(field_type)
+            end
+
+            def initialize(field_type)
+              @field_type = field_type
+            end
+
+            def scalar_type
+              @field_type.scalar_type
+            end
+
+            def to_mapping
+              @field_type.to_mapping
+            end
+          end
+        end
+      end
+    end
+  end
+end

elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/field_type/union.rb

@@ -0,0 +1,43 @@
+# Copyright 2024 - 2026 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/json_ingestion/schema_definition/indexing/field_type/union_extension"
+require "elastic_graph/schema_definition/indexing/field_type/union"
+
+module ElasticGraph
+  module JSONIngestion
+    module SchemaDefinition
+      module Indexing
+        module FieldType
+          # JSON-ingestion indexing field type wrapper for unions.
+          #
+          # @private
+          class Union
+            include UnionExtension
+
+            def self.wrap(field_type)
+              new(field_type)
+            end
+
+            def initialize(field_type)
+              @field_type = field_type
+            end
+
+            def subtypes_by_name
+              @field_type.subtypes_by_name
+            end
+
+            def to_mapping
+              @field_type.to_mapping
+            end
+          end
+        end
+      end
+    end
+  end
+end