Namespace JSON schema helpers under JSON ingestion

Author: jwils

elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/event_envelope.rb

@@ -7,17 +7,20 @@
 # frozen_string_literal: true
 
 require "elastic_graph/constants"
+require "elastic_graph/json_ingestion"
 
-module ElasticGraph
+module ElasticGraph::JSONIngestion
   module SchemaDefinition
     module Indexing
-      # Contains logic related to "event envelope"--the layer of metadata that wraps all indexing events.
+      # Contains logic related to the JSON schema for ElasticGraph's event envelope.
       #
       # @api private
       module EventEnvelope
-        # @param indexed_type_names [Array<String>] names of the indexed types
-        # @param json_schema_version [Integer] the version of the JSON schema
-        # @return [Hash<String, Object>] the JSON schema for the ElasticGraph event envelope for the given `indexed_type_names`.
+        # Builds the JSON schema definition for ElasticGraph's event envelope.
+        #
+        # @param indexed_type_names [Array<String>] names of indexed types
+        # @param json_schema_version [Integer] the JSON schema version number
+        # @return [Hash<String, Object>] the event envelope JSON schema
         def self.json_schema(indexed_type_names, json_schema_version)
           {
             "type" => "object",
@@ -31,13 +34,12 @@ def self.json_schema(indexed_type_names, json_schema_version)
               "type" => {
                 "description" => "The type of object present in `record`.",
                 "type" => "string",
-                # Sorting doesn't really matter here, but it's nice for the output in the schema artifact to be consistent.
                 "enum" => indexed_type_names.sort
               },
               "id" => {
                 "description" => "The unique identifier of the record.",
                 "type" => "string",
-                "maxLength" => DEFAULT_MAX_KEYWORD_LENGTH
+                "maxLength" => ::ElasticGraph::DEFAULT_MAX_KEYWORD_LENGTH
               },
               "version" => {
                 "description" => 'Used to handle duplicate and out-of-order events. When ElasticGraph ingests multiple events for the same `type` and `id`, the one with the largest `version` will "win".',
@@ -61,7 +63,7 @@ def self.json_schema(indexed_type_names, json_schema_version)
                   }
                 }
               },
-              JSON_SCHEMA_VERSION_KEY => {
+              ::ElasticGraph::JSON_SCHEMA_VERSION_KEY => {
                 "description" => "The version of the JSON schema the publisher was using when the event was published. ElasticGraph will use the JSON schema matching this version to process the event.",
                 "const" => json_schema_version
               },
@@ -71,7 +73,7 @@ def self.json_schema(indexed_type_names, json_schema_version)
               }
             },
             "additionalProperties" => false,
-            "required" => ["op", "type", "id", "version", JSON_SCHEMA_VERSION_KEY],
+            "required" => ["op", "type", "id", "version", ::ElasticGraph::JSON_SCHEMA_VERSION_KEY],
             "if" => {
               "properties" => {
                 "op" => {"const" => "upsert"}

elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_field_metadata.rb

@@ -6,23 +6,17 @@
 #
 # frozen_string_literal: true
 
-module ElasticGraph
+module ElasticGraph::JSONIngestion
   module SchemaDefinition
     module Indexing
       # @!parse class JSONSchemaFieldMetadata; end
       JSONSchemaFieldMetadata = ::Data.define(:type, :name_in_index)
 
-      # Metadata about an ElasticGraph field that needs to be stored in our versioned JSON schemas
-      # alongside the JSON schema fields.
-      #
-      # @!attribute [r] type
-      #   @return [String] name of the ElasticGraph type for this field
-      # @!attribute [r] name_in_index
-      #   @return [String] name of the field in the index
+      # Metadata about an ElasticGraph field that needs to be stored in our versioned JSON schemas.
       #
       # @api private
       class JSONSchemaFieldMetadata < ::Data
-        # @return [Hash<String, String>] hash form of the metadata that can be dumped in JSON schema
+        # @return [Hash<String, String>] hash representation suitable for serialization
         def to_dumpable_hash
           {"type" => type, "nameInIndex" => name_in_index}
         end

elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata.rb

@@ -7,29 +7,23 @@
 # frozen_string_literal: true
 
 require "elastic_graph/constants"
+require "elastic_graph/json_ingestion"
 
-module ElasticGraph
+module ElasticGraph::JSONIngestion
   module SchemaDefinition
     module Indexing
-      # Represents the result of merging a JSON schema with metadata. The result includes both
-      # the merged JSON schema and a list of `failed_fields` indicating which fields metadata
-      # could not be determined for.
+      # Represents the result of merging a JSON schema with ElasticGraph metadata.
       #
       # @private
       class JSONSchemaWithMetadata < ::Data.define(
-        # The JSON schema.
         :json_schema,
-        # A set of fields (in the form `Type.field`) that were needed but not found.
         :missing_fields,
-        # A set of type names that were needed but not found.
         :missing_types,
-        # A set of `DeprecatedElement` objects that create conflicting definitions.
         :definition_conflicts,
-        # A set of fields that have been deleted but that must be retained (e.g. for custom shard routing or rollover)
         :missing_necessary_fields
       )
         def json_schema_version
-          json_schema.fetch(JSON_SCHEMA_VERSION_KEY)
+          json_schema.fetch(::ElasticGraph::JSON_SCHEMA_VERSION_KEY)
         end
 
         # Responsible for building `JSONSchemaWithMetadata` instances.
@@ -60,10 +54,10 @@ def merge_metadata_into(json_schema)
             missing_fields = ::Set.new
             missing_types = ::Set.new
             definition_conflicts = ::Set.new
-            old_type_name_by_current_name = {} # : ::Hash[String, String]
+            old_type_name_by_current_name = {} # : ::Hash[::String, ::String]
 
             defs = json_schema.fetch("$defs").to_h do |type_name, type_def|
-              if type_name != EVENT_ENVELOPE_JSON_SCHEMA_NAME && (properties = type_def["properties"])
+              if type_name != ::ElasticGraph::EVENT_ENVELOPE_JSON_SCHEMA_NAME && (properties = type_def["properties"])
                 current_type_name = determine_current_type_name(
                   type_name,
                   missing_types: missing_types,
@@ -110,7 +104,6 @@ def merge_metadata_into(json_schema)
 
           private
 
-          # Given a historical `type_name`, determines (and returns) the current name for that type.
           def determine_current_type_name(type_name, missing_types:, definition_conflicts:)
             exists_currently = @field_metadata_by_type_and_field_name.key?(type_name)
             deleted = @deleted_types_by_old_name[type_name]&.tap { |elem| @unused_deprecated_elements.delete(elem) }
@@ -128,7 +121,6 @@ def determine_current_type_name(type_name, missing_types:, definition_conflicts:
             nil
           end
 
-          # Given a historical `type_name` and `field_name` determines (and returns) the field metadata for it.
           def field_metadata_for(type_name, field_name, missing_fields:, definition_conflicts:)
             full_name = "#{type_name}.#{field_name}"
 
@@ -154,27 +146,21 @@ def field_metadata_for(type_name, field_name, missing_fields:, definition_confli
 
           def identify_missing_necessary_fields(json_schema, old_type_name_by_current_name)
             json_schema_resolver = JSONSchemaResolver.new(@state, json_schema, old_type_name_by_current_name)
-            version = json_schema.fetch(JSON_SCHEMA_VERSION_KEY)
 
             @state.object_types_by_name.values
               .select { |type| type.has_own_index_def? && !@derived_indexing_type_names.include?(type.name) }
               .flat_map do |object_type|
-                identify_missing_necessary_fields_for_index_def(
-                  object_type,
-                  object_type.own_index_def, # : Indexing::Index
-                  json_schema_resolver, version
-                )
+                index_def = object_type.own_index_def # : ::ElasticGraph::SchemaDefinition::Indexing::Index
+                identify_missing_necessary_fields_for_index_def(object_type, index_def, json_schema_resolver)
               end
           end
 
-          def identify_missing_necessary_fields_for_index_def(object_type, index_def, json_schema_resolver, json_schema_version)
+          def identify_missing_necessary_fields_for_index_def(object_type, index_def, json_schema_resolver)
             {
               "routing" => index_def.routing_field_path,
               "rollover" => index_def.rollover_config&.timestamp_field_path
             }.compact.filter_map do |field_type, field_path|
               if json_schema_resolver.necessary_path_missing?(field_path)
-                # The JSON schema v # {json_schema_version} artifact has no field that maps to the #{field_type} path of `#{field_path.fully_qualified_path_in_index}`.
-
                 MissingNecessaryField.new(
                   field_type: field_type,
                   fully_qualified_path: field_path.fully_qualified_path_in_index
@@ -198,12 +184,6 @@ def initialize(state, json_schema, old_type_name_by_current_name)
               end
             end
 
-            # Indicates if the given `field_path` is (1) necessary and (2) missing from the JSON schema, indicating a problem.
-            #
-            # - Returns `false` is the given `field_path` is present in the JSON schema.
-            # - Returns `false` is the parent type of `field_path` has not been retained in this JSON schema version
-            #   (in that case, the field path is not necessary).
-            # - Otherwise, returns `true` since the field path is both necessary and missing.
             def necessary_path_missing?(field_path)
               parent_type = field_path.first_part.parent_type.name
 
@@ -227,7 +207,13 @@ def necessary_path_part_missing?(parent_type, name_in_index)
           end
         end
 
+        # @!parse class MissingNecessaryField < ::Data; end
         MissingNecessaryField = ::Data.define(:field_type, :fully_qualified_path)
+
+        # @private
+        class MissingNecessaryField < ::Data
+          # @dynamic initialize, with, field_type, fully_qualified_path
+        end
       end
     end
   end

elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/json_schema_pruner.rb

@@ -7,30 +7,29 @@
 # frozen_string_literal: true
 
 require "elastic_graph/constants"
+require "elastic_graph/json_ingestion"
 
-module ElasticGraph
+module ElasticGraph::JSONIngestion
   module SchemaDefinition
     # Prunes unused type definitions from a given JSON schema.
     #
     # @private
     class JSONSchemaPruner
       def self.prune(original_json_schema)
-        initial_type_names = [EVENT_ENVELOPE_JSON_SCHEMA_NAME] + original_json_schema
-          .dig("$defs", EVENT_ENVELOPE_JSON_SCHEMA_NAME, "properties", "type", "enum")
+        initial_type_names = [::ElasticGraph::EVENT_ENVELOPE_JSON_SCHEMA_NAME] + original_json_schema
+          .dig("$defs", ::ElasticGraph::EVENT_ENVELOPE_JSON_SCHEMA_NAME, "properties", "type", "enum")
 
         types_to_keep = referenced_type_names(initial_type_names, original_json_schema["$defs"])
 
-        # The .select will preserve the sort order of the original hash
-        # standard:disable Style/HashSlice -- https://github.com/soutaro/steep/issues/1503
-        pruned_defs = original_json_schema["$defs"].select { |k, _v| types_to_keep.include?(k) }
+        # The .select will preserve the sort order of the original hash.
+        # standard:disable Style/HashSlice -- We intentionally preserve the dumped definition order.
+        pruned_defs = original_json_schema["$defs"].select { |type_name, _type_def| types_to_keep.include?(type_name) }
         # standard:enable Style/HashSlice
 
         original_json_schema.merge("$defs" => pruned_defs)
       end
 
-      # Returns a list of type names indicating all types referenced from any type in source_type_names.
-      private_class_method
-      def self.referenced_type_names(source_type_names, original_defs)
+      private_class_method def self.referenced_type_names(source_type_names, original_defs)
         return Set.new if source_type_names.empty?
 
         referenced_type_defs = original_defs.slice(*source_type_names)
@@ -39,8 +38,7 @@ def self.referenced_type_names(source_type_names, original_defs)
         referenced_type_names(ref_names, original_defs) + source_type_names
       end
 
-      private_class_method
-      def self.collect_ref_names(hash)
+      private_class_method def self.collect_ref_names(hash)
         hash.flat_map do |key, value|
           case value
           when ::Hash

elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/event_envelope.rbs

@@ -1,4 +1,4 @@
-module ElasticGraph
+module ElasticGraph::JSONIngestion
   module SchemaDefinition
     module Indexing
       module EventEnvelope

elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_field_metadata.rbs

@@ -1,4 +1,4 @@
-module ElasticGraph
+module ElasticGraph::JSONIngestion
   module SchemaDefinition
     module Indexing
       class JSONSchemaFieldMetadata

elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata.rbs

@@ -1,26 +1,26 @@
-module ElasticGraph
+module ElasticGraph::JSONIngestion
   module SchemaDefinition
     module Indexing
       class JSONSchemaWithMetadataSupertype
         attr_reader json_schema: ::Hash[::String, untyped]
         attr_reader missing_fields: ::Set[::String]
         attr_reader missing_types: ::Set[::String]
-        attr_reader definition_conflicts: ::Set[SchemaElements::DeprecatedElement]
+        attr_reader definition_conflicts: ::Set[::ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement]
         attr_reader missing_necessary_fields: ::Array[JSONSchemaWithMetadata::MissingNecessaryField]
 
         def initialize: (
           json_schema: ::Hash[::String, untyped],
           missing_fields: ::Set[::String],
           missing_types: ::Set[::String],
-          definition_conflicts: ::Set[SchemaElements::DeprecatedElement],
+          definition_conflicts: ::Set[::ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement],
           missing_necessary_fields: ::Array[JSONSchemaWithMetadata::MissingNecessaryField]
         ) -> void
 
         def with: (
           ?json_schema: ::Hash[::String, untyped],
           ?missing_fields: ::Set[::String],
           ?missing_types: ::Set[::String],
-          ?definition_conflicts: ::Set[SchemaElements::DeprecatedElement],
+          ?definition_conflicts: ::Set[::ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement],
           ?missing_necessary_fields: ::Array[JSONSchemaWithMetadata::MissingNecessaryField]
         ) -> instance
       end
@@ -30,31 +30,31 @@ module ElasticGraph
 
         class Merger
           @field_metadata_by_type_and_field_name: ::Hash[::String, ::Hash[::String, JSONSchemaFieldMetadata]]
-          @renamed_types_by_old_name: ::Hash[::String, SchemaElements::DeprecatedElement]
-          @deleted_types_by_old_name: ::Hash[::String, SchemaElements::DeprecatedElement]
-          @renamed_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, SchemaElements::DeprecatedElement]]
-          @deleted_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, SchemaElements::DeprecatedElement]]
-          @state: State
+          @renamed_types_by_old_name: ::Hash[::String, ::ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement]
+          @deleted_types_by_old_name: ::Hash[::String, ::ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement]
+          @renamed_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, ::ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement]]
+          @deleted_fields_by_type_name_and_old_field_name: ::Hash[::String, ::Hash[::String, ::ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement]]
+          @state: ::ElasticGraph::SchemaDefinition::State
           @derived_indexing_type_names: ::Set[::String]
 
-          attr_reader unused_deprecated_elements: ::Set[SchemaElements::DeprecatedElement]
+          attr_reader unused_deprecated_elements: ::Set[::ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement]
 
-          def initialize: (Results) -> void
+          def initialize: (::ElasticGraph::SchemaDefinition::Results) -> void
           def merge_metadata_into: (::Hash[::String, untyped]) -> JSONSchemaWithMetadata
 
           private
 
           def determine_current_type_name: (
             ::String,
             missing_types: ::Set[::String],
-            definition_conflicts: ::Set[SchemaElements::DeprecatedElement]
+            definition_conflicts: ::Set[::ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement]
           ) -> ::String?
 
           def field_metadata_for: (
             ::String,
             ::String,
             missing_fields: ::Set[::String],
-            definition_conflicts: ::Set[SchemaElements::DeprecatedElement]
+            definition_conflicts: ::Set[::ElasticGraph::SchemaDefinition::SchemaElements::DeprecatedElement]
           ) -> JSONSchemaFieldMetadata?
 
           def identify_missing_necessary_fields: (
@@ -63,19 +63,18 @@ module ElasticGraph
           ) -> ::Array[MissingNecessaryField]
 
           def identify_missing_necessary_fields_for_index_def: (
-            indexableType,
-            Index,
-            JSONSchemaResolver,
-            ::Integer
+            ::ElasticGraph::SchemaDefinition::indexableType,
+            ::ElasticGraph::SchemaDefinition::Indexing::Index,
+            JSONSchemaResolver
           ) -> ::Array[MissingNecessaryField]
 
           class JSONSchemaResolver
-            @state: State
+            @state: ::ElasticGraph::SchemaDefinition::State
             @old_type_name_by_current_name: ::Hash[::String, ::String]
             @meta_by_old_type_and_name_in_index: ::Hash[::String, ::Hash[::String, ::Hash[::String, untyped]]]
 
-            def initialize: (State, ::Hash[::String, untyped], ::Hash[::String, ::String]) -> void
-            def necessary_path_missing?: (SchemaElements::FieldPath) -> bool
+            def initialize: (::ElasticGraph::SchemaDefinition::State, ::Hash[::String, untyped], ::Hash[::String, ::String]) -> void
+            def necessary_path_missing?: (::ElasticGraph::SchemaDefinition::SchemaElements::FieldPath) -> bool
 
             private
 
@@ -91,6 +90,11 @@ module ElasticGraph
             field_type: ::String,
             fully_qualified_path: ::String
           ) -> void
+
+          def with: (
+            ?field_type: ::String,
+            ?fully_qualified_path: ::String
+          ) -> MissingNecessaryField
         end
       end
     end

elasticgraph-json_ingestion/sig/elastic_graph/json_ingestion/schema_definition/json_schema_pruner.rbs

@@ -1,7 +1,10 @@
-module ElasticGraph
+module ElasticGraph::JSONIngestion
   module SchemaDefinition
     class JSONSchemaPruner
       def self.prune: (::Hash[::String, untyped]) -> ::Hash[::String, untyped]
+
+      private
+
       def self.referenced_type_names: (::Array[::String], ::Hash[::String, untyped]) -> ::Set[::String]
       def self.collect_ref_names: (::Hash[::String, untyped]) -> ::Array[::String]
     end

elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/field.rb

@@ -77,7 +77,7 @@ def json_schema
 
         # @return [JSONSchemaFieldMetadata] additional ElasticGraph metadata to be stored in the JSON schema for this field.
         def json_schema_metadata
-          JSONSchemaFieldMetadata.new(type: type.name, name_in_index: name_in_index)
+          ::ElasticGraph::JSONIngestion::SchemaDefinition::Indexing::JSONSchemaFieldMetadata.new(type: type.name, name_in_index: name_in_index)
         end
 
         # Builds a hash containing the mapping for the provided fields, normalizing it in the same way that the