Generate a protobuf schema artifact from ElasticGraph schemas

elasticgraph-proto_ingestion/README.md

@@ -1,6 +1,7 @@
 # ElasticGraph::ProtoIngestion
 
 An ElasticGraph extension that supports ingesting Protocol Buffer data into ElasticGraph.
+Currently it generates `proto3` Protocol Buffers schema artifacts from ElasticGraph schemas.
 
 ## Dependency Diagram
 
@@ -15,3 +16,130 @@ graph LR;
     elasticgraph-proto_ingestion --> elasticgraph-support;
     class elasticgraph-support otherEgGemStyle;
 ```
+
+## Usage
+
+First, add `elasticgraph-proto_ingestion` to your `Gemfile`, alongside the other ElasticGraph gems:
+
+```diff
+diff --git a/Gemfile b/Gemfile
+index 4a5ef1e..5c16c2b 100644
+--- a/Gemfile
++++ b/Gemfile
+@@ -8,6 +8,7 @@ gem "elasticgraph-query_registry", *elasticgraph_details
+
+ # Can be elasticgraph-elasticsearch or elasticgraph-opensearch based on the datastore you want to use.
+ gem "elasticgraph-opensearch", *elasticgraph_details
++gem "elasticgraph-proto_ingestion", *elasticgraph_details
+
+ gem "httpx", "~> 1.3"
+
+```
+
+Next, update your `Rakefile` so that `ElasticGraph::ProtoIngestion::SchemaDefinition::APIExtension` is
+included in the schema-definition extension modules:
+
+```diff
+diff --git a/Rakefile b/Rakefile
+index 2943335..26633c3 100644
+--- a/Rakefile
++++ b/Rakefile
+@@ -3,5 +3,6 @@
+ require "elastic_graph/json_ingestion/schema_definition/api_extension"
+ require "elastic_graph/local/rake_tasks"
++require "elastic_graph/proto_ingestion/schema_definition/api_extension"
+ require "elastic_graph/query_registry/rake_tasks"
+ require "rspec/core/rake_task"
+ require "standard/rake"
+@@ -16,6 +17,7 @@ ElasticGraph::Local::RakeTasks.new(
+   # Determines casing of field names. Can be either `:camelCase` or `:snake_case`.
+   tasks.schema_element_name_form = :camelCase
+   tasks.schema_definition_extension_modules << ElasticGraph::JSONIngestion::SchemaDefinition::APIExtension
++  tasks.schema_definition_extension_modules << ElasticGraph::ProtoIngestion::SchemaDefinition::APIExtension
+
+   # Customizes the names of fields generated by ElasticGraph.
+   tasks.schema_element_name_overrides = {
+```
+
+Then opt into proto generation from your schema definition:
+
+```ruby
+# in config/schema/protobuf.rb
+
+ElasticGraph.define_schema do |schema|
+  schema.proto_schema_artifacts package_name: "myapp.events.v1"
+end
+```
+
+After running `bundle exec rake schema_artifacts:dump`, ElasticGraph will generate `schema.proto`.
+
+## Schema Definition Options
+
+### Custom Scalar Types
+
+Built-in ElasticGraph scalar types are automatically mapped to proto scalar types.
+For custom scalar types, use `protobuf` to define the proto scalar type:
+
+```ruby
+# in config/schema/money.rb
+
+ElasticGraph.define_schema do |schema|
+  schema.scalar_type "Money" do |t|
+    t.mapping type: "long"
+    t.json_schema type: "integer"
+    t.protobuf type: "int64"
+  end
+end
+```
+
+## Type Mappings
+
+The generated `schema.proto` uses these built-in scalar mappings:
+
+| ElasticGraph Type | Protobuf Type |
+|-------------------|------------|
+| `Boolean`         | `bool`     |
+| `Cursor`          | `string`   |
+| `Date`            | `string`   |
+| `DateTime`        | `string`   |
+| `Float`           | `double`   |
+| `ID`              | `string`   |
+| `Int`             | `int32`    |
+| `JsonSafeLong`    | `int64`    |
+| `LocalTime`       | `string`   |
+| `LongString`      | `int64`    |
+| `String`          | `string`   |
+| `TimeZone`        | `string`   |
+| `Untyped`         | `string`   |
+
+Additionally:
+- List types become `repeated` fields.
+- Lists of lists (e.g. `[[Float!]!]!`) generate wrapper messages, because protobuf has no
+  `repeated repeated` equivalent. Each inner nesting level becomes a generated message with a
+  single `repeated values = 1;` field. For example, the `values` field here:
+
+  ```ruby
+  # in config/schema/matrix.rb
+
+  ElasticGraph.define_schema do |schema|
+    schema.object_type "Matrix" do |t|
+      t.field "id", "ID"
+      t.field "values", "[[Float!]!]!"
+      t.index "matrices"
+    end
+  end
+  ```
+
+  generates:
+
+  ```text
+  message Matrix {
+    string id = 1;
+    repeated MatrixValuesListLevel1 values = 2;
+  }
+
+  message MatrixValuesListLevel1 {
+    repeated double values = 1;
+  }
+  ```
+- Enum types generate `enum` definitions whose values are prefixed with the enum type name in `UPPER_SNAKE_CASE`, including a zero-valued `*_UNSPECIFIED` entry.

elasticgraph-proto_ingestion/elasticgraph-proto_ingestion.gemspec

@@ -36,5 +36,8 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "elasticgraph-support", ElasticGraph::VERSION
 
+  # This gem's schema-definition extension code references `elasticgraph-schema_definition`, but
+  # applications load it through schema-definition tasks after `elasticgraph-schema_definition` is already
+  # available. Keeping this as a development dependency avoids a runtime dependency cycle.
   spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION
 end

elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion.rb

@@ -11,8 +11,5 @@ module ElasticGraph
   module ProtoIngestion
     # The name of the generated Protocol Buffers schema file.
     PROTO_SCHEMA_FILE = "schema.proto"
-
-    # The name of the generated proto field-number mapping file.
-    PROTO_FIELD_NUMBERS_FILE = "proto_field_numbers.yaml"
   end
 end

elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb

@@ -6,7 +6,11 @@
 #
 # frozen_string_literal: true
 
+require "elastic_graph/errors"
 require "elastic_graph/proto_ingestion"
+require "elastic_graph/proto_ingestion/schema_definition/factory_extension"
+require "elastic_graph/proto_ingestion/schema_definition/schema"
+require "elastic_graph/proto_ingestion/schema_definition/state_extension"
 
 module ElasticGraph
   module ProtoIngestion
@@ -16,10 +20,68 @@ module ProtoIngestion
     module SchemaDefinition
       # Module designed to be extended onto an {ElasticGraph::SchemaDefinition::API} instance
       # to enable protobuf schema artifact generation.
-      #
-      # @note The protobuf schema artifact generation logic has not been implemented yet, so
-      #   extending this module is currently a no-op.
       module APIExtension
+        # Maps built-in ElasticGraph scalar types to proto field types.
+        PROTO_TYPES_BY_BUILT_IN_SCALAR_TYPE = {
+          "Boolean" => "bool",
+          "Cursor" => "string",
+          "Date" => "string",
+          "DateTime" => "string",
+          "Float" => "double",
+          "ID" => "string",
+          "Int" => "int32",
+          "JsonSafeLong" => "int64",
+          "LocalTime" => "string",
+          "LongString" => "int64",
+          "String" => "string",
+          "TimeZone" => "string",
+          "Untyped" => "string"
+        }.freeze
+
+        # Wires up the protobuf extensions when this module is extended onto an API instance.
+        #
+        # @param api [ElasticGraph::SchemaDefinition::API] the API instance to extend
+        # @return [void]
+        # @api private
+        def self.extended(api)
+          api.state.extend(StateExtension)
+          api.factory.extend(FactoryExtension)
+
+          api.on_built_in_types do |type|
+            if type.is_a?(SchemaElements::ScalarTypeExtension)
+              # Use the reverted (original) type name so that built-in scalars renamed via
+              # `type_name_overrides` still resolve to their proto types.
+              type.protobuf type: PROTO_TYPES_BY_BUILT_IN_SCALAR_TYPE.fetch(type.type_ref.with_reverted_override.name)
+            end
+          end
+        end
+
+        # Configures protobuf artifact generation behavior.
+        #
+        # @param package_name [String] proto package name to emit
+        # @return [void]
+        #
+        # @example Set the proto package name
+        #   ElasticGraph.define_schema do |schema|
+        #     schema.proto_schema_artifacts package_name: "myapp.events.v1"
+        #   end
+        def proto_schema_artifacts(package_name: "elasticgraph")
+          if !package_name.is_a?(String) || package_name.empty?
+            raise Errors::SchemaError, "`package_name` must be a non-empty String"
+          end
+
+          proto_ingestion_state.package_name = package_name
+          nil
+        end
+
+        private
+
+        # Returns this gem's state container. Centralizes the Steep cast that's needed because
+        # Steep can't see the `extend(StateExtension)` applied at runtime in `extended`.
+        def proto_ingestion_state
+          extension_state = state # : ElasticGraph::SchemaDefinition::State & StateExtension
+          extension_state.proto_ingestion_state
+        end
       end
     end
   end

elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/factory_extension.rb

@@ -0,0 +1,100 @@
+# Copyright 2024 - 2026 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/proto_ingestion/schema_definition/results_extension"
+require "elastic_graph/proto_ingestion/schema_definition/schema_artifact_manager_extension"
+require "elastic_graph/proto_ingestion/schema_definition/schema_elements/enum_type_extension"
+require "elastic_graph/proto_ingestion/schema_definition/schema_elements/object_interface_and_union_extension"
+require "elastic_graph/proto_ingestion/schema_definition/schema_elements/scalar_type_extension"
+
+module ElasticGraph
+  module ProtoIngestion
+    module SchemaDefinition
+      # Extension module applied to Factory to add proto support.
+      module FactoryExtension
+        # Creates a new enum type with proto extensions.
+        #
+        # @param name [String] enum type name
+        # @yield [ElasticGraph::SchemaDefinition::SchemaElements::EnumType]
+        # @return [ElasticGraph::SchemaDefinition::SchemaElements::EnumType]
+        def new_enum_type(name)
+          super(name) do |type|
+            type.extend SchemaElements::EnumTypeExtension
+            yield type if block_given?
+          end
+        end
+
+        # Creates a new interface type with proto extensions.
+        #
+        # @param name [String] interface type name
+        # @yield [ElasticGraph::SchemaDefinition::SchemaElements::InterfaceType]
+        # @return [ElasticGraph::SchemaDefinition::SchemaElements::InterfaceType]
+        def new_interface_type(name)
+          super(name) do |type|
+            type.extend SchemaElements::ObjectInterfaceAndUnionExtension
+            yield type if block_given?
+          end
+        end
+
+        # Creates a new object type with proto extensions.
+        #
+        # @param name [String] object type name
+        # @yield [ElasticGraph::SchemaDefinition::SchemaElements::ObjectType]
+        # @return [ElasticGraph::SchemaDefinition::SchemaElements::ObjectType]
+        def new_object_type(name)
+          super(name) do |type|
+            type.extend SchemaElements::ObjectInterfaceAndUnionExtension
+            yield type if block_given?
+          end
+        end
+
+        # Creates a new scalar type with proto extensions.
+        #
+        # @param name [String] scalar type name
+        # @yield [ElasticGraph::SchemaDefinition::SchemaElements::ScalarType]
+        # @return [ElasticGraph::SchemaDefinition::SchemaElements::ScalarType]
+        def new_scalar_type(name)
+          super(name) do |type|
+            type.extend SchemaElements::ScalarTypeExtension
+            yield type if block_given?
+          end
+        end
+
+        # Creates a new union type with proto extensions.
+        #
+        # @param name [String] union type name
+        # @yield [ElasticGraph::SchemaDefinition::SchemaElements::UnionType]
+        # @return [ElasticGraph::SchemaDefinition::SchemaElements::UnionType]
+        def new_union_type(name)
+          super(name) do |type|
+            type.extend SchemaElements::ObjectInterfaceAndUnionExtension
+            yield type if block_given?
+          end
+        end
+
+        # Creates a new results object and extends it with proto generation APIs.
+        #
+        # @return [ElasticGraph::SchemaDefinition::Results]
+        def new_results
+          super.tap do |results|
+            results.extend ResultsExtension
+          end
+        end
+
+        # Creates a new schema artifact manager and extends it with proto artifact support.
+        #
+        # @return [ElasticGraph::SchemaDefinition::SchemaArtifactManager]
+        def new_schema_artifact_manager(...)
+          super.tap do |manager|
+            manager.extend SchemaArtifactManagerExtension
+          end
+        end
+      end
+    end
+  end
+end