Scaffold the elasticgraph-proto_ingestion gem (#1297)

jwils · web-flow · commit 4cdc0652b1dd · 2026-07-07T12:46:18.000-05:00
Creates the new `elasticgraph-proto_ingestion` extension gem and wires it into the repo — bundle/lockfile, RBS collection, docker demo image, doctest helper, dependency diagrams, and the `run_gem_specs` flatware list — without any of the Protocol Buffers generation logic yet. The gem currently just defines its artifact file name constants and an `APIExtension` entry point that does not customize anything. Splitting the gem creation out this way keeps the follow-up PR that adds the actual generation logic (#1080) contained entirely to the gem's own directory. Second of a three-PR sequence: #1296 (artifact comment prefixes), this scaffold, then #1080 (the generation logic).
diff --git a/CODEBASE_OVERVIEW.md b/CODEBASE_OVERVIEW.md
@@ -192,13 +192,14 @@ graph LR;
     click opensearch-ruby href "https://rubygems.org/gems/opensearch-ruby" "Open on RubyGems.org" _blank;
 ```
 
-### Extensions (6 gems)
+### Extensions (7 gems)
 
 These libraries extend ElasticGraph to provide optional but commonly needed functionality.
 
 * [elasticgraph-apollo](elasticgraph-apollo/README.md): Transforms an ElasticGraph project into an Apollo subgraph.
 * [elasticgraph-health_check](elasticgraph-health_check/README.md): Provides a health check for high availability ElasticGraph deployments.
 * [elasticgraph-json_ingestion](elasticgraph-json_ingestion/README.md): JSON Schema ingestion support for ElasticGraph.
+* [elasticgraph-proto_ingestion](elasticgraph-proto_ingestion/README.md): Supports ingesting Protocol Buffer data into ElasticGraph.
 * [elasticgraph-query_interceptor](elasticgraph-query_interceptor/README.md): Intercepts ElasticGraph datastore queries.
 * [elasticgraph-query_registry](elasticgraph-query_registry/README.md): Provides a source-controlled query registry for ElasticGraph applications.
 * [elasticgraph-warehouse](elasticgraph-warehouse/README.md): Extends ElasticGraph to support ingestion into a data warehouse.
@@ -218,6 +219,7 @@ graph LR;
     elasticgraph-health_check["eg-health_check"];
     elasticgraph-datastore_core["eg-datastore_core"];
     elasticgraph-json_ingestion["eg-json_ingestion"];
+    elasticgraph-proto_ingestion["eg-proto_ingestion"];
     elasticgraph-query_interceptor["eg-query_interceptor"];
     elasticgraph-schema_artifacts["eg-schema_artifacts"];
     elasticgraph-query_registry["eg-query_registry"];
@@ -231,6 +233,7 @@ graph LR;
     elasticgraph-health_check --> elasticgraph-graphql;
     elasticgraph-health_check --> elasticgraph-support;
     elasticgraph-json_ingestion --> elasticgraph-support;
+    elasticgraph-proto_ingestion --> elasticgraph-support;
     elasticgraph-query_interceptor --> elasticgraph-graphql;
     elasticgraph-query_interceptor --> elasticgraph-schema_artifacts;
     elasticgraph-query_registry --> elasticgraph-graphql;
@@ -246,6 +249,7 @@ graph LR;
     class elasticgraph-health_check targetGemStyle;
     class elasticgraph-datastore_core otherEgGemStyle;
     class elasticgraph-json_ingestion targetGemStyle;
+    class elasticgraph-proto_ingestion targetGemStyle;
     class elasticgraph-query_interceptor targetGemStyle;
     class elasticgraph-schema_artifacts otherEgGemStyle;
     class elasticgraph-query_registry targetGemStyle;
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -140,6 +140,12 @@ PATH
       faraday-retry (~> 2.4)
       opensearch-ruby (~> 3.4)
 
+PATH
+  remote: elasticgraph-proto_ingestion
+  specs:
+    elasticgraph-proto_ingestion (1.2.1.pre)
+      elasticgraph-support (= 1.2.1.pre)
+
 PATH
   remote: elasticgraph-query_interceptor
   specs:
@@ -688,6 +694,7 @@ DEPENDENCIES
   elasticgraph-lambda_support (= 1.2.1.pre)!
   elasticgraph-local (= 1.2.1.pre)!
   elasticgraph-opensearch (= 1.2.1.pre)!
+  elasticgraph-proto_ingestion (= 1.2.1.pre)!
   elasticgraph-query_interceptor (= 1.2.1.pre)!
   elasticgraph-query_registry (= 1.2.1.pre)!
   elasticgraph-rack (= 1.2.1.pre)!
@@ -779,6 +786,7 @@ CHECKSUMS
   elasticgraph-lambda_support (1.2.1.pre)
   elasticgraph-local (1.2.1.pre)
   elasticgraph-opensearch (1.2.1.pre)
+  elasticgraph-proto_ingestion (1.2.1.pre)
   elasticgraph-query_interceptor (1.2.1.pre)
   elasticgraph-query_registry (1.2.1.pre)
   elasticgraph-rack (1.2.1.pre)
diff --git a/config/docker_demo/Dockerfile b/config/docker_demo/Dockerfile
@@ -19,6 +19,7 @@ COPY elasticgraph-indexer elasticgraph-indexer/
 COPY elasticgraph-json_ingestion elasticgraph-json_ingestion/
 COPY elasticgraph-local elasticgraph-local/
 COPY elasticgraph-opensearch elasticgraph-opensearch/
+COPY elasticgraph-proto_ingestion elasticgraph-proto_ingestion/
 COPY elasticgraph-query_registry elasticgraph-query_registry/
 COPY elasticgraph-rack elasticgraph-rack/
 COPY elasticgraph-schema_artifacts elasticgraph-schema_artifacts/
diff --git a/config/site/support/doctest_helper.rb b/config/site/support/doctest_helper.rb
@@ -8,6 +8,7 @@
 
 require "elastic_graph/apollo/schema_definition/api_extension"
 require "elastic_graph/json_ingestion/schema_definition/api_extension"
+require "elastic_graph/proto_ingestion/schema_definition/api_extension"
 require "elastic_graph/schema_artifacts/runtime_metadata/schema_element_names"
 require "elastic_graph/schema_definition/api"
 require "elastic_graph/schema_definition/schema_artifact_manager"
@@ -53,6 +54,7 @@ module ElasticGraph
       "ElasticGraph.define_schema" => [],
       "ElasticGraph::Apollo::SchemaDefinition" => [Apollo::SchemaDefinition::APIExtension],
       "ElasticGraph::JSONIngestion::SchemaDefinition" => [JSONIngestion::SchemaDefinition::APIExtension],
+      "ElasticGraph::ProtoIngestion::SchemaDefinition" => [ProtoIngestion::SchemaDefinition::APIExtension],
       "ElasticGraph::SchemaDefinition" => [],
       "ElasticGraph::Warehouse::SchemaDefinition" => [Warehouse::SchemaDefinition::APIExtension]
     }
diff --git a/elasticgraph-proto_ingestion/.rspec b/elasticgraph-proto_ingestion/.rspec
@@ -0,0 +1 @@
+../spec_support/subdir_dot_rspec
diff --git a/elasticgraph-proto_ingestion/.yardopts b/elasticgraph-proto_ingestion/.yardopts
@@ -0,0 +1 @@
+../config/site/yardopts
diff --git a/elasticgraph-proto_ingestion/Gemfile b/elasticgraph-proto_ingestion/Gemfile
@@ -0,0 +1 @@
+../Gemfile
diff --git a/elasticgraph-proto_ingestion/LICENSE.txt b/elasticgraph-proto_ingestion/LICENSE.txt
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 - 2026 Block, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/elasticgraph-proto_ingestion/README.md b/elasticgraph-proto_ingestion/README.md
@@ -0,0 +1,17 @@
+# ElasticGraph::ProtoIngestion
+
+An ElasticGraph extension that supports ingesting Protocol Buffer data into ElasticGraph.
+
+## Dependency Diagram
+
+```mermaid
+graph LR;
+    classDef targetGemStyle fill:#FADBD8,stroke:#EC7063,color:#000,stroke-width:2px;
+    classDef otherEgGemStyle fill:#A9DFBF,stroke:#2ECC71,color:#000;
+    classDef externalGemStyle fill:#E0EFFF,stroke:#70A1D7,color:#2980B9;
+    elasticgraph-proto_ingestion["elasticgraph-proto_ingestion"];
+    class elasticgraph-proto_ingestion targetGemStyle;
+    elasticgraph-support["elasticgraph-support"];
+    elasticgraph-proto_ingestion --> elasticgraph-support;
+    class elasticgraph-support otherEgGemStyle;
+```
diff --git a/elasticgraph-proto_ingestion/elasticgraph-proto_ingestion.gemspec b/elasticgraph-proto_ingestion/elasticgraph-proto_ingestion.gemspec
@@ -0,0 +1,40 @@
+# Copyright 2024 - 2026 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require_relative "../elasticgraph-support/lib/elastic_graph/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "elasticgraph-proto_ingestion"
+  spec.version = ElasticGraph::VERSION
+  spec.authors = ["Josh Wilson", "Myron Marston", "Block Engineering"]
+  spec.email = ["joshuaw@squareup.com"]
+  spec.homepage = "https://block.github.io/elasticgraph/"
+  spec.license = "MIT"
+  spec.summary = "Supports ingesting Protocol Buffer data into ElasticGraph."
+
+  spec.metadata = {
+    "bug_tracker_uri" => "https://github.com/block/elasticgraph/issues",
+    "changelog_uri" => "https://github.com/block/elasticgraph/releases/tag/v#{ElasticGraph::VERSION}",
+    "documentation_uri" => "https://block.github.io/elasticgraph/api-docs/v#{ElasticGraph::VERSION}/",
+    "homepage_uri" => "https://block.github.io/elasticgraph/",
+    "source_code_uri" => "https://github.com/block/elasticgraph/tree/v#{ElasticGraph::VERSION}/#{spec.name}",
+    "gem_category" => "extension"
+  }
+
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (f == __FILE__) || f.match(%r{\A(?:(?:test|spec|features|sig)/|\.(?:git|travis|circleci)|appveyor)})
+    end - [".rspec", "Gemfile", ".yardopts"]
+  end
+
+  spec.required_ruby_version = [">= 3.4", "< 4.1"]
+
+  spec.add_dependency "elasticgraph-support", ElasticGraph::VERSION
+
+  spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION
+end
diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion.rb
@@ -0,0 +1,18 @@
+# Copyright 2024 - 2026 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+module ElasticGraph
+  # Namespace for Protocol Buffers schema artifact generation extensions.
+  module ProtoIngestion
+    # The name of the generated Protocol Buffers schema file.
+    PROTO_SCHEMA_FILE = "schema.proto"
+
+    # The name of the generated proto field-number mapping file.
+    PROTO_FIELD_NUMBERS_FILE = "proto_field_numbers.yaml"
+  end
+end
diff --git a/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb b/elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/api_extension.rb
@@ -0,0 +1,26 @@
+# Copyright 2024 - 2026 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/proto_ingestion"
+
+module ElasticGraph
+  module ProtoIngestion
+    # Namespace for all protobuf schema definition support.
+    #
+    # {SchemaDefinition::APIExtension} is the primary entry point and should be used as a schema definition extension module.
+    module SchemaDefinition
+      # Module designed to be extended onto an {ElasticGraph::SchemaDefinition::API} instance
+      # to enable protobuf schema artifact generation.
+      #
+      # @note The protobuf schema artifact generation logic has not been implemented yet, so
+      #   extending this module is currently a no-op.
+      module APIExtension
+      end
+    end
+  end
+end
diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion.rbs
@@ -0,0 +1,6 @@
+module ElasticGraph
+  module ProtoIngestion
+    PROTO_SCHEMA_FILE: ::String
+    PROTO_FIELD_NUMBERS_FILE: ::String
+  end
+end
diff --git a/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs b/elasticgraph-proto_ingestion/sig/elastic_graph/proto_ingestion/schema_definition/api_extension.rbs
@@ -0,0 +1,8 @@
+module ElasticGraph
+  module ProtoIngestion
+    module SchemaDefinition
+      module APIExtension
+      end
+    end
+  end
+end
diff --git a/elasticgraph-proto_ingestion/spec/spec_helper.rb b/elasticgraph-proto_ingestion/spec/spec_helper.rb
@@ -0,0 +1,10 @@
+# Copyright 2024 - 2026 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+# This file contains RSpec configuration for `elasticgraph-proto_ingestion`.
+# It is loaded by the shared spec helper at `spec_support/spec_helper.rb`.
diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion/schema_definition/api_extension_spec.rb
@@ -0,0 +1,33 @@
+# Copyright 2024 - 2026 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/proto_ingestion/schema_definition/api_extension"
+require "elastic_graph/spec_support/schema_definition_helpers"
+
+module ElasticGraph
+  module ProtoIngestion
+    module SchemaDefinition
+      RSpec.describe APIExtension do
+        include_context "SchemaDefinitionHelpers"
+
+        it "can be used as a schema definition extension module, without customizing the schema definition yet" do
+          with_extension, without_extension = [[APIExtension], []].map do |extension_modules|
+            define_schema(schema_element_name_form: "snake_case", extension_modules: extension_modules) do |schema|
+              schema.object_type "Widget" do |t|
+                t.field "id", "ID!"
+                t.index "widgets"
+              end
+            end.graphql_schema_string
+          end
+
+          expect(with_extension).to eq(without_extension)
+        end
+      end
+    end
+  end
+end
diff --git a/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion_spec.rb b/elasticgraph-proto_ingestion/spec/unit/elastic_graph/proto_ingestion_spec.rb
@@ -0,0 +1,21 @@
+# Copyright 2024 - 2026 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/proto_ingestion"
+
+module ElasticGraph
+  RSpec.describe ProtoIngestion do
+    it "defines the PROTO_SCHEMA_FILE constant" do
+      expect(ProtoIngestion::PROTO_SCHEMA_FILE).to eq("schema.proto")
+    end
+
+    it "defines the PROTO_FIELD_NUMBERS_FILE constant" do
+      expect(ProtoIngestion::PROTO_FIELD_NUMBERS_FILE).to eq("proto_field_numbers.yaml")
+    end
+  end
+end
diff --git a/elasticgraph-support/README.md b/elasticgraph-support/README.md
@@ -51,6 +51,9 @@ graph LR;
     elasticgraph-opensearch["elasticgraph-opensearch"];
     elasticgraph-opensearch --> elasticgraph-support;
     class elasticgraph-opensearch otherEgGemStyle;
+    elasticgraph-proto_ingestion["elasticgraph-proto_ingestion"];
+    elasticgraph-proto_ingestion --> elasticgraph-support;
+    class elasticgraph-proto_ingestion otherEgGemStyle;
     elasticgraph-query_registry["elasticgraph-query_registry"];
     elasticgraph-query_registry --> elasticgraph-support;
     class elasticgraph-query_registry otherEgGemStyle;
diff --git a/rbs_collection.yaml b/rbs_collection.yaml
@@ -82,6 +82,8 @@ gems:
     ignore: true
   - name: elasticgraph-opensearch
     ignore: true
+  - name: elasticgraph-proto_ingestion
+    ignore: true
   - name: elasticgraph-query_interceptor
     ignore: true
   - name: elasticgraph-query_registry
