Skip to content

Commit f3a3547

Browse files
jwilsjwils
committed
Migrate JSON ingestion specs into gem
1 parent 7371130 commit f3a3547

13 files changed

Lines changed: 666 additions & 12 deletions

File tree

‎elasticgraph-json_ingestion/spec/integration/elastic_graph/json_ingestion/schema_definition/schema_artifact_manager_extension_spec.rb‎

Lines changed: 342 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,342 @@
1+
# Copyright 2024 - 2026 Block, Inc.
2+
#
3+
# Use of this source code is governed by an MIT-style
4+
# license that can be found in the LICENSE file or at
5+
# https://opensource.org/licenses/MIT.
6+
#
7+
# frozen_string_literal: true
8+
9+
require "elastic_graph/constants"
10+
require "elastic_graph/json_ingestion/schema_definition/api_extension"
11+
require "elastic_graph/schema_definition/rake_tasks"
12+
require "fileutils"
13+
require "yaml"
14+
15+
module ElasticGraph
16+
module JSONIngestion
17+
module SchemaDefinition
18+
RSpec.describe SchemaArtifactManagerExtension, :in_temp_dir, :rake_task do
19+
after do
20+
Thread.current[:eg_schema_load_count] = nil
21+
end
22+
23+
it "dumps public JSON schemas and private versioned JSON schemas with ElasticGraph metadata" do
24+
write_schema(json_schema_version: 1)
25+
output = run_rake("schema_artifacts:dump")
26+
27+
expect(output.lines).to include(
28+
a_string_including("Dumped", JSON_SCHEMAS_FILE),
29+
a_string_including("Dumped", versioned_json_schema_file(1))
30+
)
31+
32+
public_id_schema = read_yaml_artifact(JSON_SCHEMAS_FILE).dig("$defs", "Widget", "properties", "id")
33+
versioned_id_schema = read_yaml_artifact(versioned_json_schema_file(1)).dig("$defs", "Widget", "properties", "id")
34+
35+
expect(public_id_schema).to eq(json_schema_for_keyword_type("ID"))
36+
expect(versioned_id_schema).to eq(json_schema_for_keyword_type("ID", {
37+
"ElasticGraph" => {
38+
"type" => "ID!",
39+
"nameInIndex" => "id"
40+
}
41+
}))
42+
43+
expect(run_rake("schema_artifacts:dump")).to include("is already up to date", JSON_SCHEMAS_FILE)
44+
end
45+
46+
it "requires JSON schema version bumps unless enforcement is disabled" do
47+
write_schema(json_schema_version: 1)
48+
run_rake("schema_artifacts:dump")
49+
50+
write_schema(json_schema_version: 2)
51+
expect {
52+
run_rake("schema_artifacts:dump")
53+
}.to change { read_artifact(JSON_SCHEMAS_FILE) }
54+
.from(a_string_including("\njson_schema_version: 1\n"))
55+
.to(a_string_including("\njson_schema_version: 2\n"))
56+
57+
write_schema(json_schema_version: 2, extra_widget_body: "t.field 'color', 'String!'")
58+
expect {
59+
run_rake("schema_artifacts:dump")
60+
}.to abort_with a_string_including(
61+
"A change has been attempted to `json_schemas.yaml`",
62+
"`schema.json_schema_version 3`"
63+
).and matching(/line \d+ at `(\S*\/?)schema\.rb`/)
64+
65+
write_schema(
66+
json_schema_version: 2,
67+
extra_widget_body: "t.field 'color', 'String!'",
68+
enforce_json_schema_version: false
69+
)
70+
71+
expect(run_rake("schema_artifacts:dump")).to include(
72+
"WARNING: the `json_schemas.yaml` artifact is being updated without the `json_schema_version` being correspondingly incremented"
73+
)
74+
end
75+
76+
it "keeps field metadata up to date on every versioned JSON schema" do
77+
write_schema(json_schema_version: 1)
78+
run_rake("schema_artifacts:dump")
79+
80+
write_schema(json_schema_version: 2, extra_widget_body: "t.field 'color', 'String!'")
81+
run_rake("schema_artifacts:dump")
82+
83+
write_schema(
84+
json_schema_version: 2,
85+
name_field_suffix: ", name_in_index: 'name2'",
86+
extra_widget_body: "t.field 'color', 'String!'"
87+
)
88+
run_rake("schema_artifacts:dump")
89+
90+
loaded_v1 = read_yaml_artifact(versioned_json_schema_file(1))
91+
loaded_v2 = read_yaml_artifact(versioned_json_schema_file(2))
92+
93+
expect(loaded_v1.dig("$defs", "Widget", "properties", "name")).to eq(
94+
json_schema_for_keyword_type("String", {
95+
"ElasticGraph" => {
96+
"type" => "String!",
97+
"nameInIndex" => "name2"
98+
}
99+
})
100+
)
101+
expect(loaded_v1.dig("$defs", "Widget", "properties", "color")).to eq(nil)
102+
103+
expect(loaded_v2.dig("$defs", "Widget", "properties", "name")).to eq(
104+
json_schema_for_keyword_type("String", {
105+
"ElasticGraph" => {
106+
"type" => "String!",
107+
"nameInIndex" => "name2"
108+
}
109+
})
110+
)
111+
expect(loaded_v2.dig("$defs", "Widget", "properties", "color")).to eq(
112+
json_schema_for_keyword_type("String", {
113+
"ElasticGraph" => {
114+
"type" => "String!",
115+
"nameInIndex" => "color"
116+
}
117+
})
118+
)
119+
end
120+
121+
it "gives clear errors for old schema versions with missing fields or types" do
122+
write_schema(json_schema_version: 8)
123+
run_rake("schema_artifacts:dump")
124+
write_schema(json_schema_version: 9, omit_widget_name_field: true)
125+
expect { run_rake("schema_artifacts:dump") }.to abort_with a_string_including(
126+
"The `Widget.name` field (which existed in JSON schema version 8) no longer exists",
127+
"at this old version",
128+
"delete its file from `json_schemas_by_version`"
129+
)
130+
131+
write_schema(json_schema_version: 9)
132+
run_rake("schema_artifacts:dump")
133+
write_schema(json_schema_version: 10, omit_widget_name_field: true)
134+
expect { run_rake("schema_artifacts:dump") }.to abort_with a_string_including(
135+
"The `Widget.name` field (which existed in JSON schema versions 8 and 9) no longer exists",
136+
"at these old versions",
137+
"delete their files from `json_schemas_by_version`"
138+
)
139+
140+
write_schema(json_schema_version: 10)
141+
run_rake("schema_artifacts:dump")
142+
write_schema(json_schema_version: 11, omit_widget_name_field: true)
143+
expect { run_rake("schema_artifacts:dump") }.to abort_with a_string_including(
144+
"The `Widget.name` field (which existed in JSON schema versions 8, 9, and 10) no longer exists"
145+
)
146+
147+
write_schema(json_schema_version: 11, omit_widget_name_field: true, extra_widget_body: "t.field('full_name', 'String') { |f| f.renamed_from 'name' }")
148+
run_rake("schema_artifacts:dump")
149+
150+
delete_artifact(JSON_SCHEMAS_FILE)
151+
write_schema(json_schema_version: 11, omit_widget_name_field: true, extra_widget_body: "t.deleted_field 'name'")
152+
run_rake("schema_artifacts:dump")
153+
154+
delete_artifacts
155+
write_schema(json_schema_version: 1)
156+
run_rake("schema_artifacts:dump")
157+
write_schema(json_schema_version: 2, widget_type_name: "Widget2")
158+
expect { run_rake("schema_artifacts:dump") }.to abort_with a_string_including(
159+
"The `Widget` type (which existed in JSON schema version 1) no longer exists",
160+
"If the `Widget` type has been renamed"
161+
)
162+
end
163+
164+
it "reports deprecated schema element warnings, conflicts, and missing necessary fields" do
165+
::File.write("schema.rb", <<~EOS)
166+
ElasticGraph.define_schema do |schema|
167+
schema.json_schema_version 1
168+
schema.deleted_type "SomeType"
169+
170+
schema.object_type "Widget" do |t|
171+
t.renamed_from "OldWidget"
172+
t.deleted_field "old_name"
173+
t.field "id", "ID!"
174+
t.field "name", "String" do |f|
175+
f.renamed_from "old_name"
176+
end
177+
t.index "widgets"
178+
end
179+
end
180+
EOS
181+
182+
expect(run_rake("schema_artifacts:dump")).to include(
183+
"The schema definition has 4 unneeded reference(s)",
184+
"`schema.deleted_type \"SomeType\"`",
185+
"`type.renamed_from \"OldWidget\"`",
186+
"`type.deleted_field \"old_name\"`",
187+
"`field.renamed_from \"old_name\"`"
188+
)
189+
190+
delete_artifacts
191+
::File.write("schema.rb", <<~EOS)
192+
ElasticGraph.define_schema do |schema|
193+
schema.json_schema_version 1
194+
schema.deleted_type "Widget"
195+
196+
schema.object_type "Widget" do |t|
197+
t.field "id", "ID!"
198+
t.index "widgets"
199+
200+
t.field "token", "ID" do |f|
201+
f.renamed_from "id"
202+
end
203+
t.deleted_field "id"
204+
end
205+
end
206+
EOS
207+
208+
expect {
209+
run_rake("schema_artifacts:dump")
210+
}.to abort_with a_string_including(
211+
"The schema definition of `Widget` has conflicts",
212+
"The schema definition of `Widget.id` has conflicts"
213+
)
214+
215+
delete_artifacts
216+
::File.write("schema.rb", <<~EOS)
217+
ElasticGraph.define_schema do |schema|
218+
schema.json_schema_version 1
219+
220+
schema.object_type "Embedded" do |t|
221+
t.field "workspace_id", "ID"
222+
t.field "created_at", "DateTime"
223+
end
224+
225+
schema.object_type "Widget" do |t|
226+
t.field "id", "ID"
227+
t.field "embedded", "Embedded"
228+
t.index "widgets" do |i|
229+
i.route_with "embedded.workspace_id"
230+
i.rollover :yearly, "embedded.created_at"
231+
end
232+
end
233+
end
234+
EOS
235+
236+
run_rake("schema_artifacts:dump")
237+
238+
::File.write("schema.rb", <<~EOS)
239+
ElasticGraph.define_schema do |schema|
240+
schema.json_schema_version 2
241+
242+
schema.object_type "Embedded" do |t|
243+
t.field "workspace_id2", "ID", name_in_index: "workspace_id"
244+
t.deleted_field "workspace_id"
245+
246+
t.field "created_at2", "DateTime", name_in_index: "created_at"
247+
t.deleted_field "created_at"
248+
end
249+
250+
schema.object_type "Widget" do |t|
251+
t.field "id", "ID"
252+
t.field "embedded", "Embedded"
253+
t.index "widgets" do |i|
254+
i.route_with "embedded.workspace_id2"
255+
i.rollover :yearly, "embedded.created_at2"
256+
end
257+
end
258+
end
259+
EOS
260+
261+
expect {
262+
run_rake("schema_artifacts:dump")
263+
}.to abort_with a_string_including(
264+
"JSON schema version 1 has no field that maps to the routing field path of `Widget.embedded.workspace_id`",
265+
"JSON schema version 1 has no field that maps to the rollover field path of `Widget.embedded.created_at`"
266+
)
267+
end
268+
269+
def write_schema(
270+
json_schema_version:,
271+
enforce_json_schema_version: true,
272+
widget_type_name: "Widget",
273+
name_field_suffix: "",
274+
extra_widget_body: "",
275+
omit_widget_name_field: false
276+
)
277+
::File.write("schema.rb", <<~EOS)
278+
Thread.current[:eg_schema_load_count] = (Thread.current[:eg_schema_load_count] || 0) + 1
279+
raise "Schema file was loaded more than once!" if Thread.current[:eg_schema_load_count] > 1
280+
281+
ElasticGraph.define_schema do |schema|
282+
schema.json_schema_version #{json_schema_version}
283+
#{"schema.enforce_json_schema_version false" unless enforce_json_schema_version}
284+
285+
schema.object_type "#{widget_type_name}" do |t|
286+
t.field "id", "ID!"
287+
#{%(t.field "name", "String!"#{name_field_suffix}) unless omit_widget_name_field}
288+
#{extra_widget_body}
289+
t.index "widgets"
290+
end
291+
end
292+
EOS
293+
end
294+
295+
def run_rake(*args)
296+
Thread.current[:eg_schema_load_count] = nil
297+
298+
super(*args) do |output|
299+
::ElasticGraph::SchemaDefinition::RakeTasks.new(
300+
schema_element_name_form: :snake_case,
301+
index_document_sizes: true,
302+
path_to_schema: "schema.rb",
303+
schema_artifacts_directory: "config/schema/artifacts",
304+
extension_modules: [APIExtension],
305+
output: output
306+
)
307+
end
308+
end
309+
310+
def read_artifact(*name_parts)
311+
path = ::File.join("config", "schema", "artifacts", *name_parts)
312+
::File.exist?(path) && ::File.read(path)
313+
end
314+
315+
def read_yaml_artifact(*name_parts)
316+
::YAML.safe_load(read_artifact(*name_parts))
317+
end
318+
319+
def delete_artifact(*name_parts)
320+
::File.delete(::File.join("config", "schema", "artifacts", *name_parts))
321+
end
322+
323+
def delete_artifacts
324+
::FileUtils.rm_rf(::File.join("config", "schema", "artifacts"))
325+
end
326+
327+
def versioned_json_schema_file(version)
328+
::File.join(JSON_SCHEMAS_BY_VERSION_DIRECTORY, "v#{version}.yaml")
329+
end
330+
331+
def json_schema_for_keyword_type(type, extras = {})
332+
{
333+
"allOf" => [
334+
{"$ref" => "#/$defs/#{type}"},
335+
{"maxLength" => DEFAULT_MAX_KEYWORD_LENGTH}
336+
]
337+
}.merge(extras)
338+
end
339+
end
340+
end
341+
end
342+
end

‎elasticgraph-json_ingestion/spec/spec_helper.rb‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# Copyright 2024 - 2026 Block, Inc.
2+
#
3+
# Use of this source code is governed by an MIT-style
4+
# license that can be found in the LICENSE file or at
5+
# https://opensource.org/licenses/MIT.
6+
#
7+
# frozen_string_literal: true
8+
9+
# This file contains RSpec configuration for `elasticgraph-json_ingestion`.
10+
# It is loaded by the shared spec helper at `spec_support/spec_helper.rb`.

elasticgraph-schema_definition/spec/support/json_schema_matcher.rb renamed to elasticgraph-json_ingestion/spec/support/json_schema_matcher.rb

File renamed without changes.

elasticgraph-schema_definition/spec/support/json_schema_matcher_spec.rb renamed to elasticgraph-json_ingestion/spec/support/json_schema_matcher_spec.rb

File renamed without changes.

elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/indexing/json_schema_with_metadata_spec.rb renamed to elasticgraph-json_ingestion/spec/unit/elastic_graph/json_ingestion/schema_definition/indexing/json_schema_with_metadata_spec.rb

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@
1010
require "elastic_graph/spec_support/schema_definition_helpers"
1111

1212
module ElasticGraph
13-
module SchemaDefinition
13+
module JSONIngestion::SchemaDefinition
1414
module Indexing
15-
::RSpec.describe JSONIngestion::SchemaDefinition::Indexing::JSONSchemaWithMetadata do
15+
::RSpec.describe JSONSchemaWithMetadata do
1616
include_context "SchemaDefinitionHelpers"
1717

1818
it "ignores derived indexed types that do not show up in the JSON schema" do
@@ -1056,7 +1056,7 @@ def metadata_for(json_schema, type, field)
10561056
def define_schema(&schema_definition)
10571057
super(
10581058
schema_element_name_form: "snake_case",
1059-
extension_modules: [JSONIngestion::SchemaDefinition::APIExtension],
1059+
extension_modules: [APIExtension],
10601060
&schema_definition
10611061
)
10621062
end

0 commit comments

Comments
 (0)