Skip to content

Commit 10cc810

Browse files
dlvenabledlvenable
authored
Generate JSON schemas for plugins with schema definitions for nested configurations (#6814)
Support JSON schema generation with schema definitions for nested types instead of using object. Signed-off-by: David Venable <dlv@amazon.com>
1 parent ad2af4f commit 10cc810

5 files changed

Lines changed: 241 additions & 3 deletions

File tree

‎data-prepper-plugin-schema-cli/src/main/java/org/opensearch/dataprepper/schemas/DataPrepperPluginSchemaExecute.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ public class DataPrepperPluginSchemaExecute implements Runnable {
4444
@CommandLine.Option(names = {"--output_folder"})
4545
private String folderPath;
4646

47+
@CommandLine.Option(names = {"--use_definitions"}, defaultValue = "false")
48+
private boolean useDefinitions;
49+
4750
public static void main(String[] args) {
4851
final int exitCode = new CommandLine(new DataPrepperPluginSchemaExecute()).execute(args);
4952
System.exit(exitCode);
@@ -59,8 +62,9 @@ public void run() {
5962
} catch (IOException e) {
6063
throw new RuntimeException("primary fields override filepath does not exist. ", e);
6164
}
65+
final JsonSchemaConverterConfig converterConfig = new JsonSchemaConverterConfig(useDefinitions);
6266
final PluginConfigsJsonSchemaConverter pluginConfigsJsonSchemaConverter = new PluginConfigsJsonSchemaConverter(
63-
pluginProvider, new JsonSchemaConverter(DataPrepperModules.dataPrepperModules(), pluginProvider),
67+
pluginProvider, new JsonSchemaConverter(DataPrepperModules.dataPrepperModules(), pluginProvider, converterConfig),
6468
primaryFieldsOverride, siteUrl, siteBaseUrl);
6569
final Class<?> pluginType = pluginConfigsJsonSchemaConverter.pluginTypeNameToPluginType(pluginTypeName);
6670
final Map<String, String> pluginNameToJsonSchemaMap = pluginConfigsJsonSchemaConverter.convertPluginConfigsIntoJsonSchemas(

‎data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverter.java‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import com.fasterxml.jackson.databind.node.ObjectNode;
99
import com.github.victools.jsonschema.generator.FieldScope;
1010
import com.github.victools.jsonschema.generator.Module;
11+
import com.github.victools.jsonschema.generator.Option;
1112
import com.github.victools.jsonschema.generator.OptionPreset;
1213
import com.github.victools.jsonschema.generator.SchemaGenerator;
1314
import com.github.victools.jsonschema.generator.SchemaGeneratorConfig;
@@ -16,6 +17,7 @@
1617
import com.github.victools.jsonschema.generator.SchemaGeneratorGeneralConfigPart;
1718
import com.github.victools.jsonschema.generator.SchemaKeyword;
1819
import com.github.victools.jsonschema.generator.SchemaVersion;
20+
import com.github.victools.jsonschema.generator.CustomDefinition;
1921
import org.opensearch.dataprepper.model.annotations.AlsoRequired;
2022
import org.opensearch.dataprepper.model.annotations.ConditionalRequired;
2123
import org.opensearch.dataprepper.model.event.EventKey;
@@ -28,6 +30,7 @@
2830
import java.util.Collections;
2931
import java.util.Arrays;
3032
import java.util.List;
33+
import java.util.Map;
3134
import java.util.Objects;
3235
import java.util.Optional;
3336
import java.util.stream.Collectors;
@@ -38,17 +41,28 @@ public class JsonSchemaConverter {
3841
static final String DEPRECATED_SINCE_KEY = "deprecated";
3942
private final List<Module> jsonSchemaGeneratorModules;
4043
private final PluginProvider pluginProvider;
44+
private final JsonSchemaConverterConfig config;
4145

4246
public JsonSchemaConverter(final List<Module> jsonSchemaGeneratorModules, final PluginProvider pluginProvider) {
47+
this(jsonSchemaGeneratorModules, pluginProvider, JsonSchemaConverterConfig.defaultConfig());
48+
}
49+
50+
public JsonSchemaConverter(final List<Module> jsonSchemaGeneratorModules, final PluginProvider pluginProvider, final JsonSchemaConverterConfig config) {
4351
this.jsonSchemaGeneratorModules = jsonSchemaGeneratorModules;
4452
this.pluginProvider = pluginProvider;
53+
this.config = config;
4554
}
4655

4756
public ObjectNode convertIntoJsonSchema(
4857
final SchemaVersion schemaVersion, final OptionPreset optionPreset, final Class<?> clazz)
4958
throws JsonProcessingException {
5059
final SchemaGeneratorConfigBuilder configBuilder = new SchemaGeneratorConfigBuilder(
5160
schemaVersion, optionPreset);
61+
62+
if (config.isUseDefinitions()) {
63+
configBuilder.with(Option.DEFINITIONS_FOR_ALL_OBJECTS);
64+
}
65+
5266
loadJsonSchemaGeneratorModules(configBuilder);
5367
final SchemaGeneratorConfigPart<FieldScope> scopeSchemaGeneratorConfigPart = configBuilder.forFields();
5468
overrideInstanceAttributeWithDeprecated(scopeSchemaGeneratorConfigPart);
@@ -57,11 +71,12 @@ public ObjectNode convertIntoJsonSchema(
5771
resolveDependentRequiresFields(scopeSchemaGeneratorConfigPart);
5872
overrideDataPrepperPluginTypeAttribute(configBuilder.forTypesInGeneral(), schemaVersion, optionPreset);
5973
overrideTypeAttributeWithConditionalRequired(configBuilder.forTypesInGeneral());
74+
overrideMapTypesAsObjects(configBuilder.forTypesInGeneral());
6075
resolveDataPrepperTypes(scopeSchemaGeneratorConfigPart);
6176
scopeSchemaGeneratorConfigPart.withInstanceAttributeOverride(new ExampleValuesInstanceAttributeOverride());
6277

63-
final SchemaGeneratorConfig config = configBuilder.build();
64-
final SchemaGenerator generator = new SchemaGenerator(config);
78+
final SchemaGeneratorConfig generatorConfig = configBuilder.build();
79+
final SchemaGenerator generator = new SchemaGenerator(generatorConfig);
6580

6681
return generator.generateSchema(clazz);
6782
}
@@ -136,6 +151,19 @@ private void overrideTypeAttributeWithConditionalRequired(
136151
});
137152
}
138153

154+
private void overrideMapTypesAsObjects(
155+
final SchemaGeneratorGeneralConfigPart schemaGeneratorGeneralConfigPart) {
156+
schemaGeneratorGeneralConfigPart.withCustomDefinitionProvider((javaType, context) -> {
157+
if (javaType.isInstanceOf(Map.class)) {
158+
final SchemaGeneratorConfig config = context.getGeneratorConfig();
159+
final ObjectNode objectSchema = config.createObjectNode()
160+
.put(config.getKeyword(SchemaKeyword.TAG_TYPE), config.getKeyword(SchemaKeyword.TAG_TYPE_OBJECT));
161+
return new CustomDefinition(objectSchema, true);
162+
}
163+
return null;
164+
});
165+
}
166+
139167
private ObjectNode constructIfObjectNode(final SchemaGeneratorConfig config,
140168
final ConditionalRequired.SchemaProperty[] schemaProperties) {
141169
final ObjectNode ifObjectNode = config.createObjectNode();

‎data-prepper-plugin-schema/src/main/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterConfig.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*/
9+
10+
package org.opensearch.dataprepper.schemas;
11+
12+
public class JsonSchemaConverterConfig {
13+
private final boolean useDefinitions;
14+
15+
public JsonSchemaConverterConfig(final boolean useDefinitions) {
16+
this.useDefinitions = useDefinitions;
17+
}
18+
19+
public static JsonSchemaConverterConfig defaultConfig() {
20+
return new JsonSchemaConverterConfig(false);
21+
}
22+
23+
public boolean isUseDefinitions() {
24+
return useDefinitions;
25+
}
26+
}

‎data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterIT.java‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,22 +15,27 @@
1515
import com.github.victools.jsonschema.module.jakarta.validation.JakartaValidationModule;
1616
import com.github.victools.jsonschema.module.jakarta.validation.JakartaValidationOption;
1717
import org.junit.jupiter.api.BeforeEach;
18+
import org.junit.jupiter.api.Nested;
1819
import org.junit.jupiter.api.Test;
1920
import org.opensearch.dataprepper.model.annotations.ExampleValues;
2021
import org.opensearch.dataprepper.model.annotations.UsesDataPrepperPlugin;
2122
import org.opensearch.dataprepper.model.configuration.PluginModel;
2223
import org.opensearch.dataprepper.plugin.ClasspathPluginProvider;
2324
import org.opensearch.dataprepper.plugin.PluginProvider;
2425
import org.opensearch.dataprepper.plugins.processor.aggregate.AggregateAction;
26+
import org.opensearch.dataprepper.plugins.processor.grok.GrokProcessorConfig;
2527

2628
import java.util.List;
29+
import java.util.Map;
2730

2831
import static com.github.victools.jsonschema.module.jackson.JacksonOption.RESPECT_JSONPROPERTY_REQUIRED;
2932
import static org.hamcrest.CoreMatchers.equalTo;
3033
import static org.hamcrest.CoreMatchers.instanceOf;
3134
import static org.hamcrest.CoreMatchers.is;
35+
import static org.hamcrest.CoreMatchers.not;
3236
import static org.hamcrest.CoreMatchers.notNullValue;
3337
import static org.hamcrest.MatcherAssert.assertThat;
38+
import static org.hamcrest.Matchers.containsString;
3439

3540
public class JsonSchemaConverterIT {
3641
static final String PROPERTIES_KEY = "properties";
@@ -96,6 +101,52 @@ void test_examples() throws JsonProcessingException {
96101
assertThat(propertyNode.get("examples").get(1).get("description").textValue(), equalTo("This is the second value."));
97102
}
98103

104+
@Nested
105+
class UseDefinitionsTests {
106+
private ObjectNode jsonSchemaNode;
107+
108+
@BeforeEach
109+
void setUp() throws JsonProcessingException {
110+
final JsonSchemaConverterConfig config = new JsonSchemaConverterConfig(true);
111+
final JsonSchemaConverter objectUnderTestWithDefinitions = new JsonSchemaConverter(
112+
List.of(new JacksonModule(RESPECT_JSONPROPERTY_REQUIRED),
113+
new JakartaValidationModule(JakartaValidationOption.NOT_NULLABLE_FIELD_IS_REQUIRED,
114+
JakartaValidationOption.INCLUDE_PATTERN_EXPRESSIONS)),
115+
new ClasspathPluginProvider(), config);
116+
117+
jsonSchemaNode = objectUnderTestWithDefinitions.convertIntoJsonSchema(
118+
SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestConfigWithNestedObject.class);
119+
}
120+
121+
@Test
122+
void verify_the_definition_is_created() {
123+
assertThat(jsonSchemaNode.has("$defs"), is(true));
124+
final JsonNode defsNode = jsonSchemaNode.get("$defs");
125+
assertThat(defsNode, instanceOf(ObjectNode.class));
126+
assertThat(defsNode.has("NestedObject"), is(true));
127+
}
128+
129+
@Test
130+
void verify_the_definition_has_the_correct_fields() {
131+
final JsonNode defsNode = jsonSchemaNode.get("$defs");
132+
final JsonNode nestedObjectDef = defsNode.get("NestedObject");
133+
assertThat(nestedObjectDef.has("properties"), is(true));
134+
final JsonNode propertiesNode = nestedObjectDef.get("properties");
135+
assertThat(propertiesNode.has("field"), is(true));
136+
}
137+
138+
@Test
139+
void verify_the_referencing_object_has_an_actual_reference_to_the_definition() {
140+
final JsonNode rootProperties = jsonSchemaNode.get("properties");
141+
assertThat(rootProperties.has("nested_list"), is(true));
142+
final JsonNode nestedListProperty = rootProperties.get("nested_list");
143+
assertThat(nestedListProperty.has("items"), is(true));
144+
final JsonNode itemsNode = nestedListProperty.get("items");
145+
assertThat(itemsNode.has("$ref"), is(true));
146+
assertThat(itemsNode.get("$ref").textValue(), equalTo("#/$defs/NestedObject"));
147+
}
148+
}
149+
99150
@JsonClassDescription("test config")
100151
static class TestConfig {
101152
@JsonPropertyDescription("The aggregate action description")
@@ -117,4 +168,56 @@ public String getStringValueWithTwoExamples() {
117168
return stringValueWithTwoExamples;
118169
}
119170
}
171+
172+
@Nested
173+
class MapTypeTests {
174+
@Test
175+
void testMapFieldsDoNotCreateDefinitions() throws JsonProcessingException {
176+
final ObjectNode jsonSchemaNode = objectUnderTest.convertIntoJsonSchema(
177+
SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestConfigWithMapFields.class);
178+
179+
if (jsonSchemaNode.has("$defs")) {
180+
final JsonNode defsNode = jsonSchemaNode.get("$defs");
181+
assertThat(defsNode.toString(), not(containsString("Map(")));
182+
}
183+
184+
final JsonNode properties = jsonSchemaNode.get("properties");
185+
assertThat(properties.get("simple_map").get("type").asText(), is("object"));
186+
assertThat(properties.get("nested_map").get("type").asText(), is("object"));
187+
}
188+
189+
@Test
190+
void testGrokProcessorMapFieldsAreInline() throws JsonProcessingException {
191+
final ObjectNode jsonSchemaNode = objectUnderTest.convertIntoJsonSchema(
192+
SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, GrokProcessorConfig.class);
193+
194+
final JsonNode matchField = jsonSchemaNode.get("properties").get("match");
195+
assertThat(matchField.get("type").asText(), is("object"));
196+
197+
if (jsonSchemaNode.has("$defs")) {
198+
final JsonNode defsNode = jsonSchemaNode.get("$defs");
199+
assertThat(defsNode.toString(), not(containsString("Map(")));
200+
}
201+
}
202+
}
203+
204+
@JsonClassDescription("Test config with map fields")
205+
static class TestConfigWithMapFields {
206+
@JsonProperty("simple_map")
207+
private Map<String, String> simpleMap;
208+
209+
@JsonProperty("nested_map")
210+
private Map<String, List<String>> nestedMap;
211+
}
212+
213+
@JsonClassDescription("test config with nested object")
214+
static class TestConfigWithNestedObject {
215+
@JsonProperty("nested_list")
216+
private List<NestedObject> nestedList;
217+
}
218+
219+
static class NestedObject {
220+
@JsonProperty("field")
221+
private String field;
222+
}
120223
}

‎data-prepper-plugin-schema/src/test/java/org/opensearch/dataprepper/schemas/JsonSchemaConverterTest.java‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,15 @@
2222

2323
import java.util.Collections;
2424
import java.util.List;
25+
import java.util.Map;
2526

2627
import static org.hamcrest.CoreMatchers.equalTo;
2728
import static org.hamcrest.CoreMatchers.instanceOf;
2829
import static org.hamcrest.CoreMatchers.is;
30+
import static org.hamcrest.CoreMatchers.not;
2931
import static org.hamcrest.CoreMatchers.notNullValue;
3032
import static org.hamcrest.MatcherAssert.assertThat;
33+
import static org.hamcrest.Matchers.containsString;
3134

3235
@ExtendWith(MockitoExtension.class)
3336
class JsonSchemaConverterTest {
@@ -216,4 +219,78 @@ public String getTestAttributeWithGetter() {
216219

217220
private EventKey testAttributeEventKey;
218221
}
222+
223+
@Test
224+
void testConvertIntoJsonSchemaWithUseDefinitions() throws JsonProcessingException {
225+
final JsonSchemaConverterConfig config = new JsonSchemaConverterConfig(true);
226+
final JsonSchemaConverter jsonSchemaConverter = new JsonSchemaConverter(
227+
Collections.emptyList(), pluginProvider, config);
228+
final ObjectNode jsonSchemaNode = jsonSchemaConverter.convertIntoJsonSchema(
229+
SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestConfigWithNestedObject.class);
230+
231+
assertThat(jsonSchemaNode.has("$defs"), is(true));
232+
final JsonNode defsNode = jsonSchemaNode.get("$defs");
233+
assertThat(defsNode, instanceOf(ObjectNode.class));
234+
}
235+
236+
@Test
237+
void convertIntoJsonSchema_withMapField_shouldNotCreateDefinition() throws JsonProcessingException {
238+
final JsonSchemaConverter jsonSchemaConverter = createObjectUnderTest(
239+
Collections.emptyList(), pluginProvider);
240+
final ObjectNode jsonSchemaNode = jsonSchemaConverter.convertIntoJsonSchema(
241+
SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestConfigWithMap.class);
242+
243+
if (jsonSchemaNode.has("$defs")) {
244+
final JsonNode defsNode = jsonSchemaNode.get("$defs");
245+
assertThat(defsNode.toString(), not(containsString("Map(")));
246+
}
247+
248+
final JsonNode properties = jsonSchemaNode.get("properties");
249+
assertThat(properties, notNullValue());
250+
final JsonNode mapField = properties.get("mapField");
251+
assertThat(mapField, notNullValue());
252+
assertThat(mapField.get("type").asText(), is("object"));
253+
}
254+
255+
@Test
256+
void convertIntoJsonSchema_withNestedMapField_shouldNotCreateDefinition() throws JsonProcessingException {
257+
final JsonSchemaConverter jsonSchemaConverter = createObjectUnderTest(
258+
Collections.emptyList(), pluginProvider);
259+
final ObjectNode jsonSchemaNode = jsonSchemaConverter.convertIntoJsonSchema(
260+
SchemaVersion.DRAFT_2020_12, OptionPreset.PLAIN_JSON, TestConfigWithNestedMap.class);
261+
262+
if (jsonSchemaNode.has("$defs")) {
263+
final JsonNode defsNode = jsonSchemaNode.get("$defs");
264+
assertThat(defsNode.toString(), not(containsString("Map(")));
265+
}
266+
267+
final JsonNode properties = jsonSchemaNode.get("properties");
268+
assertThat(properties, notNullValue());
269+
final JsonNode nestedMapField = properties.get("nestedMap");
270+
assertThat(nestedMapField, notNullValue());
271+
assertThat(nestedMapField.get("type").asText(), is("object"));
272+
}
273+
274+
@JsonClassDescription("Test config with nested object")
275+
static class TestConfigWithNestedObject {
276+
@JsonProperty("nested_list")
277+
private List<NestedObject> nestedList;
278+
}
279+
280+
@JsonClassDescription("Test config with map field")
281+
static class TestConfigWithMap {
282+
@JsonProperty("map_field")
283+
private Map<String, String> mapField;
284+
}
285+
286+
@JsonClassDescription("Test config with nested map field")
287+
static class TestConfigWithNestedMap {
288+
@JsonProperty("nested_map")
289+
private Map<String, List<String>> nestedMap;
290+
}
291+
292+
static class NestedObject {
293+
@JsonProperty("field")
294+
private String field;
295+
}
219296
}

0 commit comments

Comments
 (0)