Backport master fixes (#1254)

* fixes #1248 validating draft4 schemas (#1249)

* fixes #1248 validating draft4 schemas

* Address regex factory review comments

* Rename regex specification version variable

* fixes #1174 TextNodes as schema seem to validate any value (#1250)

* fixes #1174 TextNodes as schema seem to validate any value

* Address loaded schema validation for issue 1174

* Address schema node validation review comments

* Preserve mapped schema override path

* Validate referenced document fragment schemas

* Remove duplicate schema type validation

* Validate loaded schemas for anchor refs

* fixes #1252 problem with additionalProperties schema in jsconfig schema (#1253)

* Add method to SpecificationVersion to determine from schema node (#1221)

* Manual adjustments for Jackson 2 compatibility and CVE fix

---------

Co-authored-by: Justin Tay <49700559+justin-tay@users.noreply.github.com>

Author: stevehu

README.md

@@ -346,7 +346,7 @@ The specification allows for the `$schema` keyword not to be specified, in which
 The following example creates a `SchemaRegistry` that does not specify a default dialect and will throw a `MissingSchemaKeywordException` if the schema does not specify a dialect using the `$schema` keyword.
 
 ```java
-SchemaRegistry registry = SchemaRegistry.builder().dialectRegistry(new BasicDialectRegistry(Dialects.getDraft202012())).build();
+SchemaRegistry registry = SchemaRegistry.withDefaultDialectId(null);
 ```
 
 ### Results and output formats

pom.xml

@@ -73,7 +73,7 @@
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 
     <version.itu>1.14.0</version.itu>
-    <version.jackson>2.18.3</version.jackson>
+    <version.jackson>2.18.4</version.jackson>
     <version.joni>2.2.6</version.joni>
     <version.logback>1.3.14</version.logback> <!-- 1.4.x and above is not Java 8 compatible -->
     <version.slf4j>2.0.17</version.slf4j>

src/main/java/com/networknt/schema/SchemaRegistry.java

@@ -282,6 +282,26 @@ public static SchemaRegistry withDefaultDialect(SpecificationVersion specificati
      * Creates a new schema registry with a default schema dialect. The schema
      * dialect will only be used if the input does not specify a $schema.
      * <p>
+     * If the dialectId is null then the $schema is mandatory.
+     * <p>
+     * This uses a dialect registry that contains all the supported standard
+     * specification dialects, Draft 4, Draft 6, Draft 7, Draft 2019-09 and Draft
+     * 2020-12.
+     * 
+     * @param dialectId  the default dialect id used when the schema does not
+     *                   specify the $schema keyword
+     * @return the factory
+     */
+    public static SchemaRegistry withDefaultDialectId(String dialectId) {
+        return withDefaultDialectId(dialectId, null);
+    }
+
+    /**
+     * Creates a new schema registry with a default schema dialect. The schema
+     * dialect will only be used if the input does not specify a $schema.
+     * <p>
+     * If the dialectId is null then the $schema is mandatory.
+     * <p>
      * This uses a dialect registry that contains all the supported standard
      * specification dialects, Draft 4, Draft 6, Draft 7, Draft 2019-09 and Draft
      * 2020-12.
@@ -449,9 +469,10 @@ public SchemaLoader getSchemaLoader() {
      * @return the schema
      */
     protected Schema newSchema(SchemaLocation schemaUri, JsonNode schemaNode) {
+        SchemaLocation schemaLocation = getSchemaLocation(schemaUri);
+        validateSchemaNodeNotNull(schemaLocation, schemaNode);
         final SchemaContext schemaContext = createSchemaContext(schemaNode);
-        Schema jsonSchema = doCreate(schemaContext, getSchemaLocation(schemaUri),
-                schemaNode, null, false);
+        Schema jsonSchema = doCreate(schemaContext, schemaLocation, schemaNode, null, false);
         preload(jsonSchema);
         return jsonSchema;
     }
@@ -487,10 +508,46 @@ public Schema create(SchemaContext schemaContext, SchemaLocation schemaLocation,
 
     private Schema doCreate(SchemaContext schemaContext, SchemaLocation schemaLocation,
             JsonNode schemaNode, Schema parentSchema, boolean suppressSubSchemaRetrieval) {
-        return Schema.from(withDialect(schemaContext, schemaNode), schemaLocation, schemaNode,
+        return doCreate(schemaContext, schemaLocation, schemaNode, parentSchema, suppressSubSchemaRetrieval, true);
+    }
+
+    private Schema doCreate(SchemaContext schemaContext, SchemaLocation schemaLocation,
+            JsonNode schemaNode, Schema parentSchema, boolean suppressSubSchemaRetrieval,
+            boolean validateSchemaNodeType) {
+        validateSchemaNodeNotNull(schemaLocation, schemaNode);
+        SchemaContext schemaContextToUse = withDialect(schemaContext, schemaNode);
+        if (validateSchemaNodeType) {
+            validateSchemaNodeType(schemaLocation, schemaNode, schemaContextToUse);
+        }
+        return Schema.from(schemaContextToUse, schemaLocation, schemaNode,
                 parentSchema, suppressSubSchemaRetrieval);
     }
 
+    private void validateSchemaNodeNotNull(SchemaLocation schemaLocation, JsonNode schemaNode) {
+        if (schemaNode == null) {
+            throw new SchemaException("Schema at " + schemaLocation + " must not be null");
+        }
+    }
+
+    private void validateSchemaNodeType(SchemaLocation schemaLocation, JsonNode schemaNode,
+            SchemaContext schemaContext) {
+        if (schemaNode.isObject()) {
+            return;
+        }
+        if (schemaNode.isBoolean() && supportsBooleanSchema(schemaContext)) {
+            return;
+        }
+        String expected = supportsBooleanSchema(schemaContext) ? "object or boolean" : "object";
+        throw new SchemaException("Schema at " + schemaLocation + " must be " + expected + " but was "
+                + schemaNode.getNodeType());
+    }
+
+    private boolean supportsBooleanSchema(SchemaContext schemaContext) {
+        return schemaContext != null
+                && schemaContext.getDialect().getSpecificationVersion().getOrder() >= SpecificationVersion.DRAFT_6
+                        .getOrder();
+    }
+
     /**
      * Determines the schema context to use for the schema given the parent schema
      * context.
@@ -647,6 +704,7 @@ public Schema getSchema(final InputStream schemaStream, InputFormat inputFormat)
      */
     public Schema getSchema(final SchemaLocation schemaUri) {
         Schema schema = loadSchema(schemaUri);
+        validateSchemaNodeType(schema.getSchemaLocation(), schema.getSchemaNode(), schema.getSchemaContext());
         preload(schema);
         return schema;
     }
@@ -722,6 +780,20 @@ public Schema getSchema(final JsonNode jsonNode) {
      * @return the schema
      */
     public Schema loadSchema(final SchemaLocation schemaUri) {
+        return loadSchema(schemaUri, true);
+    }
+
+    /**
+     * Loads the schema.
+     *
+     * @param schemaUri            the absolute IRI of the schema which can map to the
+     *                             retrieval IRI.
+     * @param validateLoadedSchema true to validate that the loaded node is a schema;
+     *                             false when loading a document container to resolve a
+     *                             fragment before using it as a schema
+     * @return the schema
+     */
+    public Schema loadSchema(final SchemaLocation schemaUri, boolean validateLoadedSchema) {
         if (schemaCacheEnabled) {
             // ConcurrentHashMap computeIfAbsent does not allow calls that result in a
             // recursive update to the map.
@@ -731,19 +803,28 @@ public Schema loadSchema(final SchemaLocation schemaUri) {
                 synchronized (this) { // acquire lock on shared registry object to prevent deadlock
                     cachedUriSchema = schemaCache.get(schemaUri);
                     if (cachedUriSchema == null) {
-                        cachedUriSchema = getMappedSchema(schemaUri);
+                        cachedUriSchema = validateLoadedSchema ? getMappedSchema(schemaUri)
+                                : getMappedSchema(schemaUri, false);
                         if (cachedUriSchema != null) {
                             schemaCache.put(schemaUri, cachedUriSchema);
                         }
                     }
                 }
             }
+            if (validateLoadedSchema && cachedUriSchema != null) {
+                validateSchemaNodeType(cachedUriSchema.getSchemaLocation(), cachedUriSchema.getSchemaNode(),
+                        cachedUriSchema.getSchemaContext());
+            }
             return cachedUriSchema;
         }
-        return getMappedSchema(schemaUri);
+        return validateLoadedSchema ? getMappedSchema(schemaUri) : getMappedSchema(schemaUri, false);
     }
 
     protected Schema getMappedSchema(final SchemaLocation schemaUri) {
+        return getMappedSchema(schemaUri, true);
+    }
+
+    protected Schema getMappedSchema(final SchemaLocation schemaUri, boolean validateLoadedSchema) {
         InputStreamSource inputStreamSource = this.schemaLoader.getSchemaResource(schemaUri.getAbsoluteIri());
         if (inputStreamSource != null) {
             try (InputStream inputStream = inputStreamSource.getInputStream()) {
@@ -762,13 +843,13 @@ protected Schema getMappedSchema(final SchemaLocation schemaUri) {
                     // Schema without fragment
                     SchemaContext schemaContext = new SchemaContext(dialect, this);
                     return doCreate(schemaContext, schemaUri, schemaNode, null,
-                            true /* retrieved via id, resolving will not change anything */);
+                            true /* retrieved via id, resolving will not change anything */, validateLoadedSchema);
                 } else {
                     // Schema with fragment pointing to sub schema
                     final SchemaContext schemaContext = createSchemaContext(schemaNode);
                     SchemaLocation documentLocation = new SchemaLocation(schemaUri.getAbsoluteIri());
                     Schema document = doCreate(schemaContext, documentLocation, schemaNode, null,
-                            false);
+                            false, false);
                     return document.getRefSchema(schemaUri.getFragment());
                 }
             } catch (IOException e) {

src/main/java/com/networknt/schema/SpecificationVersion.java

@@ -19,6 +19,8 @@
 
 import com.networknt.schema.dialect.DialectId;
 
+import com.fasterxml.jackson.databind.JsonNode;
+
 /**
  * The version of the JSON Schema specification that defines the standard
  * dialects.
@@ -90,4 +92,20 @@ public static Optional<SpecificationVersion> fromDialectId(String dialectId) {
         }
         return Optional.empty();
     }
+
+    /**
+     * Gets the specification version that matches the dialect id indicated by
+     * $schema keyword. The dialect id is an IRI that identifies the meta schema
+     * used to validate the dialect.
+     *
+     * @param schemaNode the schema
+     * @return the specification version if it matches the dialect id
+     */
+    public static Optional<SpecificationVersion> fromSchemaNode(JsonNode schemaNode) {
+        JsonNode schema = schemaNode.get("$schema");
+        if (schema != null && schema.isTextual()) {
+            return fromDialectId(schema.textValue());
+        }
+        return Optional.empty();
+    }
 }

src/main/java/com/networknt/schema/keyword/AdditionalPropertiesValidator.java

@@ -109,10 +109,6 @@ protected void validate(ExecutionContext executionContext, JsonNode node, JsonNo
         for (Iterator<Entry<String, JsonNode>> it = node.fields(); it.hasNext(); ) {
             Entry<String, JsonNode> entry = it.next();
             String pname = entry.getKey();
-            // skip the context items
-            if (pname.startsWith("#")) {
-                continue;
-            }
             if (!allowedProperties.contains(pname) && !handledByPatternProperties(pname)) {
                 if (!allowAdditionalProperties) {
                     executionContext.addError(error().instanceNode(node).property(pname)
@@ -156,10 +152,6 @@ public void walk(ExecutionContext executionContext, JsonNode node, JsonNode root
         // Else continue walking.
         for (Iterator<String> it = node.fieldNames(); it.hasNext(); ) {
             String pname = it.next();
-            // skip the context items
-            if (pname.startsWith("#")) {
-                continue;
-            }
             if (!allowedProperties.contains(pname) && !handledByPatternProperties(pname)) {
                 if (allowAdditionalProperties) {
                     if (additionalPropertiesSchema != null) {

src/main/java/com/networknt/schema/keyword/RefValidator.java

@@ -60,28 +60,30 @@ static SchemaRef getRefSchema(Schema parentSchema, SchemaContext schemaContext,
                 refUri = refValue;
             }
 
-            // This will determine the correct absolute uri for the refUri. This decision will take into
-            // account the current uri of the parent schema.
-            String schemaUriFinal = resolve(parentSchema, refUri);
-            SchemaLocation schemaLocation = SchemaLocation.of(schemaUriFinal);
-            // This should retrieve schemas regardless of the protocol that is in the uri.
-            return new SchemaRef(getSupplier(() -> {
-                Schema schemaResource = schemaContext.getSchemaResources().get(schemaUriFinal);
-                if (schemaResource == null) {
-                    schemaResource = schemaContext.getSchemaRegistry().loadSchema(schemaLocation); 
-                    if (schemaResource != null) {
-                        copySchemaResources(schemaContext, schemaResource);
-                    }
-                }
+            // This will determine the correct absolute uri for the refUri. This decision will take into
+            // account the current uri of the parent schema.
+            String schemaUriFinal = resolve(parentSchema, refUri);
+            SchemaLocation schemaLocation = SchemaLocation.of(schemaUriFinal);
+            String fragment = index < 0 ? null : refValue.substring(index);
+            boolean validateLoadedSchema = fragment == null || !SchemaLocation.Fragment.isJsonPointerFragment(fragment);
+            // This should retrieve schemas regardless of the protocol that is in the uri.
+            return new SchemaRef(getSupplier(() -> {
+                Schema schemaResource = schemaContext.getSchemaResources().get(schemaUriFinal);
+                if (schemaResource == null) {
+                    schemaResource = schemaContext.getSchemaRegistry().loadSchema(schemaLocation, validateLoadedSchema);
+                    if (schemaResource != null) {
+                        copySchemaResources(schemaContext, schemaResource);
+                    }
+                }
                 if (index < 0) {
                     if (schemaResource == null) {
                         return null;
-                    }
-                    return schemaResource;
-                } else {
-                    String newRefValue = refValue.substring(index);
-                    String find = schemaLocation.getAbsoluteIri() + newRefValue;
-                    Schema findSchemaResource = schemaContext.getSchemaResources().get(find);
+                    }
+                    return schemaResource;
+                } else {
+                    String newRefValue = fragment;
+                    String find = schemaLocation.getAbsoluteIri() + newRefValue;
+                    Schema findSchemaResource = schemaContext.getSchemaResources().get(find);
                     if (findSchemaResource == null) {
                         findSchemaResource = schemaContext.getDynamicAnchors().get(find); 
                     }

src/main/java/com/networknt/schema/regex/AllowRegularExpressionFactory.java

@@ -19,6 +19,7 @@
 
 import com.networknt.schema.InvalidSchemaException;
 import com.networknt.schema.Error;
+import com.networknt.schema.SchemaContext;
 
 /**
  * {@link RegularExpressionFactory} that allows regular expressions to be used.
@@ -34,9 +35,16 @@ public AllowRegularExpressionFactory(RegularExpressionFactory delegate, Predicat
 
     @Override
     public RegularExpression getRegularExpression(String regex) {
+        return getRegularExpression(regex, null);
+    }
+
+    @Override
+    public RegularExpression getRegularExpression(String regex, SchemaContext schemaContext) {
         if (this.allowed.test(regex)) {
             // Allowed to delegate
-            return this.delegate.getRegularExpression(regex);
+            return schemaContext != null
+                    ? this.delegate.getRegularExpression(regex, schemaContext)
+                    : this.delegate.getRegularExpression(regex);
         }
         throw new InvalidSchemaException(Error.builder()
                 .message("Regular expression ''{0}'' is not allowed to be used.").arguments(regex).build());

src/main/java/com/networknt/schema/regex/ECMAScriptRegularExpressionFactory.java

@@ -15,6 +15,7 @@
  */
 package com.networknt.schema.regex;
 
+import com.networknt.schema.SchemaContext;
 import com.networknt.schema.utils.Classes;
 
 /**
@@ -45,4 +46,9 @@ public static ECMAScriptRegularExpressionFactory getInstance() {
     public RegularExpression getRegularExpression(String regex) {
         return DELEGATE.getRegularExpression(regex);
     }
+
+    @Override
+    public RegularExpression getRegularExpression(String regex, SchemaContext schemaContext) {
+        return DELEGATE.getRegularExpression(regex, schemaContext);
+    }
 }

src/main/java/com/networknt/schema/regex/GraalJSRegularExpression.java

@@ -28,9 +28,13 @@ class GraalJSRegularExpression implements RegularExpression {
     private final Value function;
 
     GraalJSRegularExpression(String regex, GraalJSRegularExpressionContext context) {
+        this(regex, context, true);
+    }
+
+    GraalJSRegularExpression(String regex, GraalJSRegularExpressionContext context, boolean unicode) {
         this.context = context;
         synchronized(context.getContext()) {
-            this.function = context.getRegExpBuilder().execute(regex);
+            this.function = context.getRegExpBuilder(unicode).execute(regex);
         }
     }
 
@@ -40,4 +44,4 @@ public boolean matches(String value) {
             return !function.execute(value).isNull();
         }
     }
-}
+}

src/main/java/com/networknt/schema/regex/GraalJSRegularExpressionContext.java

@@ -22,12 +22,13 @@
  * GraalJSRegularExpressionContext.
  */
 public class GraalJSRegularExpressionContext {
-    private static final String SOURCE = "pattern => {\n"
-            + "    const regex = new RegExp(pattern, 'u');\n"
+    private static final String SOURCE = "flags => pattern => {\n"
+            + "    const regex = new RegExp(pattern, flags);\n"
             + "    return text => text.match(regex)\n"
             + "};";
 
     private final Context context;
+    private final Value nonUnicodeRegExpBuilder;
     private final Value regExpBuilder;
 
     /**
@@ -41,7 +42,9 @@ public class GraalJSRegularExpressionContext {
     public GraalJSRegularExpressionContext(Context context) {
         this.context = context;
         synchronized(this.context) {
-            this.regExpBuilder = this.context.eval("js", SOURCE);
+            Value builder = this.context.eval("js", SOURCE);
+            this.nonUnicodeRegExpBuilder = builder.execute("");
+            this.regExpBuilder = builder.execute("u");
         }
     }
 
@@ -63,4 +66,14 @@ public Context getContext() {
     public Value getRegExpBuilder() {
         return regExpBuilder;
     }
+
+    /**
+     * Gets the RegExp builder.
+     *
+     * @param unicode whether the unicode flag should be used
+     * @return the regexp builder
+     */
+    public Value getRegExpBuilder(boolean unicode) {
+        return unicode ? regExpBuilder : nonUnicodeRegExpBuilder;
+    }
 }