fix: reject unpaired JSON surrogates

Author: He-Pin

sjsonnet/src/sjsonnet/Evaluator.scala

@@ -1395,20 +1395,28 @@ class Evaluator(
                 if (result.isInfinite) Error.fail("Overflow", pos)
                 Val.cachedNum(pos, result)
               case _: Val.Str =>
-                Val.Str.concat(pos, Val.Str(pos, RenderUtils.renderDouble(ld)), r.asInstanceOf[Val.Str])
+                Val.Str.concat(
+                  pos,
+                  Val.Str(pos, RenderUtils.renderDouble(ld)),
+                  r.asInstanceOf[Val.Str]
+                )
               case _ => failBinOp(l, e.op, r, pos)
             }
           case ls: Val.Str =>
             r match {
-              case rs: Val.Str => Val.Str.concat(pos, ls, rs)
+              case rs: Val.Str    => Val.Str.concat(pos, ls, rs)
               case Val.Num(_, rd) =>
                 Val.Str.concat(pos, ls, Val.Str(pos, RenderUtils.renderDouble(rd)))
               case _ => Val.Str.concat(pos, ls, Val.Str(pos, Materializer.stringify(r)))
             }
           case _ =>
             r match {
               case _: Val.Str =>
-                Val.Str.concat(pos, Val.Str(pos, Materializer.stringify(l)), r.asInstanceOf[Val.Str])
+                Val.Str.concat(
+                  pos,
+                  Val.Str(pos, Materializer.stringify(l)),
+                  r.asInstanceOf[Val.Str]
+                )
               case _ =>
                 (l, r) match {
                   case (lo: Val.Obj, ro: Val.Obj) => ro.addSuper(pos, lo)

sjsonnet/src/sjsonnet/Importer.scala

@@ -377,8 +377,9 @@ object CachedResolver {
         new JsonImportVisitor(fileScope, internedStrings, settings)
       Some((ujson.ByteArrayParser.transform(content.readRawBytes(), visitor), fileScope))
     } catch {
-      case _: ujson.ParsingFailedException | _: DuplicateJsonKey | _: InvalidJsonNumber |
-          _: JsonParseDepthExceeded | _: NumberFormatException =>
+      case _: ujson.ParsingFailedException | _: DuplicateJsonKey |
+          _: ValVisitor.InvalidUnicodeString | _: InvalidJsonNumber | _: JsonParseDepthExceeded |
+          _: NumberFormatException =>
         None
     }
   }
@@ -420,7 +421,11 @@ object CachedResolver {
         private var key: String = _
         def subVisitor: upickle.core.Visitor[?, ?] = self
         def visitKey(index: Int): upickle.core.StringVisitor.type = upickle.core.StringVisitor
-        def visitKeyValue(s: Any): Unit = key = intern(s.toString)
+        def visitKeyValue(s: Any): Unit = {
+          val str = s.toString
+          ValVisitor.rejectUnpairedSurrogates(str)
+          key = intern(str)
+        }
         def visitValue(v: Val, index: Int): Unit = {
           if (!seen.add(key)) throw new DuplicateJsonKey
           keys += key
@@ -474,6 +479,7 @@ object CachedResolver {
         case str: String => str
         case _           => s.toString
       }
+      ValVisitor.rejectUnpairedSurrogates(str)
       val unique = intern(str)
       Val.Str(pos(index), unique)
     }

sjsonnet/src/sjsonnet/ValVisitor.scala

@@ -7,8 +7,31 @@ import upickle.core.{ArrVisitor, ObjVisitor, Visitor}
 
 import scala.collection.mutable
 
+object ValVisitor {
+  final class InvalidUnicodeString extends RuntimeException(null, null, false, false)
+
+  def rejectUnpairedSurrogates(s: CharSequence): Unit = {
+    var i = 0
+    val length = s.length
+    while (i < length) {
+      val c = s.charAt(i)
+      if (Character.isHighSurrogate(c)) {
+        if (i + 1 >= length || !Character.isLowSurrogate(s.charAt(i + 1))) {
+          throw new InvalidUnicodeString
+        }
+        i += 2
+      } else if (Character.isLowSurrogate(c)) {
+        throw new InvalidUnicodeString
+      } else {
+        i += 1
+      }
+    }
+  }
+}
+
 /** Parse JSON directly into a literal `Val` */
-class ValVisitor(pos: Position) extends JsVisitor[Val, Val] { self =>
+class ValVisitor(pos: Position, rejectUnpairedSurrogates: Boolean = false)
+    extends JsVisitor[Val, Val] { self =>
 
   override def visitJsonableObject(length: Int, index: Int): ObjVisitor[Val, Val] =
     visitObject(length, index)
@@ -27,7 +50,10 @@ class ValVisitor(pos: Position) extends JsVisitor[Val, Val] { self =>
     var key: String = _
     def subVisitor: Visitor[?, ?] = self
     def visitKey(index: Int): upickle.core.StringVisitor.type = upickle.core.StringVisitor
-    def visitKeyValue(s: Any): Unit = key = s.toString
+    def visitKeyValue(s: Any): Unit = {
+      key = s.toString
+      if (rejectUnpairedSurrogates) ValVisitor.rejectUnpairedSurrogates(key)
+    }
     def visitValue(v: Val, index: Int): Unit = {
       cache.put(key, v)
       allKeys.put(key, false)
@@ -52,5 +78,8 @@ class ValVisitor(pos: Position) extends JsVisitor[Val, Val] { self =>
       }
     )
 
-  def visitString(s: CharSequence, index: Int): Val = Val.Str(pos, s.toString)
+  def visitString(s: CharSequence, index: Int): Val = {
+    if (rejectUnpairedSurrogates) ValVisitor.rejectUnpairedSurrogates(s)
+    Val.Str(pos, s.toString)
+  }
 }

sjsonnet/src/sjsonnet/stdlib/ManifestModule.scala

@@ -126,8 +126,13 @@ object ManifestModule extends AbstractFunctionModule {
   private object ParseJson extends Val.Builtin1("parseJson", "str") {
     def evalRhs(str: Eval, ev: EvalScope, pos: Position): Val = {
       try {
-        ujson.StringParser.transform(str.value.asString, new ValVisitor(pos))
+        ujson.StringParser.transform(
+          str.value.asString,
+          new ValVisitor(pos, rejectUnpairedSurrogates = true)
+        )
       } catch {
+        case _: ValVisitor.InvalidUnicodeString =>
+          throw Error.fail("Invalid JSON: unpaired surrogate in string", pos)(ev)
         case e: ujson.ParseException =>
           throw Error.fail("Invalid JSON: " + e.getMessage, pos)(ev)
       }

sjsonnet/test/resources/new_test_suite/error.parsejson_lone_high_surrogate.jsonnet

@@ -0,0 +1 @@
+std.parseJson('"\\uD800"')

sjsonnet/test/resources/new_test_suite/error.parsejson_lone_high_surrogate.jsonnet.golden

@@ -0,0 +1,3 @@
+sjsonnet.Error: [std.parseJson] Invalid JSON: unpaired surrogate in string
+    at [<root>].(error.parsejson_lone_high_surrogate.jsonnet:1:14)
+

sjsonnet/test/resources/new_test_suite/error.parsejson_lone_low_surrogate.jsonnet

@@ -0,0 +1 @@
+std.parseJson('"\\uDC00"')

sjsonnet/test/resources/new_test_suite/error.parsejson_lone_low_surrogate.jsonnet.golden

@@ -0,0 +1,3 @@
+sjsonnet.Error: [std.parseJson] Invalid JSON: unpaired surrogate in string
+    at [<root>].(error.parsejson_lone_low_surrogate.jsonnet:1:14)
+

sjsonnet/test/resources/new_test_suite/error.parsejson_lone_surrogate_key.jsonnet

@@ -0,0 +1 @@
+std.parseJson('{"\\uD800": 1}')

sjsonnet/test/resources/new_test_suite/error.parsejson_lone_surrogate_key.jsonnet.golden

@@ -0,0 +1,3 @@
+sjsonnet.Error: [std.parseJson] Invalid JSON: unpaired surrogate in string
+    at [<root>].(error.parsejson_lone_surrogate_key.jsonnet:1:14)
+