fix: reject unpaired JSON surrogates

Motivation:
std.parseJson and strict .json import fast paths accepted or normalized JSON strings containing unpaired UTF-16 surrogate escapes. That allowed invalid JSON Unicode to become Jsonnet strings, diverging from cpp-jsonnet and jrsonnet behavior and from the intent of rejecting malformed JSON input.

Modification:
Add a shared surrogate validator for JSON visitors, enable it for std.parseJson, and convert strict .json import surrogate failures into normal Jsonnet errors instead of falling back or producing raw invalid strings. Keep the import fast path allocation-conscious with a private nullable return contract, leaving any reusable OptionVal abstraction for a separate PR.

Result:
Lone high/low surrogate values and keys now fail with Invalid JSON: unpaired surrogate in string, valid surrogate pairs still materialize as codepoint strings, escaped literal text such as \\uD800 remains ordinary text, and loose Jsonnet .json files can still fall back through the Jsonnet parser.

References:
https://jsonnet.org/ref/stdlib.html#std-parsejson
google/go-jsonnet#885
com-lihaoyi/upickle#722

Author: He-Pin

sjsonnet/src/sjsonnet/Importer.scala

@@ -302,24 +302,29 @@ class CachedResolver(
 
   def parse(path: Path, content: ResolvedFile)(implicit
       ev: EvalErrorScope): Either[Error, (Expr, FileScope)] = {
-    parseCache.getOrElseUpdate(
-      (path, content.contentHash()), {
-        val parsed: Either[Error, (Expr, FileScope)] = content.preParsedAst match {
-          case Some(pre) => Right(pre)
-          case None      =>
-            CachedResolver.parseJsonImport(
-              path,
-              content,
-              internedStrings,
-              settings
-            ) match {
-              case Some(parsedJson) => Right(parsedJson)
-              case None             => parseJsonnet(path, content)
-            }
+    try {
+      parseCache.getOrElseUpdate(
+        (path, content.contentHash()), {
+          val parsed: Either[Error, (Expr, FileScope)] = content.preParsedAst match {
+            case Some(pre) => Right(pre)
+            case None      =>
+              CachedResolver.parseJsonImportOrNull(
+                path,
+                content,
+                internedStrings,
+                settings
+              ) match {
+                case null       => parseJsonnet(path, content)
+                case parsedJson => Right(parsedJson)
+              }
+          }
+          parsed.flatMap { case (e, fs) => process(e, fs) }
         }
-        parsed.flatMap { case (e, fs) => process(e, fs) }
-      }
-    )
+      )
+    } catch {
+      case e: CachedResolver.InvalidJsonUnicode =>
+        Left(new Error(CachedResolver.InvalidJsonUnicodeMessage).addFrame(e.fileScope.noOffsetPos))
+    }
   }
 
   private def parseJsonnet(path: Path, content: ResolvedFile)(implicit
@@ -364,25 +369,128 @@ object CachedResolver {
   private final class DuplicateJsonKey extends RuntimeException(null, null, false, false)
   private final class InvalidJsonNumber extends RuntimeException(null, null, false, false)
   private final class JsonParseDepthExceeded extends RuntimeException(null, null, false, false)
+  private[sjsonnet] final class InvalidJsonUnicode(val fileScope: FileScope)
+      extends RuntimeException(null, null, false, false)
 
-  private[sjsonnet] def parseJsonImport(
+  private[sjsonnet] val InvalidJsonUnicodeMessage = "Invalid JSON: unpaired surrogate in string"
+
+  /**
+   * Parses strict `.json` imports through ujson's parser. Returns null when this fast path should
+   * fall back to the Jsonnet parser; the nullable result avoids Some/None allocations in the import
+   * hot path without adding a reusable OptionVal abstraction to this semantic fix.
+   */
+  private[sjsonnet] def parseJsonImportOrNull(
       path: Path,
       content: ResolvedFile,
       internedStrings: mutable.HashMap[String, String],
-      settings: Settings): Option[(Expr, FileScope)] = {
-    if (!path.last.endsWith(".json")) return None
+      settings: Settings): (Expr, FileScope) = {
+    if (!path.last.endsWith(".json")) return null
     val fileScope = new FileScope(path)
     try {
+      val bytes = content.readRawBytes()
       val visitor =
         new JsonImportVisitor(fileScope, internedStrings, settings)
-      Some((ujson.ByteArrayParser.transform(content.readRawBytes(), visitor), fileScope))
+      val expr = ujson.ByteArrayParser.transform(bytes, visitor)
+      rejectUnpairedSurrogateEscapes(bytes)
+      (expr, fileScope)
     } catch {
+      case _: ValVisitor.InvalidUnicodeString =>
+        throw new InvalidJsonUnicode(fileScope)
+      case e: Exception if isUnpairedSurrogateParserError(e) =>
+        throw new InvalidJsonUnicode(fileScope)
       case _: ujson.ParsingFailedException | _: DuplicateJsonKey | _: InvalidJsonNumber |
           _: JsonParseDepthExceeded | _: NumberFormatException =>
-        None
+        null
     }
   }
 
+  private final val Backslash = '\\'.toByte
+  private final val Quote = '"'.toByte
+  private final val U = 'u'.toByte
+  private final val UpperD = 'D'.toByte
+  private final val LowerD = 'd'.toByte
+  private final val Zero = '0'.toByte
+  private final val Nine = '9'.toByte
+  private final val UpperA = 'A'.toByte
+  private final val UpperF = 'F'.toByte
+  private final val LowerA = 'a'.toByte
+  private final val LowerF = 'f'.toByte
+
+  private def rejectUnpairedSurrogateEscapes(bytes: Array[Byte]): Unit = {
+    var i = 0
+    var foundPotentialSurrogateEscape = false
+    while (i + 5 < bytes.length && !foundPotentialSurrogateEscape) {
+      foundPotentialSurrogateEscape =
+        bytes(i) == Backslash && bytes(i + 1) == U && isD(bytes(i + 2))
+      i += 1
+    }
+    if (!foundPotentialSurrogateEscape) return
+
+    i = 0
+    var inString = false
+    var escaped = false
+    while (i < bytes.length) {
+      val b = bytes(i)
+      if (!inString) {
+        if (b == Quote) inString = true
+        i += 1
+      } else if (escaped) {
+        if (b == U && i + 4 < bytes.length && isHex4(bytes, i + 1)) {
+          val code = hex4(bytes, i + 1)
+          if (Character.isHighSurrogate(code.toChar)) {
+            val nextEscape = i + 5
+            if (
+              nextEscape + 6 > bytes.length || bytes(nextEscape) != Backslash ||
+              bytes(nextEscape + 1) != U || !isHex4(bytes, nextEscape + 2) ||
+              !Character.isLowSurrogate(hex4(bytes, nextEscape + 2).toChar)
+            ) {
+              throw new ValVisitor.InvalidUnicodeString
+            }
+            i = nextEscape + 6
+          } else if (Character.isLowSurrogate(code.toChar)) {
+            throw new ValVisitor.InvalidUnicodeString
+          } else {
+            i += 5
+          }
+        } else {
+          i += 1
+        }
+        escaped = false
+      } else if (b == Backslash) {
+        escaped = true
+        i += 1
+      } else {
+        if (b == Quote) inString = false
+        i += 1
+      }
+    }
+  }
+
+  private def isD(b: Byte): Boolean = b == UpperD || b == LowerD
+
+  private def isHex4(bytes: Array[Byte], offset: Int): Boolean =
+    isHex(bytes(offset)) && isHex(bytes(offset + 1)) && isHex(bytes(offset + 2)) &&
+    isHex(bytes(offset + 3))
+
+  private def isHex(b: Byte): Boolean =
+    (b >= Zero && b <= Nine) || (b >= UpperA && b <= UpperF) ||
+    (b >= LowerA && b <= LowerF)
+
+  private def hex4(bytes: Array[Byte], offset: Int): Int =
+    (hex(bytes(offset)) << 12) | (hex(bytes(offset + 1)) << 8) |
+    (hex(bytes(offset + 2)) << 4) | hex(bytes(offset + 3))
+
+  private def hex(b: Byte): Int =
+    if (b <= Nine) b - Zero
+    else if (b <= UpperF) b - UpperA + 10
+    else b - LowerA + 10
+
+  private def isUnpairedSurrogateParserError(e: Exception): Boolean =
+    e.getClass == classOf[Exception] &&
+    e.getMessage != null &&
+    e.getMessage.startsWith("Un-paired ") &&
+    e.getMessage.contains(" surrogate ")
+
   private final class JsonImportVisitor(
       fileScope: FileScope,
       internedStrings: mutable.HashMap[String, String],
@@ -420,7 +528,11 @@ object CachedResolver {
         private var key: String = _
         def subVisitor: upickle.core.Visitor[?, ?] = self
         def visitKey(index: Int): upickle.core.StringVisitor.type = upickle.core.StringVisitor
-        def visitKeyValue(s: Any): Unit = key = intern(s.toString)
+        def visitKeyValue(s: Any): Unit = {
+          val str = s.toString
+          ValVisitor.rejectUnpairedSurrogates(str)
+          key = intern(str)
+        }
         def visitValue(v: Val, index: Int): Unit = {
           if (!seen.add(key)) throw new DuplicateJsonKey
           keys += key
@@ -474,6 +586,7 @@ object CachedResolver {
         case str: String => str
         case _           => s.toString
       }
+      ValVisitor.rejectUnpairedSurrogates(str)
       val unique = intern(str)
       Val.Str(pos(index), unique)
     }

sjsonnet/src/sjsonnet/Preloader.scala

@@ -130,16 +130,18 @@ class Preloader(parentImporter: Importer, settings: Settings = Settings.default)
   }
 
   private def discover(path: Path, content: ResolvedFile): Either[Error, Unit] = {
-    CachedResolver.parseJsonImport(
-      path,
-      content,
-      internedStrings,
-      settings
-    ) match {
-      case Some((expr, fs)) =>
+    try {
+      val parsedJson = CachedResolver.parseJsonImportOrNull(
+        path,
+        content,
+        internedStrings,
+        settings
+      )
+      if (parsedJson != null) {
+        val (expr, fs) = parsedJson
         cache.put((path, false), PreParsedResolvedFile(content, expr, fs))
         Right(())
-      case None =>
+      } else {
         val parser = new Parser(path, internedStrings, internedFieldSets, settings)
         try {
           fastparse.parse(content.getParserInput(), parser.document(_)) match {
@@ -164,6 +166,10 @@ class Preloader(parentImporter: Importer, settings: Settings = Settings.default)
         } catch {
           case e: ParseError => Left(e)
         }
+      }
+    } catch {
+      case _: CachedResolver.InvalidJsonUnicode =>
+        Left(new ParseError(s"$path: ${CachedResolver.InvalidJsonUnicodeMessage}"))
     }
   }
 

sjsonnet/src/sjsonnet/ValVisitor.scala

@@ -7,8 +7,31 @@ import upickle.core.{ArrVisitor, ObjVisitor, Visitor}
 
 import scala.collection.mutable
 
+object ValVisitor {
+  final class InvalidUnicodeString extends RuntimeException(null, null, false, false)
+
+  def rejectUnpairedSurrogates(s: CharSequence): Unit = {
+    var i = 0
+    val length = s.length
+    while (i < length) {
+      val c = s.charAt(i)
+      if (Character.isHighSurrogate(c)) {
+        if (i + 1 >= length || !Character.isLowSurrogate(s.charAt(i + 1))) {
+          throw new InvalidUnicodeString
+        }
+        i += 2
+      } else if (Character.isLowSurrogate(c)) {
+        throw new InvalidUnicodeString
+      } else {
+        i += 1
+      }
+    }
+  }
+}
+
 /** Parse JSON directly into a literal `Val` */
-class ValVisitor(pos: Position) extends JsVisitor[Val, Val] { self =>
+class ValVisitor(pos: Position, rejectUnpairedSurrogates: Boolean = false)
+    extends JsVisitor[Val, Val] { self =>
 
   override def visitJsonableObject(length: Int, index: Int): ObjVisitor[Val, Val] =
     visitObject(length, index)
@@ -27,7 +50,10 @@ class ValVisitor(pos: Position) extends JsVisitor[Val, Val] { self =>
     var key: String = _
     def subVisitor: Visitor[?, ?] = self
     def visitKey(index: Int): upickle.core.StringVisitor.type = upickle.core.StringVisitor
-    def visitKeyValue(s: Any): Unit = key = s.toString
+    def visitKeyValue(s: Any): Unit = {
+      key = s.toString
+      if (rejectUnpairedSurrogates) ValVisitor.rejectUnpairedSurrogates(key)
+    }
     def visitValue(v: Val, index: Int): Unit = {
       cache.put(key, v)
       allKeys.put(key, false)
@@ -52,5 +78,8 @@ class ValVisitor(pos: Position) extends JsVisitor[Val, Val] { self =>
       }
     )
 
-  def visitString(s: CharSequence, index: Int): Val = Val.Str(pos, s.toString)
+  def visitString(s: CharSequence, index: Int): Val = {
+    if (rejectUnpairedSurrogates) ValVisitor.rejectUnpairedSurrogates(s)
+    Val.Str(pos, s.toString)
+  }
 }

sjsonnet/src/sjsonnet/stdlib/ManifestModule.scala

@@ -126,8 +126,13 @@ object ManifestModule extends AbstractFunctionModule {
   private object ParseJson extends Val.Builtin1("parseJson", "str") {
     def evalRhs(str: Eval, ev: EvalScope, pos: Position): Val = {
       try {
-        ujson.StringParser.transform(str.value.asString, new ValVisitor(pos))
+        ujson.StringParser.transform(
+          str.value.asString,
+          new ValVisitor(pos, rejectUnpairedSurrogates = true)
+        )
       } catch {
+        case _: ValVisitor.InvalidUnicodeString =>
+          throw Error.fail("Invalid JSON: unpaired surrogate in string", pos)(ev)
         case e: ujson.ParseException =>
           throw Error.fail("Invalid JSON: " + e.getMessage, pos)(ev)
         case _: ujson.IncompleteParseException =>

sjsonnet/test/resources/new_test_suite/error.parsejson_lone_high_surrogate.jsonnet

@@ -0,0 +1 @@
+std.parseJson('"\\uD800"')

sjsonnet/test/resources/new_test_suite/error.parsejson_lone_high_surrogate.jsonnet.golden

@@ -0,0 +1,2 @@
+sjsonnet.Error: [std.parseJson] Invalid JSON: unpaired surrogate in string
+    at [<root>].(error.parsejson_lone_high_surrogate.jsonnet:1:14)

sjsonnet/test/resources/new_test_suite/error.parsejson_lone_low_surrogate.jsonnet

@@ -0,0 +1 @@
+std.parseJson('"\\uDC00"')

sjsonnet/test/resources/new_test_suite/error.parsejson_lone_low_surrogate.jsonnet.golden

@@ -0,0 +1,2 @@
+sjsonnet.Error: [std.parseJson] Invalid JSON: unpaired surrogate in string
+    at [<root>].(error.parsejson_lone_low_surrogate.jsonnet:1:14)

sjsonnet/test/resources/new_test_suite/error.parsejson_lone_surrogate_key.jsonnet

@@ -0,0 +1 @@
+std.parseJson('{"\\uD800": 1}')

sjsonnet/test/resources/new_test_suite/error.parsejson_lone_surrogate_key.jsonnet.golden

@@ -0,0 +1,2 @@
+sjsonnet.Error: [std.parseJson] Invalid JSON: unpaired surrogate in string
+    at [<root>].(error.parsejson_lone_surrogate_key.jsonnet:1:14)