feat(parse): add unknown_fields keyword (#451)

* feat(parse): add unknown_fields keyword

Add an  keyword to typed JSON parsing and parse!
that preserves the current ignore behavior by default while allowing
callers to reject unmatched members with .

This updates the JSON read style plumbing to use StructUtils 2.8's
unknownfield hook, adds regression tests for nested and in-place
parsing, and documents the new option.

* refactor(parse): simplify unknown field policy state

Author: quinnj

Project.toml

@@ -22,7 +22,7 @@ Arrow = "2.8.0"
 ArrowTypes = "2.2"
 Parsers = "1, 2"
 PrecompileTools = "1"
-StructUtils = "2.3"
+StructUtils = "2.8"
 julia = "1.9"
 
 [extras]

docs/src/reading.md

@@ -164,6 +164,14 @@ With this approach, JSON.jl automatically:
 - Converts values to the appropriate field types
 - Constructs the struct with the parsed values
 
+By default, any extra JSON keys that don't match fields in the target type are
+ignored. If you want those to error instead, pass `unknown_fields=:error`:
+
+```julia
+JSON.parse("""{"name": "Alice", "age": 30, "admin": true}""", Person; unknown_fields=:error)
+# ERROR: ArgumentError: encountered unknown JSON member "admin" while parsing `Person`
+```
+
 This works for nested structs too:
 
 ```julia
@@ -505,4 +513,4 @@ Let's walk through some notable features of the example above:
   * The `percentages` field is a dictionary with keys of type `Percent`, which is a custom type. The `liftkey` function is defined to convert the JSON string keys to `Percent` types (parses the Float64 manually)
   * The `json_properties` field has a type of `JSONText`, which means the raw JSON will be preserved as a String of the `JSONText` type.
   * The `matrix` field is a `Matrix{Float64}`, so the JSON input array-of-arrays are materialized as such.
-  * The `extra_key` field is not defined in the `FrankenStruct` type, so it is ignored and skipped over.
+  * The `extra_key` field is not defined in the `FrankenStruct` type, so it is ignored and skipped over by default. Pass `unknown_fields=:error` if you want unknown keys to throw instead.

src/parse.jl

@@ -18,6 +18,7 @@ Currently supported keyword arguments include:
   * `isroot`: whether this is the root LazyValue encompassing the entire json buffer. If `false` parses only the first JSON value and ignores trailing characters. (default: `true`)
   * `dicttype`: a custom `AbstractDict` type to use instead of `$DEFAULT_OBJECT_TYPE` as the default type for JSON object materialization
   * `null`: a custom value to use for JSON null values (default: `nothing`)
+  * `unknown_fields`: controls how unmatched JSON object keys or positional values are handled when parsing into a target type or existing object; supported values are `:ignore` (default) and `:error`
   * `style`: a custom `StructUtils.StructStyle` subtype instance to be used in calls to `StructUtils.make` and `StructUtils.lift`. This allows overriding
     default behaviors for non-owned types.
 
@@ -37,7 +38,7 @@ of type `T` will be attempted utilizing machinery and interfaces provided by the
   * If `T` was defined with the `@noarg` macro, an empty instance will be constructed, and field values set as JSON keys match field names
   * If `T` had default field values defined using the `@defaults` or `@kwarg` macros (from StructUtils.jl package), those will be set in the value of `T` unless different values are parsed from the JSON
   * If `T` was defined with the `@nonstruct` macro, the struct will be treated as a primitive type and constructed using the `lift` function rather than from field values
-  * JSON keys that don't match field names in `T` will be ignored (skipped over)
+  * JSON keys that don't match field names in `T` will be ignored (skipped over) by default; pass `unknown_fields=:error` to reject them
   * If a field in `T` has a `name` fieldtag, the `name` value will be used to match JSON keys instead
   * If `T` or any recursive field type of `T` is abstract, an appropriate `JSON.@choosetype T x -> ...` definition should exist for "choosing" a concrete type at runtime; default type choosing exists for `Union{T, Missing}` and `Union{T, Nothing}` where the JSON value is checked if `null`. If the `Any` type is encountered, the default materialization types will be used (`JSON.Object`, `Vector{Any}`, etc.)
   * For any non-JSON-standard non-aggregate (i.e. non-object, non-array) field type of `T`, a `JSON.lift(::Type{T}, x) = ...` definition can be defined for how to "lift" the default JSON value (String, Number, Bool, `nothing`) to the type `T`; a default lift definition exists, for example, for `JSON.lift(::Type{Missing}, x) = missing` where the standard JSON value for `null` is `nothing` and it can be "lifted" to `missing`
@@ -150,12 +151,16 @@ import StructUtils: StructStyle
 
 abstract type JSONStyle <: StructStyle end
 
-# defining a custom style allows us to pass a non-default dicttype `O` through JSON.parse
-struct JSONReadStyle{O,T} <: JSONStyle
+# defining a custom style allows us to pass a non-default dicttype `O` through JSON.parse,
+# while still delegating custom behavior to an inner StructStyle if one was provided
+struct JSONReadStyle{O,T,S} <: JSONStyle
     null::T
+    style::S
+    ignore_unknown_fields::Bool
 end
 
-JSONReadStyle{O}(null::T) where {O,T} = JSONReadStyle{O,T}(null)
+JSONReadStyle{O}(null::T, style::S=StructUtils.DefaultStyle(), ignore_unknown_fields::Bool=true) where {O,T,S} =
+    JSONReadStyle{O,T,S}(null, style, ignore_unknown_fields)
 
 objecttype(::StructStyle) = DEFAULT_OBJECT_TYPE
 objecttype(::JSONReadStyle{OT}) where {OT} = OT
@@ -164,6 +169,26 @@ nullvalue(st::JSONReadStyle) = st.null
 
 # this allows struct fields to specify tags under the json key specifically to override JSON behavior
 StructUtils.fieldtagkey(::JSONStyle) = :json
+StructUtils.defaultstate(st::JSONReadStyle) = StructUtils.defaultstate(st.style)
+
+function jsonreadstyle(::Type{T}, ::Type{O}, null, style::StructStyle, unknown_fields::Symbol) where {T,O}
+    ignore_unknown_fields =
+        unknown_fields === :ignore ? true :
+        unknown_fields === :error ? false :
+        throw(ArgumentError("`unknown_fields` must be `:ignore` or `:error`, got `$(repr(unknown_fields))`"))
+    if T === Any && !ignore_unknown_fields
+        throw(ArgumentError("`unknown_fields` is only supported when parsing into a target type or existing object"))
+    end
+    return JSONReadStyle{O}(null, style, ignore_unknown_fields)
+end
+
+@noinline unknownfielderror(::Type{T}, key) where {T} =
+    ArgumentError("encountered unknown JSON member $(repr(key)) while parsing `$T`")
+
+function StructUtils.unknownfield(st::JSONReadStyle, ::Type{T}, key, value) where {T}
+    st.ignore_unknown_fields || throw(unknownfielderror(T, key))
+    return StructUtils.unknownfield(st.style, T, key, value)
+end
 
 "See [`parse`](@ref)."
 function parsefile end
@@ -180,15 +205,19 @@ parse(io::Union{IO,Base.AbstractCmd}, ::Type{T}=Any; kw...) where {T} = parse(Ba
 parse!(io::Union{IO,Base.AbstractCmd}, x::T; kw...) where {T} = parse!(Base.read(io), x; kw...)
 
 parse(buf::Union{AbstractVector{UInt8},AbstractString}, ::Type{T}=Any;
-    dicttype::Type{O}=DEFAULT_OBJECT_TYPE, null=nothing,
-    style::StructStyle=JSONReadStyle{dicttype}(null), kw...) where {T,O} =
-    @inline parse(lazy(buf; kw...), T; dicttype, null, style)
+    dicttype::Type{O}=DEFAULT_OBJECT_TYPE, null=nothing, style::StructStyle=StructUtils.DefaultStyle(),
+    unknown_fields::Symbol=:ignore, kw...) where {T,O} =
+    @inline parse(lazy(buf; kw...), T; dicttype, null, style, unknown_fields)
 
-parse!(buf::Union{AbstractVector{UInt8},AbstractString}, x::T; dicttype::Type{O}=DEFAULT_OBJECT_TYPE, null=nothing, style::StructStyle=JSONReadStyle{dicttype}(null), kw...) where {T,O} =
-    @inline parse!(lazy(buf; kw...), x; dicttype, null, style)
+parse!(buf::Union{AbstractVector{UInt8},AbstractString}, x::T;
+    dicttype::Type{O}=DEFAULT_OBJECT_TYPE, null=nothing, style::StructStyle=StructUtils.DefaultStyle(),
+    unknown_fields::Symbol=:ignore, kw...) where {T,O} =
+    @inline parse!(lazy(buf; kw...), x; dicttype, null, style, unknown_fields)
 
-parse(x::LazyValue, ::Type{T}=Any; dicttype::Type{O}=DEFAULT_OBJECT_TYPE, null=nothing, style::StructStyle=JSONReadStyle{dicttype}(null)) where {T,O} =
-    @inline _parse(x, T, dicttype, null, style)
+parse(x::LazyValue, ::Type{T}=Any;
+    dicttype::Type{O}=DEFAULT_OBJECT_TYPE, null=nothing, style::StructStyle=StructUtils.DefaultStyle(),
+    unknown_fields::Symbol=:ignore) where {T,O} =
+    @inline _parse(x, T, dicttype, null, jsonreadstyle(T, O, null, style, unknown_fields))
 
 function _parse(x::LazyValue, ::Type{T}, dicttype::Type{O}, null, style::StructStyle) where {T,O}
     y, pos = StructUtils.make(style, T, x)
@@ -210,7 +239,10 @@ function _parse(x::LazyValue, ::Type{Any}, ::Type{DEFAULT_OBJECT_TYPE}, null, ::
     return out.value
 end
 
-parse!(x::LazyValue, obj::T; dicttype::Type{O}=DEFAULT_OBJECT_TYPE, null=nothing, style::StructStyle=JSONReadStyle{dicttype}(null)) where {T,O} = StructUtils.make!(style, obj, x)
+parse!(x::LazyValue, obj::T;
+    dicttype::Type{O}=DEFAULT_OBJECT_TYPE, null=nothing, style::StructStyle=StructUtils.DefaultStyle(),
+    unknown_fields::Symbol=:ignore) where {T,O} =
+    StructUtils.make!(jsonreadstyle(T, O, null, style, unknown_fields), obj, x)
 
 # for LazyValue, if x started at the beginning of the JSON input,
 # then we want to ensure that the entire input was consumed
@@ -332,25 +364,28 @@ function StructUtils.make(st::StructStyle, ::Type{Any}, x::LazyValues)
 end
 
 # catch PtrString via lift or make! so we can ensure it never "escapes" to user-level
-StructUtils.liftkey(st::StructStyle, ::Type{T}, x::PtrString) where {T} =
+StructUtils.liftkey(st::JSONReadStyle, ::Type{T}, x::PtrString) where {T} =
     StructUtils.liftkey(st, T, convert(String, x))
-StructUtils.lift(st::StructStyle, ::Type{T}, x::PtrString, tags) where {T} =
+StructUtils.lift(st::JSONReadStyle, ::Type{T}, x::PtrString, tags) where {T} =
     StructUtils.lift(st, T, convert(String, x), tags)
-StructUtils.lift(st::StructStyle, ::Type{T}, x::PtrString) where {T} =
+StructUtils.lift(st::JSONReadStyle, ::Type{T}, x::PtrString) where {T} =
     StructUtils.lift(st, T, convert(String, x))
 
 # liftkey for numeric dict key types to enable round-tripping Dict{Int,V}, Dict{Float64,V}, etc.
 # these correspond to the lowerkey definitions in write.jl that convert numeric keys to strings
-StructUtils.liftkey(::JSONStyle, ::Type{T}, x::AbstractString) where {T<:Integer} = Base.parse(T, x)
-StructUtils.liftkey(::JSONStyle, ::Type{T}, x::AbstractString) where {T<:AbstractFloat} = Base.parse(T, x)
+StructUtils.liftkey(::JSONReadStyle, ::Type{T}, x::AbstractString) where {T<:Integer} = Base.parse(T, x)
+StructUtils.liftkey(::JSONReadStyle, ::Type{T}, x::AbstractString) where {T<:AbstractFloat} = Base.parse(T, x)
+
+StructUtils.lift(style::JSONReadStyle, ::Type{T}, x, tags) where {T} = StructUtils.lift(style.style, T, x, tags)
+StructUtils.lift(style::JSONReadStyle, ::Type{T}, x) where {T} = StructUtils.lift(style.style, T, x)
 
-function StructUtils.lift(style::StructStyle, ::Type{T}, x::LazyValues) where {T<:AbstractArray{E,0}} where {E}
+function StructUtils.lift(style::JSONReadStyle, ::Type{T}, x::LazyValues) where {T<:AbstractArray{E,0}} where {E}
     m = T(undef)
     m[1], pos = StructUtils.lift(style, E, x)
     return m, pos
 end
 
-function StructUtils.lift(style::StructStyle, ::Type{T}, x::LazyValues, tags=(;)) where {T}
+function StructUtils.lift(style::JSONReadStyle, ::Type{T}, x::LazyValues, tags=(;)) where {T}
     type = gettype(x)
     buf = getbuf(x)
     if type == JSONTypes.STRING
@@ -525,4 +560,4 @@ end
         invalid(error, buf, pos, "tuple")
     end
     return ex
-end
+end

test/parse.jl

@@ -501,6 +501,7 @@ end
     @testset "JSON.parse with types" begin
         obj = JSON.parse("""{ "a": 1,"b": 2,"c": 3,"d": 4}""", A)
         @test obj == A(1, 2, 3, 4)
+        @test JSON.parse("""{ "a": 1,"b": 2,"c": 3,"d": 4, "e": 5}""", A) == A(1, 2, 3, 4)
         # test order doesn't matter
         obj2 = JSON.parse("""{ "d": 1,"b": 2,"c": 3,"a": 4}""", A)
         @test obj2 == A(4, 2, 3, 1)
@@ -552,6 +553,13 @@ end
         @test obj.id == 1 && !isdefined(obj, :name)
         obj = JSON.parse("""{ "id": 1, "a": {"a": 1, "b": 2, "c": 3, "d": 4}}""", E)
         @test obj == E(1, A(1, 2, 3, 4))
+        @test_throws ArgumentError JSON.parse("""{ "a": 1,"b": 2,"c": 3,"d": 4, "e": 5}""", A; unknown_fields=:error)
+        @test_throws ArgumentError JSON.parse("""{ "id": 1, "a": {"a": 1, "b": 2, "c": 3, "d": 4, "e": 5}}""", E; unknown_fields=:error)
+        obj = B()
+        @test_throws ArgumentError JSON.parse!("""{ "a": 1,"b": 2,"c": 3,"d": 4, "e": 5}""", obj; unknown_fields=:error)
+        @test_throws ArgumentError JSON.parse("""[1, 2, 3, 4, 5]""", A; unknown_fields=:error)
+        @test_throws ArgumentError JSON.parse("""{ "a": 1,"b": 2,"c": 3,"d": 4}""", A; unknown_fields=:boom)
+        @test_throws ArgumentError JSON.parse("""{ "a": 1}"""; unknown_fields=:error)
         obj = JSON.parse("""{ "id": 1, "rate": 2.0, "name": "3"}""", F)
         @test obj == F(1, 2.0, "3")
         obj = JSON.parse("""{ "id": 1, "rate": 2.0, "name": "3", "f": {"id": 1, "rate": 2.0, "name": "3"}}""", G)
@@ -756,6 +764,7 @@ end
     # test custom JSONStyle overload
     JSON.lift(::CustomJSONStyle, ::Type{Rational}, x) = Rational(x.num[], x.den[])
     @test JSON.parse("{\"num\": 1,\"den\":3}", Rational; style=CustomJSONStyle()) == 1//3
+    @test JSON.parse("{\"num\": 1,\"den\":3}", Rational; style=CustomJSONStyle(), unknown_fields=:error) == 1//3
     @test isequal(JSON.parse("{\"num\": 1,\"den\":null}", @NamedTuple{num::Int, den::Union{Int, Missing}}; null=missing, style=StructUtils.DefaultStyle()), (num=1, den=missing))
     # choosetype field tag on Any struct field
     @test JSON.parse("{\"id\":1,\"any\":{\"type\":\"int\",\"value\":10}}", Q) == Q(1, (type="int", value=10))