[Repo Assist] Add ExceptionIfMissing static parameter to JsonProvider and XmlProvider (#1680)

* Add ExceptionIfMissing static parameter to JsonProvider and XmlProvider

Implements opt-in strict mode for missing field handling. When ExceptionIfMissing=true,
accessing a non-optional field that is absent in the data raises an exception instead of
silently returning a default value (empty string for string, NaN for float).

The default behavior (ExceptionIfMissing=false) is unchanged for backward compatibility.

- Added GetNonOptionalValueStrict<'T> to JsonRuntime and TextRuntime
- Added ExceptionIfMissing static parameter to JsonProvider and XmlProvider
- Updated code generation pipeline (JsonConversionsGenerator, ConversionsGenerator)
- CsvProvider and HtmlProvider always use non-strict mode (false)
- Updated TypeProviderInstantiation.fs test helper
- Added tests for ExceptionIfMissing behavior

Also fixes a pre-existing XmlProvider bug where args.[14] was used for both
UseSchemaTypeNames and PreferDateTimeOffset (should be args.[14] and args.[15]).

Closes #1241

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* ci: trigger CI checks

---------

Co-authored-by: Repo Assist <repo-assist@github.com>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Don Syme <dsyme@users.noreply.github.com>

Author: 4 people

RELEASE_NOTES.md

@@ -2,6 +2,7 @@
 
 ## 8.1.0-beta
 
+- Add `ExceptionIfMissing` static parameter to `JsonProvider` and `XmlProvider`: when true, accessing a non-optional field that is missing in the data raises an exception instead of silently returning a default value (empty string for string, NaN for float). Defaults to false for backward compatibility.
 - Add `Http.ParseLinkHeader` utility for parsing RFC 5988 `Link` response headers (used by GitHub, GitLab, and other paginated APIs) into a `Map<string, string>` from relation name to URL (closes #805)
 - Add `PreferDateTimeOffset` parameter to `CsvProvider`, `JsonProvider`, and `XmlProvider`: when true, date-time values without an explicit timezone offset are inferred as `DateTimeOffset` (using local offset) instead of `DateTime` (closes #1100, #1072)
 - Make `Http.AppendQueryToUrl` public (closes #1325)

src/FSharp.Data.DesignTime/CommonProviderImplementation/ConversionsGenerator.fs

@@ -75,7 +75,7 @@ let getBackConversionQuotation missingValuesStr cultureStr typ value : Expr<stri
 
 /// Creates a function that takes Expr<string option> and converts it to
 /// an expression of other type - the type is specified by `field`
-let internal convertStringValue missingValuesStr cultureStr (field: PrimitiveInferedProperty) =
+let internal convertStringValue missingValuesStr cultureStr exceptionIfMissing (field: PrimitiveInferedProperty) =
     let fieldName = field.Name
     let field = field.Value
 
@@ -95,16 +95,20 @@ let internal convertStringValue missingValuesStr cultureStr (field: PrimitiveInf
         let convert value =
             getConversionQuotation missingValuesStr cultureStr field.InferedType value
 
+        let getNonOptionalValueMethod =
+            if exceptionIfMissing then
+                nameof (TextRuntime.GetNonOptionalValueStrict)
+            else
+                nameof (TextRuntime.GetNonOptionalValue)
+
         match field.TypeWrapper with
         | TypeWrapper.None ->
             //prevent value being calculated twice
             let var = Var("value", typeof<string option>)
             let varExpr = Expr.Cast<string option>(Expr.Var var)
 
             let body =
-                typeof<TextRuntime>?(nameof (TextRuntime.GetNonOptionalValue))
-                    field.RuntimeType
-                    (fieldName, convert varExpr, varExpr)
+                typeof<TextRuntime>?(getNonOptionalValueMethod) field.RuntimeType (fieldName, convert varExpr, varExpr)
 
             Expr.Let(var, value, body)
         | TypeWrapper.Option -> convert value

src/FSharp.Data.DesignTime/Csv/CsvGenerator.fs

@@ -33,7 +33,7 @@ module internal CsvTypeBuilder =
             inferredFields
             |> List.mapi (fun index field ->
                 let typ, typWithoutMeasure, conv, convBack =
-                    ConversionsGenerator.convertStringValue missingValuesStr cultureStr field
+                    ConversionsGenerator.convertStringValue missingValuesStr cultureStr false field
 
                 let propertyName =
                     if useOriginalNames then

src/FSharp.Data.DesignTime/Html/HtmlGenerator.fs

@@ -67,7 +67,7 @@ module internal HtmlGenerator =
             columns
             |> List.mapi (fun index field ->
                 let typ, typWithoutMeasure, conv, _convBack =
-                    ConversionsGenerator.convertStringValue missingValuesStr cultureStr field
+                    ConversionsGenerator.convertStringValue missingValuesStr cultureStr false field
 
                 { TypeForTuple = typWithoutMeasure
                   ProvidedProperty =
@@ -165,6 +165,7 @@ module internal HtmlGenerator =
                     ConversionsGenerator.convertStringValue
                         missingValuesStr
                         cultureStr
+                        false
                         (StructuralTypes.PrimitiveInferedProperty.Create("", typ, optional, None))
 
                 typ, conv
@@ -173,6 +174,7 @@ module internal HtmlGenerator =
                     ConversionsGenerator.convertStringValue
                         missingValuesStr
                         cultureStr
+                        false
                         (StructuralTypes.PrimitiveInferedProperty.Create("", typeof<string>, false, None))
 
                 typ, conv
@@ -238,6 +240,7 @@ module internal HtmlGenerator =
                         ConversionsGenerator.convertStringValue
                             missingValuesStr
                             cultureStr
+                            false
                             (StructuralTypes.PrimitiveInferedProperty.Create("", typ, optional, None))
 
                     typ, conv
@@ -246,6 +249,7 @@ module internal HtmlGenerator =
                         ConversionsGenerator.convertStringValue
                             missingValuesStr
                             cultureStr
+                            false
                             (StructuralTypes.PrimitiveInferedProperty.Create("", typeof<String>, false, None))
 
                     typ, conv

src/FSharp.Data.DesignTime/Json/JsonConversionsGenerator.fs

@@ -56,6 +56,7 @@ let internal convertJsonValue
     missingValuesStr
     cultureStr
     canPassAllConversionCallingTypes
+    exceptionIfMissing
     (field: PrimitiveInferedValue)
     =
 
@@ -80,15 +81,21 @@ let internal convertJsonValue
         let convert value =
             getConversionQuotation missingValuesStr cultureStr field.InferedType value
 
+        let getNonOptionalValueMethod =
+            if exceptionIfMissing then
+                nameof (JsonRuntime.GetNonOptionalValueStrict)
+            else
+                nameof (JsonRuntime.GetNonOptionalValue)
+
         match field.TypeWrapper, canPassAllConversionCallingTypes with
         | TypeWrapper.None, true ->
             wrapInLetIfNeeded value (fun (varExpr: Expr<JsonValueOptionAndPath>) ->
-                typeof<JsonRuntime>?(nameof (JsonRuntime.GetNonOptionalValue))
+                typeof<JsonRuntime>?(getNonOptionalValueMethod)
                     (field.RuntimeType)
                     (<@ (%varExpr).Path @>, convert <@ (%varExpr).JsonOpt @>, <@ (%varExpr).JsonOpt @>))
         | TypeWrapper.None, false ->
             wrapInLetIfNeeded value (fun (varExpr: Expr<IJsonDocument>) ->
-                typeof<JsonRuntime>?(nameof (JsonRuntime.GetNonOptionalValue))
+                typeof<JsonRuntime>?(getNonOptionalValueMethod)
                     (field.RuntimeType)
                     (<@ (%varExpr).Path() @>, convert <@ Some (%varExpr).JsonValue @>, <@ Some (%varExpr).JsonValue @>))
         | TypeWrapper.Option, true -> convert <@ (%%value: JsonValue option) @>

src/FSharp.Data.DesignTime/Json/JsonGenerator.fs

@@ -32,7 +32,8 @@ type internal JsonGenerationContext =
       InferenceMode: InferenceMode'
       UnitsOfMeasureProvider: IUnitsOfMeasureProvider
       UseOriginalNames: bool
-      OmitNullFields: bool }
+      OmitNullFields: bool
+      ExceptionIfMissing: bool }
 
     static member Create
         (
@@ -44,7 +45,8 @@ type internal JsonGenerationContext =
             ?typeCache,
             ?preferDictionaries,
             ?useOriginalNames,
-            ?omitNullFields
+            ?omitNullFields,
+            ?exceptionIfMissing
         ) =
         let useOriginalNames = defaultArg useOriginalNames false
 
@@ -56,6 +58,7 @@ type internal JsonGenerationContext =
         let typeCache = defaultArg typeCache (Dictionary())
         let preferDictionaries = defaultArg preferDictionaries false
         let omitNullFields = defaultArg omitNullFields false
+        let exceptionIfMissing = defaultArg exceptionIfMissing false
 
         JsonGenerationContext.Create(
             cultureStr,
@@ -67,7 +70,8 @@ type internal JsonGenerationContext =
             inferenceMode,
             unitsOfMeasureProvider,
             useOriginalNames,
-            omitNullFields
+            omitNullFields,
+            exceptionIfMissing
         )
 
     static member Create
@@ -81,7 +85,8 @@ type internal JsonGenerationContext =
             inferenceMode,
             unitsOfMeasureProvider,
             useOriginalNames,
-            omitNullFields
+            omitNullFields,
+            exceptionIfMissing
         ) =
         { CultureStr = cultureStr
           TypeProviderType = tpType
@@ -95,7 +100,8 @@ type internal JsonGenerationContext =
           InferenceMode = inferenceMode
           UnitsOfMeasureProvider = unitsOfMeasureProvider
           UseOriginalNames = useOriginalNames
-          OmitNullFields = omitNullFields }
+          OmitNullFields = omitNullFields
+          ExceptionIfMissing = exceptionIfMissing }
 
     static member Create
         (
@@ -119,6 +125,7 @@ type internal JsonGenerationContext =
             inferenceMode,
             unitsOfMeasureProvider,
             useOriginalNames,
+            false,
             false
         )
 
@@ -367,7 +374,7 @@ module JsonTypeBuilder =
 
             let typ, conv, conversionCallingType =
                 PrimitiveInferedValue.Create(inferedType, optional, unit)
-                |> convertJsonValue "" ctx.CultureStr canPassAllConversionCallingTypes
+                |> convertJsonValue "" ctx.CultureStr canPassAllConversionCallingTypes ctx.ExceptionIfMissing
 
             { ConvertedType = typ
               OptionalConverter = Some conv

src/FSharp.Data.DesignTime/Json/JsonProvider.fs

@@ -62,6 +62,7 @@ type public JsonProvider(cfg: TypeProviderConfig) as this =
         let omitNullFields = args.[13] :?> bool
         let preferOptionals = args.[14] :?> bool
         let preferDateTimeOffset = args.[15] :?> bool
+        let exceptionIfMissing = args.[16] :?> bool
 
         let inferenceMode =
             InferenceMode'.FromPublicApi(inferenceMode, inferTypesFromValues)
@@ -136,7 +137,8 @@ type public JsonProvider(cfg: TypeProviderConfig) as this =
                     inferenceMode,
                     ?preferDictionaries = Some preferDictionaries,
                     ?useOriginalNames = Some useOriginalNames,
-                    ?omitNullFields = Some omitNullFields
+                    ?omitNullFields = Some omitNullFields,
+                    ?exceptionIfMissing = Some exceptionIfMissing
                 )
 
             let result = JsonTypeBuilder.generateJsonType ctx false false rootName inferedType
@@ -187,7 +189,8 @@ type public JsonProvider(cfg: TypeProviderConfig) as this =
           ProvidedStaticParameter("UseOriginalNames", typeof<bool>, parameterDefaultValue = false)
           ProvidedStaticParameter("OmitNullFields", typeof<bool>, parameterDefaultValue = false)
           ProvidedStaticParameter("PreferOptionals", typeof<bool>, parameterDefaultValue = true)
-          ProvidedStaticParameter("PreferDateTimeOffset", typeof<bool>, parameterDefaultValue = false) ]
+          ProvidedStaticParameter("PreferDateTimeOffset", typeof<bool>, parameterDefaultValue = false)
+          ProvidedStaticParameter("ExceptionIfMissing", typeof<bool>, parameterDefaultValue = false) ]
 
     let helpText =
         """<summary>Typed representation of a JSON document.</summary>
@@ -215,7 +218,8 @@ type public JsonProvider(cfg: TypeProviderConfig) as this =
            <param name='UseOriginalNames'>When true, JSON property names are used as-is for generated property names instead of being normalized to PascalCase. Defaults to false.</param>
            <param name='OmitNullFields'>When true, optional fields with value None are omitted from the generated JSON rather than serialized as null. Defaults to false.</param>
            <param name='PreferOptionals'>When set to true (default), inference will use the option type for missing or null values. When false, inference will prefer to use empty string or double.NaN for missing values where possible, matching the default CsvProvider behavior.</param>
-           <param name='PreferDateTimeOffset'>When true, date-time strings without an explicit timezone offset are inferred as DateTimeOffset (using the local offset) instead of DateTime. Defaults to false.</param>"""
+           <param name='PreferDateTimeOffset'>When true, date-time strings without an explicit timezone offset are inferred as DateTimeOffset (using the local offset) instead of DateTime. Defaults to false.</param>
+           <param name='ExceptionIfMissing'>When true, accessing a non-optional field that is missing in the JSON data raises an exception instead of returning a default value (empty string for string, NaN for float). Defaults to false for backward compatibility.</param>"""
 
     do jsonProvTy.AddXmlDoc helpText
     do jsonProvTy.DefineStaticParameters(parameters, buildTypes)

src/FSharp.Data.DesignTime/Xml/XmlGenerator.fs

@@ -30,12 +30,15 @@ type internal XmlGenerationContext =
       UnifyGlobally: bool
       XmlTypeCache: Dictionary<InferedType, XmlGenerationResult>
       JsonTypeCache: Dictionary<InferedType, ProvidedTypeDefinition>
-      UseOriginalNames: bool }
+      UseOriginalNames: bool
+      ExceptionIfMissing: bool }
 
-    static member Create(unitsOfMeasureProvider, inferenceMode, cultureStr, tpType, unifyGlobally, useOriginalNames) =
+    static member Create
+        (unitsOfMeasureProvider, inferenceMode, cultureStr, tpType, unifyGlobally, useOriginalNames, ?exceptionIfMissing) =
         let niceName = if useOriginalNames then id else NameUtils.nicePascalName
         let uniqueNiceName = NameUtils.uniqueGenerator niceName
         uniqueNiceName "XElement" |> ignore
+        let exceptionIfMissing = defaultArg exceptionIfMissing false
 
         { CultureStr = cultureStr
           UnitsOfMeasureProvider = unitsOfMeasureProvider
@@ -45,15 +48,18 @@ type internal XmlGenerationContext =
           UnifyGlobally = unifyGlobally
           XmlTypeCache = Dictionary()
           JsonTypeCache = Dictionary()
-          UseOriginalNames = useOriginalNames }
+          UseOriginalNames = useOriginalNames
+          ExceptionIfMissing = exceptionIfMissing }
 
     member x.ConvertValue prop =
-        let typ, _, conv, _ = ConversionsGenerator.convertStringValue "" x.CultureStr prop
+        let typ, _, conv, _ =
+            ConversionsGenerator.convertStringValue "" x.CultureStr x.ExceptionIfMissing prop
+
         typ, conv
 
     member x.ConvertValueBack prop =
         let typ, _, _, convBack =
-            ConversionsGenerator.convertStringValue "" x.CultureStr prop
+            ConversionsGenerator.convertStringValue "" x.CultureStr x.ExceptionIfMissing prop
 
         typ, convBack
 
@@ -155,7 +161,8 @@ module internal XmlTypeBuilder =
                           ctx.UnitsOfMeasureProvider,
                           ctx.InferenceMode,
                           ctx.UniqueNiceName,
-                          ctx.JsonTypeCache
+                          ctx.JsonTypeCache,
+                          ?exceptionIfMissing = Some ctx.ExceptionIfMissing
                       )
 
                   let result = JsonTypeBuilder.generateJsonType ctx false true "" typ

src/FSharp.Data.DesignTime/Xml/XmlProvider.fs

@@ -55,7 +55,8 @@ type public XmlProvider(cfg: TypeProviderConfig) as this =
         let useOriginalNames = args.[12] :?> bool
         let preferOptionals = args.[13] :?> bool
         let useSchemaTypeNames = args.[14] :?> bool
-        let preferDateTimeOffset = args.[14] :?> bool
+        let preferDateTimeOffset = args.[15] :?> bool
+        let exceptionIfMissing = args.[16] :?> bool
 
         let inferenceMode =
             InferenceMode'.FromPublicApi(inferenceMode, inferTypesFromValues)
@@ -110,7 +111,8 @@ type public XmlProvider(cfg: TypeProviderConfig) as this =
                         cultureStr,
                         tpType,
                         globalInference || schema <> "",
-                        useOriginalNames
+                        useOriginalNames,
+                        exceptionIfMissing
                     )
 
                 let result = XmlTypeBuilder.generateXmlType ctx inferedType
@@ -174,7 +176,8 @@ type public XmlProvider(cfg: TypeProviderConfig) as this =
                         cultureStr,
                         tpType,
                         globalInference || schema <> "",
-                        useOriginalNames
+                        useOriginalNames,
+                        exceptionIfMissing
                     )
 
                 let result = XmlTypeBuilder.generateXmlType ctx inferedType
@@ -226,7 +229,8 @@ type public XmlProvider(cfg: TypeProviderConfig) as this =
           ProvidedStaticParameter("UseOriginalNames", typeof<bool>, parameterDefaultValue = false)
           ProvidedStaticParameter("PreferOptionals", typeof<bool>, parameterDefaultValue = true)
           ProvidedStaticParameter("UseSchemaTypeNames", typeof<bool>, parameterDefaultValue = false)
-          ProvidedStaticParameter("PreferDateTimeOffset", typeof<bool>, parameterDefaultValue = false) ]
+          ProvidedStaticParameter("PreferDateTimeOffset", typeof<bool>, parameterDefaultValue = false)
+          ProvidedStaticParameter("ExceptionIfMissing", typeof<bool>, parameterDefaultValue = false) ]
 
     let helpText =
         """<summary>Typed representation of a XML file.</summary>
@@ -255,7 +259,8 @@ type public XmlProvider(cfg: TypeProviderConfig) as this =
            <param name='UseOriginalNames'>When true, XML element and attribute names are used as-is for generated property names instead of being normalized to PascalCase. Defaults to false.</param>
            <param name='PreferOptionals'>When set to true (default), inference will use the option type for missing or absent values. When false, inference will prefer to use empty string or double.NaN for missing values where possible, matching the default CsvProvider behavior.</param>
            <param name='UseSchemaTypeNames'>When true and a Schema is provided, the XSD complex type name is used for the generated F# type instead of the element name. This causes multiple elements that share the same XSD type to map to a single F# type. Defaults to false for backward compatibility.</param>
-           <param name='PreferDateTimeOffset'>When true, date-time strings without an explicit timezone offset are inferred as DateTimeOffset (using the local offset) instead of DateTime. Defaults to false.</param>"""
+           <param name='PreferDateTimeOffset'>When true, date-time strings without an explicit timezone offset are inferred as DateTimeOffset (using the local offset) instead of DateTime. Defaults to false.</param>
+           <param name='ExceptionIfMissing'>When true, accessing a non-optional field that is missing in the XML data raises an exception instead of returning a default value (empty string for string, NaN for float). Defaults to false for backward compatibility.</param>"""
 
 
     do xmlProvTy.AddXmlDoc helpText

src/FSharp.Data.Json.Core/JsonRuntime.fs

@@ -99,6 +99,27 @@ type JsonRuntime =
             failwithf "Expecting %s at '%s', got %s" (getTypeName ()) path
             <| x.ToString(JsonSaveOptions.DisableFormatting)
 
+    /// Operation that extracts the value from an option and always throws if the value is not present.
+    /// Used when ExceptionIfMissing=true to raise an exception for missing fields instead of returning defaults.
+    static member GetNonOptionalValueStrict<'T>(path: string, opt: option<'T>, originalValue) : 'T =
+        let getTypeName () =
+            let name = typeof<'T>.Name
+
+            if name.StartsWith("i", StringComparison.OrdinalIgnoreCase) then
+                "an " + name
+            else
+                "a " + name
+
+        match opt, originalValue with
+        | Some value, _ -> value
+        | None, Some((JsonValue.Array _ | JsonValue.Record _) as x) ->
+            failwithf "Expecting %s at '%s', got %s" (getTypeName ()) path
+            <| x.ToString(JsonSaveOptions.DisableFormatting)
+        | None, None -> failwithf "'%s' is missing" path
+        | None, Some x ->
+            failwithf "Expecting %s at '%s', got %s" (getTypeName ()) path
+            <| x.ToString(JsonSaveOptions.DisableFormatting)
+
     /// Converts JSON array to array of target types
     static member ConvertArray<'T>(doc: IJsonDocument, mapping: Func<IJsonDocument, 'T>) =
         match doc.JsonValue with