Fix #12796: handle arrays of user-defined types in custom attributes (#19472)

* Fix #12796: handle arrays of user-defined types in custom attributes

Empty arrays of non-encodable types (classes, records, structs) in custom
attributes now compile successfully by encoding them as System.Object[] per
ECMA-335. Non-empty arrays of such types produce FS3887 instead of the
internal error FS0192.

Author: T-Gro

docs/release-notes/.FSharp.Compiler.Service/11.0.100.md

@@ -39,6 +39,7 @@
 * Fix methods being tagged as `Member` instead of `Method` in tooltips. ([Issue #10540](https://github.com/dotnet/fsharp/issues/10540), [PR #19507](https://github.com/dotnet/fsharp/pull/19507))
 * Fix Debug-mode compilation when mixing resumable and standard computation expressions. ([Issue #19625](https://github.com/dotnet/fsharp/issues/19625), [PR #19630](https://github.com/dotnet/fsharp/pull/19630))
 * IlxGen: fix missing CompilationMapping attribute for generic values ([PR #19643](https://github.com/dotnet/fsharp/pull/19643))
+* Fix internal error `FS0192: encodeCustomAttrElemType` when using arrays of user-defined types as custom attribute arguments. Empty arrays (e.g. `[<DefaultValue([||] : A[])>]`) now compile successfully; non-empty arrays of unencodable types report a proper diagnostic (FS3887) instead of an internal error. ([Issue #12796](https://github.com/dotnet/fsharp/issues/12796), [PR #19472](https://github.com/dotnet/fsharp/pull/19472))
 * Fix internal compiler error in `use` bindings when a C#-style `Dispose` extension method is in scope alongside `IDisposable.Dispose`. ([Issue #19552](https://github.com/dotnet/fsharp/issues/19552), [PR #19568](https://github.com/dotnet/fsharp/pull/19568))
 * Fix signature generation: single-case struct DU gets spurious bar causing FS0300. ([Issue #19597](https://github.com/dotnet/fsharp/issues/19597), [PR #19609](https://github.com/dotnet/fsharp/pull/19609))
 * Fix signature generation: backticked active pattern case names lose escaping. ([Issue #19592](https://github.com/dotnet/fsharp/issues/19592), [PR #19609](https://github.com/dotnet/fsharp/pull/19609))

src/Compiler/AbstractIL/il.fs

@@ -4978,6 +4978,7 @@ let rec decodeCustomAttrElemType bytes sigptr x =
         let elemTy, sigptr = decodeCustomAttrElemType bytes sigptr et
         mkILArr1DTy elemTy, sigptr
     | x when x = 0x50uy -> PrimaryAssemblyILGlobals.typ_Type, sigptr
+    | x when x = 0x51uy -> PrimaryAssemblyILGlobals.typ_Object, sigptr // SERIALIZATION_TYPE_TAGGED_OBJECT (ECMA-335 II.23.3)
     | _ -> failwithf "decodeCustomAttrElemType ilg: unrecognized custom element type: %A" x
 
 /// Given a custom attribute element, encode it to a binary representation according to the rules in Ecma 335 Partition II.

src/Compiler/CodeGen/IlxGen.fs

@@ -10300,7 +10300,7 @@ and EmitRestoreStack cgbuf (savedStack, savedStackLocals) =
 //GenAttr: custom attribute generation
 //-------------------------------------------------------------------------
 
-and GenAttribArg amap g eenv x (ilArgTy: ILType) =
+and GenAttribArg amap (g: TcGlobals) eenv x (ilArgTy: ILType) =
 
     match stripDebugPoints x, ilArgTy with
     // Detect 'null' used for an array argument
@@ -10356,7 +10356,46 @@ and GenAttribArg amap g eenv x (ilArgTy: ILType) =
     // Detect '[| ... |]' nodes
     | Expr.Op(TOp.Array, [ elemTy ], args, m), _ ->
         let ilElemTy = GenType amap m eenv.tyenv elemTy
-        ILAttribElem.Array(ilElemTy, List.map (fun arg -> GenAttribArg amap g eenv arg ilElemTy) args)
+
+        // Check if element type can be encoded in custom attribute metadata (ECMA-335 II.23.3).
+        // Valid element types: primitives, enums, string, System.Type, System.Object.
+        let isEncodableElemType =
+            match ilElemTy with
+            | ILType.Value tspec ->
+                match tspec.Name with
+                | "System.SByte"
+                | "System.Byte"
+                | "System.Int16"
+                | "System.UInt16"
+                | "System.Int32"
+                | "System.UInt32"
+                | "System.Int64"
+                | "System.UInt64"
+                | "System.Double"
+                | "System.Single"
+                | "System.Char"
+                | "System.Boolean" -> true
+                | _ -> isEnumTy g elemTy
+            | ILType.Boxed tspec ->
+                tspec.Name = "System.String"
+                || tspec.Name = "System.Object"
+                || tspec.Name = "System.Type"
+            | _ -> false
+
+        if not isEncodableElemType then
+            if args.IsEmpty then
+                // Empty arrays: substitute System.Object as element type since no elements need encoding.
+                ILAttribElem.Array(g.ilg.typ_Object, [])
+            else
+                let elemTypeName =
+                    if ilElemTy.IsNominal then
+                        ilElemTy.TypeRef.FullName
+                    else
+                        string ilElemTy
+
+                error (Error(FSComp.SR.ilCustomAttrInvalidArrayElemType elemTypeName, m))
+        else
+            ILAttribElem.Array(ilElemTy, List.map (fun arg -> GenAttribArg amap g eenv arg ilElemTy) args)
 
     // Detect 'typeof<ty>' calls
     | TypeOfExpr g ty, _ -> ILAttribElem.Type(Some(GenType amap x.Range eenv.tyenv ty))

src/Compiler/FSComp.txt

@@ -1817,3 +1817,4 @@ featurePreprocessorElif,"#elif preprocessor directive"
 3884,tcFunctionValueUsedAsInterpolatedStringArg,"This expression is a function value. When used in an interpolated string it will be formatted using its 'ToString' method, which is likely not the intended behavior. Consider applying the function to its arguments."
 3885,parsLetBangCannotBeLastInCE,"'%s' cannot be the final expression in a computation expression. Finish with 'return', 'return!', or a simple expression."
 3886,tcListLiteralWithSingleTupleElement,"This list expression contains a single tuple element. Did you mean to use ';' instead of ',' to separate list elements?"
+3887,ilCustomAttrInvalidArrayElemType,"The type '%s' is not a valid custom attribute argument type. Custom attribute arrays must have elements of primitive types, enums, string, System.Type, or System.Object."