Reapply "Inline CORINFO_HELP_ARRADDR_ST helper call, remove WriteBarr… …ier FCall" (#126530)

src/coreclr/System.Private.CoreLib/src/System/Array.CoreCLR.cs

@@ -395,6 +395,7 @@ private ArrayInitializeCache(delegate*<ref byte, void> constructorEntrypoint)
     // array that is castable to "T[]" (i.e. for primitives and valuetypes, it will be exactly
     // "T[]" - for orefs, it may be a "U[]" where U derives from T.)
     //----------------------------------------------------------------------------------------
+    [Intrinsic]
     internal sealed class SZArrayHelper
     {
         // It is never legal to instantiate this class.

src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs

@@ -59,9 +59,6 @@ private static object ChkCastAny_NoCacheLookup(void* toTypeHnd, object obj)
             return obj;
         }
 
-        [MethodImpl(MethodImplOptions.InternalCall)]
-        private static extern void WriteBarrier(ref object? dst, object? obj);
-
         // IsInstanceOf test used for unusual cases (naked type parameters, variant generic types)
         // Unlike the IsInstanceOfInterface and IsInstanceOfClass functions,
         // this test must deal with all kinds of type tests
@@ -471,7 +468,7 @@ private static void StelemRef(object?[] array, nint index, object? obj)
                 goto notExactMatch;
 
             doWrite:
-                WriteBarrier(ref element, obj);
+                RuntimeHelpers.WriteBarrier(ref element, obj);
                 return;
 
             assigningNull:
@@ -493,7 +490,7 @@ private static void StelemRef_Helper(ref object? element, void* elementType, obj
             CastResult result = CastCache.TryGet(s_table!, (nuint)RuntimeHelpers.GetMethodTable(obj), (nuint)elementType);
             if (result == CastResult.CanCast)
             {
-                WriteBarrier(ref element, obj);
+                RuntimeHelpers.WriteBarrier(ref element, obj);
                 return;
             }
 
@@ -512,7 +509,7 @@ private static void StelemRef_Helper_NoCacheLookup(ref object? element, void* el
                 ThrowArrayMismatchException();
             }
 
-            WriteBarrier(ref element, obj2);
+            RuntimeHelpers.WriteBarrier(ref element, obj2);
         }
 
         [DebuggerHidden]

src/coreclr/jit/compiler.h

@@ -5371,6 +5371,30 @@ class Compiler
                                                             GenTree*    dereferencedAddress,
                                                             InlArgInfo* inlArgInfo);
 
+    typedef JitHashTable<CORINFO_METHOD_HANDLE, JitPtrKeyFuncs<struct CORINFO_METHOD_STRUCT_>, CORINFO_METHOD_HANDLE> HelperToManagedMap;
+    HelperToManagedMap* m_helperToManagedMap = nullptr;
+
+public:
+    HelperToManagedMap* GetHelperToManagedMap()
+    {
+        if (m_helperToManagedMap == nullptr)
+        {
+            m_helperToManagedMap = new (getAllocator()) HelperToManagedMap(getAllocator());
+        }
+        return m_helperToManagedMap;
+    }
+    bool HelperToManagedMapLookup(CORINFO_METHOD_HANDLE helperCallHnd, CORINFO_METHOD_HANDLE* userCallHnd)
+    {
+        if (m_helperToManagedMap == nullptr)
+        {
+            return false;
+        }
+        bool found = m_helperToManagedMap->Lookup(helperCallHnd, userCallHnd);
+        return found;
+    }
+private:
+
+    void impConvertToUserCallAndMarkForInlining(GenTreeCall* call);
     void impMarkInlineCandidate(GenTree*               call,
                                 CORINFO_CONTEXT_HANDLE exactContextHnd,
                                 bool                   exactContextNeedsRuntimeLookup,

src/coreclr/jit/gentree.cpp

@@ -2397,6 +2397,34 @@ bool GenTreeCall::IsHelperCall(unsigned helper) const
     return IsHelperCall(Compiler::eeFindHelper(helper));
 }
 
+//-------------------------------------------------------------------------
+// IsHelperCallOrUserEquivalent: Determine if this GT_CALL node is a specific helper call
+//     or its CT_USER equivalent.
+//
+// Arguments:
+//     compiler - the compiler instance so that we can call eeFindHelper
+//
+// Return Value:
+//     Returns true if this GT_CALL node is a call to the specified helper.
+//
+bool GenTreeCall::IsHelperCallOrUserEquivalent(Compiler* compiler, unsigned helper) const
+{
+    CORINFO_METHOD_HANDLE helperCallHnd = Compiler::eeFindHelper(helper);
+    if (IsHelperCall())
+    {
+        return helperCallHnd == gtCallMethHnd;
+    }
+
+    if (gtCallType == CT_USER_FUNC)
+    {
+        CORINFO_METHOD_HANDLE userCallHnd = NO_METHOD_HANDLE;
+        return compiler->impInlineRoot()->HelperToManagedMapLookup(helperCallHnd, &userCallHnd) &&
+               (userCallHnd == gtCallMethHnd);
+    }
+
+    return false;
+}
+
 //-------------------------------------------------------------------------
 // IsRuntimeLookupHelperCall: Determine if this GT_CALL node represents a runtime lookup helper call.
 //
@@ -12972,6 +13000,9 @@ void Compiler::gtDispTree(GenTree*                    tree,
                 case NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant:
                     printf(" isKnownConst");
                     break;
+                case NI_System_Runtime_CompilerServices_RuntimeHelpers_WriteBarrier:
+                    printf(" WriteBarrier");
+                    break;
 #if defined(FEATURE_SIMD)
                 case NI_SIMD_UpperRestore:
                     printf(" simdUpperRestore");

src/coreclr/jit/gentree.h

@@ -5870,6 +5870,8 @@ struct GenTreeCall final : public GenTree
 
     bool IsHelperCall(unsigned helper) const;
 
+    bool IsHelperCallOrUserEquivalent(Compiler* compiler, unsigned helper) const;
+
     bool IsRuntimeLookupHelperCall(Compiler* compiler) const;
 
     bool IsSpecialIntrinsic(Compiler* compiler, NamedIntrinsic ni) const;

src/coreclr/jit/importer.cpp

@@ -7268,7 +7268,12 @@ void Compiler::impImportBlockCode(BasicBlock* block)
                 // The array helper takes a native int for array length.
                 // So if we have an int, explicitly extend it to be a native int.
                 index = impImplicitIorI4Cast(index, TYP_I_IMPL);
-                op1   = gtNewHelperCallNode(CORINFO_HELP_ARRADDR_ST, TYP_VOID, array, index, value);
+
+                GenTreeCall* call = gtNewHelperCallNode(CORINFO_HELP_ARRADDR_ST, TYP_VOID, array, index, value);
+                INDEBUG(call->gtRawILOffset = opcodeOffs);
+                impConvertToUserCallAndMarkForInlining(call);
+                op1 = call;
+
                 goto SPILL_APPEND;
             }
 

src/coreclr/jit/importercalls.cpp

@@ -3411,6 +3411,8 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE    clsHnd,
             // This one is just `return true/false`
             case NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant:
 
+            case NI_System_Runtime_CompilerServices_RuntimeHelpers_WriteBarrier:
+
             // Not expanding this can lead to noticeable allocations in T0
             case NI_System_Runtime_CompilerServices_RuntimeHelpers_CreateSpan:
 
@@ -3640,6 +3642,14 @@ GenTree* Compiler::impIntrinsic(CORINFO_CLASS_HANDLE    clsHnd,
                 break;
             }
 
+            case NI_System_Runtime_CompilerServices_RuntimeHelpers_WriteBarrier:
+            {
+                GenTree* val = impPopStack().val;
+                GenTree* dst = impPopStack().val;
+                retNode      = gtNewStoreIndNode(TYP_REF, dst, val, GTF_IND_TGT_HEAP);
+                break;
+            }
+
             case NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant:
             {
                 GenTree* op1 = impPopStack().val;
@@ -7978,6 +7988,53 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall*           call,
     call->AddGDVCandidateInfo(this, pInfo);
 }
 
+//------------------------------------------------------------------------
+// impConvertToUserCallAndMarkForInlining: convert a helper call to a user call
+//   and mark it for inlining. This is used for helper calls that are
+//   known to be backed by a user method that can be inlined.
+//
+// Arguments:
+//    call - the helper call to convert
+//
+void Compiler::impConvertToUserCallAndMarkForInlining(GenTreeCall* call)
+{
+    assert(call->IsHelperCall());
+
+    if (!opts.OptEnabled(CLFLG_INLINING))
+    {
+        return;
+    }
+
+    CORINFO_METHOD_HANDLE helperCallHnd     = call->gtCallMethHnd;
+    CORINFO_METHOD_HANDLE managedCallHnd    = NO_METHOD_HANDLE;
+    CORINFO_CONST_LOOKUP  pNativeEntrypoint = {};
+    info.compCompHnd->getHelperFtn(eeGetHelperNum(helperCallHnd), &pNativeEntrypoint, &managedCallHnd);
+
+    if (managedCallHnd != NO_METHOD_HANDLE)
+    {
+        call->gtCallMethHnd = managedCallHnd;
+        call->gtCallType    = CT_USER_FUNC;
+
+        CORINFO_CALL_INFO hCallInfo = {};
+        hCallInfo.hMethod           = managedCallHnd;
+        hCallInfo.methodFlags       = info.compCompHnd->getMethodAttribs(hCallInfo.hMethod);
+        impMarkInlineCandidate(call, nullptr, false, &hCallInfo, compInlineContext);
+
+#if DEBUG
+        CORINFO_METHOD_HANDLE existingValue = NO_METHOD_HANDLE;
+        if (impInlineRoot()->HelperToManagedMapLookup(helperCallHnd, &existingValue))
+        {
+            // Let's make sure HelperToManagedMap::Overwrite behavior always overwrites the same value.
+            assert(existingValue == managedCallHnd);
+        }
+#endif
+
+        impInlineRoot()->GetHelperToManagedMap()->Set(helperCallHnd, managedCallHnd, HelperToManagedMap::Overwrite);
+        JITDUMP("Converting helperCall [%06u] to user call [%s] and marking for inlining\n", dspTreeID(call),
+                eeGetMethodFullName(managedCallHnd));
+    }
+}
+
 //------------------------------------------------------------------------
 // impMarkInlineCandidate: determine if this call can be subsequently inlined
 //
@@ -10954,6 +11011,10 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method)
                             {
                                 result = NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant;
                             }
+                            else if (strcmp(methodName, "WriteBarrier") == 0)
+                            {
+                                result = NI_System_Runtime_CompilerServices_RuntimeHelpers_WriteBarrier;
+                            }
                             else if (strcmp(methodName, "IsReferenceOrContainsReferences") == 0)
                             {
                                 result =

src/coreclr/jit/morph.cpp

@@ -6439,14 +6439,23 @@ GenTree* Compiler::fgMorphCall(GenTreeCall* call)
 
     // Morph stelem.ref helper call to store a null value, into a store into an array without the helper.
     // This needs to be done after the arguments are morphed to ensure constant propagation has already taken place.
-    if (opts.OptimizationEnabled() && call->IsHelperCall(CORINFO_HELP_ARRADDR_ST))
+    if (opts.OptimizationEnabled() && call->IsHelperCallOrUserEquivalent(this, CORINFO_HELP_ARRADDR_ST))
     {
         assert(call->gtArgs.CountUserArgs() == 3);
 
         GenTree* arr   = call->gtArgs.GetUserArgByIndex(0)->GetNode();
         GenTree* index = call->gtArgs.GetUserArgByIndex(1)->GetNode();
         GenTree* value = call->gtArgs.GetUserArgByIndex(2)->GetNode();
 
+        if (!call->IsHelperCall())
+        {
+            // Convert back to helper call if it wasn't inlined.
+            // Currently, only helper calls are eligible to be direct calls if the target has reached
+            // its final tier. TODO: remove this workaround and convert this user call to direct as well.
+            call->gtCallMethHnd = eeFindHelper(CORINFO_HELP_ARRADDR_ST);
+            call->gtCallType    = CT_HELPER;
+        }
+
         if (gtCanSkipCovariantStoreCheck(value, arr))
         {
             // Either or both of the array and index arguments may have been spilled to temps by `fgMorphArgs`. Copy

src/coreclr/jit/namedintrinsiclist.h

@@ -123,6 +123,7 @@ enum NamedIntrinsic : unsigned short
     NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant,
     NI_System_Runtime_CompilerServices_RuntimeHelpers_IsReferenceOrContainsReferences,
     NI_System_Runtime_CompilerServices_RuntimeHelpers_GetMethodTable,
+    NI_System_Runtime_CompilerServices_RuntimeHelpers_WriteBarrier,
     NI_System_Runtime_CompilerServices_RuntimeHelpers_SetNextCallGenericContext,
     NI_System_Runtime_CompilerServices_RuntimeHelpers_SetNextCallAsyncContinuation,
 

src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/CompilerServices/RuntimeHelpers.cs

@@ -18,5 +18,8 @@ public static int OffsetToStringData
 
         [Intrinsic]
         public static extern void InitializeArray(Array array, RuntimeFieldHandle fldHandle);
+
+        [Intrinsic]
+        internal static void WriteBarrier(ref object? dst, object? obj) => dst = obj;
     }
 }

src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/InternalCalls.cs

@@ -121,10 +121,6 @@ internal static class InternalCalls
         internal static extern unsafe object RhpNewFastMisalign(MethodTable * pEEType);
 #endif // FEATURE_64BIT_ALIGNMENT
 
-        [RuntimeImport(RuntimeLibrary, "RhpAssignRef")]
-        [MethodImpl(MethodImplOptions.InternalCall)]
-        internal static extern unsafe void RhpAssignRef(ref object? address, object? obj);
-
         [MethodImplAttribute(MethodImplOptions.InternalCall)]
         [RuntimeImport(RuntimeLibrary, "RhpGcSafeZeroMemory")]
         internal static extern unsafe ref byte RhpGcSafeZeroMemory(ref byte dmem, nuint size);

src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/TypeCast.cs

@@ -803,7 +803,7 @@ public static unsafe void StelemRef(object?[] array, nint index, object? obj)
                 goto notExactMatch;
 
         doWrite:
-            InternalCalls.RhpAssignRef(ref element, obj);
+            RuntimeHelpers.WriteBarrier(ref element, obj);
             return;
 
         assigningNull:
@@ -826,7 +826,7 @@ private static unsafe void StelemRef_Helper(ref object? element, MethodTable* el
             CastResult result = s_castCache.TryGet((nuint)obj.GetMethodTable() + (int)AssignmentVariation.BoxedSource, (nuint)elementType);
             if (result == CastResult.CanCast)
             {
-                InternalCalls.RhpAssignRef(ref element, obj);
+                RuntimeHelpers.WriteBarrier(ref element, obj);
                 return;
             }
 
@@ -843,7 +843,7 @@ private static unsafe void StelemRef_Helper_NoCacheLookup(ref object? element, M
                 throw elementType->GetClasslibException(ExceptionIDs.ArrayTypeMismatch);
             }
 
-            InternalCalls.RhpAssignRef(ref element, obj);
+            RuntimeHelpers.WriteBarrier(ref element, castedObj);
         }
 
         private static unsafe object IsInstanceOfArray(MethodTable* pTargetType, object obj)

src/coreclr/nativeaot/System.Private.CoreLib/src/System/Array.NativeAot.cs

@@ -512,6 +512,7 @@ private static int LastIndexOfImpl<T>(T[] array, T value, int startIndex, int co
     //
     // Note: the declared base type and interface list also determines what Reflection returns from TypeInfo.BaseType and TypeInfo.ImplementedInterfaces for array types.
     //
+    [Intrinsic]
     internal class Array<T> : Array, IEnumerable<T>, ICollection<T>, IList<T>, IReadOnlyList<T>
     {
         // Prevent the C# compiler from generating a public default constructor

src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/RuntimeHelpers.cs

@@ -15,5 +15,8 @@ public static int OffsetToStringData
                 return string.FIRST_CHAR_OFFSET;
             }
         }
+
+        [Intrinsic]
+        internal static void WriteBarrier(ref object? dst, object? obj) => dst = obj;
     }
 }

src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs

@@ -3053,6 +3053,25 @@ private bool isExactType(CORINFO_CLASS_STRUCT_* cls)
             if (type.HasTypeEquivalence || type.HasVariance)
                 return false;
 
+            // SZArrayHelper (CoreCLR) and Array<T> (NativeAOT) are phantom dispatch
+            // targets: the runtime maps generic array interface methods
+            // (IList<T>.set_Item, etc.) to methods on these sealed/effectively-sealed
+            // types, but no runtime object ever has them as its MethodTable - `this`
+            // inside those methods is always a T[]. Reporting them as exact would
+            // allow the JIT (e.g. VN-based invariant-load folding of GetMethodTable)
+            // to embed their MT as a constant and mis-read fields off it. Both types
+            // are marked [Intrinsic] so that the cheap flag check filters out
+            // non-candidates before we compare names.
+            if (type.IsIntrinsic && type is MetadataType mdType)
+            {
+                ReadOnlySpan<byte> name = mdType.Name;
+                if ((name.SequenceEqual("SZArrayHelper"u8) || name.SequenceEqual("Array`1"u8)) &&
+                    mdType.Namespace.SequenceEqual("System"u8))
+                {
+                    return false;
+                }
+            }
+
             // Valuetypes are invariant. This assumes that introducing type equivalence to an existing type
             // is not compatible change.
             if (type.IsValueType)

src/coreclr/vm/amd64/JitHelpers_Fast.asm

@@ -265,11 +265,6 @@ Section segment para 'DATA'
 JIT_WriteBarrier_Loc:
         dq 0
 
-LEAF_ENTRY  JIT_WriteBarrier_Callable, _TEXT
-        ; JIT_WriteBarrier(Object** dst, Object* src)
-        jmp     QWORD PTR [JIT_WriteBarrier_Loc]
-LEAF_END JIT_WriteBarrier_Callable, _TEXT
-
 ; There is an even more optimized version of these helpers possible which takes
 ; advantage of knowledge of which way the ephemeral heap is growing to only do 1/2
 ; that check (this is more significant in the JIT_WriteBarrier case).

src/coreclr/vm/amd64/jithelpers_fast.S

@@ -224,14 +224,6 @@ LEAF_END_MARKED JIT_ByRefWriteBarrier, _TEXT
         .text
 #endif
 
-// ------------------------------------------------------------------
-// __declspec(naked) void F_CALL_CONV JIT_WriteBarrier_Callable(Object **dst, Object* val)
-.balign 16
-LEAF_ENTRY  JIT_WriteBarrier_Callable, _TEXT
-    // JIT_WriteBarrier(Object** dst, Object* src)
-    jmp     [rip + C_FUNC(JIT_WriteBarrier_Loc)]
-LEAF_END JIT_WriteBarrier_Callable, _TEXT
-
 
 // The following helper will access ("probe") a word on each page of the stack
 // starting with the page right beneath rsp down to the one pointed to by r11.