8383997: [lworld] Buffering via SharedRuntime::store_inline_type_fields_to_buf is slow

Reviewed-by: mchevalier

Author: TobiHartmann

src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp

@@ -12685,20 +12685,30 @@ class StubGenerator: public StubCodeGenerator {
     __ ldpd(j_farg3, j_farg2, Address(__ post(sp, 2 * wordSize)));
     __ ldpd(j_farg1, j_farg0, Address(__ post(sp, 2 * wordSize)));
 
-    __ leave();
-
     // check for pending exceptions
     Label pending;
     __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
     __ cbnz(rscratch1, pending);
 
     if (has_res) {
+      // We just called SharedRuntime::store_inline_type_fields_to_buf. Check if we still
+      // need to initialize the buffer and if so, call the inline class specific pack handler.
+      Label skip_pack;
       __ get_vm_result_oop(r0, rthread);
+      __ get_vm_result_metadata(rscratch1, rthread);
+      __ cbz(rscratch1, skip_pack);
+      __ ldr(rscratch1, Address(rscratch1, InlineKlass::adr_members_offset()));
+      __ ldr(rscratch1, Address(rscratch1, InlineKlass::pack_handler_offset()));
+      __ blr(rscratch1);
+      __ membar(Assembler::StoreStore);
+      __ bind(skip_pack);
     }
 
+    __ leave();
     __ ret(lr);
 
     __ bind(pending);
+    __ leave();
     __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
 
     // -------------

src/hotspot/cpu/x86/stubGenerator_x86_64.cpp

@@ -32,6 +32,7 @@
 #include "gc/shared/barrierSetNMethod.hpp"
 #include "gc/shared/gc_globals.hpp"
 #include "memory/universe.hpp"
+#include "oops/inlineKlass.hpp"
 #include "prims/jvmtiExport.hpp"
 #include "prims/upcallLinker.hpp"
 #include "runtime/arguments.hpp"
@@ -5002,7 +5003,18 @@ address StubGenerator::generate_return_value_stub(address destination, const cha
   __ jcc(Assembler::notEqual, pending);
 
   if (has_res) {
+    // We just called SharedRuntime::store_inline_type_fields_to_buf. Check if we still
+    // need to initialize the buffer and if so, call the inline class specific pack handler.
+    Label skip_pack;
     __ get_vm_result_oop(rax);
+    __ get_vm_result_metadata(rscratch1);
+    __ testptr(rscratch1, rscratch1);
+    __ jcc(Assembler::zero, skip_pack);
+    __ movptr(rscratch1, Address(rscratch1, InlineKlass::adr_members_offset()));
+    __ movptr(rscratch1, Address(rscratch1, InlineKlass::pack_handler_offset()));
+    __ call(rscratch1);
+    __ membar(Assembler::StoreStore);
+    __ bind(skip_pack);
   }
 
   __ ret(0);

src/hotspot/share/runtime/sharedRuntime.cpp

@@ -4291,32 +4291,43 @@ JRT_END
 // from field's values in registers.
 JRT_BLOCK_ENTRY(void, SharedRuntime::store_inline_type_fields_to_buf(JavaThread* current, intptr_t res))
 {
-  ResourceMark rm;
-  RegisterMap reg_map(current,
-                      RegisterMap::UpdateMap::include,
-                      RegisterMap::ProcessFrames::include,
-                      RegisterMap::WalkContinuation::skip);
-  frame stubFrame = current->last_frame();
-  frame callerFrame = stubFrame.sender(&reg_map);
-
-#ifdef ASSERT
-  InlineKlass* verif_vk = InlineKlass::returned_inline_klass(reg_map);
-#endif
-
   if (!is_set_nth_bit(res, 0)) {
     // We're not returning with inline type fields in registers (the
     // calling convention didn't allow it for this inline klass)
     assert(!Metaspace::contains((void*)res), "should be oop or pointer in buffer area");
     current->set_vm_result_oop((oopDesc*)res);
-    assert(verif_vk == nullptr, "broken calling convention");
+    current->set_vm_result_metadata(nullptr);
     return;
   }
 
   clear_nth_bit(res, 0);
   InlineKlass* vk = (InlineKlass*)res;
-  assert(verif_vk == vk, "broken calling convention");
   assert(Metaspace::contains((void*)res), "should be klass");
 
+  if (!vk->contains_oops()) {
+    // No oop fields. Initialize the fields by calling the pack handler from
+    // the stub which is much faster (see 'generate_return_value_stub').
+    // Signal this by setting the metadata result to the value klass.
+    JRT_BLOCK;
+    {
+      oop vt = vk->allocate_instance(CHECK);
+      current->set_vm_result_oop(vt);
+      current->set_vm_result_metadata(vk);
+    }
+    JRT_BLOCK_END;
+    return;
+  }
+
+  ResourceMark rm;
+  RegisterMap reg_map(current,
+                      RegisterMap::UpdateMap::include,
+                      RegisterMap::ProcessFrames::include,
+                      RegisterMap::WalkContinuation::skip);
+  frame stubFrame = current->last_frame();
+  stubFrame.sender(&reg_map);
+
+  assert(vk == InlineKlass::returned_inline_klass(reg_map), "broken calling convention");
+
   // Allocate handles for every oop field so they are safe in case of
   // a safepoint when allocating
   GrowableArray<Handle> handles;
@@ -4325,9 +4336,9 @@ JRT_BLOCK_ENTRY(void, SharedRuntime::store_inline_type_fields_to_buf(JavaThread*
   // It's unsafe to safepoint until we are here
   JRT_BLOCK;
   {
-    JavaThread* THREAD = current;
     oop vt = vk->realloc_result(reg_map, handles, CHECK);
     current->set_vm_result_oop(vt);
+    current->set_vm_result_metadata(nullptr);
   }
   JRT_BLOCK_END;
 }