8365526: Crash with null Symbol passed to SystemDictionary::resolve_or_null

Reviewed-by: mdoerr
Backport-of: 8ed7c23fa02f7c40cceefb24f92a3df281d12c4f

Author: GoeLin

src/hotspot/share/classfile/resolutionErrors.cpp

@@ -46,7 +46,7 @@ void ResolutionErrorTable::add_entry(int index, unsigned int hash,
   entry->set_cp_index(cp_index);
   entry->set_error(error);
   entry->set_message(message);
-  entry->set_nest_host_error(NULL);
+  entry->init_nest_host_error(NULL);
   entry->set_cause(cause);
   entry->set_cause_msg(cause_msg);
 
@@ -63,7 +63,7 @@ void ResolutionErrorTable::add_entry(int index, unsigned int hash,
 
   ResolutionErrorEntry* entry = (ResolutionErrorEntry*)Hashtable<ConstantPool*, mtClass>::new_entry(hash, pool());
   entry->set_cp_index(cp_index);
-  entry->set_nest_host_error(message);
+  entry->init_nest_host_error(message);
   entry->set_error(NULL);
   entry->set_message(NULL);
   entry->set_cause(NULL);
@@ -112,6 +112,14 @@ void ResolutionErrorEntry::set_cause_msg(const char* c) {
 
 // The incoming nest host error message is already in the C-Heap.
 void ResolutionErrorEntry::set_nest_host_error(const char* message) {
+  // If a message is already set, free it.
+  if (nest_host_error() != nullptr) {
+    FREE_C_HEAP_ARRAY(char, _nest_host_error);
+  }
+  init_nest_host_error(message);
+}
+
+void ResolutionErrorEntry::init_nest_host_error(const char* message) {
   _nest_host_error = message;
 }
 

src/hotspot/share/classfile/resolutionErrors.hpp

@@ -122,6 +122,7 @@ class ResolutionErrorEntry : public HashtableEntry<ConstantPool*, mtClass> {
   void               set_cause_msg(const char* c);
 
   const char*        nest_host_error() const    { return _nest_host_error; }
+  void               init_nest_host_error(const char* message);
   // The incoming nest host error message is already in the C-Heap.
   void               set_nest_host_error(const char* message);
 

src/hotspot/share/classfile/systemDictionary.cpp

@@ -1896,14 +1896,19 @@ void SystemDictionary::add_nest_host_error(const constantPoolHandle& pool,
   {
     MutexLocker ml(Thread::current(), SystemDictionary_lock);
     ResolutionErrorEntry* entry = resolution_errors()->find_entry(index, hash, pool, which);
-    if (entry != NULL && entry->nest_host_error() == NULL) {
+    if (entry == NULL) {
+      // Only add a new entry to the resolution error table if one hasn't been found for this
+      // constant pool index. In this case resolution succeeded but there's an error in this nest host
+      // that we use the table to record.
+      assert(pool->resolved_klass_at(which) != nullptr, "klass should be resolved if there is no entry");
+      resolution_errors()->add_entry(index, hash, pool, which, message);
+    } else {
       // An existing entry means we had a true resolution failure (LinkageError) with our nest host, but we
       // still want to add the error message for the higher-level access checks to report. We should
       // only reach here under the same error condition, so we can ignore the potential race with setting
-      // the message. If we see it is already set then we can ignore it.
+      // the message, and set it again.
+      assert(entry->nest_host_error() == nullptr || strcmp(entry->nest_host_error(), message) == 0, "should be the same message");
       entry->set_nest_host_error(message);
-    } else {
-      resolution_errors()->add_entry(index, hash, pool, which, message);
     }
   }
 }