8268139: CDS ArchiveBuilder may reference unloaded classes

Reviewed-by: coleenp, ccheung

src/hotspot/share/cds/dynamicArchive.cpp

@@ -109,6 +109,7 @@ public:
     DEBUG_ONLY(SystemDictionaryShared::NoClassLoadingMark nclm);
     SystemDictionaryShared::check_excluded_classes();
 
+    MutexLocker ml(DumpTimeTable_lock, Mutex::_no_safepoint_check_flag);
     init_header();
     gather_source_objs();
     reserve_buffer();

src/hotspot/share/cds/metaspaceShared.cpp

@@ -489,6 +489,7 @@ void VM_PopulateDumpSharedSpace::doit() {
   // that so we don't have to walk the SystemDictionary again.
   SystemDictionaryShared::check_excluded_classes();
 
+  MutexLocker ml(DumpTimeTable_lock, Mutex::_no_safepoint_check_flag);
   StaticArchiveBuilder builder;
   builder.gather_source_objs();
   builder.reserve_buffer();

src/hotspot/share/classfile/systemDictionaryShared.cpp

@@ -352,6 +352,8 @@ public:
       LambdaProxyClassKey const& k1, LambdaProxyClassKey const& k2) {
     return (k1.equals(k2));
   }
+
+  InstanceKlass* caller_ik() const { return _caller_ik; }
 };
 
 
@@ -427,7 +429,7 @@ DumpTimeLambdaProxyClassDictionary* _dumptime_lambda_proxy_class_dictionary = NU
 
 static void add_to_dump_time_lambda_proxy_class_dictionary(LambdaProxyClassKey key,
                                                            InstanceKlass* proxy_klass) {
-  assert(DumpTimeTable_lock->owned_by_self(), "sanity");
+  assert_lock_strong(DumpTimeTable_lock);
   if (_dumptime_lambda_proxy_class_dictionary == NULL) {
     _dumptime_lambda_proxy_class_dictionary =
       new (ResourceObj::C_HEAP, mtClass)DumpTimeLambdaProxyClassDictionary();
@@ -1234,6 +1236,11 @@ void SystemDictionaryShared::start_dumping() {
 
 DumpTimeSharedClassInfo* SystemDictionaryShared::find_or_allocate_info_for(InstanceKlass* k) {
   MutexLocker ml(DumpTimeTable_lock, Mutex::_no_safepoint_check_flag);
+  return find_or_allocate_info_for_locked(k);
+}
+
+DumpTimeSharedClassInfo* SystemDictionaryShared::find_or_allocate_info_for_locked(InstanceKlass* k) {
+  assert_lock_strong(DumpTimeTable_lock);
   if (_dumptime_table == NULL) {
     _dumptime_table = new (ResourceObj::C_HEAP, mtClass)DumpTimeSharedClassTable();
   }
@@ -1467,8 +1474,9 @@ void SystemDictionaryShared::check_excluded_classes() {
 
 bool SystemDictionaryShared::is_excluded_class(InstanceKlass* k) {
   assert(_no_class_loading_should_happen, "sanity");
+  assert_lock_strong(DumpTimeTable_lock);
   Arguments::assert_is_dumping_archive();
-  DumpTimeSharedClassInfo* p = find_or_allocate_info_for(k);
+  DumpTimeSharedClassInfo* p = find_or_allocate_info_for_locked(k);
   return (p == NULL) ? true : p->is_excluded();
 }
 
@@ -1505,7 +1513,8 @@ public:
   IterateDumpTimeSharedClassTable(MetaspaceClosure* it) : _it(it) {}
 
   bool do_entry(InstanceKlass* k, DumpTimeSharedClassInfo& info) {
-    if (!info.is_excluded()) {
+    assert_lock_strong(DumpTimeTable_lock);
+    if (k->is_loader_alive() && !info.is_excluded()) {
       info.metaspace_pointers_do(_it);
     }
     return true; // keep on iterating
@@ -1518,14 +1527,17 @@ public:
   IterateDumpTimeLambdaProxyClassDictionary(MetaspaceClosure* it) : _it(it) {}
 
   bool do_entry(LambdaProxyClassKey& key, DumpTimeLambdaProxyClassInfo& info) {
+    assert_lock_strong(DumpTimeTable_lock);
+    if (key.caller_ik()->is_loader_alive()) {
       info.metaspace_pointers_do(_it);
       key.metaspace_pointers_do(_it);
-    return true;
+    }
+    return true; // keep on iterating
   }
 };
 
 void SystemDictionaryShared::dumptime_classes_do(class MetaspaceClosure* it) {
-  assert_locked_or_safepoint(DumpTimeTable_lock);
+  assert_lock_strong(DumpTimeTable_lock);
   IterateDumpTimeSharedClassTable iter(it);
   _dumptime_table->iterate(&iter);
   if (_dumptime_lambda_proxy_class_dictionary != NULL) {
@@ -2154,7 +2166,7 @@ void SystemDictionaryShared::write_dictionary(RunTimeSharedDictionary* dictionar
   dictionary->reset();
   CompactHashtableWriter writer(_dumptime_table->count_of(is_builtin), &stats);
   CopySharedClassInfoToArchive copy(&writer, is_builtin);
-  MutexLocker ml(DumpTimeTable_lock, Mutex::_no_safepoint_check_flag);
+  assert_lock_strong(DumpTimeTable_lock);
   _dumptime_table->iterate(&copy);
   writer.dump(dictionary, is_builtin ? "builtin dictionary" : "unregistered dictionary");
 }

src/hotspot/share/classfile/systemDictionaryShared.hpp

@@ -207,6 +207,7 @@ private:
                                  const ClassFileStream* cfs,
                                  TRAPS);
   static DumpTimeSharedClassInfo* find_or_allocate_info_for(InstanceKlass* k);
+  static DumpTimeSharedClassInfo* find_or_allocate_info_for_locked(InstanceKlass* k);
   static void write_dictionary(RunTimeSharedDictionary* dictionary,
                                bool is_builtin);
   static void write_lambda_proxy_class_dictionary(LambdaProxyClassDictionary* dictionary);