8190191: Subclasses of jdk.jfr.Event loaded from CDS breaks -XX:FlightRecorderOptions=retransform=false

Do not archive these classes

Reviewed-by: jiangli, sspitsyn

src/hotspot/share/classfile/dictionary.cpp

@@ -357,6 +357,16 @@ bool Dictionary::is_valid_protection_domain(int index, unsigned int hash,
   return entry->is_valid_protection_domain(protection_domain);
 }
 
+#if INCLUDE_CDS
+static bool is_jfr_event_class(Klass *k) {
+  while (k) {
+    if (k->name()->equals("jdk/jfr/Event")) {
+      return true;
+    }
+    k = k->super();
+  }
+  return false;
+}
 
 void Dictionary::reorder_dictionary_for_sharing() {
 
@@ -368,14 +378,22 @@ void Dictionary::reorder_dictionary_for_sharing() {
     while (p != NULL) {
       DictionaryEntry* next = p->next();
       InstanceKlass*ik = p->instance_klass();
-      // we cannot include signed classes in the archive because the certificates
-      // used during dump time may be different than those used during
-      // runtime (due to expiration, etc).
       if (ik->signers() != NULL) {
+        // We cannot include signed classes in the archive because the certificates
+        // used during dump time may be different than those used during
+        // runtime (due to expiration, etc).
         ResourceMark rm;
         tty->print_cr("Preload Warning: Skipping %s from signed JAR",
                        ik->name()->as_C_string());
         free_entry(p);
+      } else if (is_jfr_event_class(ik)) {
+        // We cannot include JFR event classes because they need runtime-specific
+        // instrumentation in order to work with -XX:FlightRecorderOptions=retransform=false.
+        // There are only a small number of these classes, so it's not worthwhile to
+        // support them and make CDS more complicated.
+        ResourceMark rm;
+        tty->print_cr("Skipping JFR event class %s", ik->name()->as_C_string());
+        free_entry(p);
       } else {
         p->set_next(master_list);
         master_list = p;
@@ -400,7 +418,7 @@ void Dictionary::reorder_dictionary_for_sharing() {
     set_entry(index, p);
   }
 }
-
+#endif
 
 SymbolPropertyTable::SymbolPropertyTable(int table_size)
   : Hashtable<Symbol*, mtSymbol>(table_size, sizeof(SymbolPropertyEntry))

src/hotspot/share/classfile/dictionary.hpp

@@ -88,7 +88,7 @@ public:
                              Handle protection_domain, TRAPS);
 
   // Sharing support
-  void reorder_dictionary_for_sharing();
+  void reorder_dictionary_for_sharing() NOT_CDS_RETURN;
 
   void print_on(outputStream* st) const;
   void verify();

src/hotspot/share/classfile/systemDictionary.cpp

@@ -2855,9 +2855,11 @@ ProtectionDomainCacheEntry* SystemDictionary::cache_get(Handle protection_domain
   return _pd_cache_table->get(protection_domain);
 }
 
+#if INCLUDE_CDS
 void SystemDictionary::reorder_dictionary_for_sharing() {
   ClassLoaderData::the_null_class_loader_data()->dictionary()->reorder_dictionary_for_sharing();
 }
+#endif
 
 size_t SystemDictionary::count_bytes_for_buckets() {
   return ClassLoaderData::the_null_class_loader_data()->dictionary()->count_bytes_for_buckets();

src/hotspot/share/classfile/systemDictionary.hpp

@@ -384,7 +384,7 @@ public:
 
 public:
   // Sharing support.
-  static void reorder_dictionary_for_sharing();
+  static void reorder_dictionary_for_sharing() NOT_CDS_RETURN;
   static void combine_shared_dictionaries();
   static size_t count_bytes_for_buckets();
   static size_t count_bytes_for_table();