6996136: VM crash in src/share/vm/runtime/virtualspace.cpp:424

Turn CDS off if compressed oops is on

Reviewed-by: ysr, kvn, jcoomes, phh

hotspot/src/share/vm/runtime/arguments.cpp

@@ -957,26 +957,65 @@ static void no_shared_spaces() {
   }
 }
 
+void Arguments::check_compressed_oops_compat() {
+#ifdef _LP64
+  assert(UseCompressedOops, "Precondition");
+#  if defined(COMPILER1) && !defined(TIERED)
+  // Until c1 supports compressed oops turn them off.
+  FLAG_SET_DEFAULT(UseCompressedOops, false);
+#  else
+  // Is it on by default or set on ergonomically
+  bool is_on_by_default = FLAG_IS_DEFAULT(UseCompressedOops) || FLAG_IS_ERGO(UseCompressedOops);
+
+  // Tiered currently doesn't work with compressed oops
+  if (TieredCompilation) {
+    if (is_on_by_default) {
+      FLAG_SET_DEFAULT(UseCompressedOops, false);
+      return;
+    } else {
+      vm_exit_during_initialization(
+        "Tiered compilation is not supported with compressed oops yet", NULL);
+    }
+  }
+
+  // XXX JSR 292 currently does not support compressed oops
+  if (EnableMethodHandles) {
+    if (is_on_by_default) {
+      FLAG_SET_DEFAULT(UseCompressedOops, false);
+      return;
+    } else {
+      vm_exit_during_initialization(
+        "JSR292 is not supported with compressed oops yet", NULL);
+    }
+  }
+
+  // If dumping an archive or forcing its use, disable compressed oops if possible
+  if (DumpSharedSpaces || RequireSharedSpaces) {
+    if (is_on_by_default) {
+      FLAG_SET_DEFAULT(UseCompressedOops, false);
+      return;
+    } else {
+      vm_exit_during_initialization(
+        "Class Data Sharing is not supported with compressed oops yet", NULL);
+    }
+  } else if (UseSharedSpaces) {
+    // UseSharedSpaces is on by default. With compressed oops, we turn it off.
+    FLAG_SET_DEFAULT(UseSharedSpaces, false);
+  }
+
+#  endif // defined(COMPILER1) && !defined(TIERED)
+#endif // _LP64
+}
+
 void Arguments::set_tiered_flags() {
   if (FLAG_IS_DEFAULT(CompilationPolicyChoice)) {
     FLAG_SET_DEFAULT(CompilationPolicyChoice, 2);
   }
-
   if (CompilationPolicyChoice < 2) {
     vm_exit_during_initialization(
       "Incompatible compilation policy selected", NULL);
   }
-
+  // Increase the code cache size - tiered compiles a lot more.
-#ifdef _LP64
-  if (FLAG_IS_DEFAULT(UseCompressedOops) || FLAG_IS_ERGO(UseCompressedOops)) {
-    UseCompressedOops = false;
-  }
-  if (UseCompressedOops) {
-    vm_exit_during_initialization(
-      "Tiered compilation is not supported with compressed oops yet", NULL);
-  }
-#endif
- // Increase the code cache size - tiered compiles a lot more.
   if (FLAG_IS_DEFAULT(ReservedCodeCacheSize)) {
     FLAG_SET_DEFAULT(ReservedCodeCacheSize, ReservedCodeCacheSize * 2);
   }
@@ -2835,6 +2874,7 @@ jint Arguments::parse_options_environment_variable(const char* name, SysClassPat
   return JNI_OK;
 }
 
+
 // Parse entry point called from JNI_CreateJavaVM
 
 jint Arguments::parse(const JavaVMInitArgs* args) {
@@ -2977,17 +3017,6 @@ jint Arguments::parse(const JavaVMInitArgs* args) {
     PrintGC = true;
   }
 
-#if defined(_LP64) && defined(COMPILER1) && !defined(TIERED)
-  UseCompressedOops = false;
-#endif
-
-#if defined(_LP64)
-  if ((DumpSharedSpaces || RequireSharedSpaces) && UseCompressedOops) {
-    // Disable compressed oops with shared spaces
-    UseCompressedOops = false;
-  }
-#endif
-
   // Set object alignment values.
   set_object_alignment();
 
@@ -3002,13 +3031,10 @@ jint Arguments::parse(const JavaVMInitArgs* args) {
   set_ergonomics_flags();
 
 #ifdef _LP64
-  // XXX JSR 292 currently does not support compressed oops.
+  if (UseCompressedOops) {
-  if (EnableMethodHandles && UseCompressedOops) {
+    check_compressed_oops_compat();
-    if (FLAG_IS_DEFAULT(UseCompressedOops) || FLAG_IS_ERGO(UseCompressedOops)) {
-      UseCompressedOops = false;
-    }
   }
-#endif // _LP64
+#endif
 
   // Check the GC selections again.
   if (!check_gc_consistency()) {

hotspot/src/share/vm/runtime/arguments.hpp

@@ -291,6 +291,8 @@ class Arguments : AllStatic {
 
   // Tiered
   static void set_tiered_flags();
+  // Check compressed oops compatibility with other flags
+  static void check_compressed_oops_compat();
   // CMS/ParNew garbage collectors
   static void set_parnew_gc_flags();
   static void set_cms_and_parnew_gc_flags();