8058818: Allocation of more then 1G of memory using Unsafe.allocateMemory is still causing a fatal error on 32bit platforms

The assert was firing for NMT_Off and minimal too even though the size isn't used.

Reviewed-by: gtriantafill, dholmes

hotspot/src/share/vm/services/mallocTracker.cpp

@@ -140,11 +140,6 @@ void* MallocTracker::record_malloc(void* malloc_base, size_t size, MEMFLAGS flag
     return NULL;
   }
 
-  // Check malloc size, size has to <= MAX_MALLOC_SIZE. This is only possible on 32-bit
-  // systems, when malloc size >= 1GB, but is is safe to assume it won't happen.
-  if (size > MAX_MALLOC_SIZE) {
-    fatal("Should not use malloc for big memory block, use virtual memory instead");
-  }
   // Uses placement global new operator to initialize malloc header
   switch(level) {
     case NMT_off:
@@ -154,10 +149,12 @@ void* MallocTracker::record_malloc(void* malloc_base, size_t size, MEMFLAGS flag
       break;
     }
     case NMT_summary: {
+      assert(size <= MAX_MALLOC_SIZE, "malloc size overrun for NMT");
       header = ::new (malloc_base) MallocHeader(size, flags);
       break;
     }
     case NMT_detail: {
+      assert(size <= MAX_MALLOC_SIZE, "malloc size overrun for NMT");
       header = ::new (malloc_base) MallocHeader(size, flags, stack);
       break;
     }

hotspot/test/runtime/NMT/UnsafeMallocLimit2.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8058818
+ * @library /testlibrary
+ * @build UnsafeMallocLimit2
+ * @run main/othervm -Xmx32m -XX:NativeMemoryTracking=off UnsafeMallocLimit2
+ */
+
+import com.oracle.java.testlibrary.*;
+import sun.misc.Unsafe;
+
+public class UnsafeMallocLimit2 {
+
+    public static void main(String args[]) throws Exception {
+        if (Platform.is32bit()) {
+            Unsafe unsafe = Utils.getUnsafe();
+            try {
+                // Allocate greater than MALLOC_MAX and likely won't fail to allocate,
+                // so it hits the NMT code that asserted.
+                // Test that this doesn't cause an assertion with NMT off.
+                // The option above overrides if all the tests are run with NMT on.
+                unsafe.allocateMemory(0x40000000);
+                System.out.println("Allocation succeeded");
+            } catch (OutOfMemoryError e) {
+                System.out.println("Allocation failed");
+            }
+        } else {
+            System.out.println("Test only valid on 32-bit platforms");
+        }
+    }
+}