8261401: Add sanity check for UseSHM large pages similar to the one used with hugetlb large pages

Reviewed-by: stuefe, tschatzl

src/hotspot/os/linux/os_linux.cpp

@@ -3564,6 +3564,30 @@ bool os::Linux::hugetlbfs_sanity_check(bool warn, size_t page_size) {
   return result;
 }
 
+bool os::Linux::shm_hugetlbfs_sanity_check(bool warn, size_t page_size) {
+  // Try to create a large shared memory segment.
+  int shmid = shmget(IPC_PRIVATE, page_size, SHM_HUGETLB|IPC_CREAT|SHM_R|SHM_W);
+  if (shmid == -1) {
+    // Possible reasons for shmget failure:
+    // 1. shmmax is too small for the request.
+    //    > check shmmax value: cat /proc/sys/kernel/shmmax
+    //    > increase shmmax value: echo "new_value" > /proc/sys/kernel/shmmax
+    // 2. not enough large page memory.
+    //    > check available large pages: cat /proc/meminfo
+    //    > increase amount of large pages:
+    //          sysctl -w vm.nr_hugepages=new_value
+    //    > For more information regarding large pages please refer to:
+    //      https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt
+    if (warn) {
+      warning("Large pages using UseSHM are not configured on this system.");
+    }
+    return false;
+  }
+  // Managed to create a segment, now delete it.
+  shmctl(shmid, IPC_RMID, NULL);
+  return true;
+}
+
 // From the coredump_filter documentation:
 //
 // - (bit 0) anonymous private memory
@@ -3748,7 +3772,18 @@ bool os::Linux::setup_large_page_type(size_t page_size) {
     UseHugeTLBFS = false;
   }
 
-  return UseSHM;
+  if (UseSHM) {
+    bool warn_on_failure = !FLAG_IS_DEFAULT(UseSHM);
+    if (shm_hugetlbfs_sanity_check(warn_on_failure, page_size)) {
+      return true;
+    }
+    UseSHM = false;
+  }
+
+  if (!FLAG_IS_DEFAULT(UseLargePages)) {
+    log_warning(pagesize)("UseLargePages disabled, no large pages configured and available on the system.");
+  }
+  return false;
 }
 
 void os::large_page_init() {
@@ -3888,13 +3923,15 @@ char* os::Linux::reserve_memory_special_shm(size_t bytes, size_t alignment,
   int shmid = shmget(IPC_PRIVATE, bytes, SHM_HUGETLB|IPC_CREAT|SHM_R|SHM_W);
   if (shmid == -1) {
     // Possible reasons for shmget failure:
-    // 1. shmmax is too small for Java heap.
+    // 1. shmmax is too small for the request.
     //    > check shmmax value: cat /proc/sys/kernel/shmmax
-    //    > increase shmmax value: echo "0xffffffff" > /proc/sys/kernel/shmmax
+    //    > increase shmmax value: echo "new_value" > /proc/sys/kernel/shmmax
     // 2. not enough large page memory.
     //    > check available large pages: cat /proc/meminfo
     //    > increase amount of large pages:
-    //          echo new_value > /proc/sys/vm/nr_hugepages
+    //          sysctl -w vm.nr_hugepages=new_value
+    //    > For more information regarding large pages please refer to:
+    //      https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt
     //      Note 1: different Linux may use different name for this property,
     //            e.g. on Redhat AS-3 it is "hugetlb_pool".
     //      Note 2: it's possible there's enough physical memory available but

src/hotspot/os/linux/os_linux.hpp

@@ -85,6 +85,7 @@ class Linux {
   static bool setup_large_page_type(size_t page_size);
   static bool transparent_huge_pages_sanity_check(bool warn, size_t pages_size);
   static bool hugetlbfs_sanity_check(bool warn, size_t page_size);
+  static bool shm_hugetlbfs_sanity_check(bool warn, size_t page_size);
 
   static char* reserve_memory_special_shm(size_t bytes, size_t alignment, char* req_addr, bool exec);
   static char* reserve_memory_special_huge_tlbfs(size_t bytes, size_t alignment, char* req_addr, bool exec);

test/hotspot/jtreg/gc/g1/TestLargePageUseForHeap.java

@@ -61,6 +61,11 @@ public class TestLargePageUseForHeap {
     }
 
     static boolean checkLargePageEnabled(OutputAnalyzer output) {
+        String lp = output.firstMatch("Large Page Support: (\\w*)", 1);
+        // Make sure large pages really are enabled.
+        if (lp == null || lp.equals("Disabled")) {
+            return false;
+        }
         // This message is printed when tried to reserve a memory with large page but it failed.
         String errorStr = "Reserve regular memory without large pages";
         String heapPattern = ".*Heap: ";
@@ -84,7 +89,7 @@ public class TestLargePageUseForHeap {
         pb = ProcessTools.createJavaProcessBuilder("-XX:+UseG1GC",
                                                    "-XX:G1HeapRegionSize=" + regionSize,
                                                    "-Xmx128m",
-                                                   "-Xlog:pagesize,gc+heap+coops=debug",
+                                                   "-Xlog:gc+init,pagesize,gc+heap+coops=debug",
                                                    "-XX:+UseLargePages",
                                                    "-version");
 
@@ -97,7 +102,7 @@ public class TestLargePageUseForHeap {
         pb = ProcessTools.createJavaProcessBuilder("-XX:+UseG1GC",
                                                    "-XX:G1HeapRegionSize=" + regionSize,
                                                    "-Xmx128m",
-                                                   "-Xlog:pagesize,gc+heap+coops=debug",
+                                                   "-Xlog:gc+init,pagesize,gc+heap+coops=debug",
                                                    "-XX:-UseLargePages",
                                                    "-version");