8176768: hotspot ignores PTHREAD_STACK_MIN when creating new threads

Use PTHREAD_STACK_MIN as a minimum, plus other stack size cleanup

Reviewed-by: dholmes, stuefe, dcubed

hotspot/src/os/aix/vm/os_aix.cpp

@@ -877,7 +877,8 @@ bool os::create_thread(Thread* thread, ThreadType thr_type,
 
   // Calculate stack size if it's not specified by caller.
   size_t stack_size = os::Posix::get_initial_stack_size(thr_type, req_stack_size);
-  pthread_attr_setstacksize(&attr, stack_size);
+  int status = pthread_attr_setstacksize(&attr, stack_size);
+  assert_status(status == 0, status, "pthread_attr_setstacksize");
 
   // Configure libc guard page.
   pthread_attr_setguardsize(&attr, os::Aix::default_guard_size(thr_type));

hotspot/src/os/bsd/vm/os_bsd.cpp

@@ -759,7 +759,8 @@ bool os::create_thread(Thread* thread, ThreadType thr_type,
 
   // calculate stack size if it's not specified by caller
   size_t stack_size = os::Posix::get_initial_stack_size(thr_type, req_stack_size);
-  pthread_attr_setstacksize(&attr, stack_size);
+  int status = pthread_attr_setstacksize(&attr, stack_size);
+  assert_status(status == 0, status, "pthread_attr_setstacksize");
 
   ThreadState state;
 

hotspot/src/os/linux/vm/os_linux.cpp

@@ -723,9 +723,16 @@ bool os::create_thread(Thread* thread, ThreadType thr_type,
   // the size of the guard pages to the stack size, instead Linux
   // takes the space out of 'stacksize'. Thus we adapt the requested
   // stack_size by the size of the guard pages to mimick proper
-  // behaviour.
-  stack_size = align_size_up(stack_size + os::Linux::default_guard_size(thr_type), vm_page_size());
-  pthread_attr_setstacksize(&attr, stack_size);
+  // behaviour. However, be careful not to end up with a size
+  // of zero due to overflow. Don't add the guard page in that case.
+  size_t guard_size = os::Linux::default_guard_size(thr_type);
+  if (stack_size <= SIZE_MAX - guard_size) {
+    stack_size += guard_size;
+  }
+  assert(is_size_aligned(stack_size, os::vm_page_size()), "stack_size not aligned");
+
+  int status = pthread_attr_setstacksize(&attr, stack_size);
+  assert_status(status == 0, status, "pthread_attr_setstacksize");
 
   // Configure glibc guard page.
   pthread_attr_setguardsize(&attr, os::Linux::default_guard_size(thr_type));

hotspot/src/os/posix/vm/os_posix.cpp

@@ -1158,11 +1158,14 @@ char* os::Posix::realpath(const char* filename, char* outbuf, size_t outbuflen)
 // page size which again depends on the concrete system the VM is running
 // on. Space for libc guard pages is not included in this size.
 jint os::Posix::set_minimum_stack_sizes() {
+  size_t os_min_stack_allowed = SOLARIS_ONLY(thr_min_stack()) NOT_SOLARIS(PTHREAD_STACK_MIN);
+
   _java_thread_min_stack_allowed = _java_thread_min_stack_allowed +
                                    JavaThread::stack_guard_zone_size() +
                                    JavaThread::stack_shadow_zone_size();
 
   _java_thread_min_stack_allowed = align_size_up(_java_thread_min_stack_allowed, vm_page_size());
+  _java_thread_min_stack_allowed = MAX2(_java_thread_min_stack_allowed, os_min_stack_allowed);
 
   size_t stack_size_in_bytes = ThreadStackSize * K;
   if (stack_size_in_bytes != 0 &&
@@ -1186,6 +1189,7 @@ jint os::Posix::set_minimum_stack_sizes() {
                                        JavaThread::stack_shadow_zone_size();
 
   _compiler_thread_min_stack_allowed = align_size_up(_compiler_thread_min_stack_allowed, vm_page_size());
+  _compiler_thread_min_stack_allowed = MAX2(_compiler_thread_min_stack_allowed, os_min_stack_allowed);
 
   stack_size_in_bytes = CompilerThreadStackSize * K;
   if (stack_size_in_bytes != 0 &&
@@ -1197,6 +1201,7 @@ jint os::Posix::set_minimum_stack_sizes() {
   }
 
   _vm_internal_thread_min_stack_allowed = align_size_up(_vm_internal_thread_min_stack_allowed, vm_page_size());
+  _vm_internal_thread_min_stack_allowed = MAX2(_vm_internal_thread_min_stack_allowed, os_min_stack_allowed);
 
   stack_size_in_bytes = VMThreadStackSize * K;
   if (stack_size_in_bytes != 0 &&
@@ -1252,6 +1257,14 @@ size_t os::Posix::get_initial_stack_size(ThreadType thr_type, size_t req_stack_s
     break;
   }
 
+  // pthread_attr_setstacksize() may require that the size be rounded up to the OS page size.
+  // Be careful not to round up to 0. Align down in that case.
+  if (stack_size <= SIZE_MAX - vm_page_size()) {
+    stack_size = align_size_up(stack_size, vm_page_size());
+  } else {
+    stack_size = align_size_down(stack_size, vm_page_size());
+  }
+
   return stack_size;
 }
 

hotspot/src/os/solaris/vm/os_solaris.cpp

@@ -4275,16 +4275,6 @@ void os::init(void) {
 
   main_thread = thr_self();
 
-  // Constant minimum stack size allowed. It must be at least
-  // the minimum of what the OS supports (thr_min_stack()), and
-  // enough to allow the thread to get to user bytecode execution.
-  Posix::_compiler_thread_min_stack_allowed = MAX2(thr_min_stack(),
-                                                   Posix::_compiler_thread_min_stack_allowed);
-  Posix::_java_thread_min_stack_allowed = MAX2(thr_min_stack(),
-                                               Posix::_java_thread_min_stack_allowed);
-  Posix::_vm_internal_thread_min_stack_allowed = MAX2(thr_min_stack(),
-                                                      Posix::_vm_internal_thread_min_stack_allowed);
-
   // dynamic lookup of functions that may not be available in our lowest
   // supported Solaris release
   void * handle = dlopen("libc.so.1", RTLD_LAZY);

hotspot/src/share/vm/prims/jvm.cpp

@@ -2807,9 +2807,11 @@ JVM_ENTRY(void, JVM_StartThread(JNIEnv* env, jobject jthread))
       jlong size =
              java_lang_Thread::stackSize(JNIHandles::resolve_non_null(jthread));
       // Allocate the C++ Thread structure and create the native thread.  The
-      // stack size retrieved from java is signed, but the constructor takes
-      // size_t (an unsigned type), so avoid passing negative values which would
-      // result in really large stacks.
+      // stack size retrieved from java is 64-bit signed, but the constructor takes
+      // size_t (an unsigned type), which may be 32 or 64-bit depending on the platform.
+      //  - Avoid truncating on 32-bit platforms if size is greater than UINT_MAX.
+      //  - Avoid passing negative values which would result in really large stacks.
+      NOT_LP64(if (size > SIZE_MAX) size = SIZE_MAX;)
       size_t sz = size > 0 ? (size_t) size : 0;
       native_thread = new JavaThread(&thread_entry, sz);
 

hotspot/test/runtime/Thread/TestThreadStackSizes.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @summary Test user threads with various stack sizes.
+ * @run main/othervm TestThreadStackSizes
+ */
+
+public class TestThreadStackSizes extends Thread {
+    static final int K = 1024;
+
+    public TestThreadStackSizes(long stackSize) {
+        super(null, null, "TestThreadStackSizes" + stackSize, stackSize);
+    }
+
+    @Override
+    public void run() {
+    }
+
+    public static void createThread(long stackSize) {
+        System.out.println("StackSize: " + stackSize);
+        try {
+            TestThreadStackSizes testThreadStackSize = new TestThreadStackSizes(stackSize);
+            testThreadStackSize.start();
+            try {
+                testThreadStackSize.join();
+            } catch (InterruptedException e) {
+                throw new Error("InterruptedException in main thread", e);
+            }
+        } catch (Error e) {  // we sometimes get OutOfMemoryError for very large stacks
+            System.out.println("Got exception for stack size " + stackSize  + ": " + e);
+        }
+    }
+
+    public static void main(String[] args) throws Error {
+        // Try every stack size from 0k to 320k.
+        for (int i = 0; i <= 320; i++) {
+            createThread(i * K);
+        }
+        // Try a few other potentially problematic stack sizes.
+        createThread(500*K);
+        createThread(501*K);
+        createThread(-1);
+        createThread(500*K*K);
+        createThread(9223372036854774784L);
+    }
+}

hotspot/test/runtime/Thread/TooSmallStackSize.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -181,12 +181,18 @@ public class TooSmallStackSize {
          */
         checkMinStackAllowed("-XX:ThreadStackSize=", ThreadStackSizeString, min_stack_allowed);
 
+        /*
+         * Now try with a stack size that is not page aligned.
+         */
+        checkMinStackAllowed("-XX:ThreadStackSize=", ThreadStackSizeString, "513");
+
         /*
          * Now redo the same tests with the compiler thread stack size:
          */
         checkStack("-XX:CompilerThreadStackSize=", CompilerThreadStackSizeString, "16");
         min_stack_allowed = checkStack("-XX:CompilerThreadStackSize=", CompilerThreadStackSizeString, "32");
         checkMinStackAllowed("-XX:CompilerThreadStackSize=", CompilerThreadStackSizeString, min_stack_allowed);
+        checkMinStackAllowed("-XX:CompilerThreadStackSize=", CompilerThreadStackSizeString, "513");
 
         /*
          * Now redo the same tests with the VM thread stack size:
@@ -194,5 +200,6 @@ public class TooSmallStackSize {
         checkStack("-XX:VMThreadStackSize=", VMThreadStackSizeString, "16");
         min_stack_allowed = checkStack("-XX:VMThreadStackSize=", VMThreadStackSizeString, "32");
         checkMinStackAllowed("-XX:VMThreadStackSize=", VMThreadStackSizeString, min_stack_allowed);
+        checkMinStackAllowed("-XX:VMThreadStackSize=", VMThreadStackSizeString, "513");
     }
 }