Merge 3b82fa4579 into 433d2ec534

src/java.base/unix/native/libjava/childproc.c

@@ -67,24 +67,39 @@ markCloseOnExec(int fd)
     return 0;
 }
 
+#if !defined(_AIX)
+  /* The /proc file System in AIX does not contain open system files
+   * like /dev/random. Therefore we use a different approach and do
+   * not need isAsciiDigit() or FD_DIR */
 static int
 isAsciiDigit(char c)
 {
   return c >= '0' && c <= '9';
 }
 
-#if defined(_AIX)
+  #if defined(_ALLBSD_SOURCE)
-  /* AIX does not understand '/proc/self' - it requires the real process ID */
-  #define FD_DIR aix_fd_dir
-#elif defined(_ALLBSD_SOURCE)
     #define FD_DIR "/dev/fd"
   #else
     #define FD_DIR "/proc/self/fd"
   #endif
+#endif
 
 static int
 markDescriptorsCloseOnExec(void)
 {
+#if defined(_AIX)
+    /* We rely on the current childProcess() functions semantic.
+     * When the parent childProcess() function reaches the call of this function
+     * only the FDs 0,1,2 and 3 are further used until the exec() or the exit(-1).
+     * So we can close all FDs beginning with 4. FD 3 is only used if the exec fails
+     * to report the reason to the JVM. It should not survive a passing exec().
+     * So we can set the close_on_exec flag for FD 3. FDs 0,1 and 2 should survive
+     * the exec(), so we do not change them. */
+    if (fcntl(STDERR_FILENO + 2, F_CLOSEM, 0) == -1 ||
+        (markCloseOnExec(STDERR_FILENO + 1) == -1 && errno != EBADF)) {
+        return -1;
+    }
+#else
     DIR *dp;
     struct dirent *dirp;
     /* This function marks all file descriptors beyond stderr as CLOEXEC.
@@ -93,12 +108,6 @@ markDescriptorsCloseOnExec(void)
      * execve. */
     const int fd_from = STDERR_FILENO + 1;
 
-#if defined(_AIX)
-    /* AIX does not understand '/proc/self' - it requires the real process ID */
-    char aix_fd_dir[32];     /* the pid has at most 19 digits */
-    snprintf(aix_fd_dir, 32, "/proc/%d/fd", getpid());
-#endif
-
     if ((dp = opendir(FD_DIR)) == NULL)
         return -1;
 
@@ -114,6 +123,7 @@ markDescriptorsCloseOnExec(void)
     }
 
     closedir(dp);
+#endif
 
     return 0;
 }
@@ -406,6 +416,10 @@ childProcess(void *arg)
     /* We moved the fail pipe fd */
     fail_pipe_fd = FAIL_FILENO;
 
+    /* For AIX: The code in markDescriptorsCloseOnExec() relies on the current
+     * semantic of this function. When this point here is reached only the
+     * FDs 0,1,2 and 3 are further used until the exec() or the exit(-1). */
+
     /* close everything */
     if (markDescriptorsCloseOnExec() == -1) { /* failed,  close the old way */
         int max_fd = (int)sysconf(_SC_OPEN_MAX);