8274548: (fc) FileChannel gathering write fails with IOException "Invalid argument" on macOS 11.6

Reviewed-by: alanb

src/java.base/share/classes/sun/nio/ch/IOUtil.java

@@ -44,6 +44,11 @@ public class IOUtil {
      */
     static final int IOV_MAX;
 
+    /**
+     * Max total number of bytes that writev supports
+     */
+    static final long WRITEV_MAX;
+
     private IOUtil() { }                // No instantiation
 
     static int write(FileDescriptor fd, ByteBuffer src, long position,
@@ -172,9 +177,10 @@ public class IOUtil {
         Runnable handleReleasers = null;
         try {
             // Iterate over buffers to populate native iovec array.
+            long writevLen = 0L;
             int count = offset + length;
             int i = offset;
-            while (i < count && iov_len < IOV_MAX) {
+            while (i < count && iov_len < IOV_MAX && writevLen < WRITEV_MAX) {
                 ByteBuffer buf = bufs[i];
                 var h = acquireScope(buf, async);
                 if (h != null) {
@@ -188,6 +194,10 @@ public class IOUtil {
                     Util.checkRemainingBufferSizeAligned(rem, alignment);
 
                 if (rem > 0) {
+                    long headroom = WRITEV_MAX - writevLen;
+                    if (headroom < rem)
+                        rem = (int)headroom;
+
                     vec.setBuffer(iov_len, buf, pos, rem);
 
                     // allocate shadow buffer to ensure I/O is done with direct buffer
@@ -197,10 +207,9 @@ public class IOUtil {
                             shadow = Util.getTemporaryAlignedDirectBuffer(rem, alignment);
                         else
                             shadow = Util.getTemporaryDirectBuffer(rem);
-                        shadow.put(buf);
+                        shadow.put(shadow.position(), buf, pos, rem);
                         shadow.flip();
                         vec.setShadow(iov_len, shadow);
-                        buf.position(pos);  // temporarily restore position in user buffer
                         buf = shadow;
                         pos = shadow.position();
                     }
@@ -208,6 +217,7 @@ public class IOUtil {
                     vec.putBase(iov_len, bufferAddress(buf) + pos);
                     vec.putLen(iov_len, rem);
                     iov_len++;
+                    writevLen += rem;
                 }
                 i++;
             }
@@ -580,6 +590,8 @@ public class IOUtil {
 
     static native int iovMax();
 
+    static native long writevMax();
+
     static native void initIDs();
 
     /**
@@ -593,6 +605,7 @@ public class IOUtil {
         initIDs();
 
         IOV_MAX = iovMax();
+        WRITEV_MAX = writevMax();
     }
 
 }

src/java.base/unix/native/libnio/ch/IOUtil.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -33,6 +33,7 @@
 #include "jlong.h"
 #include "sun_nio_ch_IOUtil.h"
 #include "java_lang_Integer.h"
+#include "java_lang_Long.h"
 #include "nio.h"
 #include "nio_util.h"
 
@@ -173,6 +174,29 @@ Java_sun_nio_ch_IOUtil_iovMax(JNIEnv *env, jclass this)
     return (jint)iov_max;
 }
 
+JNIEXPORT jlong JNICALL
+Java_sun_nio_ch_IOUtil_writevMax(JNIEnv *env, jclass this)
+{
+#if defined(MACOSX) || defined(__linux__)
+    //
+    // The man pages of writev() on both Linux and macOS specify this
+    // constraint on the sum of all byte lengths in the iovec array:
+    //
+    // [EINVAL] The sum of the iov_len values in the iov array
+    //          overflows a 32-bit integer.
+    //
+    // As of macOS 11 Big Sur, Darwin version 20, writev() started to
+    // actually enforce the constraint which had been previously ignored.
+    //
+    // In practice on Linux writev() has been observed not to write more
+    // than 0x7fff0000 (aarch64) or 0x7ffff000 (x64) bytes in one call.
+    //
+    return java_lang_Integer_MAX_VALUE;
+#else
+    return java_lang_Long_MAX_VALUE;
+#endif
+}
+
 /* Declared in nio_util.h for use elsewhere in NIO */
 
 jint

src/java.base/windows/native/libnio/ch/IOUtil.c

@@ -31,6 +31,7 @@
 #include "jvm.h"
 #include "jlong.h"
 
+#include "java_lang_Long.h"
 #include "nio.h"
 #include "nio_util.h"
 #include "net_util.h"
@@ -77,6 +78,11 @@ Java_sun_nio_ch_IOUtil_iovMax(JNIEnv *env, jclass this)
     return 16;
 }
 
+JNIEXPORT jlong JNICALL
+Java_sun_nio_ch_IOUtil_writevMax(JNIEnv *env, jclass this)
+{
+    return java_lang_Long_MAX_VALUE;
+}
 
 jint
 convertReturnVal(JNIEnv *env, jint n, jboolean reading)

test/jdk/java/nio/channels/FileChannel/LargeGatheringWrite.java

@@ -0,0 +1,125 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8274548
+ * @summary Test gathering write of more than INT_MAX bytes
+ * @library ..
+ * @library /test/lib
+ * @build jdk.test.lib.RandomFactory
+ * @run main/othervm -Xmx4G LargeGatheringWrite
+ * @key randomness
+ */
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Random;
+
+import jdk.test.lib.RandomFactory;
+
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.READ;
+import static java.nio.file.StandardOpenOption.WRITE;
+
+public class LargeGatheringWrite {
+    private static final int GB = 1024*1024*1024;
+
+    private static final Random RND = RandomFactory.getRandom();
+
+    public static void main(String[] args) throws IOException {
+        // Create direct and heap buffers
+        ByteBuffer direct = ByteBuffer.allocateDirect(GB);
+        ByteBuffer heap   = ByteBuffer.allocate(GB);
+
+        // Load buffers with random values
+        assert heap.hasArray();
+        RND.nextBytes(heap.array());
+        direct.put(0, heap, 0, heap.capacity());
+
+        // Create an array of buffers derived from direct and heap
+        ByteBuffer[] bigBuffers = new ByteBuffer[] {
+            direct,
+            heap,
+            direct.slice(0, GB/2),
+            heap.slice(0, GB/2),
+            direct.slice(GB/2, GB/2),
+            heap.slice(GB/2, GB/2),
+            direct.slice(GB/4, GB/2),
+            heap.slice(GB/4, GB/2),
+            direct.slice(0, 1),
+            heap.slice(GB - 2, 1)
+        };
+
+        // Calculate the sum of all buffer capacities
+        long totalLength = 0L;
+        for(ByteBuffer buf : bigBuffers)
+            totalLength += buf.capacity();
+
+        // Write the data to a temporary file
+        Path tempFile = Files.createTempFile("LargeGatheringWrite", ".dat");
+
+        System.out.printf("Writing %d bytes of data...%n", totalLength);
+        try (FileChannel fcw = FileChannel.open(tempFile, CREATE, WRITE);) {
+            // Print size of individual writes and total number written
+            long bytesWritten = 0;
+            long n;
+            while ((n = fcw.write(bigBuffers)) > 0) {
+                System.out.printf("Wrote %d bytes\n", n);
+                bytesWritten += n;
+            }
+            System.out.printf("Total of %d bytes written\n", bytesWritten);
+
+            // Verify the content written
+            try (FileChannel fcr = FileChannel.open(tempFile, READ);) {
+                byte[] bytes = null;
+                for (ByteBuffer buf : bigBuffers) {
+                    // For each buffer read the corresponding number of bytes
+                    buf.rewind();
+                    int length = buf.remaining();
+                    System.out.printf("Checking length %d%n", length);
+                    if (bytes == null || bytes.length < length)
+                        bytes = new byte[length];
+                    ByteBuffer dst = ByteBuffer.wrap(bytes).slice(0, length);
+                    if (dst.remaining() != length)
+                        throw new RuntimeException("remaining");
+                    if (fcr.read(dst) != length)
+                        throw new RuntimeException("length");
+                    dst.rewind();
+
+                    // Verify that the bytes read from the file match the buffer
+                    int mismatch;
+                    if ((mismatch = dst.mismatch(buf)) != -1) {
+                        String msg = String.format("mismatch: %d%n", mismatch);
+                        throw new RuntimeException(msg);
+                    }
+                }
+            }
+        } finally {
+            Files.delete(tempFile);
+        }
+    }
+}