8140241: (fc) Data transfer from FileChannel to itself causes hang in case of overlap

Reviewed-by: alanb

src/java.base/share/classes/sun/nio/ch/FileChannelImpl.java

@@ -585,6 +585,15 @@ public class FileChannelImpl
         if (!((target instanceof FileChannelImpl) || isSelChImpl))
             return IOStatus.UNSUPPORTED;
 
+        if (target == this) {
+            long posThis = position();
+            if (posThis - count + 1 <= position &&
+                position - count + 1 <= posThis &&
+                !nd.canTransferToFromOverlappedMap()) {
+                return IOStatus.UNSUPPORTED_CASE;
+            }
+        }
+
         // Trusted target: Use a mapped buffer
         long remaining = count;
         while (remaining > 0L) {
@@ -677,7 +686,7 @@ public class FileChannelImpl
             return 0;
 
         if ((sz - position) < count)
-            count = (int)(sz - position);
+            count = sz - position;
 
         // Attempt a direct transfer, if the kernel supports it, limiting
         // the number of bytes according to which platform
@@ -704,6 +713,14 @@ public class FileChannelImpl
             long pos = src.position();
             long max = Math.min(count, src.size() - pos);
 
+            if (src == this) {
+                if (position() - max + 1 <= pos &&
+                    pos - max + 1 <= position() &&
+                    !nd.canTransferToFromOverlappedMap()) {
+                    return IOStatus.UNSUPPORTED_CASE;
+                }
+            }
+
             long remaining = max;
             long p = pos;
             while (remaining > 0L) {
@@ -779,9 +796,12 @@ public class FileChannelImpl
             throw new IllegalArgumentException();
         if (position > size())
             return 0;
-        if (src instanceof FileChannelImpl)
-           return transferFromFileChannel((FileChannelImpl)src,
-                                          position, count);
+
+        if (src instanceof FileChannelImpl fci) {
+            long n = transferFromFileChannel(fci, position, count);
+            if (n >= 0)
+                return n;
+        }
 
         return transferFromArbitraryChannel(src, position, count);
     }

src/java.base/share/classes/sun/nio/ch/FileDispatcher.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2007, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2007, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -67,5 +67,7 @@ abstract class FileDispatcher extends NativeDispatcher {
 
     abstract boolean transferToDirectlyNeedsPositionLock();
 
+    abstract boolean canTransferToFromOverlappedMap();
+
     abstract int setDirectIO(FileDescriptor fd, String path);
 }

src/java.base/unix/classes/sun/nio/ch/FileDispatcherImpl.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -126,6 +126,10 @@ class FileDispatcherImpl extends FileDispatcher {
         return false;
     }
 
+    boolean canTransferToFromOverlappedMap() {
+        return canTransferToFromOverlappedMap0();
+    }
+
     int setDirectIO(FileDescriptor fd, String path) {
         int result = -1;
         try {
@@ -184,6 +188,8 @@ class FileDispatcherImpl extends FileDispatcher {
 
     static native void closeIntFD(int fd) throws IOException;
 
+    static native boolean canTransferToFromOverlappedMap0();
+
     static native int setDirect0(FileDescriptor fd) throws IOException;
 
     static native void init();

src/java.base/unix/native/libnio/ch/FileDispatcherImpl.c

@@ -338,6 +338,16 @@ Java_sun_nio_ch_FileDispatcherImpl_closeIntFD(JNIEnv *env, jclass clazz, jint fd
     closeFileDescriptor(env, fd);
 }
 
+JNIEXPORT jboolean JNICALL
+Java_sun_nio_ch_FileDispatcherImpl_canTransferToFromOverlappedMap0(JNIEnv *env, jclass clazz)
+{
+#ifdef MACOSX
+    return JNI_FALSE;
+#else
+    return JNI_TRUE;
+#endif
+}
+
 JNIEXPORT jint JNICALL
 Java_sun_nio_ch_FileDispatcherImpl_setDirect0(JNIEnv *env, jclass clazz,
                                            jobject fdo)

src/java.base/windows/classes/sun/nio/ch/FileDispatcherImpl.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -125,6 +125,10 @@ class FileDispatcherImpl extends FileDispatcher {
         return true;
     }
 
+    boolean canTransferToFromOverlappedMap() {
+        return true;
+    }
+
     int setDirectIO(FileDescriptor fd, String path) {
         int result = -1;
         String filePath = path.substring(0, path.lastIndexOf(File.separator));

test/jdk/java/nio/channels/FileChannel/TransferOverlappedFileChannel.java

@@ -0,0 +1,118 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/* @test
+ * @bug 8140241
+ * @summary Test transferring to and from same file channel
+ */
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.OutputStream;
+import java.io.RandomAccessFile;
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.util.Random;
+
+public class TransferOverlappedFileChannel {
+
+    public static void main(String[] args) throws Exception {
+        File file = File.createTempFile("readingin", null);
+        file.deleteOnExit();
+        generateBigFile(file);
+        RandomAccessFile raf = new RandomAccessFile(file, "rw");
+        try (FileChannel channel = raf.getChannel()) {
+            transferToNoOverlap(file, channel);
+            transferToOverlap(file, channel);
+            transferFromNoOverlap(file, channel);
+            transferFromOverlap(file, channel);
+        } finally {
+            file.delete();
+        }
+    }
+
+    private static void transferToNoOverlap(File file, FileChannel channel)
+        throws IOException {
+        final long length = file.length();
+
+        // position at three quarters
+        channel.position(length*3/4);
+        // copy last quarter to third quarter
+        // (copied and overwritten regions do NOT overlap)
+        // So: 1 2 3 4 -> 1 2 4 4
+        channel.transferTo(length / 2, length / 4, channel);
+        System.out.println("transferToNoOverlap: OK");
+    }
+
+    private static void transferToOverlap(File file, FileChannel channel)
+        throws IOException {
+        final long length = file.length();
+
+        // position at half
+        channel.position(length/2);
+        // copy last half to second quarter
+        // (copied and overwritten regions DO overlap)
+        // So: 1 2 3 4 -> 1 3 4 4
+        channel.transferTo(length / 4, length / 2, channel);
+        System.out.println("transferToOverlap: OK");
+    }
+
+    private static void transferFromNoOverlap(File file, FileChannel channel)
+        throws IOException {
+        final long length = file.length();
+
+        // position at three quarters
+        channel.position(length*3/4);
+        // copy last quarter to third quarter
+        // (copied and overwritten regions do NOT overlap)
+        // So: 1 2 3 4 -> 1 2 4 4
+        channel.transferFrom(channel, length / 2, length / 4);
+        System.out.println("transferFromNoOverlap: OK");
+    }
+
+    private static void transferFromOverlap(File file, FileChannel channel)
+        throws IOException {
+        final long length = file.length();
+
+        // position at half
+        channel.position(length/2);
+        // copy last half to second quarter
+        // (copied and overwritten regions DO overlap)
+        // So: 1 2 3 4 -> 1 3 4 4
+        channel.transferFrom(channel, length / 4, length / 2);
+        System.out.println("transferFromOverlap: OK");
+    }
+
+    private static void generateBigFile(File file) throws Exception {
+        try (OutputStream out = new BufferedOutputStream(
+                new FileOutputStream(file))) {
+            byte[] randomBytes = new byte[1024];
+            Random rand = new Random(0);
+            rand.nextBytes(randomBytes);
+            for (int i = 0; i < 1024; i++) {
+                out.write(randomBytes);
+            }
+            out.flush();
+        }
+    }
+}