8140241: (fc) Data transfer from FileChannel to itself causes hang in case of overlap

Reviewed-by: alanb

src/java.base/share/classes/sun/nio/ch/FileChannelImpl.java

@@ -585,6 +585,15 @@ public class FileChannelImpl
         if (!((target instanceof FileChannelImpl) || isSelChImpl))
             return IOStatus.UNSUPPORTED;
 
+        if (target == this) {
+            long posThis = position();
+            if (posThis - count + 1 <= position &&
+                position - count + 1 <= posThis &&
+                !nd.canTransferToFromOverlappedMap()) {
+                return IOStatus.UNSUPPORTED_CASE;
+            }
+        }
+
         // Trusted target: Use a mapped buffer
         long remaining = count;
         while (remaining > 0L) {
@@ -677,7 +686,7 @@ public class FileChannelImpl
             return 0;
 
         if ((sz - position) < count)
-            count = (int)(sz - position);
+            count = sz - position;
 
         // Attempt a direct transfer, if the kernel supports it, limiting
         // the number of bytes according to which platform
@@ -704,6 +713,14 @@ public class FileChannelImpl
             long pos = src.position();
             long max = Math.min(count, src.size() - pos);
 
+            if (src == this) {
+                if (position() - max + 1 <= pos &&
+                    pos - max + 1 <= position() &&
+                    !nd.canTransferToFromOverlappedMap()) {
+                    return IOStatus.UNSUPPORTED_CASE;
+                }
+            }
+
             long remaining = max;
             long p = pos;
             while (remaining > 0L) {
@@ -779,9 +796,12 @@ public class FileChannelImpl
             throw new IllegalArgumentException();
         if (position > size())
             return 0;
-        if (src instanceof FileChannelImpl)
-           return transferFromFileChannel((FileChannelImpl)src,
-                                          position, count);
+
+        if (src instanceof FileChannelImpl fci) {
+            long n = transferFromFileChannel(fci, position, count);
+            if (n >= 0)
+                return n;
+        }
 
         return transferFromArbitraryChannel(src, position, count);
     }

src/java.base/share/classes/sun/nio/ch/FileDispatcher.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2007, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2007, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -67,5 +67,7 @@ abstract class FileDispatcher extends NativeDispatcher {
 
     abstract boolean transferToDirectlyNeedsPositionLock();
 
+    abstract boolean canTransferToFromOverlappedMap();
+
     abstract int setDirectIO(FileDescriptor fd, String path);
 }

src/java.base/unix/classes/sun/nio/ch/FileDispatcherImpl.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -126,6 +126,10 @@ class FileDispatcherImpl extends FileDispatcher {
         return false;
     }
 
+    boolean canTransferToFromOverlappedMap() {
+        return canTransferToFromOverlappedMap0();
+    }
+
     int setDirectIO(FileDescriptor fd, String path) {
         int result = -1;
         try {
@@ -184,6 +188,8 @@ class FileDispatcherImpl extends FileDispatcher {
 
     static native void closeIntFD(int fd) throws IOException;
 
+    static native boolean canTransferToFromOverlappedMap0();
+
     static native int setDirect0(FileDescriptor fd) throws IOException;
 
     static native void init();

src/java.base/unix/native/libnio/ch/FileDispatcherImpl.c

@@ -338,6 +338,16 @@ Java_sun_nio_ch_FileDispatcherImpl_closeIntFD(JNIEnv *env, jclass clazz, jint fd
     closeFileDescriptor(env, fd);
 }
 
+JNIEXPORT jboolean JNICALL
+Java_sun_nio_ch_FileDispatcherImpl_canTransferToFromOverlappedMap0(JNIEnv *env, jclass clazz)
+{
+#ifdef MACOSX
+    return JNI_FALSE;
+#else
+    return JNI_TRUE;
+#endif
+}
+
 JNIEXPORT jint JNICALL
 Java_sun_nio_ch_FileDispatcherImpl_setDirect0(JNIEnv *env, jclass clazz,
                                            jobject fdo)

src/java.base/windows/classes/sun/nio/ch/FileDispatcherImpl.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -125,6 +125,10 @@ class FileDispatcherImpl extends FileDispatcher {
         return true;
     }
 
+    boolean canTransferToFromOverlappedMap() {
+        return true;
+    }
+
     int setDirectIO(FileDescriptor fd, String path) {
         int result = -1;
         String filePath = path.substring(0, path.lastIndexOf(File.separator));