8236184: (dc) IP_MULTICAST_* and IP_TOS socket options not effective

Reviewed-by: dfuchs

src/java.base/share/classes/sun/nio/ch/DatagramChannelImpl.java

@@ -338,23 +338,41 @@ class DatagramChannelImpl
 
             ProtocolFamily family = familyFor(name);
 
+            // Some platforms require both IPV6_XXX and IP_XXX socket options to
+            // be set when the channel's socket is IPv6 and it is used to send
+            // IPv4 multicast datagrams. The IP_XXX socket options are set on a
+            // best effort basis.
+            boolean needToSetIPv4Option = (family != Net.UNSPEC)
+                    && (this.family == StandardProtocolFamily.INET6)
+                    && Net.shouldSetBothIPv4AndIPv6Options();
+
+            // outgoing multicast interface
             if (name == StandardSocketOptions.IP_MULTICAST_IF) {
-                NetworkInterface interf = (NetworkInterface)value;
+                assert family != Net.UNSPEC;
+                NetworkInterface interf = (NetworkInterface) value;
                 if (family == StandardProtocolFamily.INET6) {
                     int index = interf.getIndex();
                     if (index == -1)
                         throw new IOException("Network interface cannot be identified");
                     Net.setInterface6(fd, index);
-                } else {
+                }
+                if (family == StandardProtocolFamily.INET || needToSetIPv4Option) {
                     // need IPv4 address to identify interface
                     Inet4Address target = Net.anyInet4Address(interf);
-                    if (target == null)
+                    if (target != null) {
+                        try {
+                            Net.setInterface4(fd, Net.inet4AsInt(target));
+                        } catch (IOException ioe) {
+                            if (family == StandardProtocolFamily.INET) throw ioe;
+                        }
+                    } else if (family == StandardProtocolFamily.INET) {
                         throw new IOException("Network interface not configured for IPv4");
-                    int targetAddress = Net.inet4AsInt(target);
-                    Net.setInterface4(fd, targetAddress);
+                    }
                 }
                 return this;
             }
+
+            // SO_REUSEADDR needs special handling as it may be emulated
             if (name == StandardSocketOptions.SO_REUSEADDR
                 && Net.useExclusiveBind() && localAddress != null) {
                 reuseAddressEmulated = true;
@@ -363,6 +381,12 @@ class DatagramChannelImpl
 
             // remaining options don't need any special handling
             Net.setSocketOption(fd, family, name, value);
+            if (needToSetIPv4Option && family != StandardProtocolFamily.INET) {
+                try {
+                    Net.setSocketOption(fd, StandardProtocolFamily.INET, name, value);
+                } catch (IOException ignore) { }
+            }
+
             return this;
         }
     }

src/java.base/share/classes/sun/nio/ch/Net.java

@@ -106,6 +106,16 @@ public class Net {
         return exclusiveBind;
     }
 
+    /**
+     * Tells whether both IPV6_XXX and IP_XXX socket options should be set on
+     * IPv6 sockets. On some kernels, both IPV6_XXX and IP_XXX socket options
+     * need to be set so that the settings are effective for IPv4 multicast
+     * datagrams sent using the socket.
+     */
+    static boolean shouldSetBothIPv4AndIPv6Options() {
+        return shouldSetBothIPv4AndIPv6Options0();
+    }
+
     /**
      * Tells whether IPv6 sockets can join IPv4 multicast groups
      */
@@ -438,6 +448,8 @@ public class Net {
      */
     private static native int isExclusiveBindAvailable();
 
+    private static native boolean shouldSetBothIPv4AndIPv6Options0();
+
     private static native boolean canIPv6SocketJoinIPv4Group0();
 
     private static native boolean canJoin6WithIPv4Group0();

src/java.base/unix/native/libnio/ch/Net.c

@@ -155,6 +155,18 @@ Java_sun_nio_ch_Net_isExclusiveBindAvailable(JNIEnv *env, jclass clazz) {
     return -1;
 }
 
+JNIEXPORT jboolean JNICALL
+Java_sun_nio_ch_Net_shouldSetBothIPv4AndIPv6Options0(JNIEnv* env, jclass cl)
+{
+#if defined(__linux__)
+    /* Set both IPv4 and IPv6 socket options when setting multicast options */
+    return JNI_TRUE;
+#else
+    /* Do not set both IPv4 and IPv6 socket options when setting multicast options */
+    return JNI_FALSE;
+#endif
+}
+
 JNIEXPORT jboolean JNICALL
 Java_sun_nio_ch_Net_canIPv6SocketJoinIPv4Group0(JNIEnv* env, jclass cl)
 {
@@ -540,12 +552,6 @@ Java_sun_nio_ch_Net_setIntOption0(JNIEnv *env, jclass clazz, jobject fdo,
                                      JNU_JAVANETPKG "SocketException",
                                      "sun.nio.ch.Net.setIntOption");
     }
-#ifdef __linux__
-    if (level == IPPROTO_IPV6 && opt == IPV6_TCLASS && isIPv6) {
-        // set the V4 option also
-        setsockopt(fdval(env, fdo), IPPROTO_IP, IP_TOS, parg, arglen);
-    }
-#endif
 }
 
 JNIEXPORT jint JNICALL

src/java.base/windows/native/libnio/ch/Net.c

@@ -25,11 +25,11 @@
 
 #include <windows.h>
 #include <winsock2.h>
+
 #include "jni.h"
 #include "jni_util.h"
 #include "jvm.h"
 #include "jlong.h"
-
 #include "nio.h"
 #include "nio_util.h"
 #include "net_util.h"
@@ -123,6 +123,12 @@ Java_sun_nio_ch_Net_isExclusiveBindAvailable(JNIEnv *env, jclass clazz) {
     return 1;
 }
 
+JNIEXPORT jboolean JNICALL
+Java_sun_nio_ch_Net_shouldSetBothIPv4AndIPv6Options0(JNIEnv* env, jclass cl)
+{
+    /* Set both IPv4 and IPv6 socket options when setting multicast options */
+    return JNI_TRUE;
+}
 
 JNIEXPORT jboolean JNICALL
 Java_sun_nio_ch_Net_canIPv6SocketJoinIPv4Group0(JNIEnv* env, jclass cl)