8303509: Socket setTrafficClass does not work for IPv4 connections when IPv6 enabled

Reviewed-by: djelinski, michaelm

src/java.base/share/classes/sun/nio/ch/Net.java

@@ -55,7 +55,6 @@ import sun.net.util.IPAddressUtil;
 import sun.security.action.GetPropertyAction;
 
 public class Net {
-
     private Net() { }
 
     // unspecified protocol family
@@ -100,8 +99,8 @@ public class Net {
     /**
      * Tells whether both IPV6_XXX and IP_XXX socket options should be set on
      * IPv6 sockets. On some kernels, both IPV6_XXX and IP_XXX socket options
-     * need to be set so that the settings are effective for IPv4 multicast
+     * need to be set so that the settings are effective for IPv4 connections
-     * datagrams sent using the socket.
+     * and datagrams.
      */
     static boolean shouldSetBothIPv4AndIPv6Options() {
         return shouldSetBothIPv4AndIPv6Options0();
@@ -448,6 +447,23 @@ public class Net {
         setIntOption0(fd, mayNeedConversion, key.level(), key.name(), arg, isIPv6);
     }
 
+    /**
+     * Sets a IPPROTO_IPV6/IPPROTO level socket. Some platforms require both
+     * IPPROTO_IPV6 and IPPROTO socket options to be set when the socket is IPv6.
+     * In that case, the IPPROTO socket option is set on a best effort basis.
+     */
+    static <T> void setIpSocketOption(FileDescriptor fd, ProtocolFamily family,
+                                      SocketOption<T> opt, T value)
+        throws IOException
+    {
+        setSocketOption(fd, family, opt, value);
+        if (family == StandardProtocolFamily.INET6 && shouldSetBothIPv4AndIPv6Options()) {
+            try {
+                setSocketOption(fd, StandardProtocolFamily.INET, opt, value);
+            } catch (IOException ignore) { }
+        }
+    }
+
     static Object getSocketOption(FileDescriptor fd, SocketOption<?> name)
         throws IOException
     {
@@ -483,7 +499,7 @@ public class Net {
         }
     }
 
-    public static boolean isFastTcpLoopbackRequested() {
+    private static boolean isFastTcpLoopbackRequested() {
         String loopbackProp = GetPropertyAction
                 .privilegedGetProperty("jdk.net.useFastTcpLoopback", "false");
         return loopbackProp.isEmpty() || Boolean.parseBoolean(loopbackProp);

src/java.base/share/classes/sun/nio/ch/NioSocketImpl.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -957,8 +957,8 @@ public final class NioSocketImpl extends SocketImpl implements PlatformSocketImp
         synchronized (stateLock) {
             ensureOpen();
             if (opt == StandardSocketOptions.IP_TOS) {
-                // maps to IP_TOS or IPV6_TCLASS
+                // maps to IPV6_TCLASS and/or IP_TOS
-                Net.setSocketOption(fd, family(), opt, value);
+                Net.setIpSocketOption(fd, family(), opt, value);
             } else if (opt == StandardSocketOptions.SO_REUSEADDR) {
                 boolean b = (boolean) value;
                 if (Net.useExclusiveBind()) {
@@ -1032,7 +1032,7 @@ public final class NioSocketImpl extends SocketImpl implements PlatformSocketImp
                 }
                 case IP_TOS: {
                     int i = intValue(value, "IP_TOS");
-                    Net.setSocketOption(fd, family(), StandardSocketOptions.IP_TOS, i);
+                    Net.setIpSocketOption(fd, family(), StandardSocketOptions.IP_TOS, i);
                     break;
                 }
                 case TCP_NODELAY: {

src/java.base/share/classes/sun/nio/ch/SocketChannelImpl.java

@@ -272,8 +272,8 @@ class SocketChannelImpl
 
             if (isNetSocket()) {
                 if (name == StandardSocketOptions.IP_TOS) {
-                    // special handling for IP_TOS
+                    // maps to IPV6_TCLASS and/or IP_TOS
-                    Net.setSocketOption(fd, family, name, value);
+                    Net.setIpSocketOption(fd, family, name, value);
                     return this;
                 }
                 if (name == StandardSocketOptions.SO_REUSEADDR && Net.useExclusiveBind()) {

src/java.base/unix/native/libnio/ch/Net.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -159,10 +159,10 @@ JNIEXPORT jboolean JNICALL
 Java_sun_nio_ch_Net_shouldSetBothIPv4AndIPv6Options0(JNIEnv* env, jclass cl)
 {
 #if defined(__linux__)
-    /* Set both IPv4 and IPv6 socket options when setting multicast options */
+    /* Set both IPv4 and IPv6 socket options when setting IPPROTO_IPV6 options */
     return JNI_TRUE;
 #else
-    /* Do not set both IPv4 and IPv6 socket options when setting multicast options */
+    /* Do not set both IPv4 and IPv6 socket options when setting IPPROTO_IPV6 options */
     return JNI_FALSE;
 #endif
 }

src/java.base/windows/native/libnio/ch/Net.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -126,7 +126,7 @@ Java_sun_nio_ch_Net_isExclusiveBindAvailable(JNIEnv *env, jclass clazz) {
 JNIEXPORT jboolean JNICALL
 Java_sun_nio_ch_Net_shouldSetBothIPv4AndIPv6Options0(JNIEnv* env, jclass cl)
 {
-    /* Set both IPv4 and IPv6 socket options when setting multicast options */
+    /* Set both IPv4 and IPv6 socket options when setting IPPROTO_IPV6 options */
     return JNI_TRUE;
 }