8248655: Support supplementary characters in String case insensitive operations

8248434: some newly added locale cannot parse uppercased date string

Reviewed-by: jlaskey, joehw, rriggs, bchristi

src/java.base/share/classes/java/lang/String.java

@@ -43,7 +43,6 @@ import java.util.Optional;
 import java.util.Spliterator;
 import java.util.StringJoiner;
 import java.util.function.Function;
-import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
 import java.util.stream.Collectors;
@@ -1134,16 +1133,16 @@ public final class String
     /**
      * Compares this {@code String} to another {@code String}, ignoring case
      * considerations.  Two strings are considered equal ignoring case if they
-     * are of the same length and corresponding characters in the two strings
+     * are of the same length and corresponding Unicode code points in the two
-     * are equal ignoring case.
+     * strings are equal ignoring case.
      *
-     * <p> Two characters {@code c1} and {@code c2} are considered the same
+     * <p> Two Unicode code points are considered the same
      * ignoring case if at least one of the following is true:
      * <ul>
-     *   <li> The two characters are the same (as compared by the
+     *   <li> The two Unicode code points are the same (as compared by the
      *        {@code ==} operator)
-     *   <li> Calling {@code Character.toLowerCase(Character.toUpperCase(char))}
+     *   <li> Calling {@code Character.toLowerCase(Character.toUpperCase(int))}
-     *        on each character produces the same result
+     *        on each Unicode code point produces the same result
      * </ul>
      *
      * <p>Note that this method does <em>not</em> take locale into account, and
@@ -1158,6 +1157,7 @@ public final class String
      *          false} otherwise
      *
      * @see  #equals(Object)
+     * @see  #codePoints()
      */
     public boolean equalsIgnoreCase(String anotherString) {
         return (this == anotherString) ? true
@@ -1224,7 +1224,8 @@ public final class String
 
     /**
      * A Comparator that orders {@code String} objects as by
-     * {@code compareToIgnoreCase}. This comparator is serializable.
+     * {@link #compareToIgnoreCase(String) compareToIgnoreCase}.
+     * This comparator is serializable.
      * <p>
      * Note that this Comparator does <em>not</em> take locale into account,
      * and will result in an unsatisfactory ordering for certain locales.
@@ -1261,10 +1262,10 @@ public final class String
     /**
      * Compares two strings lexicographically, ignoring case
      * differences. This method returns an integer whose sign is that of
-     * calling {@code compareTo} with normalized versions of the strings
+     * calling {@code compareTo} with case folded versions of the strings
      * where case differences have been eliminated by calling
-     * {@code Character.toLowerCase(Character.toUpperCase(character))} on
+     * {@code Character.toLowerCase(Character.toUpperCase(int))} on
-     * each character.
+     * each Unicode code point.
      * <p>
      * Note that this method does <em>not</em> take locale into account,
      * and will result in an unsatisfactory ordering for certain locales.
@@ -1275,6 +1276,7 @@ public final class String
      *          specified String is greater than, equal to, or less
      *          than this String, ignoring case considerations.
      * @see     java.text.Collator
+     * @see     #codePoints()
      * @since   1.2
      */
     public int compareToIgnoreCase(String str) {
@@ -1362,30 +1364,26 @@ public final class String
      * <p>
      * A substring of this {@code String} object is compared to a substring
      * of the argument {@code other}. The result is {@code true} if these
-     * substrings represent character sequences that are the same, ignoring
+     * substrings represent Unicode code point sequences that are the same,
-     * case if and only if {@code ignoreCase} is true. The substring of
+     * ignoring case if and only if {@code ignoreCase} is true.
-     * this {@code String} object to be compared begins at index
+     * The sequences {@code tsequence} and {@code osequence} are compared,
-     * {@code toffset} and has length {@code len}. The substring of
+     * where {@code tsequence} is the sequence produced as if by calling
-     * {@code other} to be compared begins at index {@code ooffset} and
+     * {@code this.substring(toffset, len).codePoints()} and {@code osequence}
-     * has length {@code len}. The result is {@code false} if and only if
+     * is the sequence produced as if by calling
-     * at least one of the following is true:
+     * {@code other.substring(ooffset, len).codePoints()}.
-     * <ul><li>{@code toffset} is negative.
+     * The result is {@code true} if and only if all of the following
-     * <li>{@code ooffset} is negative.
+     * are true:
-     * <li>{@code toffset+len} is greater than the length of this
+     * <ul><li>{@code toffset} is non-negative.
+     * <li>{@code ooffset} is non-negative.
+     * <li>{@code toffset+len} is less than or equal to the length of this
      * {@code String} object.
-     * <li>{@code ooffset+len} is greater than the length of the other
+     * <li>{@code ooffset+len} is less than or equal to the length of the other
      * argument.
-     * <li>{@code ignoreCase} is {@code false} and there is some nonnegative
+     * <li>if {@code ignoreCase} is {@code false}, all pairs of corresponding Unicode
-     * integer <i>k</i> less than {@code len} such that:
+     * code points are equal integer values; or if {@code ignoreCase} is {@code true},
-     * <blockquote><pre>
+     * {@link Character#toLowerCase(int) Character.toLowerCase(}
-     * this.charAt(toffset+k) != other.charAt(ooffset+k)
+     * {@link Character#toUpperCase(int)}{@code )} on all pairs of Unicode code points
-     * </pre></blockquote>
+     * results in equal integer values.
-     * <li>{@code ignoreCase} is {@code true} and there is some nonnegative
-     * integer <i>k</i> less than {@code len} such that:
-     * <blockquote><pre>
-     * Character.toLowerCase(Character.toUpperCase(this.charAt(toffset+k))) !=
-     * Character.toLowerCase(Character.toUpperCase(other.charAt(ooffset+k)))
-     * </pre></blockquote>
      * </ul>
      *
      * <p>Note that this method does <em>not</em> take locale into account,
@@ -1400,12 +1398,14 @@ public final class String
      * @param   other        the string argument.
      * @param   ooffset      the starting offset of the subregion in the string
      *                       argument.
-     * @param   len          the number of characters to compare.
+     * @param   len          the number of characters (Unicode code units -
+     *                       16bit {@code char} value) to compare.
      * @return  {@code true} if the specified subregion of this string
      *          matches the specified subregion of the string argument;
      *          {@code false} otherwise. Whether the matching is exact
      *          or case insensitive depends on the {@code ignoreCase}
      *          argument.
+     * @see     #codePoints()
      */
     public boolean regionMatches(boolean ignoreCase, int toffset,
             String other, int ooffset, int len) {

src/java.base/share/classes/java/lang/StringUTF16.java

@@ -319,25 +319,92 @@ final class StringUTF16 {
     }
 
     public static int compareToCI(byte[] value, byte[] other) {
-        int len1 = length(value);
+        return compareToCIImpl(value, 0, length(value), other, 0, length(other));
-        int len2 = length(other);
+    }
-        int lim = Math.min(len1, len2);
+
-        for (int k = 0; k < lim; k++) {
+    private static int compareToCIImpl(byte[] value, int toffset, int tlen,
-            char c1 = getChar(value, k);
+                                      byte[] other, int ooffset, int olen) {
-            char c2 = getChar(other, k);
+        int tlast = toffset + tlen;
-            if (c1 != c2) {
+        int olast = ooffset + olen;
-                c1 = Character.toUpperCase(c1);
+        assert toffset >= 0 && ooffset >= 0;
-                c2 = Character.toUpperCase(c2);
+        assert tlast <= length(value);
-                if (c1 != c2) {
+        assert olast <= length(other);
-                    c1 = Character.toLowerCase(c1);
+
-                    c2 = Character.toLowerCase(c2);
+        for (int k1 = toffset, k2 = ooffset; k1 < tlast && k2 < olast; k1++, k2++) {
-                    if (c1 != c2) {
+            int cp1 = (int)getChar(value, k1);
-                        return c1 - c2;
+            int cp2 = (int)getChar(other, k2);
-                    }
+
-                }
+            if (cp1 == cp2 || compareCodePointCI(cp1, cp2) == 0) {
+                continue;
+            }
+
+            // Check for supplementary characters case
+            cp1 = codePointIncluding(value, cp1, k1, toffset, tlast);
+            if (cp1 < 0) {
+                k1++;
+                cp1 -= cp1;
+            }
+            cp2 = codePointIncluding(other, cp2, k2, ooffset, olast);
+            if (cp2 < 0) {
+                k2++;
+                cp2 -= cp2;
+            }
+
+            int diff = compareCodePointCI(cp1, cp2);
+            if (diff != 0) {
+                return diff;
             }
         }
-        return len1 - len2;
+        return tlen - olen;
+    }
+
+    // Case insensitive comparison of two code points
+    private static int compareCodePointCI(int cp1, int cp2) {
+        // try converting both characters to uppercase.
+        // If the results match, then the comparison scan should
+        // continue.
+        cp1 = Character.toUpperCase(cp1);
+        cp2 = Character.toUpperCase(cp2);
+        if (cp1 != cp2) {
+            // Unfortunately, conversion to uppercase does not work properly
+            // for the Georgian alphabet, which has strange rules about case
+            // conversion.  So we need to make one last check before
+            // exiting.
+            cp1 = Character.toLowerCase(cp1);
+            cp2 = Character.toLowerCase(cp2);
+            if (cp1 != cp2) {
+                return cp1 - cp2;
+            }
+        }
+        return 0;
+    }
+
+    // Returns a code point from the code unit pointed by "index". If it is
+    // not a surrogate or an unpaired surrogate, then the code unit is
+    // returned as is. Otherwise, it is combined with the code unit before
+    // or after, depending on the type of the surrogate at index, to make a
+    // supplementary code point. The return value will be negated if the code
+    // unit pointed by index is a high surrogate, and index + 1 is a low surrogate.
+    private static int codePointIncluding(byte[] ba, int cp, int index, int start, int end) {
+        // fast check
+        if (!Character.isSurrogate((char)cp)) {
+            return cp;
+        }
+        if (Character.isLowSurrogate((char)cp)) {
+            if (index > start) {
+                char c = getChar(ba, index - 1);
+                if (Character.isHighSurrogate(c)) {
+                    return Character.toCodePoint(c, (char)cp);
+                }
+            }
+        } else if (index + 1 < end) { // cp == high surrogate
+            char c = getChar(ba, index + 1);
+            if (Character.isLowSurrogate(c)) {
+                // negate the code point
+                return - Character.toCodePoint((char)cp, c);
+            }
+        }
+        return cp;
     }
 
     public static int compareToCI_Latin1(byte[] value, byte[] other) {
@@ -716,34 +783,7 @@ final class StringUTF16 {
 
     public static boolean regionMatchesCI(byte[] value, int toffset,
                                           byte[] other, int ooffset, int len) {
-        int last = toffset + len;
+        return compareToCIImpl(value, toffset, len, other, ooffset, len) == 0;
-        assert toffset >= 0 && ooffset >= 0;
-        assert ooffset + len <= length(other);
-        assert last <= length(value);
-        while (toffset < last) {
-            char c1 = getChar(value, toffset++);
-            char c2 = getChar(other, ooffset++);
-            if (c1 == c2) {
-                continue;
-            }
-            // try converting both characters to uppercase.
-            // If the results match, then the comparison scan should
-            // continue.
-            char u1 = Character.toUpperCase(c1);
-            char u2 = Character.toUpperCase(c2);
-            if (u1 == u2) {
-                continue;
-            }
-            // Unfortunately, conversion to uppercase does not work properly
-            // for the Georgian alphabet, which has strange rules about case
-            // conversion.  So we need to make one last check before
-            // exiting.
-            if (Character.toLowerCase(u1) == Character.toLowerCase(u2)) {
-                continue;
-            }
-            return false;
-        }
-        return true;
     }
 
     public static boolean regionMatchesCI_Latin1(byte[] value, int toffset,

test/jdk/java/lang/String/CompactString/CompareToIgnoreCase.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,7 +28,7 @@ import static org.testng.Assert.assertEquals;
 
 /*
  * @test
- * @bug 8077559
+ * @bug 8077559 8248655
  * @summary Tests Compact String. This one is for String.compareToIgnoreCase.
  * @run testng/othervm -XX:+CompactStrings CompareToIgnoreCase
  * @run testng/othervm -XX:-CompactStrings CompareToIgnoreCase
@@ -39,7 +39,6 @@ public class CompareToIgnoreCase extends CompactString {
     @DataProvider
     public Object[][] provider() {
         return new Object[][] {
-
                 new Object[] { STRING_EMPTY, "A", -1 },
                 new Object[] { STRING_L1, "a", 0 },
                 new Object[] { STRING_L1, "A", 0 },
@@ -67,7 +66,10 @@ public class CompareToIgnoreCase extends CompactString {
                 new Object[] { STRING_M12, "\uFF41a", 0 },
                 new Object[] { STRING_M12, "\uFF41\uFF42", -65249 },
                 new Object[] { STRING_M11, "a\uFF41", 0 },
-                new Object[] { STRING_M11, "a\uFF42", -1 }, };
+                new Object[] { STRING_M11, "a\uFF42", -1 },
+                new Object[] { STRING_SUPPLEMENTARY, STRING_SUPPLEMENTARY_LOWERCASE, 0 },
+                new Object[] { STRING_SUPPLEMENTARY, "\uD801\uDC28\uD801\uDC27\uFF41a", -38 },
+        };
     }
 
     @Test(dataProvider = "provider")

test/jdk/java/lang/String/CompactString/EqualsIgnoreCase.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,7 +28,7 @@ import static org.testng.Assert.assertEquals;
 
 /*
  * @test
- * @bug 8077559
+ * @bug 8077559 8248655
  * @summary Tests Compact String. This one is for String.equalsIgnoreCase.
  * @run testng/othervm -XX:+CompactStrings EqualsIgnoreCase
  * @run testng/othervm -XX:-CompactStrings EqualsIgnoreCase
@@ -54,6 +54,7 @@ public class EqualsIgnoreCase extends CompactString {
                 new Object[] { STRING_M12, "\uFF21A", true },
                 new Object[] { STRING_M11, "a\uFF41", true },
                 new Object[] { STRING_M11, "A\uFF21", true },
+                new Object[] { STRING_SUPPLEMENTARY, STRING_SUPPLEMENTARY_LOWERCASE, true },
 
         };
     }

test/jdk/java/lang/String/CompactString/RegionMatches.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,7 +28,7 @@ import static org.testng.Assert.assertEquals;
 
 /*
  * @test
- * @bug 8077559
+ * @bug 8077559 8248655
  * @summary Tests Compact String. This one is for String.regionMatches.
  * @run testng/othervm -XX:+CompactStrings RegionMatches
  * @run testng/othervm -XX:-CompactStrings RegionMatches
@@ -66,15 +66,23 @@ public class RegionMatches extends CompactString {
                         3, true },
                 new Object[] { STRING_MDUPLICATE1, false, 0, "\uFF21a\uFF21",
                         0, 3, false },
+                new Object[] { STRING_SUPPLEMENTARY, true, 0, "\uD801\uDC28\uD801\uDC29",
+                        0, 4, true },
                 new Object[] { STRING_SUPPLEMENTARY, true, 1, "\uDC00\uD801",
                         0, 2, true },
+                new Object[] { STRING_SUPPLEMENTARY, true, 1, "\uDC28",
+                        0, 1, false },
                 new Object[] { STRING_SUPPLEMENTARY, true, 4, "\uFF21", 0, 1,
                         true },
                 new Object[] { STRING_SUPPLEMENTARY, true, 5, "A", 0, 1, true },
                 new Object[] { STRING_SUPPLEMENTARY_LOWERCASE, false, 0,
                         "\uD801\uDC28\uD801\uDC29", 0, 4, true },
+                new Object[] { STRING_SUPPLEMENTARY_LOWERCASE, true, 0,
+                        "\uD801\uDC00\uD801\uDC01", 0, 4, true },
                 new Object[] { STRING_SUPPLEMENTARY_LOWERCASE, true, 1,
                         "\uDC28\uD801", 0, 2, true },
+                new Object[] { STRING_SUPPLEMENTARY_LOWERCASE, true, 1,
+                        "\uDC00", 0, 1, false },
                 new Object[] { STRING_SUPPLEMENTARY_LOWERCASE, true, 1,
                         "\uDC00\uD801", 0, 2, false },
                 new Object[] { STRING_SUPPLEMENTARY_LOWERCASE, true, 4,

test/jdk/java/text/Format/DateFormat/CaseInsensitiveParseTest.java

@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @bug 8248434
+ * @modules jdk.localedata
+ * @run testng/othervm CaseInsensitiveParseTest
+ * @summary Checks format/parse round trip in case-insensitive manner.
+ */
+
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.Locale;
+import java.util.stream.Stream;
+
+import static org.testng.Assert.assertEquals;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class CaseInsensitiveParseTest {
+
+    private final static String PATTERN = "GGGG/yyyy/MMMM/dddd/hhhh/mmmm/ss/aaaa";
+    private final static Date EPOCH = new Date(0L);
+
+    @DataProvider
+    private Object[][] locales() {
+        return (Object[][])Arrays.stream(DateFormat.getAvailableLocales())
+            .map(Stream::of)
+            .map(Stream::toArray)
+            .toArray(Object[][]::new);
+    }
+
+    @Test(dataProvider = "locales")
+    public void testUpperCase(Locale loc) throws ParseException {
+        SimpleDateFormat sdf = new SimpleDateFormat(PATTERN, loc);
+        String formatted = sdf.format(EPOCH);
+        assertEquals(sdf.parse(formatted.toUpperCase(Locale.ROOT)), EPOCH,
+                "roundtrip failed for string '" + formatted + "', locale: " + loc);
+    }
+
+    @Test(dataProvider = "locales")
+    public void testLowerCase(Locale loc) throws ParseException {
+        SimpleDateFormat sdf = new SimpleDateFormat(PATTERN, loc);
+        String formatted = sdf.format(EPOCH);
+        assertEquals(sdf.parse(formatted.toLowerCase(Locale.ROOT)), EPOCH,
+                "roundtrip failed for string '" + formatted + "', locale: " + loc);
+    }
+}

test/micro/org/openjdk/bench/java/lang/StringCompareToIgnoreCase.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.bench.java.lang;
+
+import org.openjdk.jmh.annotations.*;
+import java.util.concurrent.TimeUnit;
+
+/*
+ * This benchmark naively explores String::compareToIgnoreCase performance
+ */
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@State(Scope.Benchmark)
+public class StringCompareToIgnoreCase {
+
+    public String upper = new String("\u0100\u0102\u0104\u0106\u0108");
+    public String upperLower = new String("\u0100\u0102\u0104\u0106\u0109");
+    public String lower = new String("\u0101\u0103\u0105\u0107\u0109");
+    public String supUpper = new String("\ud801\udc00\ud801\udc01\ud801\udc02\ud801\udc03\ud801\udc04");
+    public String supUpperLower = new String("\ud801\udc00\ud801\udc01\ud801\udc02\ud801\udc03\ud801\udc2c");
+    public String supLower = new String("\ud801\udc28\ud801\udc29\ud801\udc2a\ud801\udc2b\ud801\udc2c");
+
+    @Benchmark
+    public int upperLower() {
+        return upper.compareToIgnoreCase(upperLower);
+    }
+
+    @Benchmark
+    public int lower() {
+        return upper.compareToIgnoreCase(lower);
+    }
+
+    @Benchmark
+    public int supUpperLower() {
+        return supUpper.compareToIgnoreCase(supUpperLower);
+    }
+
+    @Benchmark
+    public int supLower() {
+        return supUpper.compareToIgnoreCase(supLower);
+    }
+}