8318306: java/util/Arrays/Sorting.java fails with "Array is not sorted at 8228-th position: 8251.0 and 8153.0"

Reviewed-by: thartmann, jbhateja

src/hotspot/share/opto/library_call.cpp

@@ -5367,8 +5367,6 @@ void LibraryCallKit::create_new_uncommon_trap(CallStaticJavaNode* uncommon_trap_
 //------------------------------inline_array_partition-----------------------
 bool LibraryCallKit::inline_array_partition() {
 
-  const char *stubName = "array_partition_stub";
-
   Node* elementType     = null_check(argument(0));
   Node* obj             = argument(1);
   Node* offset          = argument(2);
@@ -5377,6 +5375,13 @@ bool LibraryCallKit::inline_array_partition() {
   Node* indexPivot1     = argument(6);
   Node* indexPivot2     = argument(7);
 
+  Node* pivotIndices = nullptr;
+
+  // Set the original stack and the reexecute bit for the interpreter to reexecute
+  // the bytecode that invokes DualPivotQuicksort.partition() if deoptimization happens.
+  { PreserveReexecuteState preexecs(this);
+    jvms()->set_should_reexecute(true);
+
     const TypeInstPtr* elem_klass = gvn().type(elementType)->isa_instptr();
     ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type();
     BasicType bt = elem_type->basic_type();
@@ -5396,7 +5401,7 @@ bool LibraryCallKit::inline_array_partition() {
     // create the pivotIndices array of type int and size = 2
     Node* size = intcon(2);
     Node* klass_node = makecon(TypeKlassPtr::make(ciTypeArrayKlass::make(T_INT)));
-  Node* pivotIndices = new_array(klass_node, size, 0);  // no arguments to push
+    pivotIndices = new_array(klass_node, size, 0);  // no arguments to push
     AllocateArrayNode* alloc = tightly_coupled_allocation(pivotIndices);
     guarantee(alloc != nullptr, "created above");
     Node* pivotIndices_adr = basic_plus_adr(pivotIndices, arrayOopDesc::base_offset_in_bytes(T_INT));
@@ -5405,11 +5410,14 @@ bool LibraryCallKit::inline_array_partition() {
     Node* elemType = intcon(bt);
 
     // Call the stub
+    const char *stubName = "array_partition_stub";
     make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::array_partition_Type(),
                       stubAddr, stubName, TypePtr::BOTTOM,
                       obj_adr, elemType, fromIndex, toIndex, pivotIndices_adr,
                       indexPivot1, indexPivot2);
 
+  } // original reexecute is set back here
+
   if (!stopped()) {
     set_result(pivotIndices);
   }
@@ -5421,9 +5429,6 @@ bool LibraryCallKit::inline_array_partition() {
 //------------------------------inline_array_sort-----------------------
 bool LibraryCallKit::inline_array_sort() {
 
-  const char *stubName;
-  stubName = "arraysort_stub";
-
   Node* elementType     = null_check(argument(0));
   Node* obj             = argument(1);
   Node* offset          = argument(2);
@@ -5451,6 +5456,7 @@ bool LibraryCallKit::inline_array_sort() {
   Node* elemType = intcon(bt);
 
   // Call the stub.
+  const char *stubName = "arraysort_stub";
   make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::array_sort_Type(),
                     stubAddr, stubName, TypePtr::BOTTOM,
                     obj_adr, elemType, fromIndex, toIndex);

test/hotspot/jtreg/compiler/intrinsics/SortingDeoptimizationTest.java

@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8318306
+ * @run main/othervm/timeout=200 -XX:+IgnoreUnrecognizedVMOptions -Xcomp -ea -esa -XX:CompileThreshold=100 -XX:+UnlockExperimentalVMOptions -server -XX:-TieredCompilation -XX:+DeoptimizeALot SortingDeoptimizationTest 1e-2 100 50
+ * @summary Exercise Arrays.parallelSort when -XX:+DeoptimizeALot is enabled
+ *
+ */
+
+import java.io.PrintStream;
+import java.util.Arrays;
+import java.util.Random;
+
+public class SortingDeoptimizationTest {
+
+    private static final PrintStream err = System.err;
+    private static final PrintStream out = System.out;
+
+    public static void main(String[] args) {
+        int MAX = 2147483647; // 2^32 - 1
+        float fraction = Float.parseFloat(args[0]);
+        int size = (int) (fraction * MAX); // size is a fraction of the MAX size
+        int iters = Integer.parseInt(args[1]); // number of iterations
+        int max = args.length > 2 ? Integer.parseInt(args[2]) : -1 ; // max value for the array elements
+        long seed = 0xC0FFEE;
+        Random rand = new Random(seed);
+
+        for (int i = 0; i < iters; i++) {
+            boolean isSorted = runSort(size, max, rand);
+            out.println("Iteration " + i + ": is sorted? -> "+ isSorted);
+            if (!isSorted) fail("Array is not correctly sorted.");
+        }
+    }
+
+    private static void fail(String message) {
+        err.format("\n*** TEST FAILED ***\n\n%s\n\n", message);
+        throw new RuntimeException("Test failed");
+    }
+
+    private static boolean runSort(int size, int max, Random rand) {
+        int[] a = new int[size];
+        for (int i = 0; i < a.length; i++) a[i] =  max > 0 ? rand.nextInt(max) : rand.nextInt();
+        // call parallel sort
+        Arrays.parallelSort(a);
+        // check if sorted
+        boolean isSorted = true;
+        for (int i = 0; i < (a.length -1); i++) isSorted = isSorted && (a[i] <= a[i+1]);
+        return isSorted;
+    }
+}