8279204: [BACKOUT] JDK-8278413: C2 crash when allocating array of size too large

Reviewed-by: chagedorn, kvn

src/hotspot/share/opto/callnode.cpp

@@ -1656,7 +1656,6 @@ AllocateNode::AllocateNode(Compile* C, const TypeFunc *atype,
   init_req( KlassNode          , klass_node);
   init_req( InitialTest        , initial_test);
   init_req( ALength            , topnode);
-  init_req( ValidLengthTest    , topnode);
   C->add_macro_node(this);
 }
 
@@ -1683,6 +1682,54 @@ Node *AllocateNode::make_ideal_mark(PhaseGVN *phase, Node* obj, Node* control, N
   return mark_node;
 }
 
+//=============================================================================
+Node* AllocateArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
+  if (remove_dead_region(phase, can_reshape))  return this;
+  // Don't bother trying to transform a dead node
+  if (in(0) && in(0)->is_top())  return NULL;
+
+  const Type* type = phase->type(Ideal_length());
+  if (type->isa_int() && type->is_int()->_hi < 0) {
+    if (can_reshape) {
+      PhaseIterGVN *igvn = phase->is_IterGVN();
+      // Unreachable fall through path (negative array length),
+      // the allocation can only throw so disconnect it.
+      Node* proj = proj_out_or_null(TypeFunc::Control);
+      Node* catchproj = NULL;
+      if (proj != NULL) {
+        for (DUIterator_Fast imax, i = proj->fast_outs(imax); i < imax; i++) {
+          Node *cn = proj->fast_out(i);
+          if (cn->is_Catch()) {
+            catchproj = cn->as_Multi()->proj_out_or_null(CatchProjNode::fall_through_index);
+            break;
+          }
+        }
+      }
+      if (catchproj != NULL && catchproj->outcnt() > 0 &&
+          (catchproj->outcnt() > 1 ||
+           catchproj->unique_out()->Opcode() != Op_Halt)) {
+        assert(catchproj->is_CatchProj(), "must be a CatchProjNode");
+        Node* nproj = catchproj->clone();
+        igvn->register_new_node_with_optimizer(nproj);
+
+        Node *frame = new ParmNode( phase->C->start(), TypeFunc::FramePtr );
+        frame = phase->transform(frame);
+        // Halt & Catch Fire
+        Node* halt = new HaltNode(nproj, frame, "unexpected negative array length");
+        phase->C->root()->add_req(halt);
+        phase->transform(halt);
+
+        igvn->replace_node(catchproj, phase->C->top());
+        return this;
+      }
+    } else {
+      // Can't correct it during regular GVN so register for IGVN
+      phase->C->record_for_igvn(this);
+    }
+  }
+  return NULL;
+}
+
 // Retrieve the length from the AllocateArrayNode. Narrow the type with a
 // CastII, if appropriate.  If we are not allowed to create new nodes, and
 // a CastII is appropriate, return NULL.

src/hotspot/share/opto/callnode.hpp

@@ -913,7 +913,6 @@ public:
     KlassNode,                        // type (maybe dynamic) of the obj.
     InitialTest,                      // slow-path test (may be constant)
     ALength,                          // array length (or TOP if none)
-    ValidLengthTest,
     ParmLimit
   };
 
@@ -923,7 +922,6 @@ public:
     fields[KlassNode]   = TypeInstPtr::NOTNULL;
     fields[InitialTest] = TypeInt::BOOL;
     fields[ALength]     = t;  // length (can be a bad length)
-    fields[ValidLengthTest] = TypeInt::BOOL;
 
     const TypeTuple *domain = TypeTuple::make(ParmLimit, fields);
 
@@ -1018,16 +1016,18 @@ public:
 //
 class AllocateArrayNode : public AllocateNode {
 public:
-  AllocateArrayNode(Compile* C, const TypeFunc* atype, Node* ctrl, Node* mem, Node* abio, Node* size, Node* klass_node,
+  AllocateArrayNode(Compile* C, const TypeFunc *atype, Node *ctrl, Node *mem, Node *abio,
-                    Node* initial_test, Node* count_val, Node* valid_length_test)
+                    Node* size, Node* klass_node, Node* initial_test,
+                    Node* count_val
+                    )
     : AllocateNode(C, atype, ctrl, mem, abio, size, klass_node,
                    initial_test)
   {
     init_class_id(Class_AllocateArray);
     set_req(AllocateNode::ALength,        count_val);
-    set_req(AllocateNode::ValidLengthTest, valid_length_test);
   }
   virtual int Opcode() const;
+  virtual Node *Ideal(PhaseGVN *phase, bool can_reshape);
 
   // Dig the length operand out of a array allocation site.
   Node* Ideal_length() {

src/hotspot/share/opto/cfgnode.cpp

@@ -2687,20 +2687,6 @@ const Type* CatchNode::Value(PhaseGVN* phase) const {
       // Rethrows always throw exceptions, never return
       if (call->entry_point() == OptoRuntime::rethrow_stub()) {
         f[CatchProjNode::fall_through_index] = Type::TOP;
-      } else if (call->is_AllocateArray()) {
-        Node* klass_node = call->in(AllocateNode::KlassNode);
-        Node *length = call->in(AllocateNode::ALength);
-        const Type* length_type = phase->type(length);
-        const Type* klass_type = phase->type(klass_node);
-        if (length_type == Type::TOP || klass_type == Type::TOP) {
-          f[CatchProjNode::fall_through_index] = Type::TOP;
-        } else {
-          Node* valid_length_test = call->in(AllocateNode::ValidLengthTest);
-          const Type* valid_length_test_t = phase->type(valid_length_test);
-          if (valid_length_test_t->isa_int() && valid_length_test_t->is_int()->is_con(0)) {
-            f[CatchProjNode::fall_through_index] = Type::TOP;
-          }
-        }
       } else if( call->req() > TypeFunc::Parms ) {
         const Type *arg0 = phase->type( call->in(TypeFunc::Parms) );
         // Check for null receiver to virtual or interface calls

src/hotspot/share/opto/compile.cpp

@@ -3756,20 +3756,17 @@ bool Compile::final_graph_reshaping() {
                 arg0->as_Type()->type()->higher_equal(TypePtr::NULL_PTR)) {
               required_outcnt--;
             }
-          } else if (call->entry_point() == OptoRuntime::new_array_Java() ||
+          } else if (call->entry_point() == OptoRuntime::new_array_Java() &&
-                     call->entry_point() == OptoRuntime::new_array_nozero_Java()) {
+                     call->req() > TypeFunc::Parms+1 &&
+                     call->is_CallStaticJava()) {
             // Check for negative array length. In such case, the optimizer has
             // detected that the allocation attempt will always result in an
             // exception. There is no fall-through projection of this CatchNode .
-            assert(call->is_CallStaticJava(), "static call expected");
+            Node *arg1 = call->in(TypeFunc::Parms+1);
-            assert(call->len() > call->req() && call->in(call->req()) != NULL, "no precendent edge");
+            if (arg1->is_Type() &&
-            Node* valid_length_test = call->in(call->req());
+                arg1->as_Type()->type()->join(TypeInt::POS)->empty()) {
-            call->rm_prec(call->req());
-            if (valid_length_test->find_int_con(1) == 0) {
               required_outcnt--;
             }
-            assert(n->outcnt() == required_outcnt, "malformed control flow");
-            continue;
           }
         }
       }
@@ -3778,13 +3775,6 @@ bool Compile::final_graph_reshaping() {
         record_method_not_compilable("malformed control flow");
         return true;            // Not all targets reachable!
       }
-    } else if (n->is_PCTable() && n->in(0) && n->in(0)->in(0) && n->in(0)->in(0)->is_Call()) {
-      CallNode* call = n->in(0)->in(0)->as_Call();
-      if (call->entry_point() == OptoRuntime::new_array_Java() ||
-          call->entry_point() == OptoRuntime::new_array_nozero_Java()) {
-        assert(call->len() > call->req() && call->in(call->req()) != NULL, "precedent edge expected");
-        call->rm_prec(call->req());
-      }
     }
     // Check that I actually visited all kids.  Unreached kids
     // must be infinite loops.

src/hotspot/share/opto/graphKit.cpp

@@ -2742,9 +2742,7 @@ void GraphKit::make_slow_call_ex(Node* call, ciInstanceKlass* ex_klass, bool sep
   // Make a catch node with just two handlers:  fall-through and catch-all
   Node* i_o  = _gvn.transform( new ProjNode(call, TypeFunc::I_O, separate_io_proj) );
   Node* catc = _gvn.transform( new CatchNode(control(), i_o, 2) );
-  Node* norm = new CatchProjNode(catc, CatchProjNode::fall_through_index, CatchProjNode::no_handler_bci);
+  Node* norm = _gvn.transform( new CatchProjNode(catc, CatchProjNode::fall_through_index, CatchProjNode::no_handler_bci) );
-  _gvn.set_type_bottom(norm);
-  C->record_for_igvn(norm);
   Node* excp = _gvn.transform( new CatchProjNode(catc, CatchProjNode::catch_all_index,    CatchProjNode::no_handler_bci) );
 
   { PreserveJVMState pjvms(this);
@@ -3985,28 +3983,20 @@ Node* GraphKit::new_array(Node* klass_node,     // array klass (maybe variable)
     initial_slow_test = initial_slow_test->as_Bool()->as_int_value(&_gvn);
   }
 
-  const TypeOopPtr* ary_type = _gvn.type(klass_node)->is_klassptr()->as_instance_type();
-  Node* valid_length_test = C->top();
-  if (ary_type->klass()->is_array_klass()) {
-    BasicType bt = ary_type->klass()->as_array_klass()->element_type()->basic_type();
-    jint max = TypeAryPtr::max_array_length(bt);
-    Node* valid_length_cmp  = _gvn.transform(new CmpUNode(length, intcon(max)));
-    valid_length_test = _gvn.transform(new BoolNode(valid_length_cmp, BoolTest::le));
-  }
-
   // Create the AllocateArrayNode and its result projections
   AllocateArrayNode* alloc
     = new AllocateArrayNode(C, AllocateArrayNode::alloc_type(TypeInt::INT),
                             control(), mem, i_o(),
                             size, klass_node,
                             initial_slow_test,
-                            length, valid_length_test);
+                            length);
 
   // Cast to correct type.  Note that the klass_node may be constant or not,
   // and in the latter case the actual array type will be inexact also.
   // (This happens via a non-constant argument to inline_native_newArray.)
   // In any case, the value of klass_node provides the desired array type.
   const TypeInt* length_type = _gvn.find_int_type(length);
+  const TypeOopPtr* ary_type = _gvn.type(klass_node)->is_klassptr()->as_instance_type();
   if (ary_type->isa_aryptr() && length_type != NULL) {
     // Try to get a better type than POS for the size
     ary_type = ary_type->is_aryptr()->cast_to_size(length_type);

src/hotspot/share/opto/macro.cpp

@@ -1208,8 +1208,7 @@ void PhaseMacroExpand::expand_allocate_common(
             AllocateNode* alloc, // allocation node to be expanded
             Node* length,  // array length for an array allocation
             const TypeFunc* slow_call_type, // Type of slow call
-            address slow_call_address,  // Address of slow call
+            address slow_call_address  // Address of slow call
-            Node* valid_length_test // whether length is valid or not
     )
 {
   Node* ctrl = alloc->in(TypeFunc::Control);
@@ -1394,9 +1393,6 @@ void PhaseMacroExpand::expand_allocate_common(
   // Copy debug information and adjust JVMState information, then replace
   // allocate node with the call
   call->copy_call_debug_info(&_igvn, alloc);
-  if (valid_length_test != NULL) {
-    call->add_prec(valid_length_test);
-  }
   if (expand_fast_path) {
     call->set_cnt(PROB_UNLIKELY_MAG(4));  // Same effect as RC_UNCOMMON.
   } else {
@@ -1879,12 +1875,11 @@ Node* PhaseMacroExpand::prefetch_allocation(Node* i_o, Node*& needgc_false,
 void PhaseMacroExpand::expand_allocate(AllocateNode *alloc) {
   expand_allocate_common(alloc, NULL,
                          OptoRuntime::new_instance_Type(),
-                         OptoRuntime::new_instance_Java(), NULL);
+                         OptoRuntime::new_instance_Java());
 }
 
 void PhaseMacroExpand::expand_allocate_array(AllocateArrayNode *alloc) {
   Node* length = alloc->in(AllocateNode::ALength);
-  Node* valid_length_test = alloc->in(AllocateNode::ValidLengthTest);
   InitializeNode* init = alloc->initialization();
   Node* klass_node = alloc->in(AllocateNode::KlassNode);
   ciKlass* k = _igvn.type(klass_node)->is_klassptr()->klass();
@@ -1899,7 +1894,7 @@ void PhaseMacroExpand::expand_allocate_array(AllocateArrayNode *alloc) {
   }
   expand_allocate_common(alloc, length,
                          OptoRuntime::new_array_Type(),
-                         slow_call_address, valid_length_test);
+                         slow_call_address);
 }
 
 //-------------------mark_eliminated_box----------------------------------

src/hotspot/share/opto/macro.hpp

@@ -92,8 +92,8 @@ private:
   void expand_allocate_common(AllocateNode* alloc,
                               Node* length,
                               const TypeFunc* slow_call_type,
-                              address slow_call_address,
+                              address slow_call_address);
-                              Node* valid_length_test);
+  void yank_initalize_node(InitializeNode* node);
   void yank_alloc_node(AllocateNode* alloc);
   Node *value_from_mem(Node *mem, Node *ctl, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, AllocateNode *alloc);
   Node *value_from_mem_phi(Node *mem, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, AllocateNode *alloc, Node_Stack *value_phis, int level);

src/hotspot/share/opto/split_if.cpp

@@ -128,8 +128,8 @@ bool PhaseIdealLoop::split_up( Node *n, Node *blk1, Node *blk2 ) {
               }
             } else {
               // We might see an Opaque1 from a loop limit check here
-              assert(use->is_If() || use->is_CMove() || use->Opcode() == Op_Opaque1 || use->is_AllocateArray(), "unexpected node type");
+              assert(use->is_If() || use->is_CMove() || use->Opcode() == Op_Opaque1, "unexpected node type");
-              Node *use_c = (use->is_If() || use->is_AllocateArray()) ? use->in(0) : get_ctrl(use);
+              Node *use_c = use->is_If() ? use->in(0) : get_ctrl(use);
               if (use_c == blk1 || use_c == blk2) {
                 assert(use->is_CMove(), "unexpected node type");
                 continue;
@@ -166,15 +166,14 @@ bool PhaseIdealLoop::split_up( Node *n, Node *blk1, Node *blk2 ) {
                 --j;
               } else {
                 // We might see an Opaque1 from a loop limit check here
-                assert(u->is_If() || u->is_CMove() || u->Opcode() == Op_Opaque1 || u->is_AllocateArray(), "unexpected node type");
+                assert(u->is_If() || u->is_CMove() || u->Opcode() == Op_Opaque1, "unexpected node type");
-                assert(u->is_AllocateArray() || u->in(1) == bol, "");
+                assert(u->in(1) == bol, "");
-                assert(!u->is_AllocateArray() || u->in(AllocateNode::ValidLengthTest) == bol, "wrong input to AllocateArray");
                 // Get control block of either the CMove or the If input
-                Node *u_ctrl = (u->is_If() || u->is_AllocateArray()) ? u->in(0) : get_ctrl(u);
+                Node *u_ctrl = u->is_If() ? u->in(0) : get_ctrl(u);
                 assert((u_ctrl != blk1 && u_ctrl != blk2) || u->is_CMove(), "won't converge");
                 Node *x = bol->clone();
                 register_new_node(x, u_ctrl);
-                _igvn.replace_input_of(u, u->is_AllocateArray() ? AllocateNode::ValidLengthTest : 1, x);
+                _igvn.replace_input_of(u, 1, x);
                 --j;
               }
             }

test/hotspot/jtreg/compiler/allocation/TestFailedAllocationBadGraph.java

@@ -1,79 +0,0 @@
-/*
- * Copyright (c) 2021, Red Hat, Inc. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-/*
- * @test
- * bug 8278413
- * @summary C2 crash when allocating array of size too large
- * @library /test/lib /
- * @build sun.hotspot.WhiteBox
- * @run driver jdk.test.lib.helpers.ClassFileInstaller sun.hotspot.WhiteBox
- * @run main/othervm  -ea -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:-BackgroundCompilation TestFailedAllocationBadGraph
- */
-
-import sun.hotspot.WhiteBox;
-import java.lang.reflect.Method;
-import compiler.whitebox.CompilerWhiteBoxTest;
-
-public class TestFailedAllocationBadGraph {
-    private static final WhiteBox WHITE_BOX = WhiteBox.getWhiteBox();
-
-    private static long[] array;
-    private static int field;
-    private static volatile int barrier;
-
-    public static void main(String[] args) throws Exception {
-        run("test1");
-        run("test2");
-    }
-
-    private static void run(String method) throws Exception {
-        Method m = TestFailedAllocationBadGraph.class.getDeclaredMethod(method);
-        WHITE_BOX.enqueueMethodForCompilation(m, CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION);
-        if (!WHITE_BOX.isMethodCompiled(m) || WHITE_BOX.getMethodCompilationLevel(m) != CompilerWhiteBoxTest.COMP_LEVEL_FULL_OPTIMIZATION) {
-            throw new RuntimeException("should still be compiled");
-        }
-    }
-
-    private static int test1() {
-        int length = Integer.MAX_VALUE;
-        try {
-            array = new long[length];
-        } catch (OutOfMemoryError outOfMemoryError) {
-            barrier = 0x42;
-            length = field;
-        }
-        return length;
-    }
-
-    private static int test2() {
-        int length = -1;
-        try {
-            array = new long[length];
-        } catch (OutOfMemoryError outOfMemoryError) {
-            barrier = 0x42;
-            length = field;
-        }
-        return length;
-    }
-}