8316414: C2: large byte array clone triggers "failed: malformed control flow" assertion failure on linux-x86

Co-authored-by: Roberto Castañeda Lozano <rcastanedalo@openjdk.org>
Reviewed-by: thartmann

Author: rwestrel

src/hotspot/share/opto/type.cpp

@@ -4750,7 +4750,7 @@ const TypeInt* TypeAryPtr::narrow_size_type(const TypeInt* size) const {
   jint hi = size->_hi;
   jint lo = size->_lo;
   jint min_lo = 0;
-  jint max_hi = max_array_length(elem()->basic_type());
+  jint max_hi = max_array_length(elem()->array_element_basic_type());
   //if (index_not_size)  --max_hi;     // type of a valid array index, FTR
   bool chg = false;
   if (lo < min_lo) {

test/hotspot/jtreg/compiler/allocation/TestNewMaxLengthArray.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8316414
+ * @summary C2: large byte array clone triggers "failed: malformed control flow" assertion failure on linux-x86
+ * @run main/othervm  -Xcomp -XX:CompileOnly=TestNewMaxLengthArray::createAndClone TestNewMaxLengthArray
+ */
+
+public class TestNewMaxLengthArray {
+
+    // Maximum length of a byte array on a 32-bits platform using default object
+    // alignment (8 bytes).
+    static final int MAX_BYTE_ARRAY_LENGTH = 0x7ffffffc;
+
+    public static byte[] createAndClone() {
+        byte[] array = new byte[MAX_BYTE_ARRAY_LENGTH];
+        return array.clone();
+    }
+
+    public static void main(String[] a) {
+        try {
+            createAndClone();
+        } catch (OutOfMemoryError oome) {
+        }
+    }
+}