8330853: Add missing checks for ConnectionGraph::can_reduce_cmp() call

Reviewed-by: iveresov, dlong, cslucas

Author: Vladimir Kozlov

src/hotspot/share/opto/escape.cpp

@@ -492,11 +492,11 @@ bool ConnectionGraph::can_reduce_phi_check_inputs(PhiNode* ophi) const {
 // I require the 'other' input to be a constant so that I can move the Cmp
 // around safely.
 bool ConnectionGraph::can_reduce_cmp(Node* n, Node* cmp) const {
+  assert(cmp->Opcode() == Op_CmpP || cmp->Opcode() == Op_CmpN, "not expected node: %s", cmp->Name());
   Node* left = cmp->in(1);
   Node* right = cmp->in(2);
 
-  return (cmp->Opcode() == Op_CmpP || cmp->Opcode() == Op_CmpN) &&
-         (left == n || right == n) &&
+  return (left == n || right == n) &&
          (left->is_Con() || right->is_Con()) &&
          cmp->outcnt() == 1;
 }
@@ -559,8 +559,12 @@ bool ConnectionGraph::can_reduce_check_users(Node* n, uint nesting) const {
     } else if (use->is_CastPP()) {
       const Type* cast_t = _igvn->type(use);
       if (cast_t == nullptr || cast_t->make_ptr()->isa_instptr() == nullptr) {
-        NOT_PRODUCT(use->dump();)
-        NOT_PRODUCT(if (TraceReduceAllocationMerges) tty->print_cr("Can NOT reduce Phi %d on invocation %d. CastPP is not to an instance.", n->_idx, _invocation);)
+#ifndef PRODUCT
+        if (TraceReduceAllocationMerges) {
+          tty->print_cr("Can NOT reduce Phi %d on invocation %d. CastPP is not to an instance.", n->_idx, _invocation);
+          use->dump();
+        }
+#endif
         return false;
       }
 
@@ -570,11 +574,18 @@ bool ConnectionGraph::can_reduce_check_users(Node* n, uint nesting) const {
         // CmpP/N used by the If controlling the cast.
         if (use->in(0)->is_IfTrue() || use->in(0)->is_IfFalse()) {
           Node* iff = use->in(0)->in(0);
-          Node* iff_cmp = iff->in(1)->in(1); // if->bool->cmp
-          if (!can_reduce_cmp(n, iff_cmp)) {
-            NOT_PRODUCT(if (TraceReduceAllocationMerges) tty->print_cr("Can NOT reduce Phi %d on invocation %d. CastPP %d doesn't have simple control.", n->_idx, _invocation, use->_idx);)
-            NOT_PRODUCT(n->dump(5);)
-            return false;
+          if (iff->Opcode() == Op_If && iff->in(1)->is_Bool() && iff->in(1)->in(1)->is_Cmp()) {
+            Node* iff_cmp = iff->in(1)->in(1);
+            int opc = iff_cmp->Opcode();
+            if ((opc == Op_CmpP || opc == Op_CmpN) && !can_reduce_cmp(n, iff_cmp)) {
+#ifndef PRODUCT
+              if (TraceReduceAllocationMerges) {
+                tty->print_cr("Can NOT reduce Phi %d on invocation %d. CastPP %d doesn't have simple control.", n->_idx, _invocation, use->_idx);
+                n->dump(5);
+              }
+#endif
+              return false;
+            }
           }
         }
       }