Arm64: Combine if conditions into compare chains

src/coreclr/jit/codegen.h

@@ -887,6 +887,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
     void genCodeForContainedCompareChain(GenTree* tree, bool* inchain, GenCondition* prevCond);
 #endif
     void genCodeForSelect(GenTreeOp* select);
+    void genCodeForConditionalCompare(GenTreeOp* select);
     void genIntrinsic(GenTreeIntrinsic* treeNode);
     void genPutArgStk(GenTreePutArgStk* treeNode);
     void genPutArgReg(GenTreeOp* tree);

src/coreclr/jit/codegenarm64.cpp

@@ -4664,7 +4664,7 @@ void CodeGen::genCodeForContainedCompareChain(GenTree* tree, bool* inChain, GenC
 {
     assert(tree->isContained());
 
-    if (tree->OperIs(GT_AND))
+    if (tree->OperIs(GT_AND) || tree->OperIsConditionalCompare())
     {
         GenTree* op1 = tree->gtGetOp1();
         GenTree* op2 = tree->gtGetOp2();
@@ -4734,28 +4734,53 @@ void CodeGen::genCodeForSelect(GenTreeOp* tree)
     assert(genTypeSize(op1Type) == genTypeSize(op2Type));
 
     GenCondition prevCond;
-    genConsumeRegs(opcond);
     if (opcond->isContained())
     {
         // Generate the contained condition.
         if (opcond->OperIsCompare())
         {
+            genConsumeRegs(opcond);
             genCodeForCompare(opcond->AsOp());
             prevCond = GenCondition::FromRelop(opcond);
         }
         else
         {
-            // Condition is a compare chain. Try to contain it.
-            assert(opcond->OperIs(GT_AND));
+            assert(opcond->OperIsConditionalCompare());
+
+            // Condition is a compare chain. Generate it.
             bool chain = false;
             JITDUMP("Generating compare chain:\n");
-            genCodeForContainedCompareChain(opcond, &chain, &prevCond);
+
+            GenTree* op1 = opcond->gtGetOp1();
+            GenTree* op2 = opcond->gtGetOp2();
+
+            genConsumeRegs(op1);
+            genConsumeRegs(op2);
+
+            // If Op1 is contained, generate it into flags.
+            if (op1->isContained())
+            {
+                genCodeForContainedCompareChain(op1, &chain, &prevCond);
+                assert(chain);
+                assert(op2->isContained());
+            }
+
+            // Generate op2 into flags.
+            assert(op2->isContained());
+            genCodeForContainedCompareChain(op2, &chain, &prevCond);
             assert(chain);
+
+            // Reverse condition for NE.
+            if (opcond->OperIs(GT_CCMP_NE))
+            {
+                prevCond = GenCondition::Reverse(prevCond);
+            }
         }
     }
     else
     {
         // Condition has been generated into a register - move it into flags.
+        genConsumeRegs(opcond);
         emit->emitIns_R_I(INS_cmp, emitActualTypeSize(opcond), opcond->GetRegNum(), 0);
         prevCond = GenCondition::NE;
     }
@@ -4784,6 +4809,75 @@ void CodeGen::genCodeForSelect(GenTreeOp* tree)
     genProduceReg(tree);
 }
 
+//------------------------------------------------------------------------
+// genCodeForConditionalCompare: Generates code for CCMP node.
+//
+// Arguments:
+//    tree - the node
+//
+void CodeGen::genCodeForConditionalCompare(GenTreeOp* tree)
+{
+    var_types targetType = tree->TypeGet();
+    emitter*  emit       = GetEmitter();
+
+    assert(tree->OperIsConditionalCompare());
+
+    GenTree* op1 = tree->gtGetOp1();
+    GenTree* op2 = tree->gtGetOp2();
+
+    assert(tree->isContainedCompareChainSegment(op2));
+
+    GenCondition cond;
+    bool         chain = false;
+
+    JITDUMP("Generating compare chain:\n");
+    if (op1->isContained())
+    {
+        // Generate Op1 into flags.
+        genCodeForContainedCompareChain(op1, &chain, &cond);
+        assert(chain);
+    }
+    else
+    {
+        // Op1 is not contained, move it from a register into flags.
+        emit->emitIns_R_I(INS_cmp, emitActualTypeSize(op1), op1->GetRegNum(), 0);
+        cond  = GenCondition::NE;
+        chain = true;
+    }
+
+    // AHTODO: not sure this is always true
+    assert(op2->isContained());
+
+    // Gen Op2 into flags.
+    genCodeForContainedCompareChain(op2, &chain, &cond);
+    assert(chain);
+
+    // Are we evaluating this into a register?
+    regNumber targetReg = tree->GetRegNum();
+    if (targetReg != REG_NA)
+    {
+        // AHTODO: merge this into helper function with genCodeForJumpTrue()
+        // Find the last contained compare in the chain.
+        GenCondition condition;
+        GenTreeOp*   lastCompare = tree->gtGetOp2()->AsOp();
+        assert(lastCompare->isContained());
+        while (!lastCompare->OperIsCompare())
+        {
+            assert(lastCompare->OperIs(GT_AND) || lastCompare->OperIsConditionalCompare());
+            lastCompare = lastCompare->gtGetOp2()->AsOp();
+            assert(lastCompare->isContained());
+        }
+        condition = GenCondition::FromRelop(lastCompare);
+        if (tree->OperIs(GT_CCMP_NE))
+        {
+            condition = GenCondition::Reverse(condition);
+        }
+
+        inst_SETCC(condition, tree->TypeGet(), targetReg);
+        genProduceReg(tree);
+    }
+}
+
 //------------------------------------------------------------------------
 // genCodeForJumpCompare: Generates code for jmpCompare statement.
 //

src/coreclr/jit/codegenarmarch.cpp

@@ -366,6 +366,12 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode)
         case GT_SELECT:
             genCodeForSelect(treeNode->AsConditional());
             break;
+
+        case GT_CCMP_EQ:
+        case GT_CCMP_NE:
+            genConsumeOperands(treeNode->AsOp());
+            genCodeForConditionalCompare(treeNode->AsOp());
+            break;
 #endif
 
         case GT_JTRUE:

src/coreclr/jit/codegenlinear.cpp

@@ -1610,10 +1610,10 @@ void CodeGen::genConsumeRegs(GenTree* tree)
             assert(cast->isContained());
             genConsumeAddress(cast->CastOp());
         }
-        else if (tree->OperIsCompare() || tree->OperIs(GT_AND))
+        else if (tree->OperIsCompare() || tree->OperIs(GT_AND) || tree->OperIsConditionalCompare())
         {
             // Compares can be contained by a SELECT.
-            // ANDs and Cmp Compares may be contained in a chain.
+            // Compares, ANDs and conditional compares may be contained in a chain.
             genConsumeRegs(tree->gtGetOp1());
             genConsumeRegs(tree->gtGetOp2());
         }
@@ -2592,8 +2592,37 @@ void CodeGen::genCodeForJumpTrue(GenTreeOp* jtrue)
     assert(compiler->compCurBB->bbJumpKind == BBJ_COND);
     assert(jtrue->OperIs(GT_JTRUE));
 
-    GenTreeOp*   relop     = jtrue->gtGetOp1()->AsOp();
-    GenCondition condition = GenCondition::FromRelop(relop);
+    GenTreeOp*   relop = jtrue->gtGetOp1()->AsOp();
+    GenCondition condition;
+
+    // Operands should never be contained inside a jtrue.
+    assert(!relop->isContained());
+
+#if defined(TARGET_ARM64)
+    if (relop->OperIsConditionalCompare())
+    {
+        // Find the last contained compare in the chain.
+        assert(relop->gtType == TYP_VOID);
+        GenTreeOp* lastCompare = relop->gtGetOp2()->AsOp();
+        assert(lastCompare->isContained());
+        while (!lastCompare->OperIsCompare())
+        {
+            assert(lastCompare->OperIs(GT_AND) || lastCompare->OperIsConditionalCompare());
+            lastCompare = lastCompare->gtGetOp2()->AsOp();
+            assert(lastCompare->isContained());
+        }
+        condition = GenCondition::FromRelop(lastCompare);
+        if (relop->OperIs(GT_CCMP_NE))
+        {
+            condition = GenCondition::Reverse(condition);
+        }
+    }
+    else
+#endif
+    {
+        assert(relop->OperIsCompare());
+        condition = GenCondition::FromRelop(relop);
+    }
 
     if (condition.PreferSwap())
     {

src/coreclr/jit/gentree.cpp

@@ -3454,6 +3454,16 @@ GenTree* Compiler::gtReverseCond(GenTree* tree)
         //     tbz <=> tbnz
         tree->gtFlags ^= GTF_JCMP_EQ;
     }
+#if defined(TARGET_ARM64)
+    else if (tree->OperIs(GT_CCMP_EQ))
+    {
+        tree->SetOper(GT_CCMP_NE);
+    }
+    else if (tree->OperIs(GT_CCMP_NE))
+    {
+        tree->SetOper(GT_CCMP_EQ);
+    }
+#endif
     else
     {
         tree = gtNewOperNode(GT_NOT, TYP_INT, tree);
@@ -17132,8 +17142,7 @@ bool GenTree::canBeContained() const
         return false;
     }
 
-    // It is not possible for nodes that do not produce values or that are not containable values to be contained.
-    if (!IsValue() || ((DebugOperKind() & DBK_NOCONTAIN) != 0) || (OperIsHWIntrinsic() && !isContainableHWIntrinsic()))
+    if (((DebugOperKind() & DBK_NOCONTAIN) != 0) || (OperIsHWIntrinsic() && !isContainableHWIntrinsic()))
     {
         return false;
     }

src/coreclr/jit/gentree.h

@@ -910,11 +910,14 @@ struct GenTree
         return isContained() && IsCnsIntOrI() && !isUsedFromSpillTemp();
     }
 
+#if defined(TARGET_ARM64)
     // Node and its child in isolation form a contained compare chain.
     bool isContainedCompareChainSegment(GenTree* child) const
     {
-        return (OperIs(GT_AND) && child->isContained() && (child->OperIs(GT_AND) || child->OperIsCmpCompare()));
+        return ((OperIs(GT_AND) || OperIsConditionalCompare()) && child->isContained() &&
+                (child->OperIs(GT_AND) || child->OperIsCmpCompare() || child->OperIsConditionalCompare()));
     }
+#endif
 
     bool isContainedFltOrDblImmed() const
     {
@@ -1090,7 +1093,7 @@ struct GenTree
         {
             // These are the only operators which can produce either VOID or non-VOID results.
             assert(OperIs(GT_NOP, GT_CALL, GT_COMMA) || OperIsCompare() || OperIsLong() || OperIsHWIntrinsic() ||
-                   IsCnsVec());
+                   IsCnsVec() || OperIsConditionalCompare());
             return false;
         }
 
@@ -1376,6 +1379,20 @@ struct GenTree
         return OperIsConditional(OperGet());
     }
 
+    static bool OperIsConditionalCompare(genTreeOps gtOper)
+    {
+#if defined(TARGET_ARM64)
+        return (GT_CCMP_EQ == gtOper || GT_CCMP_NE == gtOper);
+#else
+        return false;
+#endif
+    }
+
+    bool OperIsConditionalCompare() const
+    {
+        return OperIsConditionalCompare(OperGet());
+    }
+
     static bool OperIsCC(genTreeOps gtOper)
     {
         return (gtOper == GT_JCC) || (gtOper == GT_SETCC);

src/coreclr/jit/gtlist.h

@@ -242,6 +242,11 @@ GTNODE(SETCC            , GenTreeCC          ,0,GTK_LEAF|DBK_NOTHIR)
 // The XARCH BT instruction. Like CMP, this sets the condition flags (CF to be precise) and does not produce a value.
 GTNODE(BT               , GenTreeOp          ,0,(GTK_BINOP|GTK_NOVALUE|DBK_NOTHIR))
 #endif
+// Sets the condition flags according to the combined results of its children.
+#if defined(TARGET_ARM64)
+GTNODE(CCMP_EQ          , GenTreeOp          ,0,GTK_BINOP|DBK_NOTHIR)
+GTNODE(CCMP_NE          , GenTreeOp          ,0,GTK_BINOP|DBK_NOTHIR)
+#endif
 
 //-----------------------------------------------------------------------------
 //  Other nodes that look like unary/binary operators:

src/coreclr/jit/lower.cpp

@@ -2868,15 +2868,6 @@ GenTree* Lowering::OptimizeConstCompare(GenTree* cmp)
     GenTreeIntCon* op2      = cmp->gtGetOp2()->AsIntCon();
     ssize_t        op2Value = op2->IconValue();
 
-#ifdef TARGET_ARM64
-    // Do not optimise further if op1 has a contained chain.
-    if (op1->OperIs(GT_AND) &&
-        (op1->isContainedCompareChainSegment(op1->gtGetOp1()) || op1->isContainedCompareChainSegment(op1->gtGetOp2())))
-    {
-        return cmp;
-    }
-#endif
-
 #ifdef TARGET_XARCH
     var_types op1Type = op1->TypeGet();
     if (IsContainableMemoryOp(op1) && varTypeIsSmall(op1Type) && FitsIn(op1Type, op2Value))
@@ -7164,10 +7155,30 @@ void Lowering::ContainCheckRet(GenTreeUnOp* ret)
 //
 void Lowering::ContainCheckJTrue(GenTreeOp* node)
 {
-    // The compare does not need to be generated into a register.
-    GenTree* cmp = node->gtGetOp1();
-    cmp->gtType  = TYP_VOID;
-    cmp->gtFlags |= GTF_SET_FLAGS;
+    GenTree* op1 = node->gtGetOp1();
+
+    if (op1->OperIsCompare())
+    {
+        // The compare does not need to be generated into a register.
+        op1->gtType = TYP_VOID;
+        op1->gtFlags |= GTF_SET_FLAGS;
+    }
+#if defined(TARGET_ARM64)
+    else if (op1->OperIsConditionalCompare())
+    {
+        // If the second op of the CCMP is contained, then the CCMP does not need to be generated
+        // into a register.
+        if (op1->gtGetOp2()->isContained())
+        {
+            op1->gtType = TYP_VOID;
+            op1->gtFlags |= GTF_SET_FLAGS;
+        }
+    }
+#endif
+    else
+    {
+        unreached();
+    }
 }
 
 //------------------------------------------------------------------------

src/coreclr/jit/lower.h

@@ -88,8 +88,8 @@ class Lowering final : public Phase
 #ifdef TARGET_ARM64
     bool IsValidCompareChain(GenTree* child, GenTree* parent);
     bool ContainCheckCompareChain(GenTree* child, GenTree* parent, GenTree** earliestValid);
-    void ContainCheckCompareChainForAnd(GenTree* tree);
-    void ContainCheckConditionalCompare(GenTreeOp* cmp);
+    bool ContainCheckCompareChainForAnd(GenTree* tree);
+    void ContainCheckChainedCompare(GenTreeOp* cmp);
     void ContainCheckNeg(GenTreeOp* neg);
 #endif
     void ContainCheckSelect(GenTreeOp* select);