From f841d2822c5c28937d2e456b5552fd8d0ad9a99f Mon Sep 17 00:00:00 2001 From: Aman Khalid Date: Fri, 20 Oct 2023 11:37:12 -0400 Subject: [PATCH 1/3] Add explicit block fallthrough successor --- src/coreclr/jit/assertionprop.cpp | 2 +- src/coreclr/jit/block.cpp | 2 +- src/coreclr/jit/block.h | 39 +++++++++++++++- src/coreclr/jit/codegenarm64.cpp | 15 +++--- src/coreclr/jit/codegenloongarch64.cpp | 15 +++--- src/coreclr/jit/codegenriscv64.cpp | 15 +++--- src/coreclr/jit/fgbasic.cpp | 42 ++++++++--------- src/coreclr/jit/fgdiagnostic.cpp | 7 +-- src/coreclr/jit/fgehopt.cpp | 9 ++-- src/coreclr/jit/fgflow.cpp | 10 ++-- src/coreclr/jit/fgopt.cpp | 62 ++++++++++++++----------- src/coreclr/jit/fgprofile.cpp | 14 +++--- src/coreclr/jit/fgprofilesynthesis.cpp | 4 +- src/coreclr/jit/flowgraph.cpp | 6 +-- src/coreclr/jit/ifconversion.cpp | 7 +-- src/coreclr/jit/importer.cpp | 33 +++++++------ src/coreclr/jit/jiteh.cpp | 2 +- src/coreclr/jit/liveness.cpp | 14 +++--- src/coreclr/jit/loopcloning.cpp | 37 +++++++++------ src/coreclr/jit/lower.cpp | 10 ++-- src/coreclr/jit/lsra.cpp | 3 +- src/coreclr/jit/morph.cpp | 27 +++++------ src/coreclr/jit/optimizebools.cpp | 41 +++++++++------- src/coreclr/jit/optimizer.cpp | 51 ++++++++++---------- src/coreclr/jit/rangecheck.cpp | 2 +- src/coreclr/jit/redundantbranchopts.cpp | 29 ++++++++---- src/coreclr/jit/switchrecognition.cpp | 15 ++++-- 27 files changed, 298 insertions(+), 215 deletions(-) diff --git a/src/coreclr/jit/assertionprop.cpp b/src/coreclr/jit/assertionprop.cpp index 69343cf0965b4..d24d434d179d1 100644 --- a/src/coreclr/jit/assertionprop.cpp +++ b/src/coreclr/jit/assertionprop.cpp @@ -5297,7 +5297,7 @@ class AssertionPropFlowCallback { // Scenario where next block and conditional block, both point to the same block. // In such case, intersect the assertions present on both the out edges of predBlock. - assert(predBlock->NextIs(block)); + assert(predBlock->FallsInto(block)); BitVecOps::IntersectionD(apTraits, pAssertionOut, predBlock->bbAssertionOut); if (VerboseDataflow()) diff --git a/src/coreclr/jit/block.cpp b/src/coreclr/jit/block.cpp index 2407afea59571..493bbc0085b96 100644 --- a/src/coreclr/jit/block.cpp +++ b/src/coreclr/jit/block.cpp @@ -1278,7 +1278,7 @@ BasicBlock* BasicBlock::GetSucc(unsigned i, Compiler* comp) return bbJumpDest; case BBJ_NONE: - return bbNext; + return GetFallThroughSucc(); case BBJ_COND: if (i == 0) diff --git a/src/coreclr/jit/block.h b/src/coreclr/jit/block.h index ecdce490528f6..cb4b11048100c 100644 --- a/src/coreclr/jit/block.h +++ b/src/coreclr/jit/block.h @@ -537,6 +537,8 @@ struct BasicBlock : private LIR::Range BBehfDesc* bbJumpEhf; // BBJ_EHFINALLYRET descriptor }; + BasicBlock* bbFallThroughSucc; + public: #ifdef DEBUG // When creating a block with a jump, we require its jump kind and target be initialized simultaneously. @@ -581,7 +583,8 @@ struct BasicBlock : private LIR::Range void SetNext(BasicBlock* next) { - bbNext = next; + bbNext = next; + bbFallThroughSucc = bbNext; if (next) { next->bbPrev = this; @@ -666,6 +669,38 @@ struct BasicBlock : private LIR::Range SetJumpKindAndTarget(jumpKind, jumpDest DEBUG_ARG(compiler)); } + BasicBlock* GetFallThroughSucc() const + { + assert(bbFallsThrough()); + // For now, bbFallThroughSucc cannot diverge from bbNext + assert(FallsIntoNext()); + return bbFallThroughSucc; + } + + void SetFallThroughSucc(BasicBlock* fallThroughSucc) + { + // TODO: Once we allow bbFallThroughSucc to diverge from bbNext, ensure we only set bbFallThroughSucc + // to something other than bbNext when this block is a BBJ_COND. + // BBJ_CALLFINALLY can fall through too, but we don't allow its fallthrough successor to diverge from bbNext + // because we aren't interested in splitting up call-always pairs + assert(KindIs(BBJ_COND, BBJ_NONE)); + bbFallThroughSucc = fallThroughSucc; + // For now, bbFallThroughSucc cannot diverge from bbNext + assert(FallsIntoNext()); + } + + bool FallsInto(const BasicBlock* dest) const + { + assert(bbFallsThrough()); + return (bbFallThroughSucc == dest); + } + + bool FallsIntoNext() const + { + assert(bbFallsThrough()); + return (bbFallThroughSucc == bbNext); + } + bool HasJump() const { return (bbJumpDest != nullptr); @@ -681,7 +716,7 @@ struct BasicBlock : private LIR::Range bool JumpsToNext() const { assert(HasJump()); - return (bbJumpDest == bbNext); + return (bbJumpDest == bbFallThroughSucc); } BBswtDesc* GetJumpSwt() const diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 4d1a2fd36f43f..3792f0258419e 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -2176,6 +2176,8 @@ BasicBlock* CodeGen::genCallFinally(BasicBlock* block) } else { + assert(block->isBBCallAlwaysPair()); + // Because of the way the flowgraph is connected, the liveness info for this one instruction // after the call is not (can not be) correct in cases where a variable has a last use in the // handler. So turn off GC reporting for this single instruction. @@ -2198,17 +2200,14 @@ BasicBlock* CodeGen::genCallFinally(BasicBlock* block) } GetEmitter()->emitEnableGC(); - } - // The BBJ_ALWAYS is used because the BBJ_CALLFINALLY can't point to the - // jump target using bbJumpDest - that is already used to point - // to the finally block. So just skip past the BBJ_ALWAYS unless the - // block is RETLESS. - if (!(block->bbFlags & BBF_RETLESS_CALL)) - { - assert(block->isBBCallAlwaysPair()); + // The BBJ_ALWAYS is used because the BBJ_CALLFINALLY can't point to the + // jump target using bbJumpDest - that is already used to point + // to the finally block. So just skip past the BBJ_ALWAYS unless the + // block is RETLESS. block = nextBlock; } + return block; } diff --git a/src/coreclr/jit/codegenloongarch64.cpp b/src/coreclr/jit/codegenloongarch64.cpp index d77532365e661..66dc71b179249 100644 --- a/src/coreclr/jit/codegenloongarch64.cpp +++ b/src/coreclr/jit/codegenloongarch64.cpp @@ -1536,6 +1536,8 @@ BasicBlock* CodeGen::genCallFinally(BasicBlock* block) } else { + assert(block->isBBCallAlwaysPair()); + // Because of the way the flowgraph is connected, the liveness info for this one instruction // after the call is not (can not be) correct in cases where a variable has a last use in the // handler. So turn off GC reporting for this single instruction. @@ -1558,17 +1560,14 @@ BasicBlock* CodeGen::genCallFinally(BasicBlock* block) } GetEmitter()->emitEnableGC(); - } - // The BBJ_ALWAYS is used because the BBJ_CALLFINALLY can't point to the - // jump target using bbJumpDest - that is already used to point - // to the finally block. So just skip past the BBJ_ALWAYS unless the - // block is RETLESS. - if (!(block->bbFlags & BBF_RETLESS_CALL)) - { - assert(block->isBBCallAlwaysPair()); + // The BBJ_ALWAYS is used because the BBJ_CALLFINALLY can't point to the + // jump target using bbJumpDest - that is already used to point + // to the finally block. So just skip past the BBJ_ALWAYS unless the + // block is RETLESS. block = nextBlock; } + return block; } diff --git a/src/coreclr/jit/codegenriscv64.cpp b/src/coreclr/jit/codegenriscv64.cpp index de589268e5ade..ec2d20ac640cb 100644 --- a/src/coreclr/jit/codegenriscv64.cpp +++ b/src/coreclr/jit/codegenriscv64.cpp @@ -1174,6 +1174,8 @@ BasicBlock* CodeGen::genCallFinally(BasicBlock* block) } else { + assert(block->isBBCallAlwaysPair()); + // Because of the way the flowgraph is connected, the liveness info for this one instruction // after the call is not (can not be) correct in cases where a variable has a last use in the // handler. So turn off GC reporting for this single instruction. @@ -1196,17 +1198,14 @@ BasicBlock* CodeGen::genCallFinally(BasicBlock* block) } GetEmitter()->emitEnableGC(); - } - // The BBJ_ALWAYS is used because the BBJ_CALLFINALLY can't point to the - // jump target using bbJumpDest - that is already used to point - // to the finally block. So just skip past the BBJ_ALWAYS unless the - // block is RETLESS. - if (!(block->bbFlags & BBF_RETLESS_CALL)) - { - assert(block->isBBCallAlwaysPair()); + // The BBJ_ALWAYS is used because the BBJ_CALLFINALLY can't point to the + // jump target using bbJumpDest - that is already used to point + // to the finally block. So just skip past the BBJ_ALWAYS unless the + // block is RETLESS. block = nextBlock; } + return block; } diff --git a/src/coreclr/jit/fgbasic.cpp b/src/coreclr/jit/fgbasic.cpp index d6ba319a9631a..21485d8189bd7 100644 --- a/src/coreclr/jit/fgbasic.cpp +++ b/src/coreclr/jit/fgbasic.cpp @@ -374,7 +374,7 @@ void Compiler::fgConvertBBToThrowBB(BasicBlock* block) // Must do this after we update bbJumpKind of block. if (isCallAlwaysPair) { - BasicBlock* leaveBlk = block->Next(); + BasicBlock* leaveBlk = block->GetFallThroughSucc(); noway_assert(leaveBlk->KindIs(BBJ_ALWAYS)); leaveBlk->bbFlags &= ~BBF_DONT_REMOVE; @@ -2958,7 +2958,7 @@ void Compiler::fgLinkBasicBlocks() FALLTHROUGH; case BBJ_NONE: - fgAddRefPred(curBBdesc->Next(), curBBdesc, oldEdge); + fgAddRefPred(curBBdesc->GetFallThroughSucc(), curBBdesc, oldEdge); break; case BBJ_EHFILTERRET: @@ -4180,7 +4180,7 @@ void Compiler::fgFixEntryFlowForOSR() // fgEnsureFirstBBisScratch(); assert(fgFirstBB->KindIs(BBJ_NONE)); - fgRemoveRefPred(fgFirstBB->Next(), fgFirstBB); + fgRemoveRefPred(fgFirstBB->GetFallThroughSucc(), fgFirstBB); fgFirstBB->SetJumpKindAndTarget(BBJ_ALWAYS, fgOSREntryBB DEBUG_ARG(this)); FlowEdge* const edge = fgAddRefPred(fgOSREntryBB, fgFirstBB); edge->setLikelihood(1.0); @@ -4224,7 +4224,7 @@ void Compiler::fgCheckBasicBlockControlFlow() { case BBJ_NONE: // block flows into the next one (no jump) - fgControlFlowPermitted(blk, blk->Next()); + fgControlFlowPermitted(blk, blk->GetFallThroughSucc()); break; @@ -4236,7 +4236,7 @@ void Compiler::fgCheckBasicBlockControlFlow() case BBJ_COND: // block conditionally jumps to the target - fgControlFlowPermitted(blk, blk->Next()); + fgControlFlowPermitted(blk, blk->GetFallThroughSucc()); fgControlFlowPermitted(blk, blk->GetJumpDest()); @@ -5042,6 +5042,10 @@ BasicBlock* Compiler::fgSplitEdge(BasicBlock* curr, BasicBlock* succ) curr->SetJumpDest(newBlock); } fgAddRefPred(newBlock, curr); + + // This isn't accurate, but it is complex to compute a reasonable number so just assume that we take the + // branch 50% of the time. + newBlock->inheritWeightPercentage(curr, 50); } else if (curr->KindIs(BBJ_SWITCH)) { @@ -5050,6 +5054,10 @@ BasicBlock* Compiler::fgSplitEdge(BasicBlock* curr, BasicBlock* succ) // And 'succ' has 'newBlock' as a new predecessor. fgAddRefPred(succ, newBlock); + + // This isn't accurate, but it is complex to compute a reasonable number so just assume that we take the + // branch 50% of the time. + newBlock->inheritWeightPercentage(curr, 50); } else { @@ -5063,14 +5071,6 @@ BasicBlock* Compiler::fgSplitEdge(BasicBlock* curr, BasicBlock* succ) fgFixFinallyTargetFlags(curr, succ, newBlock); #endif - // This isn't accurate, but it is complex to compute a reasonable number so just assume that we take the - // branch 50% of the time. - // - if (!curr->KindIs(BBJ_ALWAYS)) - { - newBlock->inheritWeightPercentage(curr, 50); - } - // The bbLiveIn and bbLiveOut are both equal to the bbLiveIn of 'succ' if (fgLocalVarLivenessDone) { @@ -5218,7 +5218,7 @@ void Compiler::fgRemoveBlock(BasicBlock* block, bool unreachable) NO_WAY("No retless call finally blocks; need unwind target instead"); #endif // defined(FEATURE_EH_FUNCLETS) && defined(TARGET_ARM) } - else if (bPrev->KindIs(BBJ_ALWAYS) && block->NextIs(bPrev->GetJumpDest()) && + else if (bPrev->KindIs(BBJ_ALWAYS) && bPrev->HasJumpTo(block->Next()) && !(bPrev->bbFlags & BBF_KEEP_BBJ_ALWAYS) && !block->IsFirstColdBlock(this) && !block->IsLastHotBlock(this)) { @@ -5246,7 +5246,7 @@ void Compiler::fgRemoveBlock(BasicBlock* block, bool unreachable) */ if (block->isBBCallAlwaysPair()) { - BasicBlock* leaveBlk = block->Next(); + BasicBlock* leaveBlk = block->GetFallThroughSucc(); noway_assert(leaveBlk->KindIs(BBJ_ALWAYS)); leaveBlk->bbFlags &= ~BBF_DONT_REMOVE; @@ -5325,7 +5325,7 @@ void Compiler::fgRemoveBlock(BasicBlock* block, bool unreachable) } else { - succBlock = block->Next(); + succBlock = block->GetFallThroughSucc(); } bool skipUnmarkLoop = false; @@ -5437,7 +5437,7 @@ void Compiler::fgRemoveBlock(BasicBlock* block, bool unreachable) } /* Check if both side of the BBJ_COND now jump to the same block */ - if (predBlock->NextIs(succBlock)) + if (predBlock->FallsInto(succBlock)) { // Make sure we are replacing "block" with "succBlock" in predBlock->bbJumpDest. noway_assert(predBlock->HasJumpTo(block)); @@ -5502,7 +5502,7 @@ void Compiler::fgRemoveBlock(BasicBlock* block, bool unreachable) case BBJ_COND: /* Check for branch to next block */ - if (bPrev->JumpsToNext()) + if (bPrev->HasJumpTo(bPrev->GetFallThroughSucc())) { fgRemoveConditionalJump(bPrev); } @@ -5538,7 +5538,7 @@ BasicBlock* Compiler::fgConnectFallThrough(BasicBlock* bSrc, BasicBlock* bDst) { /* If bSrc falls through to a block that is not bDst, we will insert a jump to bDst */ - if (bSrc->bbFallsThrough() && !bSrc->NextIs(bDst)) + if (bSrc->bbFallsThrough() && !bSrc->FallsInto(bDst)) { switch (bSrc->GetJumpKind()) { @@ -5622,7 +5622,7 @@ BasicBlock* Compiler::fgConnectFallThrough(BasicBlock* bSrc, BasicBlock* bDst) bSrc->SetJumpKindAndTarget(BBJ_NONE DEBUG_ARG(this)); JITDUMP("Changed an unconditional jump from " FMT_BB " to the next block " FMT_BB " into a BBJ_NONE block\n", - bSrc->bbNum, bSrc->Next()->bbNum); + bSrc->bbNum, bSrc->GetFallThroughSucc()->bbNum); } } } @@ -6413,7 +6413,7 @@ bool Compiler::fgIsBetterFallThrough(BasicBlock* bCur, BasicBlock* bAlt) } // Currently bNext is the fall through for bCur - BasicBlock* bNext = bCur->Next(); + BasicBlock* bNext = bCur->GetFallThroughSucc(); noway_assert(bNext != nullptr); // We will set result to true if bAlt is a better fall through than bCur diff --git a/src/coreclr/jit/fgdiagnostic.cpp b/src/coreclr/jit/fgdiagnostic.cpp index 835c6f401fe9b..3bbc457738f04 100644 --- a/src/coreclr/jit/fgdiagnostic.cpp +++ b/src/coreclr/jit/fgdiagnostic.cpp @@ -2697,11 +2697,12 @@ bool BBPredsChecker::CheckJump(BasicBlock* blockPred, BasicBlock* block) switch (blockPred->GetJumpKind()) { case BBJ_COND: - assert(blockPred->NextIs(block) || blockPred->HasJumpTo(block)); + assert(blockPred->FallsInto(block) || blockPred->HasJumpTo(block)); return true; case BBJ_NONE: - assert(blockPred->NextIs(block)); + assert(blockPred->FallsInto(block)); + assert(blockPred->FallsIntoNext()); return true; case BBJ_CALLFINALLY: @@ -4871,7 +4872,7 @@ void Compiler::fgDebugCheckLoopTable() else { assert(h->KindIs(BBJ_NONE)); - assert(h->NextIs(e)); + assert(h->FallsInto(e)); assert(loop.lpTop == e); assert(loop.lpIsTopEntry()); } diff --git a/src/coreclr/jit/fgehopt.cpp b/src/coreclr/jit/fgehopt.cpp index 309bc9a5a5557..bb7ae2649a103 100644 --- a/src/coreclr/jit/fgehopt.cpp +++ b/src/coreclr/jit/fgehopt.cpp @@ -807,7 +807,7 @@ PhaseStatus Compiler::fgCloneFinally() if (block->KindIs(BBJ_NONE) && (block == lastTryBlock)) { - jumpDest = block->Next(); + jumpDest = block->GetFallThroughSucc(); } else if (block->KindIs(BBJ_ALWAYS)) { @@ -1032,7 +1032,7 @@ PhaseStatus Compiler::fgCloneFinally() // If the clone ends up just after the finally, adjust // the stopping point for finally traversal. - if (newBlock->NextIs(nextBlock)) + if (newBlock->FallsInto(nextBlock)) { assert(newBlock->PrevIs(lastBlock)); nextBlock = newBlock; @@ -2178,7 +2178,7 @@ PhaseStatus Compiler::fgTailMergeThrows() case BBJ_COND: { // Flow to non canonical block could be via fall through or jump or both. - if (predBlock->NextIs(nonCanonicalBlock)) + if (predBlock->FallsInto(nonCanonicalBlock)) { fgTailMergeThrowsFallThroughHelper(predBlock, nonCanonicalBlock, canonicalBlock, predEdge); } @@ -2254,7 +2254,8 @@ void Compiler::fgTailMergeThrowsFallThroughHelper(BasicBlock* predBlock, BasicBlock* canonicalBlock, FlowEdge* predEdge) { - assert(predBlock->NextIs(nonCanonicalBlock)); + assert(predBlock->bbFallsThrough()); + assert(predBlock->FallsInto(nonCanonicalBlock)); BasicBlock* const newBlock = fgNewBBafter(BBJ_ALWAYS, predBlock, true, canonicalBlock); diff --git a/src/coreclr/jit/fgflow.cpp b/src/coreclr/jit/fgflow.cpp index 363ad7c2a17dc..fc04c4cccd05d 100644 --- a/src/coreclr/jit/fgflow.cpp +++ b/src/coreclr/jit/fgflow.cpp @@ -350,13 +350,13 @@ void Compiler::fgRemoveBlockAsPred(BasicBlock* block) fgRemoveRefPred(block->GetJumpDest(), block); break; - case BBJ_NONE: - fgRemoveRefPred(block->Next(), block); - break; - case BBJ_COND: fgRemoveRefPred(block->GetJumpDest(), block); - fgRemoveRefPred(block->Next(), block); + + FALLTHROUGH; + + case BBJ_NONE: + fgRemoveRefPred(block->GetFallThroughSucc(), block); break; case BBJ_EHFINALLYRET: diff --git a/src/coreclr/jit/fgopt.cpp b/src/coreclr/jit/fgopt.cpp index 79b098547523b..cda8398d9d778 100644 --- a/src/coreclr/jit/fgopt.cpp +++ b/src/coreclr/jit/fgopt.cpp @@ -134,7 +134,7 @@ bool Compiler::fgReachable(BasicBlock* b1, BasicBlock* b2) { noway_assert(b1->KindIs(BBJ_NONE, BBJ_ALWAYS, BBJ_COND)); - if (b1->KindIs(BBJ_NONE, BBJ_COND) && fgReachable(b1->Next(), b2)) + if (b1->KindIs(BBJ_NONE, BBJ_COND) && fgReachable(b1->GetFallThroughSucc(), b2)) { return true; } @@ -1946,13 +1946,16 @@ bool Compiler::fgCanCompactBlocks(BasicBlock* block, BasicBlock* bNext) return false; } - noway_assert(block->NextIs(bNext)); - if (!block->KindIs(BBJ_NONE)) { return false; } + // If block is BBJ_NONE, then its fallthrough successor must be block->bbNext. + // If not, compacting the two will mess up the flowgraph. + noway_assert(block->FallsInto(bNext)); + assert(block->FallsIntoNext()); + // If the next block has multiple incoming edges, we can still compact if the first block is empty. // However, not if it is the beginning of a handler. if (bNext->countOfInEdges() != 1 && @@ -2060,7 +2063,7 @@ void Compiler::fgCompactBlocks(BasicBlock* block, BasicBlock* bNext) noway_assert((block->bbFlags & BBF_REMOVED) == 0); noway_assert((bNext->bbFlags & BBF_REMOVED) == 0); noway_assert(block->KindIs(BBJ_NONE)); - noway_assert(block->NextIs(bNext)); + noway_assert(block->FallsInto(bNext)); noway_assert(bNext->countOfInEdges() == 1 || block->isEmpty()); noway_assert(bNext->bbPreds != nullptr); @@ -2365,25 +2368,26 @@ void Compiler::fgCompactBlocks(BasicBlock* block, BasicBlock* bNext) FALLTHROUGH; case BBJ_COND: + /* Update the predecessor list for bNext->bbFallThroughSucc if it is different than 'bNext->bbJumpDest' */ + if (!bNext->JumpsToNext()) + { + fgReplacePred(bNext->GetFallThroughSucc(), bNext, block); + } + + FALLTHROUGH; + case BBJ_ALWAYS: case BBJ_EHCATCHRET: case BBJ_EHFILTERRET: block->SetJumpKindAndTarget(bNext->GetJumpKind(), bNext->GetJumpDest() DEBUG_ARG(this)); - /* Update the predecessor list for 'bNext->bbJumpDest' */ fgReplacePred(bNext->GetJumpDest(), bNext, block); - - /* Update the predecessor list for 'bNext->bbNext' if it is different than 'bNext->bbJumpDest' */ - if (bNext->KindIs(BBJ_COND) && !bNext->JumpsToNext()) - { - fgReplacePred(bNext->Next(), bNext, block); - } break; case BBJ_NONE: block->SetJumpKindAndTarget(BBJ_NONE DEBUG_ARG(this)); /* Update the predecessor list for 'bNext->bbNext' */ - fgReplacePred(bNext->Next(), bNext, block); + fgReplacePred(bNext->GetFallThroughSucc(), bNext, block); break; case BBJ_EHFINALLYRET: @@ -2634,15 +2638,16 @@ void Compiler::fgUnreachableBlock(BasicBlock* block) // void Compiler::fgRemoveConditionalJump(BasicBlock* block) { - noway_assert(block->KindIs(BBJ_COND) && block->JumpsToNext()); + noway_assert(block->KindIs(BBJ_COND) && block->HasJumpTo(block->GetFallThroughSucc())); assert(compRationalIRForm == block->IsLIR()); FlowEdge* flow = fgGetPredForBlock(block->Next(), block); noway_assert(flow->getDupCount() == 2); // Change the BBJ_COND to BBJ_NONE, and adjust the refCount and dupCount. + // TODO: Once bbFallThroughSucc can diverge from bbNext, we may have to use BBJ_ALWAYS instead of BBJ_NONE. block->SetJumpKindAndTarget(BBJ_NONE DEBUG_ARG(this)); - --block->Next()->bbRefs; + --block->GetFallThroughSucc()->bbRefs; flow->decrementDupCount(); #ifdef DEBUG @@ -2650,7 +2655,7 @@ void Compiler::fgRemoveConditionalJump(BasicBlock* block) { printf("Block " FMT_BB " becoming a BBJ_NONE to " FMT_BB " (jump target is the same whether the condition" " is true or false)\n", - block->bbNum, block->Next()->bbNum); + block->bbNum, block->GetFallThroughSucc()->bbNum); } #endif @@ -2992,7 +2997,7 @@ bool Compiler::fgOptimizeEmptyBlock(BasicBlock* block) } else { - succBlock = block->Next(); + succBlock = block->GetFallThroughSucc(); } if ((succBlock != nullptr) && !BasicBlock::sameEHRegion(block, succBlock)) @@ -3746,10 +3751,10 @@ bool Compiler::fgOptimizeUncondBranchToSimpleCond(BasicBlock* block, BasicBlock* { assert(target->KindIs(BBJ_COND)); - if ((target->Next()->bbFlags & BBF_BACKWARD_JUMP_TARGET) != 0) + if ((target->GetFallThroughSucc()->bbFlags & BBF_BACKWARD_JUMP_TARGET) != 0) { JITDUMP("Deferring: " FMT_BB " --> " FMT_BB "; latter looks like loop top\n", target->bbNum, - target->Next()->bbNum); + target->GetFallThroughSucc()->bbNum); return false; } @@ -4041,8 +4046,8 @@ bool Compiler::fgOptimizeBranch(BasicBlock* bJump) } // do not jump into another try region - BasicBlock* bDestNext = bDest->Next(); - if (bDestNext->hasTryIndex() && !BasicBlock::sameTryRegion(bJump, bDestNext)) + BasicBlock* bDestFallThrough = bDest->GetFallThroughSucc(); + if (bDestFallThrough->hasTryIndex() && !BasicBlock::sameTryRegion(bJump, bDestFallThrough)) { return false; } @@ -4234,21 +4239,21 @@ bool Compiler::fgOptimizeBranch(BasicBlock* bJump) // We need to update the following flags of the bJump block if they were set in the bDest block bJump->bbFlags |= bDest->bbFlags & BBF_COPY_PROPAGATE; - bJump->SetJumpKindAndTarget(BBJ_COND, bDest->Next() DEBUG_ARG(this)); + bJump->SetJumpKindAndTarget(BBJ_COND, bDestFallThrough DEBUG_ARG(this)); /* Update bbRefs and bbPreds */ // bJump now falls through into the next block // - fgAddRefPred(bJump->Next(), bJump); + fgAddRefPred(bJump->GetFallThroughSucc(), bJump); // bJump no longer jumps to bDest // fgRemoveRefPred(bDest, bJump); - // bJump now jumps to bDest->bbNext + // bJump now jumps to bDest->bbFallThroughSucc // - fgAddRefPred(bDest->Next(), bJump); + fgAddRefPred(bDestFallThrough, bJump); if (weightJump > 0) { @@ -6123,7 +6128,7 @@ bool Compiler::fgUpdateFlowGraph(bool doTailDuplication, bool isPhase) change = true; modified = true; bDest = block->GetJumpDest(); - bNext = block->Next(); + bNext = block->GetFallThroughSucc(); } } @@ -7114,7 +7119,8 @@ bool Compiler::fgTryOneHeadMerge(BasicBlock* block, bool early) Statement* nextFirstStmt; Statement* destFirstStmt; - if (!getSuccCandidate(block->Next(), &nextFirstStmt) || !getSuccCandidate(block->GetJumpDest(), &destFirstStmt)) + if (!getSuccCandidate(block->GetFallThroughSucc(), &nextFirstStmt) || + !getSuccCandidate(block->GetJumpDest(), &destFirstStmt)) { return false; } @@ -7142,10 +7148,10 @@ bool Compiler::fgTryOneHeadMerge(BasicBlock* block, bool early) JITDUMP("We can; moving statement\n"); - fgUnlinkStmt(block->Next(), nextFirstStmt); + fgUnlinkStmt(block->GetFallThroughSucc(), nextFirstStmt); fgInsertStmtNearEnd(block, nextFirstStmt); fgUnlinkStmt(block->GetJumpDest(), destFirstStmt); - block->bbFlags |= block->Next()->bbFlags & BBF_COPY_PROPAGATE; + block->bbFlags |= block->GetFallThroughSucc()->bbFlags & BBF_COPY_PROPAGATE; return true; } diff --git a/src/coreclr/jit/fgprofile.cpp b/src/coreclr/jit/fgprofile.cpp index a274f99b8596c..bfa26af05531c 100644 --- a/src/coreclr/jit/fgprofile.cpp +++ b/src/coreclr/jit/fgprofile.cpp @@ -529,7 +529,7 @@ void BlockCountInstrumentor::RelocateProbes() // Handle case where we had a fall through critical edge // - if (pred->NextIs(intermediary)) + if (pred->FallsInto(intermediary)) { m_comp->fgRemoveRefPred(pred, block); m_comp->fgAddRefPred(intermediary, block); @@ -984,7 +984,7 @@ void Compiler::WalkSpanningTree(SpanningTreeVisitor* visitor) break; } - __fallthrough; + FALLTHROUGH; case BBJ_RETURN: { @@ -2688,7 +2688,7 @@ PhaseStatus Compiler::fgIncorporateProfileData() } } - __fallthrough; + FALLTHROUGH; default: JITDUMP("Unknown PGO record type 0x%x in schema entry %u (offset 0x%x count 0x%x other 0x%x)\n", @@ -4428,7 +4428,7 @@ bool Compiler::fgComputeMissingBlockWeights(weight_t* returnWeight) // Does this block flow into only one other block if (bSrc->KindIs(BBJ_NONE)) { - bOnlyNext = bSrc->Next(); + bOnlyNext = bSrc->GetFallThroughSucc(); } else if (bSrc->KindIs(BBJ_ALWAYS)) { @@ -4449,7 +4449,7 @@ bool Compiler::fgComputeMissingBlockWeights(weight_t* returnWeight) // Does this block flow into only one other block if (bDst->KindIs(BBJ_NONE)) { - bOnlyNext = bDst->Next(); + bOnlyNext = bDst->GetFallThroughSucc(); } else if (bDst->KindIs(BBJ_ALWAYS)) { @@ -4758,13 +4758,13 @@ PhaseStatus Compiler::fgComputeEdgeWeights() weight_t diff; FlowEdge* otherEdge; BasicBlock* otherDst; - if (bSrc->NextIs(bDst)) + if (bSrc->FallsInto(bDst)) { otherDst = bSrc->GetJumpDest(); } else { - otherDst = bSrc->Next(); + otherDst = bSrc->GetFallThroughSucc(); } otherEdge = fgGetPredForBlock(otherDst, bSrc); diff --git a/src/coreclr/jit/fgprofilesynthesis.cpp b/src/coreclr/jit/fgprofilesynthesis.cpp index fbea28ed4107c..ed395beaf123b 100644 --- a/src/coreclr/jit/fgprofilesynthesis.cpp +++ b/src/coreclr/jit/fgprofilesynthesis.cpp @@ -317,7 +317,7 @@ void ProfileSynthesis::AssignLikelihoodJump(BasicBlock* block) void ProfileSynthesis::AssignLikelihoodCond(BasicBlock* block) { BasicBlock* const jump = block->GetJumpDest(); - BasicBlock* const next = block->Next(); + BasicBlock* const next = block->GetFallThroughSucc(); // Watch for degenerate case // @@ -1221,7 +1221,7 @@ void ProfileSynthesis::ComputeCyclicProbabilities(SimpleLoop* loop) exitBlock->bbNum, exitEdge->getLikelihood()); BasicBlock* const jump = exitBlock->GetJumpDest(); - BasicBlock* const next = exitBlock->Next(); + BasicBlock* const next = exitBlock->GetFallThroughSucc(); FlowEdge* const jumpEdge = m_comp->fgGetPredForBlock(jump, exitBlock); FlowEdge* const nextEdge = m_comp->fgGetPredForBlock(next, exitBlock); weight_t const exitLikelihood = (missingExitWeight + currentExitWeight) / exitBlockWeight; diff --git a/src/coreclr/jit/flowgraph.cpp b/src/coreclr/jit/flowgraph.cpp index 1a9818ef08897..3f0b3bbdac001 100644 --- a/src/coreclr/jit/flowgraph.cpp +++ b/src/coreclr/jit/flowgraph.cpp @@ -256,7 +256,7 @@ BasicBlock* Compiler::fgCreateGCPoll(GCPollType pollType, BasicBlock* block) if (top->KindIs(BBJ_COND)) { - topFallThrough = top->Next(); + topFallThrough = top->GetFallThroughSucc(); lpIndexFallThrough = topFallThrough->bbNatLoopNum; } @@ -380,7 +380,7 @@ BasicBlock* Compiler::fgCreateGCPoll(GCPollType pollType, BasicBlock* block) switch (oldJumpKind) { case BBJ_NONE: - fgReplacePred(bottom->Next(), top, bottom); + fgReplacePred(bottom->GetFallThroughSucc(), top, bottom); break; case BBJ_RETURN: case BBJ_THROW: @@ -389,7 +389,7 @@ BasicBlock* Compiler::fgCreateGCPoll(GCPollType pollType, BasicBlock* block) case BBJ_COND: // replace predecessor in the fall through block. noway_assert(!bottom->IsLast()); - fgReplacePred(bottom->Next(), top, bottom); + fgReplacePred(bottom->GetFallThroughSucc(), top, bottom); // fall through for the jump target FALLTHROUGH; diff --git a/src/coreclr/jit/ifconversion.cpp b/src/coreclr/jit/ifconversion.cpp index e6aa34c07736d..7d5e360d3d2ac 100644 --- a/src/coreclr/jit/ifconversion.cpp +++ b/src/coreclr/jit/ifconversion.cpp @@ -121,8 +121,9 @@ bool OptIfConversionDsc::IfConvertCheckInnerBlockFlow(BasicBlock* block) // bool OptIfConversionDsc::IfConvertCheckThenFlow() { + assert(m_startBlock->KindIs(BBJ_COND)); m_flowFound = false; - BasicBlock* thenBlock = m_startBlock->Next(); + BasicBlock* thenBlock = m_startBlock->GetFallThroughSucc(); for (int thenLimit = 0; thenLimit < m_checkLimit; thenLimit++) { @@ -575,7 +576,7 @@ bool OptIfConversionDsc::optIfConvert() } // Check the Then and Else blocks have a single operation each. - if (!IfConvertCheckStmts(m_startBlock->Next(), &m_thenOperation)) + if (!IfConvertCheckStmts(m_startBlock->GetFallThroughSucc(), &m_thenOperation)) { return false; } @@ -742,7 +743,7 @@ bool OptIfConversionDsc::optIfConvert() } // Update the flow from the original block. - m_comp->fgRemoveAllRefPreds(m_startBlock->Next(), m_startBlock); + m_comp->fgRemoveAllRefPreds(m_startBlock->GetFallThroughSucc(), m_startBlock); assert(m_startBlock->HasJump()); m_startBlock->SetJumpKind(BBJ_ALWAYS); diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 0b24cbda69bba..823e04575985d 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -7325,7 +7325,7 @@ void Compiler::impImportBlockCode(BasicBlock* block) if (block->KindIs(BBJ_COND)) { JITDUMP(FMT_BB " both branches and falls through to " FMT_BB ", changing to BBJ_NONE\n", - block->bbNum, block->Next()->bbNum); + block->bbNum, block->GetFallThroughSucc()->bbNum); fgRemoveRefPred(block->GetJumpDest(), block); block->SetJumpKindAndTarget(BBJ_NONE DEBUG_ARG(this)); } @@ -7391,7 +7391,8 @@ void Compiler::impImportBlockCode(BasicBlock* block) { if (foldedJumpKind == BBJ_NONE) { - JITDUMP("\nThe block falls through into the next " FMT_BB "\n", block->Next()->bbNum); + JITDUMP("\nThe block falls through into the next " FMT_BB "\n", + block->GetFallThroughSucc()->bbNum); fgRemoveRefPred(block->GetJumpDest(), block); block->SetJumpKindAndTarget(BBJ_NONE DEBUG_ARG(this)); } @@ -7399,7 +7400,7 @@ void Compiler::impImportBlockCode(BasicBlock* block) { JITDUMP("\nThe conditional jump becomes an unconditional jump to " FMT_BB "\n", block->GetJumpDest()->bbNum); - fgRemoveRefPred(block->Next(), block); + fgRemoveRefPred(block->GetFallThroughSucc(), block); assert(foldedJumpKind == BBJ_ALWAYS); block->SetJumpKind(BBJ_ALWAYS); } @@ -7578,7 +7579,7 @@ void Compiler::impImportBlockCode(BasicBlock* block) if (block->KindIs(BBJ_COND)) { JITDUMP(FMT_BB " both branches and falls through to " FMT_BB ", changing to BBJ_NONE\n", - block->bbNum, block->Next()->bbNum); + block->bbNum, block->GetFallThroughSucc()->bbNum); fgRemoveRefPred(block->GetJumpDest(), block); block->SetJumpKindAndTarget(BBJ_NONE DEBUG_ARG(this)); } @@ -11257,14 +11258,15 @@ void Compiler::impImportBlock(BasicBlock* block) assert(addStmt->GetRootNode()->gtOper == GT_JTRUE); + tgtBlock = block->GetFallThroughSucc(); + /* Note if the next block has more than one ancestor */ - multRef |= block->Next()->bbRefs; + multRef |= tgtBlock->bbRefs; /* Does the next block have temps assigned? */ - baseTmp = block->Next()->bbStkTempsIn; - tgtBlock = block->Next(); + baseTmp = tgtBlock->bbStkTempsIn; if (baseTmp != NO_BASE_TMP) { @@ -11273,21 +11275,18 @@ void Compiler::impImportBlock(BasicBlock* block) /* Try the target of the jump then */ - multRef |= block->GetJumpDest()->bbRefs; - baseTmp = block->GetJumpDest()->bbStkTempsIn; - tgtBlock = block->GetJumpDest(); - break; + FALLTHROUGH; case BBJ_ALWAYS: - multRef |= block->GetJumpDest()->bbRefs; - baseTmp = block->GetJumpDest()->bbStkTempsIn; tgtBlock = block->GetJumpDest(); + multRef |= tgtBlock->bbRefs; + baseTmp = tgtBlock->bbStkTempsIn; break; case BBJ_NONE: - multRef |= block->Next()->bbRefs; - baseTmp = block->Next()->bbStkTempsIn; - tgtBlock = block->Next(); + tgtBlock = block->GetFallThroughSucc(); + multRef |= tgtBlock->bbRefs; + baseTmp = tgtBlock->bbStkTempsIn; break; case BBJ_SWITCH: @@ -12096,7 +12095,7 @@ void Compiler::impImport() if (entryBlock->KindIs(BBJ_NONE)) { - entryBlock = entryBlock->Next(); + entryBlock = entryBlock->GetFallThroughSucc(); } else if (opts.IsOSR() && entryBlock->KindIs(BBJ_ALWAYS)) { diff --git a/src/coreclr/jit/jiteh.cpp b/src/coreclr/jit/jiteh.cpp index c245cdb6ee026..399a0681ff488 100644 --- a/src/coreclr/jit/jiteh.cpp +++ b/src/coreclr/jit/jiteh.cpp @@ -2281,7 +2281,7 @@ bool Compiler::fgNormalizeEHCase2() fgReplaceJumpTarget(predBlock, newTryStart, insertBeforeBlk); } - if (predBlock->NextIs(newTryStart) && predBlock->bbFallsThrough()) + if (predBlock->bbFallsThrough() && predBlock->FallsInto(newTryStart)) { fgRemoveRefPred(insertBeforeBlk, predBlock); fgAddRefPred(newTryStart, predBlock); diff --git a/src/coreclr/jit/liveness.cpp b/src/coreclr/jit/liveness.cpp index 3c914b2875f68..712508715071e 100644 --- a/src/coreclr/jit/liveness.cpp +++ b/src/coreclr/jit/liveness.cpp @@ -888,9 +888,13 @@ void Compiler::fgExtendDbgLifetimes() switch (block->GetJumpKind()) { + case BBJ_COND: + VarSetOps::UnionD(this, initVars, block->GetJumpDest()->bbScope); + FALLTHROUGH; + case BBJ_NONE: PREFIX_ASSUME(!block->IsLast()); - VarSetOps::UnionD(this, initVars, block->Next()->bbScope); + VarSetOps::UnionD(this, initVars, block->GetFallThroughSucc()->bbScope); break; case BBJ_ALWAYS: @@ -904,17 +908,11 @@ void Compiler::fgExtendDbgLifetimes() { assert(block->isBBCallAlwaysPair()); PREFIX_ASSUME(!block->IsLast()); - VarSetOps::UnionD(this, initVars, block->Next()->bbScope); + VarSetOps::UnionD(this, initVars, block->GetFallThroughSucc()->bbScope); } VarSetOps::UnionD(this, initVars, block->GetJumpDest()->bbScope); break; - case BBJ_COND: - PREFIX_ASSUME(!block->IsLast()); - VarSetOps::UnionD(this, initVars, block->Next()->bbScope); - VarSetOps::UnionD(this, initVars, block->GetJumpDest()->bbScope); - break; - case BBJ_SWITCH: for (BasicBlock* const bTarget : block->SwitchTargets()) { diff --git a/src/coreclr/jit/loopcloning.cpp b/src/coreclr/jit/loopcloning.cpp index 2cdf1906295c0..42f876dbabaf3 100644 --- a/src/coreclr/jit/loopcloning.cpp +++ b/src/coreclr/jit/loopcloning.cpp @@ -2184,8 +2184,12 @@ void Compiler::optCloneLoop(unsigned loopInd, LoopCloneContext* context) // Add predecessor edges for the new successors, as well as the fall-through paths. switch (newblk->GetJumpKind()) { + case BBJ_COND: + fgAddRefPred(newblk->GetJumpDest(), newblk); + FALLTHROUGH; + case BBJ_NONE: - fgAddRefPred(newblk->Next(), newblk); + fgAddRefPred(newblk->GetFallThroughSucc(), newblk); break; case BBJ_ALWAYS: @@ -2193,11 +2197,6 @@ void Compiler::optCloneLoop(unsigned loopInd, LoopCloneContext* context) fgAddRefPred(newblk->GetJumpDest(), newblk); break; - case BBJ_COND: - fgAddRefPred(newblk->Next(), newblk); - fgAddRefPred(newblk->GetJumpDest(), newblk); - break; - case BBJ_SWITCH: for (BasicBlock* const switchDest : newblk->SwitchTargets()) { @@ -2242,17 +2241,17 @@ void Compiler::optCloneLoop(unsigned loopInd, LoopCloneContext* context) assert(context->HasBlockConditions(loopInd)); assert(h->KindIs(BBJ_NONE)); - assert(h->NextIs(h2)); + assert(h->FallsInto(h2)); // If any condition is false, go to slowHead (which branches or falls through to e2). BasicBlock* e2 = nullptr; bool foundIt = blockMap->Lookup(loop.lpEntry, &e2); assert(foundIt && e2 != nullptr); - if (!slowHead->NextIs(e2)) + assert(slowHead->KindIs(BBJ_NONE)); + if (!slowHead->FallsInto(e2)) { // We can't just fall through to the slow path entry, so make it an unconditional branch. - assert(slowHead->KindIs(BBJ_NONE)); // This is how we created it above. slowHead->SetJumpKindAndTarget(BBJ_ALWAYS, e2 DEBUG_ARG(this)); } @@ -2264,8 +2263,8 @@ void Compiler::optCloneLoop(unsigned loopInd, LoopCloneContext* context) // Add the fall-through path pred (either to T/E for fall-through from conditions to fast path, // or H2 if branch to E of fast path). assert(condLast->KindIs(BBJ_COND)); - JITDUMP("Adding " FMT_BB " -> " FMT_BB "\n", condLast->bbNum, condLast->Next()->bbNum); - fgAddRefPred(condLast->Next(), condLast); + JITDUMP("Adding " FMT_BB " -> " FMT_BB "\n", condLast->bbNum, condLast->GetFallThroughSucc()->bbNum); + fgAddRefPred(condLast->GetFallThroughSucc(), condLast); // Don't unroll loops that we've cloned -- the unroller expects any loop it should unroll to // initialize the loop counter immediately before entering the loop, but we've left a shared @@ -2917,8 +2916,20 @@ bool Compiler::optCheckLoopCloningGDVTestProfitable(GenTreeOp* guard, LoopCloneV // Check for (4) // - BasicBlock* const hotSuccessor = guard->OperIs(GT_EQ) ? typeTestBlock->GetJumpDest() : typeTestBlock->Next(); - BasicBlock* const coldSuccessor = guard->OperIs(GT_EQ) ? typeTestBlock->Next() : typeTestBlock->GetJumpDest(); + assert(typeTestBlock->KindIs(BBJ_COND)); + BasicBlock* hotSuccessor; + BasicBlock* coldSuccessor; + + if (guard->OperIs(GT_EQ)) + { + hotSuccessor = typeTestBlock->GetJumpDest(); + coldSuccessor = typeTestBlock->GetFallThroughSucc(); + } + else + { + hotSuccessor = typeTestBlock->GetFallThroughSucc(); + coldSuccessor = typeTestBlock->GetJumpDest(); + } if (!hotSuccessor->hasProfileWeight() || !coldSuccessor->hasProfileWeight()) { diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 5c25441db55ca..b150dca1e7b0b 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -891,7 +891,7 @@ GenTree* Lowering::LowerSwitch(GenTree* node) // originalSwitchBB is now a BBJ_NONE, and there is a predecessor edge in afterDefaultCondBlock // representing the fall-through flow from originalSwitchBB. assert(originalSwitchBB->KindIs(BBJ_NONE)); - assert(originalSwitchBB->NextIs(afterDefaultCondBlock)); + assert(originalSwitchBB->FallsInto(afterDefaultCondBlock)); assert(afterDefaultCondBlock->KindIs(BBJ_SWITCH)); assert(afterDefaultCondBlock->GetJumpSwt()->bbsHasDefault); assert(afterDefaultCondBlock->isEmpty()); // Nothing here yet. @@ -1056,7 +1056,8 @@ GenTree* Lowering::LowerSwitch(GenTree* node) // There is a fall-through to the following block. In the loop // above, we deleted all the predecessor edges from the switch. // In this case, we need to add one back. - comp->fgAddRefPred(currentBlock->Next(), currentBlock); + assert(currentBlock->bbFallsThrough()); + comp->fgAddRefPred(currentBlock->GetFallThroughSucc(), currentBlock); } if (!fUsedAfterDefaultCondBlock) @@ -1239,6 +1240,7 @@ bool Lowering::TryLowerSwitchToBitTest( // GenCondition bbSwitchCondition; + ssize_t tstCnsIconValue; comp->fgRemoveAllRefPreds(bbCase1, bbSwitch); comp->fgRemoveAllRefPreds(bbCase0, bbSwitch); @@ -1250,6 +1252,7 @@ bool Lowering::TryLowerSwitchToBitTest( comp->fgAddRefPred(bbCase0, bbSwitch); comp->fgAddRefPred(bbCase1, bbSwitch); + tstCnsIconValue = 1; } else { @@ -1261,6 +1264,7 @@ bool Lowering::TryLowerSwitchToBitTest( comp->fgAddRefPred(bbCase0, bbSwitch); comp->fgAddRefPred(bbCase1, bbSwitch); + tstCnsIconValue = 0; } var_types bitTableType = (bitCount <= (genTypeSize(TYP_INT) * 8)) ? TYP_INT : TYP_LONG; @@ -1278,7 +1282,7 @@ bool Lowering::TryLowerSwitchToBitTest( // // Fallback to AND(RSZ(bitTable, switchValue), 1) // - GenTree* tstCns = comp->gtNewIconNode(bbSwitch->NextIs(bbCase0) ? 1 : 0, bitTableType); + GenTree* tstCns = comp->gtNewIconNode(tstCnsIconValue, bitTableType); GenTree* shift = comp->gtNewOperNode(GT_RSZ, bitTableType, bitTableIcon, switchValue); GenTree* one = comp->gtNewIconNode(1, bitTableType); GenTree* andOp = comp->gtNewOperNode(GT_AND, bitTableType, shift, one); diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 391682418ed1e..15c0d3e8dbe3e 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -2545,7 +2545,8 @@ BasicBlock* LinearScan::findPredBlockForLiveIn(BasicBlock* block, if (predBlock->KindIs(BBJ_COND)) { // Special handling to improve matching on backedges. - BasicBlock* otherBlock = predBlock->NextIs(block) ? predBlock->GetJumpDest() : predBlock->Next(); + BasicBlock* otherBlock = + predBlock->FallsInto(block) ? predBlock->GetJumpDest() : predBlock->GetFallThroughSucc(); noway_assert(otherBlock != nullptr); if (isBlockVisited(otherBlock) && !blockInfo[otherBlock->bbNum].hasEHBoundaryIn) { diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 4bae7722294e0..a7f733043eaa5 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -13125,7 +13125,8 @@ Compiler::FoldResult Compiler::fgFoldConditional(BasicBlock* block) * Remove the conditional statement */ noway_assert(cond->gtOper == GT_CNS_INT); - noway_assert((block->Next()->countOfInEdges() > 0) && (block->GetJumpDest()->countOfInEdges() > 0)); + noway_assert((block->GetFallThroughSucc()->countOfInEdges() > 0) && + (block->GetJumpDest()->countOfInEdges() > 0)); if (condTree != cond) { @@ -13150,9 +13151,9 @@ Compiler::FoldResult Compiler::fgFoldConditional(BasicBlock* block) if (cond->AsIntCon()->gtIconVal != 0) { /* JTRUE 1 - transform the basic block into a BBJ_ALWAYS */ - block->SetJumpKind(BBJ_ALWAYS); bTaken = block->GetJumpDest(); - bNotTaken = block->Next(); + bNotTaken = block->GetFallThroughSucc(); + block->SetJumpKind(BBJ_ALWAYS); } else { @@ -13166,7 +13167,7 @@ Compiler::FoldResult Compiler::fgFoldConditional(BasicBlock* block) } /* JTRUE 0 - transform the basic block into a BBJ_NONE */ - bTaken = block->Next(); + bTaken = block->GetFallThroughSucc(); bNotTaken = block->GetJumpDest(); block->SetJumpKindAndTarget(BBJ_NONE DEBUG_ARG(this)); } @@ -13224,17 +13225,17 @@ Compiler::FoldResult Compiler::fgFoldConditional(BasicBlock* block) switch (bUpdated->GetJumpKind()) { case BBJ_NONE: - edge = fgGetPredForBlock(bUpdated->Next(), bUpdated); + edge = fgGetPredForBlock(bUpdated->GetFallThroughSucc(), bUpdated); newMaxWeight = bUpdated->bbWeight; newMinWeight = min(edge->edgeWeightMin(), newMaxWeight); - edge->setEdgeWeights(newMinWeight, newMaxWeight, bUpdated->Next()); + edge->setEdgeWeights(newMinWeight, newMaxWeight, bUpdated->GetFallThroughSucc()); break; case BBJ_COND: - edge = fgGetPredForBlock(bUpdated->Next(), bUpdated); + edge = fgGetPredForBlock(bUpdated->GetFallThroughSucc(), bUpdated); newMaxWeight = bUpdated->bbWeight; newMinWeight = min(edge->edgeWeightMin(), newMaxWeight); - edge->setEdgeWeights(newMinWeight, newMaxWeight, bUpdated->Next()); + edge->setEdgeWeights(newMinWeight, newMaxWeight, bUpdated->GetFallThroughSucc()); FALLTHROUGH; case BBJ_ALWAYS: @@ -13392,15 +13393,15 @@ Compiler::FoldResult Compiler::fgFoldConditional(BasicBlock* block) if ((val == switchVal) || (!foundVal && (val == jumpCnt - 1))) { - if (!block->NextIs(curJump)) + if (block->NextIs(curJump)) { - // transform the basic block into a BBJ_ALWAYS - block->SetJumpKindAndTarget(BBJ_ALWAYS, curJump DEBUG_ARG(this)); + // transform the basic block into a BBJ_NONE + block->SetJumpKindAndTarget(BBJ_NONE DEBUG_ARG(this)); } else { - // transform the basic block into a BBJ_NONE - block->SetJumpKindAndTarget(BBJ_NONE DEBUG_ARG(this)); + // transform the basic block into a BBJ_ALWAYS + block->SetJumpKindAndTarget(BBJ_ALWAYS, curJump DEBUG_ARG(this)); } foundVal = true; } diff --git a/src/coreclr/jit/optimizebools.cpp b/src/coreclr/jit/optimizebools.cpp index 3c55cef198ab5..f5544d5b54343 100644 --- a/src/coreclr/jit/optimizebools.cpp +++ b/src/coreclr/jit/optimizebools.cpp @@ -107,7 +107,7 @@ class OptBoolsDsc // B3: GT_RETURN (BBJ_RETURN) // B4: GT_RETURN (BBJ_RETURN) // -// Case 2: if B2->NextIs(B1.bbJumpDest), it transforms +// Case 2: if B2->FallsInto(B1.bbJumpDest), it transforms // B1 : brtrue(t1, B3) // B2 : brtrue(t2, Bx) // B3 : @@ -117,7 +117,9 @@ class OptBoolsDsc // bool OptBoolsDsc::optOptimizeBoolsCondBlock() { - assert(m_b1 != nullptr && m_b2 != nullptr && m_b3 == nullptr); + assert((m_b1 != nullptr) && (m_b2 != nullptr) && (m_b3 == nullptr)); + assert(m_b1->KindIs(BBJ_COND)); + assert(m_b2->KindIs(BBJ_COND)); // Check if m_b1 and m_b2 jump to the same target and get back pointers to m_testInfo1 and t2 tree nodes @@ -137,7 +139,7 @@ bool OptBoolsDsc::optOptimizeBoolsCondBlock() m_sameTarget = true; } - else if (m_b2->NextIs(m_b1->GetJumpDest())) + else if (m_b2->FallsInto(m_b1->GetJumpDest())) { // Given the following sequence of blocks : // B1: brtrue(t1, B3) @@ -637,7 +639,7 @@ bool OptBoolsDsc::optOptimizeRangeTests() { // At this point we have two consecutive conditional blocks (BBJ_COND): m_b1 and m_b2 assert((m_b1 != nullptr) && (m_b2 != nullptr) && (m_b3 == nullptr)); - assert(m_b1->KindIs(BBJ_COND) && m_b2->KindIs(BBJ_COND) && m_b1->NextIs(m_b2)); + assert(m_b1->KindIs(BBJ_COND) && m_b2->KindIs(BBJ_COND) && m_b1->FallsInto(m_b2)); if (m_b2->isRunRarely()) { @@ -662,7 +664,7 @@ bool OptBoolsDsc::optOptimizeRangeTests() // BasicBlock* notInRangeBb = m_b1->GetJumpDest(); BasicBlock* inRangeBb; - if (notInRangeBb == m_b2->GetJumpDest()) + if (m_b2->HasJumpTo(notInRangeBb)) { // Shape 1: both conditions jump to NotInRange // @@ -674,9 +676,9 @@ bool OptBoolsDsc::optOptimizeRangeTests() // // InRange: // ... - inRangeBb = m_b2->Next(); + inRangeBb = m_b2->GetFallThroughSucc(); } - else if (notInRangeBb == m_b2->Next()) + else if (m_b2->FallsInto(notInRangeBb)) { // Shape 2: 2nd block jumps to InRange // @@ -809,16 +811,18 @@ bool OptBoolsDsc::optOptimizeRangeTests() bool OptBoolsDsc::optOptimizeCompareChainCondBlock() { assert((m_b1 != nullptr) && (m_b2 != nullptr) && (m_b3 == nullptr)); + assert(m_b1->KindIs(BBJ_COND)); + assert(m_b2->KindIs(BBJ_COND)); m_t3 = nullptr; bool foundEndOfOrConditions = false; - if (m_b1->NextIs(m_b2) && m_b2->NextIs(m_b1->GetJumpDest())) + if (m_b1->FallsInto(m_b2) && m_b2->FallsInto(m_b1->GetJumpDest())) { // Found the end of two (or more) conditions being ORed together. // The final condition has been inverted. foundEndOfOrConditions = true; } - else if (m_b1->NextIs(m_b2) && m_b1->HasJumpTo(m_b2->GetJumpDest())) + else if (m_b1->FallsInto(m_b2) && m_b1->HasJumpTo(m_b2->GetJumpDest())) { // Found two conditions connected together. } @@ -1180,7 +1184,7 @@ void OptBoolsDsc::optOptimizeBoolsUpdateTrees() } else { - edge2 = m_comp->fgGetPredForBlock(m_b2->Next(), m_b2); + edge2 = m_comp->fgGetPredForBlock(m_b2->GetFallThroughSucc(), m_b2); m_comp->fgRemoveRefPred(m_b1->GetJumpDest(), m_b1); @@ -1206,19 +1210,19 @@ void OptBoolsDsc::optOptimizeBoolsUpdateTrees() /* Modify the target of the conditional jump and update bbRefs and bbPreds */ + assert(m_b1->KindIs(BBJ_COND)); + assert(m_b1->FallsInto(m_b2)); + if (optReturnBlock) { m_b1->SetJumpKindAndTarget(BBJ_RETURN DEBUG_ARG(m_comp)); assert(m_b2->KindIs(BBJ_RETURN)); - assert(m_b1->NextIs(m_b2)); assert(m_b3 != nullptr); } else { - assert(m_b1->KindIs(BBJ_COND)); assert(m_b2->KindIs(BBJ_COND)); assert(m_b1->HasJumpTo(m_b2->GetJumpDest())); - assert(m_b1->NextIs(m_b2)); assert(!m_b2->IsLast()); } @@ -1228,7 +1232,7 @@ void OptBoolsDsc::optOptimizeBoolsUpdateTrees() // // Replace pred 'm_b2' for 'm_b2->bbNext' with 'm_b1' // Remove pred 'm_b2' for 'm_b2->bbJumpDest' - m_comp->fgReplacePred(m_b2->Next(), m_b2, m_b1); + m_comp->fgReplacePred(m_b2->GetFallThroughSucc(), m_b2, m_b1); m_comp->fgRemoveRefPred(m_b2->GetJumpDest(), m_b2); } @@ -1288,6 +1292,9 @@ void OptBoolsDsc::optOptimizeBoolsUpdateTrees() bool OptBoolsDsc::optOptimizeBoolsReturnBlock(BasicBlock* b3) { assert(m_b1 != nullptr && m_b2 != nullptr); + assert(m_b1->KindIs(BBJ_COND)); + assert(m_b2->KindIs(BBJ_RETURN)); + assert(b3->KindIs(BBJ_RETURN)); // m_b3 is set for cond/return/return case m_b3 = b3; @@ -1802,9 +1809,9 @@ PhaseStatus Compiler::optOptimizeBools() continue; } - // If there is no next block, we're done + // If there is no fallthrough block, we're done - BasicBlock* b2 = b1->Next(); + BasicBlock* b2 = b1->GetFallThroughSucc(); if (b2 == nullptr) { break; @@ -1822,7 +1829,7 @@ PhaseStatus Compiler::optOptimizeBools() if (b2->KindIs(BBJ_COND)) { - if (!b1->HasJumpTo(b2->GetJumpDest()) && !b2->NextIs(b1->GetJumpDest())) + if (!b1->HasJumpTo(b2->GetJumpDest()) && !b2->FallsInto(b1->GetJumpDest())) { continue; } diff --git a/src/coreclr/jit/optimizer.cpp b/src/coreclr/jit/optimizer.cpp index 355e0f198d5a0..bc880322ecbca 100644 --- a/src/coreclr/jit/optimizer.cpp +++ b/src/coreclr/jit/optimizer.cpp @@ -742,7 +742,7 @@ bool Compiler::optPopulateInitInfo(unsigned loopInd, BasicBlock* initBlock, GenT bool initBlockOk = (predBlock == initBlock); if (!initBlockOk) { - if (predBlock->KindIs(BBJ_NONE) && predBlock->NextIs(optLoopTable[loopInd].lpEntry) && + if (predBlock->KindIs(BBJ_NONE) && predBlock->FallsInto(optLoopTable[loopInd].lpEntry) && (predBlock->countOfInEdges() == 1) && (predBlock->firstStmt() == nullptr) && !predBlock->IsFirst() && predBlock->Prev()->bbFallsThrough()) { @@ -1151,7 +1151,7 @@ bool Compiler::optExtractInitTestIncr( // If we are rebuilding the loop table, we would already have the pre-header block introduced // the first time, which might be empty if no hoisting has yet occurred. In this case, look a // little harder for the possible loop initialization statement. - if (initBlock->KindIs(BBJ_NONE) && initBlock->NextIs(top) && (initBlock->countOfInEdges() == 1) && + if (initBlock->KindIs(BBJ_NONE) && initBlock->FallsInto(top) && (initBlock->countOfInEdges() == 1) && !initBlock->IsFirst() && initBlock->Prev()->bbFallsThrough()) { initBlock = initBlock->Prev(); @@ -1395,7 +1395,7 @@ void Compiler::optCheckPreds() } FALLTHROUGH; case BBJ_NONE: - noway_assert(bb->NextIs(block)); + noway_assert(bb->FallsInto(block)); break; case BBJ_EHFILTERRET: case BBJ_ALWAYS: @@ -2463,7 +2463,7 @@ class LoopSearch break; } - if (block->bbFallsThrough() && !loopBlocks.IsMember(block->Next()->bbNum)) + if (block->bbFallsThrough() && !loopBlocks.IsMember(block->GetFallThroughSucc()->bbNum)) { // Found a fall-through exit. lastExit = block; @@ -2733,7 +2733,7 @@ void Compiler::optRedirectBlock(BasicBlock* blk, BlockToBlockMap* redirectMap, R if (addPreds && blk->bbFallsThrough()) { - fgAddRefPred(blk->Next(), blk); + fgAddRefPred(blk->GetFallThroughSucc(), blk); } BasicBlock* newJumpDest = nullptr; @@ -3052,7 +3052,7 @@ bool Compiler::optCanonicalizeLoop(unsigned char loopInd) // BasicBlock* const t = optLoopTable[loopInd].lpTop; assert(siblingB->KindIs(BBJ_COND)); - assert(siblingB->NextIs(t)); + assert(siblingB->FallsInto(t)); JITDUMP(FMT_LP " head " FMT_BB " is also " FMT_LP " bottom\n", loopInd, h->bbNum, sibling); @@ -3226,9 +3226,9 @@ bool Compiler::optCanonicalizeLoopCore(unsigned char loopInd, LoopCanonicalizati // Because of this, introducing a block before t automatically gives us // the right flow out of h. // - assert(h->NextIs(t)); assert(h->bbFallsThrough()); assert(h->KindIs(BBJ_NONE, BBJ_COND)); + assert(h->FallsInto(t)); if (h->KindIs(BBJ_COND)) { BasicBlock* const hj = h->GetJumpDest(); @@ -3350,8 +3350,8 @@ bool Compiler::optCanonicalizeLoopCore(unsigned char loopInd, LoopCanonicalizati } } - assert(h->NextIs(newT)); - assert(newT->NextIs(t)); + assert(h->FallsInto(newT)); + assert(newT->FallsInto(t)); // With the Option::Current we are changing which block is loop top. // Make suitable updates. @@ -3381,7 +3381,7 @@ bool Compiler::optCanonicalizeLoopCore(unsigned char loopInd, LoopCanonicalizati childLoop = optLoopTable[childLoop].lpSibling) { if ((optLoopTable[childLoop].lpEntry == origE) && (optLoopTable[childLoop].lpHead == h) && - newT->KindIs(BBJ_NONE) && newT->NextIs(origE)) + newT->KindIs(BBJ_NONE) && newT->FallsInto(origE)) { optUpdateLoopHead(childLoop, h, newT); @@ -3455,7 +3455,7 @@ BasicBlock* Compiler::optLoopEntry(BasicBlock* preHeader) if (preHeader->KindIs(BBJ_NONE)) { - return preHeader->Next(); + return preHeader->GetFallThroughSucc(); } else { @@ -4890,7 +4890,7 @@ bool Compiler::optInvertWhileLoop(BasicBlock* block) // Since bTest is a BBJ_COND it will have a bbNext // - BasicBlock* const bJoin = bTest->Next(); + BasicBlock* const bJoin = bTest->GetFallThroughSucc(); noway_assert(bJoin != nullptr); // 'block' must be in the same try region as the condition, since we're going to insert a duplicated condition @@ -5237,15 +5237,15 @@ bool Compiler::optInvertWhileLoop(BasicBlock* block) weight_t const testToAfterWeight = weightTop * testToAfterLikelihood; FlowEdge* const edgeTestToNext = fgGetPredForBlock(bTop, bTest); - FlowEdge* const edgeTestToAfter = fgGetPredForBlock(bTest->Next(), bTest); + FlowEdge* const edgeTestToAfter = fgGetPredForBlock(bTest->GetFallThroughSucc(), bTest); JITDUMP("Setting weight of " FMT_BB " -> " FMT_BB " to " FMT_WT " (iterate loop)\n", bTest->bbNum, bTop->bbNum, testToNextWeight); JITDUMP("Setting weight of " FMT_BB " -> " FMT_BB " to " FMT_WT " (exit loop)\n", bTest->bbNum, - bTest->Next()->bbNum, testToAfterWeight); + bTest->GetFallThroughSucc()->bbNum, testToAfterWeight); edgeTestToNext->setEdgeWeights(testToNextWeight, testToNextWeight, bTop); - edgeTestToAfter->setEdgeWeights(testToAfterWeight, testToAfterWeight, bTest->Next()); + edgeTestToAfter->setEdgeWeights(testToAfterWeight, testToAfterWeight, bTest->GetFallThroughSucc()); // Adjust edges out of block, using the same distribution. // @@ -5257,15 +5257,15 @@ bool Compiler::optInvertWhileLoop(BasicBlock* block) weight_t const blockToNextWeight = weightBlock * blockToNextLikelihood; weight_t const blockToAfterWeight = weightBlock * blockToAfterLikelihood; - FlowEdge* const edgeBlockToNext = fgGetPredForBlock(bNewCond->Next(), bNewCond); + FlowEdge* const edgeBlockToNext = fgGetPredForBlock(bNewCond->GetFallThroughSucc(), bNewCond); FlowEdge* const edgeBlockToAfter = fgGetPredForBlock(bNewCond->GetJumpDest(), bNewCond); JITDUMP("Setting weight of " FMT_BB " -> " FMT_BB " to " FMT_WT " (enter loop)\n", bNewCond->bbNum, - bNewCond->Next()->bbNum, blockToNextWeight); + bNewCond->GetFallThroughSucc()->bbNum, blockToNextWeight); JITDUMP("Setting weight of " FMT_BB " -> " FMT_BB " to " FMT_WT " (avoid loop)\n", bNewCond->bbNum, bNewCond->GetJumpDest()->bbNum, blockToAfterWeight); - edgeBlockToNext->setEdgeWeights(blockToNextWeight, blockToNextWeight, bNewCond->Next()); + edgeBlockToNext->setEdgeWeights(blockToNextWeight, blockToNextWeight, bNewCond->GetFallThroughSucc()); edgeBlockToAfter->setEdgeWeights(blockToAfterWeight, blockToAfterWeight, bNewCond->GetJumpDest()); #ifdef DEBUG @@ -5274,7 +5274,7 @@ bool Compiler::optInvertWhileLoop(BasicBlock* block) if ((activePhaseChecks & PhaseChecks::CHECK_PROFILE) == PhaseChecks::CHECK_PROFILE) { const ProfileChecks checks = (ProfileChecks)JitConfig.JitProfileChecks(); - const bool nextProfileOk = fgDebugCheckIncomingProfileData(bNewCond->Next(), checks); + const bool nextProfileOk = fgDebugCheckIncomingProfileData(bNewCond->GetFallThroughSucc(), checks); const bool jumpProfileOk = fgDebugCheckIncomingProfileData(bNewCond->GetJumpDest(), checks); if (hasFlag(checks, ProfileChecks::RAISE_ASSERT)) @@ -5290,7 +5290,7 @@ bool Compiler::optInvertWhileLoop(BasicBlock* block) if (verbose) { printf("\nDuplicated loop exit block at " FMT_BB " for loop (" FMT_BB " - " FMT_BB ")\n", bNewCond->bbNum, - bNewCond->Next()->bbNum, bTest->bbNum); + bNewCond->GetFallThroughSucc()->bbNum, bTest->bbNum); printf("Estimated code size expansion is %d\n", estDupCostSz); fgDumpBlock(bNewCond); @@ -8008,6 +8008,7 @@ bool Compiler::optVNIsLoopInvariant(ValueNum vn, unsigned lnum, VNSet* loopVnInv // void Compiler::fgSetEHRegionForNewLoopHead(BasicBlock* newHead, BasicBlock* top) { + assert(newHead->KindIs(BBJ_ALWAYS, BBJ_NONE)); assert(newHead->NextIs(top)); assert(!fgIsFirstBlockOfFilterOrHandler(top)); @@ -8225,13 +8226,13 @@ bool Compiler::fgCreateLoopPreHeader(unsigned lnum) { // Allow for either the fall-through or branch to target 'entry'. BasicBlock* skipLoopBlock; - if (head->NextIs(entry)) + if (head->FallsInto(entry)) { skipLoopBlock = head->GetJumpDest(); } else { - skipLoopBlock = head->Next(); + skipLoopBlock = head->GetFallThroughSucc(); } assert(skipLoopBlock != entry); @@ -8327,7 +8328,7 @@ bool Compiler::fgCreateLoopPreHeader(unsigned lnum) case BBJ_NONE: // This 'entry' predecessor that isn't dominated by 'entry' must be outside the loop, // meaning it must be fall-through to 'entry', and we must have a top-entry loop. - noway_assert((entry == top) && (predBlock == head) && predBlock->NextIs(preHead)); + noway_assert((entry == top) && (predBlock == head) && predBlock->FallsInto(preHead)); fgRemoveRefPred(entry, predBlock); fgAddRefPred(preHead, predBlock); break; @@ -8336,11 +8337,11 @@ bool Compiler::fgCreateLoopPreHeader(unsigned lnum) if (predBlock->HasJumpTo(entry)) { predBlock->SetJumpDest(preHead); - noway_assert(!predBlock->NextIs(preHead)); + noway_assert(!predBlock->FallsInto(preHead)); } else { - noway_assert((entry == top) && (predBlock == head) && predBlock->NextIs(preHead)); + noway_assert((entry == top) && (predBlock == head) && predBlock->FallsInto(preHead)); } fgRemoveRefPred(entry, predBlock); fgAddRefPred(preHead, predBlock); diff --git a/src/coreclr/jit/rangecheck.cpp b/src/coreclr/jit/rangecheck.cpp index 89720f6c7748b..f6e9b16f13d3c 100644 --- a/src/coreclr/jit/rangecheck.cpp +++ b/src/coreclr/jit/rangecheck.cpp @@ -935,7 +935,7 @@ void RangeCheck::MergeAssertion(BasicBlock* block, GenTree* op, Range* pRange DE { GenTreePhiArg* arg = (GenTreePhiArg*)op; BasicBlock* pred = arg->gtPredBB; - if (pred->bbFallsThrough() && pred->NextIs(block)) + if (pred->bbFallsThrough() && pred->FallsInto(block)) { assertions = pred->bbAssertionOut; JITDUMP("Merge assertions from pred " FMT_BB " edge: ", pred->bbNum); diff --git a/src/coreclr/jit/redundantbranchopts.cpp b/src/coreclr/jit/redundantbranchopts.cpp index 5c149bd35a141..2c7c6b161d881 100644 --- a/src/coreclr/jit/redundantbranchopts.cpp +++ b/src/coreclr/jit/redundantbranchopts.cpp @@ -48,7 +48,7 @@ PhaseStatus Compiler::optRedundantBranches() { bool madeChangesThisBlock = m_compiler->optRedundantRelop(block); - BasicBlock* const bbNext = block->Next(); + BasicBlock* const bbNext = block->GetFallThroughSucc(); BasicBlock* const bbJump = block->GetJumpDest(); madeChangesThisBlock |= m_compiler->optRedundantBranch(block); @@ -568,7 +568,7 @@ bool Compiler::optRedundantBranch(BasicBlock* const block) (rii.vnRelation == ValueNumStore::VN_RELATION_KIND::VRK_Swap); BasicBlock* const trueSuccessor = domBlock->GetJumpDest(); - BasicBlock* const falseSuccessor = domBlock->Next(); + BasicBlock* const falseSuccessor = domBlock->GetFallThroughSucc(); // If we can trace the flow from the dominating relop, we can infer its value. // @@ -613,7 +613,7 @@ bool Compiler::optRedundantBranch(BasicBlock* const block) // const bool relopIsFalse = rii.reverseSense ^ (domIsSameRelop | domIsInferredRelop); JITDUMP("Fall through successor " FMT_BB " of " FMT_BB " reaches, relop [%06u] must be %s\n", - domBlock->Next()->bbNum, domBlock->bbNum, dspTreeID(tree), + domBlock->GetFallThroughSucc()->bbNum, domBlock->bbNum, dspTreeID(tree), relopIsFalse ? "false" : "true"); relopValue = relopIsFalse ? 0 : 1; break; @@ -711,7 +711,7 @@ struct JumpThreadInfo JumpThreadInfo(Compiler* comp, BasicBlock* block) : m_block(block) , m_trueTarget(block->GetJumpDest()) - , m_falseTarget(block->Next()) + , m_falseTarget(block->GetFallThroughSucc()) , m_fallThroughPred(nullptr) , m_ambiguousVNBlock(nullptr) , m_truePreds(BlockSetOps::MakeEmpty(comp)) @@ -1072,9 +1072,20 @@ bool Compiler::optJumpThreadDom(BasicBlock* const block, BasicBlock* const domBl // latter should prove useful in subsequent work, where we aim to enable jump // threading in cases where block has side effects. // - BasicBlock* const domTrueSuccessor = domIsSameRelop ? domBlock->GetJumpDest() : domBlock->Next(); - BasicBlock* const domFalseSuccessor = domIsSameRelop ? domBlock->Next() : domBlock->GetJumpDest(); - JumpThreadInfo jti(this, block); + BasicBlock* domTrueSuccessor; + BasicBlock* domFalseSuccessor; + JumpThreadInfo jti(this, block); + + if (domIsSameRelop) + { + domTrueSuccessor = domBlock->GetJumpDest(); + domFalseSuccessor = domBlock->GetFallThroughSucc(); + } + else + { + domTrueSuccessor = domBlock->GetFallThroughSucc(); + domFalseSuccessor = domBlock->GetJumpDest(); + } for (BasicBlock* const predBlock : block->PredBlocks()) { @@ -1143,7 +1154,7 @@ bool Compiler::optJumpThreadDom(BasicBlock* const block, BasicBlock* const domBl // Note if the true or false pred is the fall through pred. // - if (predBlock->NextIs(block)) + if (predBlock->bbFallsThrough() && predBlock->FallsInto(block)) { JITDUMP(FMT_BB " is the fall-through pred\n", predBlock->bbNum); assert(jti.m_fallThroughPred == nullptr); @@ -1403,7 +1414,7 @@ bool Compiler::optJumpThreadPhi(BasicBlock* block, GenTree* tree, ValueNum treeN // Note if the true or false pred is the fall through pred. // - if (predBlock->NextIs(block)) + if (predBlock->bbFallsThrough() && predBlock->FallsInto(block)) { JITDUMP(FMT_BB " is the fall-through pred\n", predBlock->bbNum); assert(jti.m_fallThroughPred == nullptr); diff --git a/src/coreclr/jit/switchrecognition.cpp b/src/coreclr/jit/switchrecognition.cpp index bc1ca0474e427..eab848a2f2907 100644 --- a/src/coreclr/jit/switchrecognition.cpp +++ b/src/coreclr/jit/switchrecognition.cpp @@ -94,9 +94,18 @@ bool IsConstantTestCondBlock(const BasicBlock* block, return false; } - *isReversed = rootNode->gtGetOp1()->OperIs(GT_NE); - *blockIfTrue = *isReversed ? block->Next() : block->GetJumpDest(); - *blockIfFalse = *isReversed ? block->GetJumpDest() : block->Next(); + *isReversed = rootNode->gtGetOp1()->OperIs(GT_NE); + + if (*isReversed) + { + *blockIfTrue = block->GetFallThroughSucc(); + *blockIfFalse = block->GetJumpDest(); + } + else + { + *blockIfTrue = block->GetJumpDest(); + *blockIfFalse = block->GetFallThroughSucc(); + } if (block->JumpsToNext() || block->HasJumpTo(block)) { From 9acad8604c6aed83a3a271d1537c5d5cfc285c49 Mon Sep 17 00:00:00 2001 From: Aman Khalid Date: Fri, 20 Oct 2023 11:37:46 -0400 Subject: [PATCH 2/3] Fix assert --- src/coreclr/jit/fgbasic.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/fgbasic.cpp b/src/coreclr/jit/fgbasic.cpp index 21485d8189bd7..1b2b077db8064 100644 --- a/src/coreclr/jit/fgbasic.cpp +++ b/src/coreclr/jit/fgbasic.cpp @@ -374,7 +374,7 @@ void Compiler::fgConvertBBToThrowBB(BasicBlock* block) // Must do this after we update bbJumpKind of block. if (isCallAlwaysPair) { - BasicBlock* leaveBlk = block->GetFallThroughSucc(); + BasicBlock* leaveBlk = block->Next(); noway_assert(leaveBlk->KindIs(BBJ_ALWAYS)); leaveBlk->bbFlags &= ~BBF_DONT_REMOVE; From a1ce47efcbccaf7ed76cd3b630e6ac8be360ed41 Mon Sep 17 00:00:00 2001 From: Aman Khalid Date: Fri, 20 Oct 2023 15:02:53 -0400 Subject: [PATCH 3/3] [skip ci] Update SetFallThroughSucc comment --- src/coreclr/jit/block.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/block.h b/src/coreclr/jit/block.h index cb4b11048100c..e3337a38c5116 100644 --- a/src/coreclr/jit/block.h +++ b/src/coreclr/jit/block.h @@ -681,9 +681,9 @@ struct BasicBlock : private LIR::Range { // TODO: Once we allow bbFallThroughSucc to diverge from bbNext, ensure we only set bbFallThroughSucc // to something other than bbNext when this block is a BBJ_COND. - // BBJ_CALLFINALLY can fall through too, but we don't allow its fallthrough successor to diverge from bbNext - // because we aren't interested in splitting up call-always pairs - assert(KindIs(BBJ_COND, BBJ_NONE)); + // For BBJ_CALLFINALLY blocks part of call-always pairs, bbFallThroughSucc points to the BBJ_ALWAYS block. + // For now, call-always pairs are always contiguous, so bbFallThroughSucc cannot diverge from bbNext. + assert(KindIs(BBJ_COND)); bbFallThroughSucc = fallThroughSucc; // For now, bbFallThroughSucc cannot diverge from bbNext assert(FallsIntoNext());