OSchip
/
llvm-project
mirror of https://github.com/THU-DSP-LAB/llvm-project.git
13
0
Fork 2
Code Issues Packages Projects Releases Wiki Activity

[VPlan] Replace CondBit with BranchOnCond VPInstruction. 

This patch removes CondBit and Predicate from VPBasicBlock. To do so,
the patch introduces a new branch-on-cond VPInstruction opcode to model
a branch on a condition explicitly.

This addresses a long-standing TODO/FIXME that blocks shouldn't be users
of VPValues. Those extra users can cause issues for VPValue-based
analyses that don't expect blocks. Addressing this fixme should allow us
to re-introduce 266ea446ab.

The generic branch opcode can also be used in follow-up patches.

Depends on D123005.

Reviewed By: Ayal

Differential Revision: https://reviews.llvm.org/D126618
This commit is contained in:
Florian Hahn 2022-06-03 11:47:16 +01:00
parent 39f28397e2
commit a5bb4a3b4d
No known key found for this signature in database
GPG Key ID: EEF712BB5E80EBBA
14 changed files with 122 additions and 199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View File

@ -8528,7 +8528,7 @@ VPRegionBlock *VPRecipeBuilder::createReplicateRegion(Instruction *Instr,
// Note: first set Entry as region entry and then connect successors starting
// from it in order, to propagate the "parent" of each VPBasicBlock.
VPBlockUtils::insertTwoBlocksAfter(Pred, Exiting, BlockInMask, Entry);
VPBlockUtils::insertTwoBlocksAfter(Pred, Exiting, Entry);
VPBlockUtils::connectBlocks(Pred, Exiting);
return Region;
@ -8661,7 +8661,7 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
// CanonicalIVIncrement{NUW} VPInstruction to increment it by VF * UF and a
// BranchOnCount VPInstruction to the latch.
static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, DebugLoc DL,
bool HasNUW, bool IsVPlanNative) {
bool HasNUW) {
Value *StartIdx = ConstantInt::get(IdxTy, 0);
auto *StartV = Plan.getOrAddVPValue(StartIdx);
@ -8677,8 +8677,6 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, DebugLoc DL,
CanonicalIVPHI->addOperand(CanonicalIVIncrement);
VPBasicBlock *EB = TopRegion->getExitingBasicBlock();
if (IsVPlanNative)
EB->setCondBit(nullptr);
EB->appendRecipe(CanonicalIVIncrement);
auto *BranchOnCount =
@ -8785,7 +8783,7 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
getDebugLocFromInstOrOperands(Legal->getPrimaryInduction());
addCanonicalIVRecipes(*Plan, Legal->getWidestInductionType(),
DLInst ? DLInst->getDebugLoc() : DebugLoc(),
!CM.foldTailByMasking(), false);
!CM.foldTailByMasking());
// Scan the body of the loop in a topological order to visit each basic block
// after having visited its predecessor basic blocks.
@ -9089,8 +9087,14 @@ VPlanPtr LoopVectorizationPlanner::buildVPlan(VFRange &Range) {
[this](PHINode *P) { return Legal->getIntOrFpInductionDescriptor(P); },
DeadInstructions, *PSE.getSE());
// Remove the existing terminator of the exiting block of the top-most region.
// A BranchOnCount will be added instead when adding the canonical IV recipes.
auto *Term =
Plan->getVectorLoopRegion()->getExitingBasicBlock()->getTerminator();
Term->eraseFromParent();
addCanonicalIVRecipes(*Plan, Legal->getWidestInductionType(), DebugLoc(),
true, true);
true);
return Plan;
}

127
llvm/lib/Transforms/Vectorize/VPlan.cpp
View File

@ -188,28 +188,6 @@ VPBlockBase *VPBlockBase::getEnclosingBlockWithPredecessors() {
return Parent->getEnclosingBlockWithPredecessors();
}
VPValue *VPBlockBase::getCondBit() {
return CondBitUser.getSingleOperandOrNull();
}
const VPValue *VPBlockBase::getCondBit() const {
return CondBitUser.getSingleOperandOrNull();
}
void VPBlockBase::setCondBit(VPValue *CV) { CondBitUser.resetSingleOpUser(CV); }
VPValue *VPBlockBase::getPredicate() {
return PredicateUser.getSingleOperandOrNull();
}
const VPValue *VPBlockBase::getPredicate() const {
return PredicateUser.getSingleOperandOrNull();
}
void VPBlockBase::setPredicate(VPValue *CV) {
PredicateUser.resetSingleOpUser(CV);
}
void VPBlockBase::deleteCFG(VPBlockBase *Entry) {
SmallVector<VPBlockBase *, 8> Blocks(depth_first(Entry));
@ -357,29 +335,6 @@ void VPBasicBlock::execute(VPTransformState *State) {
for (VPRecipeBase &Recipe : Recipes)
Recipe.execute(*State);
VPValue *CBV;
if (EnableVPlanNativePath && (CBV = getCondBit())) {
assert(CBV->getUnderlyingValue() &&
"Unexpected null underlying value for condition bit");
// Condition bit value in a VPBasicBlock is used as the branch selector. In
// the VPlan-native path case, since all branches are uniform we generate a
// branch instruction using the condition value from vector lane 0 and dummy
// successors. The successors are fixed later when the successor blocks are
// visited.
Value *NewCond = State->get(CBV, {0, 0});
// Replace the temporary unreachable terminator with the new conditional
// branch.
auto *CurrentTerminator = NewBB->getTerminator();
assert(isa<UnreachableInst>(CurrentTerminator) &&
"Expected to replace unreachable terminator with conditional "
"branch.");
auto *CondBr = BranchInst::Create(NewBB, nullptr, NewCond);
CondBr->setSuccessor(0, nullptr);
ReplaceInstWithInst(CurrentTerminator, CondBr);
}
LLVM_DEBUG(dbgs() << "LV: filled BB:" << *NewBB);
}
@ -428,6 +383,50 @@ VPRegionBlock *VPBasicBlock::getEnclosingLoopRegion() {
return P;
}
static bool hasConditionalTerminator(const VPBasicBlock *VPBB) {
if (VPBB->empty()) {
assert(
VPBB->getNumSuccessors() < 2 &&
"block with multiple successors doesn't have a recipe as terminator");
return false;
}
const VPRecipeBase *R = &VPBB->back();
auto *VPI = dyn_cast<VPInstruction>(R);
bool IsCondBranch =
isa<VPBranchOnMaskRecipe>(R) ||
(VPI && (VPI->getOpcode() == VPInstruction::BranchOnCond ||
VPI->getOpcode() == VPInstruction::BranchOnCount));
if (VPBB->getNumSuccessors() >= 2 || VPBB->isExiting()) {
assert(IsCondBranch && "block with multiple successors not terminated by "
"conditional branch recipe");
return true;
}
assert(
!IsCondBranch &&
"block with 0 or 1 successors terminated by conditional branch recipe");
return false;
}
VPRecipeBase *VPBasicBlock::getTerminator() {
if (hasConditionalTerminator(this))
return &back();
return nullptr;
}
const VPRecipeBase *VPBasicBlock::getTerminator() const {
if (hasConditionalTerminator(this))
return &back();
return nullptr;
}
bool VPBasicBlock::isExiting() const {
return getParent()->getExitingBasicBlock() == this;
}
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
void VPBlockBase::printSuccessors(raw_ostream &O, const Twine &Indent) const {
if (getSuccessors().empty()) {
@ -444,13 +443,6 @@ void VPBlockBase::printSuccessors(raw_ostream &O, const Twine &Indent) const {
void VPBasicBlock::print(raw_ostream &O, const Twine &Indent,
VPSlotTracker &SlotTracker) const {
O << Indent << getName() << ":\n";
if (const VPValue *Pred = getPredicate()) {
O << Indent << "BlockPredicate:";
Pred->printAsOperand(O, SlotTracker);
if (const auto *PredInst = dyn_cast<VPInstruction>(Pred))
O << " (" << PredInst->getParent()->getName() << ")";
O << '\n';
}
auto RecipeIndent = Indent + " ";
for (const VPRecipeBase &Recipe : *this) {
@ -459,14 +451,6 @@ void VPBasicBlock::print(raw_ostream &O, const Twine &Indent,
}
printSuccessors(O, Indent);
if (const VPValue *CBV = getCondBit()) {
O << Indent << "CondBit: ";
CBV->printAsOperand(O, SlotTracker);
if (const auto *CBI = dyn_cast<VPInstruction>(CBV))
O << " (" << CBI->getParent()->getName() << ")";
O << '\n';
}
}
#endif
@ -781,6 +765,26 @@ void VPInstruction::generateInstruction(VPTransformState &State,
State.set(this, Next, Part);
break;
}
case VPInstruction::BranchOnCond: {
if (Part != 0)
break;
Value *Cond = State.get(getOperand(0), VPIteration(Part, 0));
VPRegionBlock *ParentRegion = getParent()->getParent();
VPBasicBlock *Header = ParentRegion->getEntryBasicBlock();
// Replace the temporary unreachable terminator with a new conditional
// branch, hooking it up to backward destination for exiting blocks now and
// to forward destination(s) later when they are created.
BranchInst *CondBr =
Builder.CreateCondBr(Cond, Builder.GetInsertBlock(), nullptr);
if (getParent()->isExiting())
CondBr->setSuccessor(1, State.CFG.VPBB2IRBB[Header]);
CondBr->setSuccessor(0, nullptr);
Builder.GetInsertBlock()->getTerminator()->eraseFromParent();
break;
}
case VPInstruction::BranchOnCount: {
if (Part != 0)
break;
@ -854,6 +858,9 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
case VPInstruction::CanonicalIVIncrementNUW:
O << "VF * UF +(nuw) ";
break;
case VPInstruction::BranchOnCond:
O << "branch-on-cond";
break;
case VPInstruction::BranchOnCount:
O << "branch-on-count ";
break;

105
llvm/lib/Transforms/Vectorize/VPlan.h
View File

@ -351,41 +351,6 @@ struct VPTransformState {
Loop *CurrentVectorLoop = nullptr;
};
/// VPUsers instance used by VPBlockBase to manage CondBit and the block
/// predicate. Currently VPBlockUsers are used in VPBlockBase for historical
/// reasons, but in the future the only VPUsers should either be recipes or
/// live-outs.VPBlockBase uses.
struct VPBlockUser : public VPUser {
VPBlockUser() : VPUser({}, VPUserID::Block) {}
VPValue *getSingleOperandOrNull() {
if (getNumOperands() == 1)
return getOperand(0);
return nullptr;
}
const VPValue *getSingleOperandOrNull() const {
if (getNumOperands() == 1)
return getOperand(0);
return nullptr;
}
void resetSingleOpUser(VPValue *NewVal) {
assert(getNumOperands() <= 1 && "Didn't expect more than one operand!");
if (!NewVal) {
if (getNumOperands() == 1)
removeLastOperand();
return;
}
if (getNumOperands() == 1)
setOperand(0, NewVal);
else
addOperand(NewVal);
}
};
/// VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
/// A VPBlockBase can be either a VPBasicBlock or a VPRegionBlock.
class VPBlockBase {
@ -406,16 +371,6 @@ class VPBlockBase {
/// List of successor blocks.
SmallVector<VPBlockBase *, 1> Successors;
/// Successor selector managed by a VPUser. For blocks with zero or one
/// successors, there is no operand. Otherwise there is exactly one operand
/// which is the branch condition.
VPBlockUser CondBitUser;
/// If the block is predicated, its predicate is stored as an operand of this
/// VPUser to maintain the def-use relations. Otherwise there is no operand
/// here.
VPBlockUser PredicateUser;
/// VPlan containing the block. Can only be set on the entry block of the
/// plan.
VPlan *Plan = nullptr;
@ -561,20 +516,6 @@ public:
return getEnclosingBlockWithPredecessors()->getSinglePredecessor();
}
/// \return the condition bit selecting the successor.
VPValue *getCondBit();
/// \return the condition bit selecting the successor.
const VPValue *getCondBit() const;
/// Set the condition bit selecting the successor.
void setCondBit(VPValue *CV);
/// \return the block's predicate.
VPValue *getPredicate();
/// \return the block's predicate.
const VPValue *getPredicate() const;
/// Set the block's predicate.
void setPredicate(VPValue *Pred);
/// Set a given VPBlockBase \p Successor as the single successor of this
/// VPBlockBase. This VPBlockBase is not added as predecessor of \p Successor.
/// This VPBlockBase must have no successors.
@ -584,14 +525,11 @@ public:
}
/// Set two given VPBlockBases \p IfTrue and \p IfFalse to be the two
/// successors of this VPBlockBase. \p Condition is set as the successor
/// selector. This VPBlockBase is not added as predecessor of \p IfTrue or \p
/// IfFalse. This VPBlockBase must have no successors.
void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse,
VPValue *Condition) {
/// successors of this VPBlockBase. This VPBlockBase is not added as
/// predecessor of \p IfTrue or \p IfFalse. This VPBlockBase must have no
/// successors.
void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse) {
assert(Successors.empty() && "Setting two successors when others exist.");
assert(Condition && "Setting two successors without condition!");
setCondBit(Condition);
appendSuccessor(IfTrue);
appendSuccessor(IfFalse);
}
@ -608,11 +546,8 @@ public:
/// Remove all the predecessor of this block.
void clearPredecessors() { Predecessors.clear(); }
/// Remove all the successors of this block and set to null its condition bit
void clearSuccessors() {
Successors.clear();
setCondBit(nullptr);
}
/// Remove all the successors of this block.
void clearSuccessors() { Successors.clear(); }
/// The method which generates the output IR that correspond to this
/// VPBlockBase, thereby "executing" the VPlan.
@ -821,6 +756,7 @@ public:
CanonicalIVIncrement,
CanonicalIVIncrementNUW,
BranchOnCount,
BranchOnCond
};
private:
@ -909,6 +845,7 @@ public:
case Instruction::Unreachable:
case Instruction::Fence:
case Instruction::AtomicRMW:
case VPInstruction::BranchOnCond:
case VPInstruction::BranchOnCount:
return false;
default:
@ -2098,6 +2035,14 @@ public:
using VPBlockBase::print; // Get the print(raw_stream &O) version.
#endif
/// If the block has multiple successors, return the branch recipe terminating
/// the block. If there are no or only a single successor, return nullptr;
VPRecipeBase *getTerminator();
const VPRecipeBase *getTerminator() const;
/// Returns true if the block is exiting it's parent region.
bool isExiting() const;
private:
/// Create an IR BasicBlock to hold the output instructions generated by this
/// VPBasicBlock, and return it. Update the CFGState accordingly.
@ -2785,9 +2730,8 @@ public:
/// Insert disconnected VPBlockBase \p NewBlock after \p BlockPtr. Add \p
/// NewBlock as successor of \p BlockPtr and \p BlockPtr as predecessor of \p
/// NewBlock, and propagate \p BlockPtr parent to \p NewBlock. \p BlockPtr's
/// successors are moved from \p BlockPtr to \p NewBlock and \p BlockPtr's
/// conditional bit is propagated to \p NewBlock. \p NewBlock must have
/// neither successors nor predecessors.
/// successors are moved from \p BlockPtr to \p NewBlock. \p NewBlock must
/// have neither successors nor predecessors.
static void insertBlockAfter(VPBlockBase *NewBlock, VPBlockBase *BlockPtr) {
assert(NewBlock->getSuccessors().empty() &&
NewBlock->getPredecessors().empty() &&
@ -2798,24 +2742,22 @@ public:
disconnectBlocks(BlockPtr, Succ);
connectBlocks(NewBlock, Succ);
}
NewBlock->setCondBit(BlockPtr->getCondBit());
BlockPtr->setCondBit(nullptr);
connectBlocks(BlockPtr, NewBlock);
}
/// Insert disconnected VPBlockBases \p IfTrue and \p IfFalse after \p
/// BlockPtr. Add \p IfTrue and \p IfFalse as succesors of \p BlockPtr and \p
/// BlockPtr as predecessor of \p IfTrue and \p IfFalse. Propagate \p BlockPtr
/// parent to \p IfTrue and \p IfFalse. \p Condition is set as the successor
/// selector. \p BlockPtr must have no successors and \p IfTrue and \p IfFalse
/// must have neither successors nor predecessors.
/// parent to \p IfTrue and \p IfFalse. \p BlockPtr must have no successors
/// and \p IfTrue and \p IfFalse must have neither successors nor
/// predecessors.
static void insertTwoBlocksAfter(VPBlockBase *IfTrue, VPBlockBase *IfFalse,
VPValue *Condition, VPBlockBase *BlockPtr) {
VPBlockBase *BlockPtr) {
assert(IfTrue->getSuccessors().empty() &&
"Can't insert IfTrue with successors.");
assert(IfFalse->getSuccessors().empty() &&
"Can't insert IfFalse with successors.");
BlockPtr->setTwoSuccessors(IfTrue, IfFalse, Condition);
BlockPtr->setTwoSuccessors(IfTrue, IfFalse);
IfTrue->setPredecessors({BlockPtr});
IfFalse->setPredecessors({BlockPtr});
IfTrue->setParent(BlockPtr->getParent());
@ -3035,7 +2977,6 @@ bool onlyFirstLaneUsed(VPValue *Def);
/// create a new one.
VPValue *getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr,
ScalarEvolution &SE);
} // end namespace vputils
} // end namespace llvm

16
llvm/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
View File

@ -216,10 +216,13 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
"Instruction shouldn't have been visited.");
if (auto *Br = dyn_cast<BranchInst>(Inst)) {
// Branch instruction is not explicitly represented in VPlan but we need
// to represent its condition bit when it's conditional.
if (Br->isConditional())
getOrCreateVPOperand(Br->getCondition());
// Conditional branch instruction are represented using BranchOnCond
// recipes.
if (Br->isConditional()) {
VPValue *Cond = getOrCreateVPOperand(Br->getCondition());
VPBB->appendRecipe(
new VPInstruction(VPInstruction::BranchOnCond, {Cond}));
}
// Skip the rest of the Instruction processing for Branch instructions.
continue;
@ -310,10 +313,9 @@ VPBasicBlock *PlainCFGBuilder::buildPlainCFG() {
// representing the condition bit in VPlan (which may be in another VPBB).
assert(IRDef2VPValue.count(BrCond) &&
"Missing condition bit in IRDef2VPValue!");
VPValue *VPCondBit = IRDef2VPValue[BrCond];
// Link successors using condition bit.
VPBB->setTwoSuccessors(SuccVPBB0, SuccVPBB1, VPCondBit);
// Link successors.
VPBB->setTwoSuccessors(SuccVPBB0, SuccVPBB1);
} else
llvm_unreachable("Number of successors not supported.");

6
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
View File

@ -27,8 +27,12 @@ void VPlanTransforms::VPInstructionsToVPRecipes(
ReversePostOrderTraversal<VPBlockRecursiveTraversalWrapper<VPBlockBase *>>
RPOT(Plan->getEntry());
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
VPRecipeBase *Term = VPBB->getTerminator();
auto EndIter = Term ? Term->getIterator() : VPBB->end();
// Introduce each ingredient into VPlan.
for (VPRecipeBase &Ingredient : llvm::make_early_inc_range(*VPBB)) {
for (VPRecipeBase &Ingredient :
make_early_inc_range(make_range(VPBB->begin(), EndIter))) {
VPValue *VPV = Ingredient.getVPSingleValue();
Instruction *Inst = cast<Instruction>(VPV->getUnderlyingValue());
if (DeadInstructions.count(Inst)) {

3
llvm/lib/Transforms/Vectorize/VPlanValue.h
View File

@ -209,9 +209,6 @@ public:
enum class VPUserID {
Recipe,
LiveOut,
// TODO: Currently VPUsers are used in VPBlockBase, but in the future the
// only VPUsers should either be recipes or live-outs.
Block
};
private:

9
llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp
View File

@ -49,11 +49,14 @@ static void verifyBlocksInRegion(const VPRegionBlock *Region) {
// Check block's parent.
assert(VPB->getParent() == Region && "VPBlockBase has wrong parent");
auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
// Check block's condition bit.
if (VPB->getNumSuccessors() > 1 || Region->getExitingBasicBlock() == VPB)
assert(VPB->getCondBit() && "Missing condition bit!");
if (VPB->getNumSuccessors() > 1 || (VPBB && VPBB->isExiting()))
assert(VPBB && VPBB->getTerminator() &&
"Block has multiple successors but doesn't "
"have a proper branch recipe!");
else
assert(!VPB->getCondBit() && "Unexpected condition bit!");
assert((!VPBB || !VPBB->getTerminator()) && "Unexpected branch recipe!");
// Check block's successors.
const auto &Successors = VPB->getSuccessors();

8
llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll
View File

@ -32,7 +32,6 @@ define void @sink_replicate_region_1(i32 %x, i8* %ptr) optsize {
; CHECK-NEXT: pred.load.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body)
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>
@ -54,7 +53,6 @@ define void @sink_replicate_region_1(i32 %x, i8* %ptr) optsize {
; CHECK-NEXT: pred.srem.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>
; CHECK-NEXT: Successor(s): pred.srem.if, pred.srem.continue
; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body)
; CHECK-EMPTY:
; CHECK-NEXT: pred.srem.if:
; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> (S->V)
@ -128,7 +126,6 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, i32* %ptr) optsize {
; CHECK-NEXT: pred.store.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body)
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x>
@ -201,7 +198,6 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, i32* %ptr) optsize
; CHECK-NEXT: pred.srem.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>
; CHECK-NEXT: Successor(s): pred.srem.if, pred.srem.continue
; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body)
; CHECK-EMPTY:
; CHECK-NEXT: pred.srem.if:
; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> (S->V)
@ -278,7 +274,6 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, i8*
; CHECK-NEXT: pred.load.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body)
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep> (S->V)
@ -302,7 +297,6 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, i8*
; CHECK-NEXT: pred.load.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body)
; CHECK: pred.load.if:
; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x> (S->V)
@ -383,7 +377,6 @@ define void @sink_replicate_region_after_replicate_region(i32* %ptr, i32 %x, i8
; CHECK-NEXT: pred.srem.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>
; CHECK-NEXT: Successor(s): pred.srem.if, pred.srem.continue
; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body)
; CHECK-EMPTY:
; CHECK-NEXT: pred.srem.if:
; CHECK-NEXT: REPLICATE ir<%rem> = srem vp<[[SPLICE]]>, ir<%x>
@ -402,7 +395,6 @@ define void @sink_replicate_region_after_replicate_region(i32* %ptr, i32 %x, i8
; CHECK-NEXT: pred.store.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body)
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: REPLICATE ir<%rem.div> = sdiv ir<20>, vp<[[PRED]]>

1
llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll
View File

@ -58,7 +58,6 @@ for.end:
; CHECK-NEXT: pred.store.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[COND]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-NEXT: CondBit: vp<[[COND]]> (vector.body)
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEPS]]>

4
llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll
View File

@ -26,16 +26,16 @@ define void @foo(i64 %n) {
; CHECK-NEXT: EMIT store ir<%add> ir<%gep.2>
; CHECK-NEXT: EMIT ir<%inner.iv.next> = add ir<%inner.iv> ir<1>
; CHECK-NEXT: EMIT ir<%inner.ec> = icmp ir<%inner.iv.next> ir<8>
; CHECK-NEXT: EMIT branch-on-cond ir<%inner.ec>
; CHECK-NEXT: No successors
; CHECK-NEXT: CondBit: ir<%inner.ec> (inner)
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): outer.latch
; CHECK-EMPTY:
; CHECK-NEXT: outer.latch:
; CHECK-NEXT: EMIT ir<%outer.iv.next> = add ir<%outer.iv> ir<1>
; CHECK-NEXT: EMIT ir<%outer.ec> = icmp ir<%outer.iv.next> ir<8>
; CHECK-NEXT: EMIT branch-on-cond ir<%outer.ec>
; CHECK-NEXT: No successors
; CHECK-NEXT: CondBit: ir<%outer.ec> (outer.latch)
; CHECK-NEXT: }
; CHECK-NEXT: Successor(s): exit
; CHECK-EMPTY:

2
llvm/test/Transforms/LoopVectorize/vplan-printing.ll
View File

@ -215,7 +215,6 @@ define void @print_replicate_predicated_phi(i64 %n, i64* %x) {
; CHECK-NEXT: pred.udiv.entry:
; CHECK-NEXT: BRANCH-ON-MASK ir<%cmp>
; CHECK-NEXT: Successor(s): pred.udiv.if, pred.udiv.continue
; CHECK-NEXT: CondBit: ir<%cmp>
; CHECK-EMPTY:
; CHECK-NEXT: pred.udiv.if:
; CHECK-NEXT: REPLICATE ir<%tmp4> = udiv ir<%n>, vp<[[STEPS]]> (S->V)
@ -420,7 +419,6 @@ define void @debug_loc_vpinstruction(i32* nocapture %asd, i32* nocapture %bsd) !
; CHECK-NEXT: pred.sdiv.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[OR1]]>
; CHECK-NEXT: Successor(s): pred.sdiv.if, pred.sdiv.continue
; CHECK-NEXT: CondBit: vp<[[OR1]]> (if.then)
; CHECK-EMPTY:
; CHECK-NEXT: pred.sdiv.if:
; CHECK-NEXT: REPLICATE ir<%sd1> = sdiv ir<%psd>, ir<%lsd> (S->V)

1
llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll
View File

@ -31,7 +31,6 @@ define void @sink_with_sideeffects(i1 %c, i8* %ptr) {
; CHECK-NEXT: pred.store.entry:
; CHECK-NEXT: BRANCH-ON-MASK ir<%c>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-NEXT: CondBit: ir<%c>
; CHECK: pred.store.if:
; CHECK-NEXT: CLONE store ir<%tmp5>, ir<%tmp2>

20
llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
View File

@ -33,7 +33,6 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
; CHECK-NEXT: pred.store.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body)
; CHECK: pred.store.if:
; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<[[STEPS]]>
@ -100,7 +99,6 @@ exit:
; CHECK-NEXT: pred.load.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body)
; CHECK: pred.load.if:
; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<[[STEPS]]>
@ -120,7 +118,6 @@ exit:
; CHECK-NEXT: pred.store.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body)
; CHECK: pred.store.if:
; CHECK-NEXT: REPLICATE ir<%add> = add vp<[[PRED]]>, ir<10>
@ -183,7 +180,6 @@ exit:
; CHECK-NEXT: pred.load.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body)
; CHECK: pred.load.if:
; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<[[STEPS]]>
@ -204,7 +200,6 @@ exit:
; CHECK-NEXT: pred.store.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body)
; CHECK: pred.store.if:
; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, ir<%mul>
@ -270,7 +265,6 @@ define void @uniform_gep(i64 %k, i16* noalias %A, i16* noalias %B) {
; CHECK-NEXT: pred.load.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body)
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: REPLICATE ir<%lv> = load ir<%gep.A.uniform>
@ -295,7 +289,6 @@ define void @uniform_gep(i64 %k, i16* noalias %A, i16* noalias %B) {
; CHECK-NEXT: pred.store.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK2]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-NEXT: CondBit: vp<[[MASK2]]> (loop.then)
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: REPLICATE ir<%gep.B> = getelementptr ir<%B>, vp<[[STEPS]]>
@ -368,7 +361,6 @@ define void @pred_cfg1(i32 %k, i32 %j) {
; CHECK-NEXT: pred.load.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK2]]>
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-NEXT: CondBit: vp<[[MASK2]]> (then.0)
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<[[STEPS]]>
@ -395,7 +387,6 @@ define void @pred_cfg1(i32 %k, i32 %j) {
; CHECK-NEXT: pred.store.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[OR]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-NEXT: CondBit: vp<[[OR]]> (next.0)
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, ir<%mul>
@ -474,7 +465,6 @@ define void @pred_cfg2(i32 %k, i32 %j) {
; CHECK-NEXT: pred.load.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK2]]>
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-NEXT: CondBit: vp<[[MASK2]]> (then.0)
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<[[STEPS]]>
@ -505,7 +495,6 @@ define void @pred_cfg2(i32 %k, i32 %j) {
; CHECK-NEXT: pred.store.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK4]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-NEXT: CondBit: vp<[[MASK4]]> (then.1)
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, ir<%mul>
@ -593,7 +582,6 @@ define void @pred_cfg3(i32 %k, i32 %j) {
; CHECK-NEXT: pred.load.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK2]]>
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-NEXT: CondBit: vp<[[MASK2]]> (then.0)
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: REPLICATE ir<%gep.b> = getelementptr ir<@b>, ir<0>, vp<[[STEPS]]>
@ -623,7 +611,6 @@ define void @pred_cfg3(i32 %k, i32 %j) {
; CHECK-NEXT: pred.store.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK5]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-NEXT: CondBit: vp<[[MASK5]]> (then.1)
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: REPLICATE ir<%gep.a> = getelementptr ir<@a>, ir<0>, ir<%mul>
@ -711,7 +698,6 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) {
; CHECK-NEXT: pred.store.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body)
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a>
@ -742,7 +728,6 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) {
; CHECK-NEXT: pred.store.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK2]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-NEXT: CondBit: vp<[[MASK2]]> (then.0)
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: REPLICATE ir<%gep.c.1> = getelementptr ir<@c>, ir<0>, vp<[[STEPS]]>
@ -827,7 +812,6 @@ define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) {
; CHECK-NEXT: pred.store.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body)
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a>
@ -893,7 +877,6 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) {
; CHECK-NEXT: pred.load.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>
; CHECK-NEXT: Successor(s): pred.load.if, pred.load.continue
; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body)
; CHECK-EMPTY:
; CHECK-NEXT: pred.load.if:
; CHECK-NEXT: REPLICATE ir<%lv.a> = load ir<%gep.a>
@ -916,7 +899,6 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) {
; CHECK-NEXT: pred.store.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-NEXT: CondBit: vp<[[MASK]]> (vector.body)
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: REPLICATE ir<%div> = sdiv vp<[[SPLICE]]>, vp<[[PRED]]>
@ -980,7 +962,6 @@ define void @update_multiple_users(i16* noalias %src, i8* noalias %dst, i1 %c) {
; CHECK-NEXT: pred.store.entry:
; CHECK-NEXT: BRANCH-ON-MASK ir<%c>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-NEXT: CondBit: ir<%c>
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: REPLICATE ir<%l1> = load ir<%src>
@ -1059,7 +1040,6 @@ define void @sinking_requires_duplication(float* %addr) {
; CHECK-NEXT: pred.store.entry:
; CHECK-NEXT: BRANCH-ON-MASK vp<[[MASK]]>
; CHECK-NEXT: Successor(s): pred.store.if, pred.store.continue
; CHECK-NEXT: CondBit: vp<[[MASK]]> (then)
; CHECK-EMPTY:
; CHECK-NEXT: pred.store.if:
; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr ir<%addr>, vp<[[STEPS]]>

3
llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
View File

@ -45,7 +45,6 @@ TEST_F(VPlanHCFGTest, testBuildHCFGInnerLoop) {
EXPECT_NE(nullptr, Entry->getSingleSuccessor());
EXPECT_EQ(0u, Entry->getNumPredecessors());
EXPECT_EQ(1u, Entry->getNumSuccessors());
EXPECT_EQ(nullptr, Entry->getCondBit());
// Check that the region following the preheader is a single basic-block
// region (loop).
@ -91,7 +90,6 @@ TEST_F(VPlanHCFGTest, testBuildHCFGInnerLoop) {
EXPECT_EQ(Instruction::ICmp, ICmp->getOpcode());
EXPECT_EQ(2u, ICmp->getNumOperands());
EXPECT_EQ(IndvarAdd, ICmp->getOperand(0));
EXPECT_EQ(VecBB->getCondBit(), ICmp);
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
// Add an external value to check we do not print the list of external values,
@ -123,7 +121,6 @@ compound=true
" EMIT ir\<%indvars.iv.next\> = add ir\<%indvars.iv\> ir\<1\>\l" +
" EMIT ir\<%exitcond\> = icmp ir\<%indvars.iv.next\> ir\<%N\>\l" +
"No successors\l" +
"CondBit: ir\<%exitcond\> (vector.body)\l"
]
}
N1 -> N3 [ label="" ltail=cluster_N2]