[VPlan] Use region for each loop in native path.
This patch updates the VPlan native path to use VPRegionBlocks for all loops in a loop nest. Up to now, only the outermost loop used a region. This is a step towards unifying both paths and keep things consistent between them. It also prepares various code-gen parts for modeling the pre-header in the inner loop vectorizer (D121624). Reviewed By: Ayal Differential Revision: https://reviews.llvm.org/D123005
This commit is contained in:
12 files changed
+128
-145
No files matched your search
@@ -9089,31 +9089,6 @@ VPlanPtr LoopVectorizationPlanner::buildVPlan(VFRange &Range) {
|
||||
[this](PHINode *P) { return Legal->getIntOrFpInductionDescriptor(P); },
|
||||
DeadInstructions, *PSE.getSE());
|
||||
|
||||
// Update plan to be compatible with the inner loop vectorizer for
|
||||
// code-generation.
|
||||
VPRegionBlock *LoopRegion = Plan->getVectorLoopRegion();
|
||||
VPBasicBlock *Preheader = LoopRegion->getEntryBasicBlock();
|
||||
VPBasicBlock *Exiting = LoopRegion->getExitingBasicBlock();
|
||||
VPBlockBase *Latch = Exiting->getSinglePredecessor();
|
||||
VPBlockBase *Header = Preheader->getSingleSuccessor();
|
||||
|
||||
// 1. Move preheader block out of main vector loop.
|
||||
Preheader->setParent(LoopRegion->getParent());
|
||||
VPBlockUtils::disconnectBlocks(Preheader, Header);
|
||||
VPBlockUtils::connectBlocks(Preheader, LoopRegion);
|
||||
Plan->setEntry(Preheader);
|
||||
|
||||
// 2. Disconnect backedge and exiting block.
|
||||
VPBlockUtils::disconnectBlocks(Latch, Header);
|
||||
VPBlockUtils::disconnectBlocks(Latch, Exiting);
|
||||
|
||||
// 3. Update entry and exiting of main vector loop region.
|
||||
LoopRegion->setEntry(Header);
|
||||
LoopRegion->setExiting(Latch);
|
||||
|
||||
// 4. Remove exiting block.
|
||||
delete Exiting;
|
||||
|
||||
addCanonicalIVRecipes(*Plan, Legal->getWidestInductionType(), DebugLoc(),
|
||||
true, true);
|
||||
return Plan;
|
||||
|
||||
@@ -265,19 +265,6 @@ VPBasicBlock::createEmptyBasicBlock(VPTransformState::CFGState &CFG) {
|
||||
auto &PredVPSuccessors = PredVPBB->getSuccessors();
|
||||
BasicBlock *PredBB = CFG.VPBB2IRBB[PredVPBB];
|
||||
|
||||
// In outer loop vectorization scenario, the predecessor BBlock may not yet
|
||||
// be visited(backedge). Mark the VPBasicBlock for fixup at the end of
|
||||
// vectorization. We do not encounter this case in inner loop vectorization
|
||||
// as we start out by building a loop skeleton with the vector loop header
|
||||
// and latch blocks. As a result, we never enter this function for the
|
||||
// header block in the non VPlan-native path.
|
||||
if (!PredBB) {
|
||||
assert(EnableVPlanNativePath &&
|
||||
"Unexpected null predecessor in non VPlan-native path");
|
||||
CFG.VPBBsToFix.push_back(PredVPBB);
|
||||
continue;
|
||||
}
|
||||
|
||||
assert(PredBB && "Predecessor basic-block not found building successor.");
|
||||
auto *PredBBTerminator = PredBB->getTerminator();
|
||||
LLVM_DEBUG(dbgs() << "LV: draw edge from" << PredBB->getName() << '\n');
|
||||
@@ -968,24 +955,6 @@ void VPlan::execute(VPTransformState *State) {
|
||||
for (VPBlockBase *Block : depth_first(Entry))
|
||||
Block->execute(State);
|
||||
|
||||
// Setup branch terminator successors for VPBBs in VPBBsToFix based on
|
||||
// VPBB's successors.
|
||||
for (auto VPBB : State->CFG.VPBBsToFix) {
|
||||
assert(EnableVPlanNativePath &&
|
||||
"Unexpected VPBBsToFix in non VPlan-native path");
|
||||
BasicBlock *BB = State->CFG.VPBB2IRBB[VPBB];
|
||||
assert(BB && "Unexpected null basic block for VPBB");
|
||||
|
||||
unsigned Idx = 0;
|
||||
auto *BBTerminator = BB->getTerminator();
|
||||
|
||||
for (VPBlockBase *SuccVPBlock : VPBB->getHierarchicalSuccessors()) {
|
||||
VPBasicBlock *SuccVPBB = SuccVPBlock->getEntryBasicBlock();
|
||||
BBTerminator->setSuccessor(Idx, State->CFG.VPBB2IRBB[SuccVPBB]);
|
||||
++Idx;
|
||||
}
|
||||
}
|
||||
|
||||
VPBasicBlock *LatchVPBB = getVectorLoopRegion()->getExitingBasicBlock();
|
||||
BasicBlock *VectorLatchBB = State->CFG.VPBB2IRBB[LatchVPBB];
|
||||
|
||||
|
||||
@@ -316,10 +316,6 @@ struct VPTransformState {
|
||||
/// of replication, maps the BasicBlock of the last replica created.
|
||||
SmallDenseMap<VPBasicBlock *, BasicBlock *> VPBB2IRBB;
|
||||
|
||||
/// Vector of VPBasicBlocks whose terminator instruction needs to be fixed
|
||||
/// up at the end of vector code generation.
|
||||
SmallVector<VPBasicBlock *, 8> VPBBsToFix;
|
||||
|
||||
CFGState() = default;
|
||||
|
||||
/// Returns the BasicBlock* mapped to the pre-header of the loop region
|
||||
@@ -2667,13 +2663,9 @@ public:
|
||||
|
||||
/// Returns the VPRegionBlock of the vector loop.
|
||||
VPRegionBlock *getVectorLoopRegion() {
|
||||
if (auto *R = dyn_cast<VPRegionBlock>(getEntry()))
|
||||
return R;
|
||||
return cast<VPRegionBlock>(getEntry()->getSingleSuccessor());
|
||||
}
|
||||
const VPRegionBlock *getVectorLoopRegion() const {
|
||||
if (auto *R = dyn_cast<VPRegionBlock>(getEntry()))
|
||||
return R;
|
||||
return cast<VPRegionBlock>(getEntry()->getSingleSuccessor());
|
||||
}
|
||||
|
||||
|
||||
@@ -42,9 +42,6 @@ private:
|
||||
// Vectorization plan that we are working on.
|
||||
VPlan &Plan;
|
||||
|
||||
// Output Top Region.
|
||||
VPRegionBlock *TopRegion = nullptr;
|
||||
|
||||
// Builder of the VPlan instruction-level representation.
|
||||
VPBuilder VPIRBuilder;
|
||||
|
||||
@@ -59,6 +56,9 @@ private:
|
||||
// Hold phi node's that need to be fixed once the plain CFG has been built.
|
||||
SmallVector<PHINode *, 8> PhisToFix;
|
||||
|
||||
/// Maps loops in the original IR to their corresponding region.
|
||||
DenseMap<Loop *, VPRegionBlock *> Loop2Region;
|
||||
|
||||
// Utility functions.
|
||||
void setVPBBPredsFromBB(VPBasicBlock *VPBB, BasicBlock *BB);
|
||||
void fixPhiNodes();
|
||||
@@ -73,8 +73,9 @@ public:
|
||||
PlainCFGBuilder(Loop *Lp, LoopInfo *LI, VPlan &P)
|
||||
: TheLoop(Lp), LI(LI), Plan(P) {}
|
||||
|
||||
// Build the plain CFG and return its Top Region.
|
||||
VPRegionBlock *buildPlainCFG();
|
||||
/// Build plain CFG for TheLoop. Return the pre-header VPBasicBlock connected
|
||||
/// to a new VPRegionBlock (TopRegion) enclosing the plain CFG.
|
||||
VPBasicBlock *buildPlainCFG();
|
||||
};
|
||||
} // anonymous namespace
|
||||
|
||||
@@ -106,19 +107,32 @@ void PlainCFGBuilder::fixPhiNodes() {
|
||||
}
|
||||
}
|
||||
|
||||
// Create a new empty VPBasicBlock for an incoming BasicBlock or retrieve an
|
||||
// existing one if it was already created.
|
||||
// Create a new empty VPBasicBlock for an incoming BasicBlock in the region
|
||||
// corresponding to the containing loop or retrieve an existing one if it was
|
||||
// already created. If no region exists yet for the loop containing \p BB, a new
|
||||
// one is created.
|
||||
VPBasicBlock *PlainCFGBuilder::getOrCreateVPBB(BasicBlock *BB) {
|
||||
auto BlockIt = BB2VPBB.find(BB);
|
||||
if (BlockIt != BB2VPBB.end())
|
||||
// Retrieve existing VPBB.
|
||||
return BlockIt->second;
|
||||
|
||||
// Get or create a region for the loop containing BB.
|
||||
Loop *CurrentLoop = LI->getLoopFor(BB);
|
||||
VPRegionBlock *ParentR = nullptr;
|
||||
if (CurrentLoop) {
|
||||
auto Iter = Loop2Region.insert({CurrentLoop, nullptr});
|
||||
if (Iter.second)
|
||||
Iter.first->second = new VPRegionBlock(
|
||||
CurrentLoop->getHeader()->getName().str(), false /*isReplicator*/);
|
||||
ParentR = Iter.first->second;
|
||||
}
|
||||
|
||||
// Create new VPBB.
|
||||
LLVM_DEBUG(dbgs() << "Creating VPBasicBlock for " << BB->getName() << "\n");
|
||||
VPBasicBlock *VPBB = new VPBasicBlock(BB->getName());
|
||||
BB2VPBB[BB] = VPBB;
|
||||
VPBB->setParent(TopRegion);
|
||||
VPBB->setParent(ParentR);
|
||||
return VPBB;
|
||||
}
|
||||
|
||||
@@ -237,11 +251,8 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
|
||||
}
|
||||
|
||||
// Main interface to build the plain CFG.
|
||||
VPRegionBlock *PlainCFGBuilder::buildPlainCFG() {
|
||||
// 1. Create the Top Region. It will be the parent of all VPBBs.
|
||||
TopRegion = new VPRegionBlock("TopRegion", false /*isReplicator*/);
|
||||
|
||||
// 2. Scan the body of the loop in a topological order to visit each basic
|
||||
VPBasicBlock *PlainCFGBuilder::buildPlainCFG() {
|
||||
// 1. Scan the body of the loop in a topological order to visit each basic
|
||||
// block after having visited its predecessor basic blocks. Create a VPBB for
|
||||
// each BB and link it to its successor and predecessor VPBBs. Note that
|
||||
// predecessors must be set in the same order as they are in the incomming IR.
|
||||
@@ -250,11 +261,12 @@ VPRegionBlock *PlainCFGBuilder::buildPlainCFG() {
|
||||
|
||||
// Loop PH needs to be explicitly visited since it's not taken into account by
|
||||
// LoopBlocksDFS.
|
||||
BasicBlock *PreheaderBB = TheLoop->getLoopPreheader();
|
||||
assert((PreheaderBB->getTerminator()->getNumSuccessors() == 1) &&
|
||||
BasicBlock *ThePreheaderBB = TheLoop->getLoopPreheader();
|
||||
assert((ThePreheaderBB->getTerminator()->getNumSuccessors() == 1) &&
|
||||
"Unexpected loop preheader");
|
||||
VPBasicBlock *PreheaderVPBB = getOrCreateVPBB(PreheaderBB);
|
||||
for (auto &I : *PreheaderBB) {
|
||||
VPBasicBlock *ThePreheaderVPBB = getOrCreateVPBB(ThePreheaderBB);
|
||||
ThePreheaderVPBB->setName("vector.ph");
|
||||
for (auto &I : *ThePreheaderBB) {
|
||||
if (I.getType()->isVoidTy())
|
||||
continue;
|
||||
IRDef2VPValue[&I] = Plan.getOrAddExternalDef(&I);
|
||||
@@ -262,8 +274,7 @@ VPRegionBlock *PlainCFGBuilder::buildPlainCFG() {
|
||||
// Create empty VPBB for Loop H so that we can link PH->H.
|
||||
VPBlockBase *HeaderVPBB = getOrCreateVPBB(TheLoop->getHeader());
|
||||
HeaderVPBB->setName("vector.body");
|
||||
// Preheader's predecessors will be set during the loop RPO traversal below.
|
||||
PreheaderVPBB->setOneSuccessor(HeaderVPBB);
|
||||
ThePreheaderVPBB->setOneSuccessor(HeaderVPBB);
|
||||
|
||||
LoopBlocksRPO RPO(TheLoop);
|
||||
RPO.perform(LI);
|
||||
@@ -310,30 +321,61 @@ VPRegionBlock *PlainCFGBuilder::buildPlainCFG() {
|
||||
setVPBBPredsFromBB(VPBB, BB);
|
||||
}
|
||||
|
||||
// 3. Process outermost loop exit. We created an empty VPBB for the loop
|
||||
// 2. Process outermost loop exit. We created an empty VPBB for the loop
|
||||
// single exit BB during the RPO traversal of the loop body but Instructions
|
||||
// weren't visited because it's not part of the the loop.
|
||||
BasicBlock *LoopExitBB = TheLoop->getUniqueExitBlock();
|
||||
assert(LoopExitBB && "Loops with multiple exits are not supported.");
|
||||
VPBasicBlock *LoopExitVPBB = BB2VPBB[LoopExitBB];
|
||||
createVPInstructionsForVPBB(LoopExitVPBB, LoopExitBB);
|
||||
// Loop exit was already set as successor of the loop exiting BB.
|
||||
// We only set its predecessor VPBB now.
|
||||
setVPBBPredsFromBB(LoopExitVPBB, LoopExitBB);
|
||||
|
||||
// 3. Fix up region blocks for loops. For each loop,
|
||||
// * use the header block as entry to the corresponding region,
|
||||
// * use the latch block as exit of the corresponding region,
|
||||
// * set the region as successor of the loop pre-header, and
|
||||
// * set the exit block as successor to the region.
|
||||
SmallVector<Loop *> LoopWorkList;
|
||||
LoopWorkList.push_back(TheLoop);
|
||||
while (!LoopWorkList.empty()) {
|
||||
Loop *L = LoopWorkList.pop_back_val();
|
||||
BasicBlock *Header = L->getHeader();
|
||||
BasicBlock *Exiting = L->getLoopLatch();
|
||||
assert(Exiting == L->getExitingBlock() &&
|
||||
"Latch must be the only exiting block");
|
||||
VPRegionBlock *Region = Loop2Region[L];
|
||||
VPBasicBlock *HeaderVPBB = getOrCreateVPBB(Header);
|
||||
VPBasicBlock *ExitingVPBB = getOrCreateVPBB(Exiting);
|
||||
|
||||
// Disconnect backedge and pre-header from header.
|
||||
VPBasicBlock *PreheaderVPBB = getOrCreateVPBB(L->getLoopPreheader());
|
||||
VPBlockUtils::disconnectBlocks(PreheaderVPBB, HeaderVPBB);
|
||||
VPBlockUtils::disconnectBlocks(ExitingVPBB, HeaderVPBB);
|
||||
|
||||
Region->setParent(PreheaderVPBB->getParent());
|
||||
Region->setEntry(HeaderVPBB);
|
||||
VPBlockUtils::connectBlocks(PreheaderVPBB, Region);
|
||||
|
||||
// Disconnect exit block from exiting (=latch) block, set exiting block and
|
||||
// connect region to exit block.
|
||||
VPBasicBlock *ExitVPBB = getOrCreateVPBB(L->getExitBlock());
|
||||
VPBlockUtils::disconnectBlocks(ExitingVPBB, ExitVPBB);
|
||||
Region->setExiting(ExitingVPBB);
|
||||
VPBlockUtils::connectBlocks(Region, ExitVPBB);
|
||||
|
||||
// Queue sub-loops for processing.
|
||||
LoopWorkList.append(L->begin(), L->end());
|
||||
}
|
||||
// 4. The whole CFG has been built at this point so all the input Values must
|
||||
// have a VPlan couterpart. Fix VPlan phi nodes by adding their corresponding
|
||||
// VPlan operands.
|
||||
fixPhiNodes();
|
||||
|
||||
// 5. Final Top Region setup. Set outermost loop pre-header and single exit as
|
||||
// Top Region entry and exit.
|
||||
TopRegion->setEntry(PreheaderVPBB);
|
||||
TopRegion->setExiting(LoopExitVPBB);
|
||||
return TopRegion;
|
||||
return ThePreheaderVPBB;
|
||||
}
|
||||
|
||||
VPRegionBlock *VPlanHCFGBuilder::buildPlainCFG() {
|
||||
VPBasicBlock *VPlanHCFGBuilder::buildPlainCFG() {
|
||||
PlainCFGBuilder PCFGBuilder(TheLoop, LI, Plan);
|
||||
return PCFGBuilder.buildPlainCFG();
|
||||
}
|
||||
@@ -341,10 +383,11 @@ VPRegionBlock *VPlanHCFGBuilder::buildPlainCFG() {
|
||||
// Public interface to build a H-CFG.
|
||||
void VPlanHCFGBuilder::buildHierarchicalCFG() {
|
||||
// Build Top Region enclosing the plain CFG and set it as VPlan entry.
|
||||
VPRegionBlock *TopRegion = buildPlainCFG();
|
||||
Plan.setEntry(TopRegion);
|
||||
VPBasicBlock *EntryVPBB = buildPlainCFG();
|
||||
Plan.setEntry(EntryVPBB);
|
||||
LLVM_DEBUG(Plan.setName("HCFGBuilder: Plain CFG\n"); dbgs() << Plan);
|
||||
|
||||
VPRegionBlock *TopRegion = Plan.getVectorLoopRegion();
|
||||
Verifier.verifyHierarchicalCFG(TopRegion);
|
||||
|
||||
// Compute plain CFG dom tree for VPLInfo.
|
||||
|
||||
@@ -57,9 +57,9 @@ private:
|
||||
// are introduced.
|
||||
VPDominatorTree VPDomTree;
|
||||
|
||||
/// Build plain CFG for TheLoop. Return a new VPRegionBlock (TopRegion)
|
||||
/// enclosing the plain CFG.
|
||||
VPRegionBlock *buildPlainCFG();
|
||||
/// Build plain CFG for TheLoop. Return the pre-header VPBasicBlock connected
|
||||
/// to a new VPRegionBlock (TopRegion) enclosing the plain CFG.
|
||||
VPBasicBlock *buildPlainCFG();
|
||||
|
||||
public:
|
||||
VPlanHCFGBuilder(Loop *Lp, LoopInfo *LI, VPlan &P)
|
||||
|
||||
@@ -24,15 +24,9 @@ void VPlanTransforms::VPInstructionsToVPRecipes(
|
||||
GetIntOrFpInductionDescriptor,
|
||||
SmallPtrSetImpl<Instruction *> &DeadInstructions, ScalarEvolution &SE) {
|
||||
|
||||
auto *TopRegion = cast<VPRegionBlock>(Plan->getEntry());
|
||||
ReversePostOrderTraversal<VPBlockBase *> RPOT(TopRegion->getEntry());
|
||||
|
||||
for (VPBlockBase *Base : RPOT) {
|
||||
// Do not widen instructions in pre-header and exit blocks.
|
||||
if (Base->getNumPredecessors() == 0 || Base->getNumSuccessors() == 0)
|
||||
continue;
|
||||
|
||||
VPBasicBlock *VPBB = Base->getEntryBasicBlock();
|
||||
ReversePostOrderTraversal<VPBlockRecursiveTraversalWrapper<VPBlockBase *>>
|
||||
RPOT(Plan->getEntry());
|
||||
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
|
||||
// Introduce each ingredient into VPlan.
|
||||
for (VPRecipeBase &Ingredient : llvm::make_early_inc_range(*VPBB)) {
|
||||
VPValue *VPV = Ingredient.getVPSingleValue();
|
||||
|
||||
@@ -50,7 +50,7 @@ static void verifyBlocksInRegion(const VPRegionBlock *Region) {
|
||||
assert(VPB->getParent() == Region && "VPBlockBase has wrong parent");
|
||||
|
||||
// Check block's condition bit.
|
||||
if (VPB->getNumSuccessors() > 1)
|
||||
if (VPB->getNumSuccessors() > 1 || Region->getExitingBasicBlock() == VPB)
|
||||
assert(VPB->getCondBit() && "Missing condition bit!");
|
||||
else
|
||||
assert(!VPB->getCondBit() && "Unexpected condition bit!");
|
||||
|
||||
@@ -8,10 +8,10 @@
|
||||
define void @foo(i64 %n) {
|
||||
; CHECK: VPlan 'HCFGBuilder: Plain CFG
|
||||
; CHECK-NEXT: {
|
||||
; CHECK-NEXT: <x1> TopRegion: {
|
||||
; CHECK-NEXT: entry:
|
||||
; CHECK-NEXT: Successor(s): vector.body
|
||||
; CHECK-NEXT: vector.ph:
|
||||
; CHECK-NEXT: Successor(s): outer.header
|
||||
; CHECK-EMPTY:
|
||||
; CHECK-NEXT: <x1> outer.header: {
|
||||
; CHECK-NEXT: vector.body:
|
||||
; CHECK-NEXT: WIDEN-PHI ir<%outer.iv> = phi ir<0>, ir<%outer.iv.next>
|
||||
; CHECK-NEXT: EMIT ir<%gep.1> = getelementptr ir<@arr2> ir<0> ir<%outer.iv>
|
||||
@@ -19,25 +19,27 @@ define void @foo(i64 %n) {
|
||||
; CHECK-NEXT: EMIT ir<%add> = add ir<%outer.iv> ir<%n>
|
||||
; CHECK-NEXT: Successor(s): inner
|
||||
; CHECK-EMPTY:
|
||||
; CHECK-NEXT: inner:
|
||||
; CHECK-NEXT: WIDEN-PHI ir<%inner.iv> = phi ir<0>, ir<%inner.iv.next>
|
||||
; CHECK-NEXT: EMIT ir<%gep.2> = getelementptr ir<@arr> ir<0> ir<%inner.iv> ir<%outer.iv>
|
||||
; CHECK-NEXT: EMIT store ir<%add> ir<%gep.2>
|
||||
; CHECK-NEXT: EMIT ir<%inner.iv.next> = add ir<%inner.iv> ir<1>
|
||||
; CHECK-NEXT: EMIT ir<%inner.ec> = icmp ir<%inner.iv.next> ir<8>
|
||||
; CHECK-NEXT: Successor(s): outer.latch, inner
|
||||
; CHECK-NEXT: <x1> inner: {
|
||||
; CHECK-NEXT: inner:
|
||||
; CHECK-NEXT: WIDEN-PHI ir<%inner.iv> = phi ir<0>, ir<%inner.iv.next>
|
||||
; CHECK-NEXT: EMIT ir<%gep.2> = getelementptr ir<@arr> ir<0> ir<%inner.iv> ir<%outer.iv>
|
||||
; CHECK-NEXT: EMIT store ir<%add> ir<%gep.2>
|
||||
; CHECK-NEXT: EMIT ir<%inner.iv.next> = add ir<%inner.iv> ir<1>
|
||||
; CHECK-NEXT: EMIT ir<%inner.ec> = icmp ir<%inner.iv.next> ir<8>
|
||||
; CHECK-NEXT: No successors
|
||||
; CHECK-NEXT: CondBit: ir<%inner.ec> (inner)
|
||||
; CHECK-NEXT: }
|
||||
; CHECK-NEXT: Successor(s): outer.latch
|
||||
; CHECK-EMPTY:
|
||||
; CHECK-NEXT: outer.latch:
|
||||
; CHECK-NEXT: EMIT ir<%outer.iv.next> = add ir<%outer.iv> ir<1>
|
||||
; CHECK-NEXT: EMIT ir<%outer.ec> = icmp ir<%outer.iv.next> ir<8>
|
||||
; CHECK-NEXT: Successor(s): exit, vector.body
|
||||
; CHECK-NEXT: CondBit: ir<%outer.ec> (outer.latch)
|
||||
; CHECK-EMPTY:
|
||||
; CHECK-NEXT: exit:
|
||||
; CHECK-NEXT: EMIT ret
|
||||
; CHECK-NEXT: No successors
|
||||
; CHECK-NEXT: }
|
||||
; CHECK-NEXT: CondBit: ir<%outer.ec> (outer.latch)
|
||||
; CHECK-NEXT: }
|
||||
; CHECK-NEXT: Successor(s): exit
|
||||
; CHECK-EMPTY:
|
||||
; CHECK-NEXT: exit:
|
||||
; CHECK-NEXT: No successors
|
||||
; CHECK-NEXT: }
|
||||
entry:
|
||||
|
||||
@@ -47,10 +47,14 @@ TEST_F(VPlanHCFGTest, testBuildHCFGInnerLoop) {
|
||||
EXPECT_EQ(1u, Entry->getNumSuccessors());
|
||||
EXPECT_EQ(nullptr, Entry->getCondBit());
|
||||
|
||||
// Check that the region following the preheader is a single basic-block
|
||||
// region (loop).
|
||||
VPBasicBlock *VecBB = Entry->getSingleSuccessor()->getEntryBasicBlock();
|
||||
EXPECT_EQ(7u, VecBB->size());
|
||||
EXPECT_EQ(2u, VecBB->getNumPredecessors());
|
||||
EXPECT_EQ(2u, VecBB->getNumSuccessors());
|
||||
EXPECT_EQ(0u, VecBB->getNumPredecessors());
|
||||
EXPECT_EQ(0u, VecBB->getNumSuccessors());
|
||||
EXPECT_EQ(VecBB->getParent()->getEntryBasicBlock(), VecBB);
|
||||
EXPECT_EQ(VecBB->getParent()->getExitingBasicBlock(), VecBB);
|
||||
EXPECT_EQ(&*Plan, VecBB->getPlan());
|
||||
|
||||
auto Iter = VecBB->begin();
|
||||
@@ -101,15 +105,15 @@ graph [labelloc=t, fontsize=30; label="Vectorization Plan"]
|
||||
node [shape=rect, fontname=Courier, fontsize=30]
|
||||
edge [fontname=Courier, fontsize=30]
|
||||
compound=true
|
||||
subgraph cluster_N0 {
|
||||
N0 [label =
|
||||
"vector.ph:\l" +
|
||||
"Successor(s): for.body\l"
|
||||
]
|
||||
N0 -> N1 [ label="" lhead=cluster_N2]
|
||||
subgraph cluster_N2 {
|
||||
fontname=Courier
|
||||
label="\<x1\> TopRegion"
|
||||
label="\<x1\> for.body"
|
||||
N1 [label =
|
||||
"entry:\l" +
|
||||
"Successor(s): vector.body\l"
|
||||
]
|
||||
N1 -> N2 [ label=""]
|
||||
N2 [label =
|
||||
"vector.body:\l" +
|
||||
" WIDEN-PHI ir\<%indvars.iv\> = phi ir\<0\>, ir\<%indvars.iv.next\>\l" +
|
||||
" EMIT ir\<%arr.idx\> = getelementptr ir\<%A\> ir\<%indvars.iv\>\l" +
|
||||
@@ -118,17 +122,15 @@ compound=true
|
||||
" EMIT store ir\<%res\> ir\<%arr.idx\>\l" +
|
||||
" EMIT ir\<%indvars.iv.next\> = add ir\<%indvars.iv\> ir\<1\>\l" +
|
||||
" EMIT ir\<%exitcond\> = icmp ir\<%indvars.iv.next\> ir\<%N\>\l" +
|
||||
"Successor(s): vector.body, for.end\l" +
|
||||
"No successors\l" +
|
||||
"CondBit: ir\<%exitcond\> (vector.body)\l"
|
||||
]
|
||||
N2 -> N2 [ label="T"]
|
||||
N2 -> N3 [ label="F"]
|
||||
N3 [label =
|
||||
"for.end:\l" +
|
||||
" EMIT ret\l" +
|
||||
"No successors\l"
|
||||
]
|
||||
}
|
||||
N1 -> N3 [ label="" ltail=cluster_N2]
|
||||
N3 [label =
|
||||
"for.end:\l" +
|
||||
"No successors\l"
|
||||
]
|
||||
}
|
||||
)";
|
||||
EXPECT_EQ(ExpectedStr, FullDump);
|
||||
@@ -174,10 +176,14 @@ TEST_F(VPlanHCFGTest, testVPInstructionToVPRecipesInner) {
|
||||
EXPECT_EQ(0u, Entry->getNumPredecessors());
|
||||
EXPECT_EQ(1u, Entry->getNumSuccessors());
|
||||
|
||||
// Check that the region following the preheader is a single basic-block
|
||||
// region (loop).
|
||||
VPBasicBlock *VecBB = Entry->getSingleSuccessor()->getEntryBasicBlock();
|
||||
EXPECT_EQ(7u, VecBB->size());
|
||||
EXPECT_EQ(2u, VecBB->getNumPredecessors());
|
||||
EXPECT_EQ(2u, VecBB->getNumSuccessors());
|
||||
EXPECT_EQ(0u, VecBB->getNumPredecessors());
|
||||
EXPECT_EQ(0u, VecBB->getNumSuccessors());
|
||||
EXPECT_EQ(VecBB->getParent()->getEntryBasicBlock(), VecBB);
|
||||
EXPECT_EQ(VecBB->getParent()->getExitingBasicBlock(), VecBB);
|
||||
|
||||
auto Iter = VecBB->begin();
|
||||
EXPECT_NE(nullptr, dyn_cast<VPWidenPHIRecipe>(&*Iter++));
|
||||
|
||||
@@ -269,7 +269,7 @@ TEST_F(VPlanSlpTest, testSlpReuse_1) {
|
||||
TEST_F(VPlanSlpTest, testSlpReuse_2) {
|
||||
const char *ModuleString =
|
||||
"%struct.Test = type { i32, i32 }\n"
|
||||
"define i32 @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
|
||||
"define void @add_x2(%struct.Test* nocapture readonly %A, %struct.Test* "
|
||||
"nocapture readonly %B, %struct.Test* nocapture %C) {\n"
|
||||
"entry:\n"
|
||||
" br label %for.body\n"
|
||||
@@ -290,11 +290,12 @@ TEST_F(VPlanSlpTest, testSlpReuse_2) {
|
||||
" %C1 = getelementptr inbounds %struct.Test, %struct.Test* %C, i64 "
|
||||
"%indvars.iv, i32 1\n"
|
||||
" store i32 %add1, i32* %C1, align 4\n"
|
||||
" %use = add i32 %vA1, 1\n"
|
||||
" %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
|
||||
" %exitcond = icmp eq i64 %indvars.iv.next, 1024\n"
|
||||
" br i1 %exitcond, label %for.cond.cleanup, label %for.body\n"
|
||||
"for.cond.cleanup: ; preds = %for.body\n"
|
||||
" ret i32 %vA1\n"
|
||||
" ret void\n"
|
||||
"}\n";
|
||||
|
||||
Module &M = parseModule(ModuleString);
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include "../lib/Transforms/Vectorize/VPlan.h"
|
||||
#include "llvm/ADT/DepthFirstIterator.h"
|
||||
#include "llvm/ADT/PostOrderIterator.h"
|
||||
#include "llvm/Analysis/VectorUtils.h"
|
||||
#include "llvm/IR/Instruction.h"
|
||||
|
||||
@@ -76,8 +76,8 @@ protected:
|
||||
|
||||
auto Plan = std::make_unique<VPlan>();
|
||||
VPlanHCFGBuilder HCFGBuilder(LI->getLoopFor(LoopHeader), LI.get(), *Plan);
|
||||
VPRegionBlock *TopRegion = HCFGBuilder.buildPlainCFG();
|
||||
Plan->setEntry(TopRegion);
|
||||
VPBasicBlock *EntryVPBB = HCFGBuilder.buildPlainCFG();
|
||||
Plan->setEntry(EntryVPBB);
|
||||
return Plan;
|
||||
}
|
||||
};
|
||||
|
||||
Reference in new issue
Block a user