[VPlan] Only store single vector per VPValue in VPTransformState. (NFC)
After 8ec406757c (https://github.com/llvm/llvm-project/pull/95842),
VPTransformState only stores a single vector value per VPValue.
Simplify the code by replacing the SmallVector in PerPartOutput with a
single Value * and rename to VPV2Vector for clarity.
Also remove the redundant Part argument from various accessors.
This commit is contained in:
4 files changed
+145
-162
No files matched your search
@@ -3075,13 +3075,13 @@ void InnerLoopVectorizer::fixNonInductionPHIs(VPlan &Plan,
|
||||
VPWidenPHIRecipe *VPPhi = dyn_cast<VPWidenPHIRecipe>(&P);
|
||||
if (!VPPhi)
|
||||
continue;
|
||||
PHINode *NewPhi = cast<PHINode>(State.get(VPPhi, 0));
|
||||
PHINode *NewPhi = cast<PHINode>(State.get(VPPhi));
|
||||
// Make sure the builder has a valid insert point.
|
||||
Builder.SetInsertPoint(NewPhi);
|
||||
for (unsigned Idx = 0; Idx < VPPhi->getNumOperands(); ++Idx) {
|
||||
VPValue *Inc = VPPhi->getIncomingValue(Idx);
|
||||
VPBasicBlock *VPBB = VPPhi->getIncomingBlock(Idx);
|
||||
NewPhi->addIncoming(State.get(Inc, 0), State.CFG.VPBB2IRBB[VPBB]);
|
||||
NewPhi->addIncoming(State.get(Inc), State.CFG.VPBB2IRBB[VPBB]);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9445,7 +9445,7 @@ void VPReplicateRecipe::execute(VPTransformState &State) {
|
||||
assert(!State.VF.isScalable() && "VF is assumed to be non scalable.");
|
||||
Value *Poison = PoisonValue::get(
|
||||
VectorType::get(UI->getType(), State.VF));
|
||||
State.set(this, Poison, State.Instance->Part);
|
||||
State.set(this, Poison);
|
||||
}
|
||||
State.packScalarIntoVectorValue(this, *State.Instance);
|
||||
}
|
||||
|
||||
@@ -242,8 +242,8 @@ Value *VPTransformState::get(VPValue *Def, const VPIteration &Instance) {
|
||||
return Data.PerPartScalars[Def][Instance.Part][0];
|
||||
}
|
||||
|
||||
assert(hasVectorValue(Def, Instance.Part));
|
||||
auto *VecPart = Data.PerPartOutput[Def][Instance.Part];
|
||||
assert(hasVectorValue(Def));
|
||||
auto *VecPart = Data.VPV2Vector[Def];
|
||||
if (!VecPart->getType()->isVectorTy()) {
|
||||
assert(Instance.Lane.isFirstLane() && "cannot get lane > 0 for scalar");
|
||||
return VecPart;
|
||||
@@ -255,20 +255,20 @@ Value *VPTransformState::get(VPValue *Def, const VPIteration &Instance) {
|
||||
return Extract;
|
||||
}
|
||||
|
||||
Value *VPTransformState::get(VPValue *Def, unsigned Part, bool NeedsScalar) {
|
||||
Value *VPTransformState::get(VPValue *Def, bool NeedsScalar) {
|
||||
if (NeedsScalar) {
|
||||
assert((VF.isScalar() || Def->isLiveIn() || hasVectorValue(Def, Part) ||
|
||||
assert((VF.isScalar() || Def->isLiveIn() || hasVectorValue(Def) ||
|
||||
!vputils::onlyFirstLaneUsed(Def) ||
|
||||
(hasScalarValue(Def, VPIteration(Part, 0)) &&
|
||||
Data.PerPartScalars[Def][Part].size() == 1)) &&
|
||||
(hasScalarValue(Def, VPIteration(0, 0)) &&
|
||||
Data.PerPartScalars[Def][0].size() == 1)) &&
|
||||
"Trying to access a single scalar per part but has multiple scalars "
|
||||
"per part.");
|
||||
return get(Def, VPIteration(Part, 0));
|
||||
return get(Def, VPIteration(0, 0));
|
||||
}
|
||||
|
||||
// If Values have been set for this Def return the one relevant for \p Part.
|
||||
if (hasVectorValue(Def, Part))
|
||||
return Data.PerPartOutput[Def][Part];
|
||||
if (hasVectorValue(Def))
|
||||
return Data.VPV2Vector[Def];
|
||||
|
||||
auto GetBroadcastInstrs = [this, Def](Value *V) {
|
||||
bool SafeToHoist = Def->isDefinedOutsideLoopRegions();
|
||||
@@ -290,21 +290,19 @@ Value *VPTransformState::get(VPValue *Def, unsigned Part, bool NeedsScalar) {
|
||||
return Shuf;
|
||||
};
|
||||
|
||||
if (!hasScalarValue(Def, {Part, 0})) {
|
||||
if (!hasScalarValue(Def, {0, 0})) {
|
||||
assert(Def->isLiveIn() && "expected a live-in");
|
||||
if (Part != 0)
|
||||
return get(Def, 0);
|
||||
Value *IRV = Def->getLiveInIRValue();
|
||||
Value *B = GetBroadcastInstrs(IRV);
|
||||
set(Def, B, Part);
|
||||
set(Def, B);
|
||||
return B;
|
||||
}
|
||||
|
||||
Value *ScalarValue = get(Def, {Part, 0});
|
||||
Value *ScalarValue = get(Def, {0, 0});
|
||||
// If we aren't vectorizing, we can just copy the scalar map values over
|
||||
// to the vector map.
|
||||
if (VF.isScalar()) {
|
||||
set(Def, ScalarValue, Part);
|
||||
set(Def, ScalarValue);
|
||||
return ScalarValue;
|
||||
}
|
||||
|
||||
@@ -312,7 +310,7 @@ Value *VPTransformState::get(VPValue *Def, unsigned Part, bool NeedsScalar) {
|
||||
|
||||
unsigned LastLane = IsUniform ? 0 : VF.getKnownMinValue() - 1;
|
||||
// Check if there is a scalar value for the selected lane.
|
||||
if (!hasScalarValue(Def, {Part, LastLane})) {
|
||||
if (!hasScalarValue(Def, {0, LastLane})) {
|
||||
// At the moment, VPWidenIntOrFpInductionRecipes, VPScalarIVStepsRecipes and
|
||||
// VPExpandSCEVRecipes can also be uniform.
|
||||
assert((isa<VPWidenIntOrFpInductionRecipe>(Def->getDefiningRecipe()) ||
|
||||
@@ -323,7 +321,7 @@ Value *VPTransformState::get(VPValue *Def, unsigned Part, bool NeedsScalar) {
|
||||
LastLane = 0;
|
||||
}
|
||||
|
||||
auto *LastInst = cast<Instruction>(get(Def, {Part, LastLane}));
|
||||
auto *LastInst = cast<Instruction>(get(Def, {0, LastLane}));
|
||||
// Set the insert point after the last scalarized instruction or after the
|
||||
// last PHI, if LastInst is a PHI. This ensures the insertelement sequence
|
||||
// will directly follow the scalar definitions.
|
||||
@@ -343,15 +341,15 @@ Value *VPTransformState::get(VPValue *Def, unsigned Part, bool NeedsScalar) {
|
||||
Value *VectorValue = nullptr;
|
||||
if (IsUniform) {
|
||||
VectorValue = GetBroadcastInstrs(ScalarValue);
|
||||
set(Def, VectorValue, Part);
|
||||
set(Def, VectorValue);
|
||||
} else {
|
||||
// Initialize packing with insertelements to start from undef.
|
||||
assert(!VF.isScalable() && "VF is assumed to be non scalable.");
|
||||
Value *Undef = PoisonValue::get(VectorType::get(LastInst->getType(), VF));
|
||||
set(Def, Undef, Part);
|
||||
set(Def, Undef);
|
||||
for (unsigned Lane = 0; Lane < VF.getKnownMinValue(); ++Lane)
|
||||
packScalarIntoVectorValue(Def, {Part, Lane});
|
||||
VectorValue = get(Def, Part);
|
||||
packScalarIntoVectorValue(Def, {0, Lane});
|
||||
VectorValue = get(Def);
|
||||
}
|
||||
Builder.restoreIP(OldIP);
|
||||
return VectorValue;
|
||||
@@ -406,10 +404,10 @@ void VPTransformState::setDebugLocFrom(DebugLoc DL) {
|
||||
void VPTransformState::packScalarIntoVectorValue(VPValue *Def,
|
||||
const VPIteration &Instance) {
|
||||
Value *ScalarInst = get(Def, Instance);
|
||||
Value *VectorValue = get(Def, Instance.Part);
|
||||
Value *VectorValue = get(Def);
|
||||
VectorValue = Builder.CreateInsertElement(
|
||||
VectorValue, ScalarInst, Instance.Lane.getAsRuntimeExpr(Builder, VF));
|
||||
set(Def, VectorValue, Instance.Part);
|
||||
set(Def, VectorValue);
|
||||
}
|
||||
|
||||
BasicBlock *
|
||||
@@ -1074,12 +1072,12 @@ void VPlan::execute(VPTransformState *State) {
|
||||
isa<VPWidenIntOrFpInductionRecipe>(&R)) {
|
||||
PHINode *Phi = nullptr;
|
||||
if (isa<VPWidenIntOrFpInductionRecipe>(&R)) {
|
||||
Phi = cast<PHINode>(State->get(R.getVPSingleValue(), 0));
|
||||
Phi = cast<PHINode>(State->get(R.getVPSingleValue()));
|
||||
} else {
|
||||
auto *WidenPhi = cast<VPWidenPointerInductionRecipe>(&R);
|
||||
assert(!WidenPhi->onlyScalarsGenerated(State->VF.isScalable()) &&
|
||||
"recipe generating only scalars should have been replaced");
|
||||
auto *GEP = cast<GetElementPtrInst>(State->get(WidenPhi, 0));
|
||||
auto *GEP = cast<GetElementPtrInst>(State->get(WidenPhi));
|
||||
Phi = cast<PHINode>(GEP->getPointerOperand());
|
||||
}
|
||||
|
||||
@@ -1092,7 +1090,7 @@ void VPlan::execute(VPTransformState *State) {
|
||||
|
||||
// Use the steps for the last part as backedge value for the induction.
|
||||
if (auto *IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R))
|
||||
Inc->setOperand(0, State->get(IV->getLastUnrolledPartOperand(), 0));
|
||||
Inc->setOperand(0, State->get(IV->getLastUnrolledPartOperand()));
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1101,8 +1099,8 @@ void VPlan::execute(VPTransformState *State) {
|
||||
isa<VPCanonicalIVPHIRecipe, VPEVLBasedIVPHIRecipe>(PhiR) ||
|
||||
(isa<VPReductionPHIRecipe>(PhiR) &&
|
||||
cast<VPReductionPHIRecipe>(PhiR)->isInLoop());
|
||||
Value *Phi = State->get(PhiR, 0, NeedsScalar);
|
||||
Value *Val = State->get(PhiR->getBackedgeValue(), 0, NeedsScalar);
|
||||
Value *Phi = State->get(PhiR, NeedsScalar);
|
||||
Value *Val = State->get(PhiR->getBackedgeValue(), NeedsScalar);
|
||||
cast<PHINode>(Phi)->addIncoming(Val, VectorLatchBB);
|
||||
}
|
||||
|
||||
|
||||
@@ -263,30 +263,22 @@ struct VPTransformState {
|
||||
std::optional<VPIteration> Instance;
|
||||
|
||||
struct DataState {
|
||||
/// A type for vectorized values in the new loop. Each value from the
|
||||
/// original loop, when vectorized, is represented by UF vector values in
|
||||
/// the new unrolled loop, where UF is the unroll factor.
|
||||
typedef SmallVector<Value *, 2> PerPartValuesTy;
|
||||
|
||||
DenseMap<VPValue *, PerPartValuesTy> PerPartOutput;
|
||||
// Each value from the original loop, when vectorized, is represented by a
|
||||
// vector value in the map.
|
||||
DenseMap<VPValue *, Value *> VPV2Vector;
|
||||
|
||||
using ScalarsPerPartValuesTy = SmallVector<SmallVector<Value *, 4>, 2>;
|
||||
DenseMap<VPValue *, ScalarsPerPartValuesTy> PerPartScalars;
|
||||
} Data;
|
||||
|
||||
/// Get the generated vector Value for a given VPValue \p Def and a given \p
|
||||
/// Part if \p IsScalar is false, otherwise return the generated scalar
|
||||
/// for \p Part. \See set.
|
||||
Value *get(VPValue *Def, unsigned Part, bool IsScalar = false);
|
||||
/// Get the generated vector Value for a given VPValue \p Def if \p IsScalar
|
||||
/// is false, otherwise return the generated scalar. \See set.
|
||||
Value *get(VPValue *Def, bool IsScalar = false);
|
||||
|
||||
/// Get the generated Value for a given VPValue and given Part and Lane.
|
||||
Value *get(VPValue *Def, const VPIteration &Instance);
|
||||
|
||||
bool hasVectorValue(VPValue *Def, unsigned Part) {
|
||||
auto I = Data.PerPartOutput.find(Def);
|
||||
return I != Data.PerPartOutput.end() && Part < I->second.size() &&
|
||||
I->second[Part];
|
||||
}
|
||||
bool hasVectorValue(VPValue *Def) { return Data.VPV2Vector.contains(Def); }
|
||||
|
||||
bool hasScalarValue(VPValue *Def, VPIteration Instance) {
|
||||
auto I = Data.PerPartScalars.find(Def);
|
||||
@@ -298,28 +290,22 @@ struct VPTransformState {
|
||||
I->second[Instance.Part][CacheIdx];
|
||||
}
|
||||
|
||||
/// Set the generated vector Value for a given VPValue and a given Part, if \p
|
||||
/// IsScalar is false. If \p IsScalar is true, set the scalar in (Part, 0).
|
||||
void set(VPValue *Def, Value *V, unsigned Part, bool IsScalar = false) {
|
||||
/// Set the generated vector Value for a given VPValue, if \p
|
||||
/// IsScalar is false. If \p IsScalar is true, set the scalar in lane 0.
|
||||
void set(VPValue *Def, Value *V, bool IsScalar = false) {
|
||||
if (IsScalar) {
|
||||
set(Def, V, VPIteration(Part, 0));
|
||||
set(Def, V, VPIteration(0, 0));
|
||||
return;
|
||||
}
|
||||
assert((VF.isScalar() || V->getType()->isVectorTy()) &&
|
||||
"scalar values must be stored as (Part, 0)");
|
||||
if (!Data.PerPartOutput.count(Def)) {
|
||||
DataState::PerPartValuesTy Entry(1);
|
||||
Data.PerPartOutput[Def] = Entry;
|
||||
}
|
||||
Data.PerPartOutput[Def][Part] = V;
|
||||
"scalar values must be stored as (0, 0)");
|
||||
Data.VPV2Vector[Def] = V;
|
||||
}
|
||||
|
||||
/// Reset an existing vector value for \p Def and a given \p Part.
|
||||
void reset(VPValue *Def, Value *V, unsigned Part) {
|
||||
auto Iter = Data.PerPartOutput.find(Def);
|
||||
assert(Iter != Data.PerPartOutput.end() &&
|
||||
"need to overwrite existing value");
|
||||
Iter->second[Part] = V;
|
||||
void reset(VPValue *Def, Value *V) {
|
||||
assert(Data.VPV2Vector.contains(Def) && "need to overwrite existing value");
|
||||
Data.VPV2Vector[Def] = V;
|
||||
}
|
||||
|
||||
/// Set the generated scalar \p V for \p Def and the given \p Instance.
|
||||
|
||||
@@ -404,8 +404,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
|
||||
|
||||
if (Instruction::isBinaryOp(getOpcode())) {
|
||||
bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
|
||||
Value *A = State.get(getOperand(0), 0, OnlyFirstLaneUsed);
|
||||
Value *B = State.get(getOperand(1), 0, OnlyFirstLaneUsed);
|
||||
Value *A = State.get(getOperand(0), OnlyFirstLaneUsed);
|
||||
Value *B = State.get(getOperand(1), OnlyFirstLaneUsed);
|
||||
auto *Res =
|
||||
Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(), A, B, Name);
|
||||
if (auto *I = dyn_cast<Instruction>(Res))
|
||||
@@ -415,19 +415,19 @@ Value *VPInstruction::generate(VPTransformState &State) {
|
||||
|
||||
switch (getOpcode()) {
|
||||
case VPInstruction::Not: {
|
||||
Value *A = State.get(getOperand(0), 0);
|
||||
Value *A = State.get(getOperand(0));
|
||||
return Builder.CreateNot(A, Name);
|
||||
}
|
||||
case Instruction::ICmp: {
|
||||
bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
|
||||
Value *A = State.get(getOperand(0), 0, OnlyFirstLaneUsed);
|
||||
Value *B = State.get(getOperand(1), 0, OnlyFirstLaneUsed);
|
||||
Value *A = State.get(getOperand(0), OnlyFirstLaneUsed);
|
||||
Value *B = State.get(getOperand(1), OnlyFirstLaneUsed);
|
||||
return Builder.CreateCmp(getPredicate(), A, B, Name);
|
||||
}
|
||||
case Instruction::Select: {
|
||||
Value *Cond = State.get(getOperand(0), 0);
|
||||
Value *Op1 = State.get(getOperand(1), 0);
|
||||
Value *Op2 = State.get(getOperand(2), 0);
|
||||
Value *Cond = State.get(getOperand(0));
|
||||
Value *Op1 = State.get(getOperand(1));
|
||||
Value *Op2 = State.get(getOperand(2));
|
||||
return Builder.CreateSelect(Cond, Op1, Op2, Name);
|
||||
}
|
||||
case VPInstruction::ActiveLaneMask: {
|
||||
@@ -461,10 +461,10 @@ Value *VPInstruction::generate(VPTransformState &State) {
|
||||
// v2 = a[i, i+1, i+2, i+3];
|
||||
// v3 = vector(v1(3), v2(0, 1, 2))
|
||||
|
||||
auto *V1 = State.get(getOperand(0), 0);
|
||||
auto *V1 = State.get(getOperand(0));
|
||||
if (!V1->getType()->isVectorTy())
|
||||
return V1;
|
||||
Value *V2 = State.get(getOperand(1), 0);
|
||||
Value *V2 = State.get(getOperand(1));
|
||||
return Builder.CreateVectorSplice(V1, V2, -1, Name);
|
||||
}
|
||||
case VPInstruction::CalculateTripCountMinusVF: {
|
||||
@@ -530,8 +530,8 @@ Value *VPInstruction::generate(VPTransformState &State) {
|
||||
}
|
||||
case VPInstruction::BranchOnCount: {
|
||||
// First create the compare.
|
||||
Value *IV = State.get(getOperand(0), 0, /*IsScalar*/ true);
|
||||
Value *TC = State.get(getOperand(1), 0, /*IsScalar*/ true);
|
||||
Value *IV = State.get(getOperand(0), /*IsScalar*/ true);
|
||||
Value *TC = State.get(getOperand(1), /*IsScalar*/ true);
|
||||
Value *Cond = Builder.CreateICmpEQ(IV, TC);
|
||||
|
||||
// Now create the branch.
|
||||
@@ -566,7 +566,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
|
||||
unsigned UF = getNumOperands() - 1;
|
||||
VectorParts RdxParts(UF);
|
||||
for (unsigned Part = 0; Part < UF; ++Part)
|
||||
RdxParts[Part] = State.get(getOperand(1 + Part), 0, PhiR->isInLoop());
|
||||
RdxParts[Part] = State.get(getOperand(1 + Part), PhiR->isInLoop());
|
||||
|
||||
// If the vector reduction can be performed in a smaller type, we truncate
|
||||
// then extend the loop exit value to enable InstCombine to evaluate the
|
||||
@@ -637,30 +637,29 @@ Value *VPInstruction::generate(VPTransformState &State) {
|
||||
VPIteration(0, VPLane::getLaneFromEnd(State.VF, Offset)));
|
||||
} else {
|
||||
assert(Offset <= 1 && "invalid offset to extract from");
|
||||
// When loop is unrolled without vectorizing, retrieve UF - Offset.
|
||||
Res = State.get(getOperand(0), 1 - Offset);
|
||||
Res = State.get(getOperand(0));
|
||||
}
|
||||
if (isa<ExtractElementInst>(Res))
|
||||
Res->setName(Name);
|
||||
return Res;
|
||||
}
|
||||
case VPInstruction::LogicalAnd: {
|
||||
Value *A = State.get(getOperand(0), 0);
|
||||
Value *B = State.get(getOperand(1), 0);
|
||||
Value *A = State.get(getOperand(0));
|
||||
Value *B = State.get(getOperand(1));
|
||||
return Builder.CreateLogicalAnd(A, B, Name);
|
||||
}
|
||||
case VPInstruction::PtrAdd: {
|
||||
assert(vputils::onlyFirstLaneUsed(this) &&
|
||||
"can only generate first lane for PtrAdd");
|
||||
Value *Ptr = State.get(getOperand(0), 0, /* IsScalar */ true);
|
||||
Value *Addend = State.get(getOperand(1), 0, /* IsScalar */ true);
|
||||
Value *Ptr = State.get(getOperand(0), /* IsScalar */ true);
|
||||
Value *Addend = State.get(getOperand(1), /* IsScalar */ true);
|
||||
return Builder.CreatePtrAdd(Ptr, Addend, Name);
|
||||
}
|
||||
case VPInstruction::ResumePhi: {
|
||||
Value *IncomingFromVPlanPred =
|
||||
State.get(getOperand(0), 0, /* IsScalar */ true);
|
||||
State.get(getOperand(0), /* IsScalar */ true);
|
||||
Value *IncomingFromOtherPreds =
|
||||
State.get(getOperand(1), 0, /* IsScalar */ true);
|
||||
State.get(getOperand(1), /* IsScalar */ true);
|
||||
auto *NewPhi =
|
||||
Builder.CreatePHI(IncomingFromOtherPreds->getType(), 2, Name);
|
||||
BasicBlock *VPlanPred =
|
||||
@@ -731,7 +730,7 @@ void VPInstruction::execute(VPTransformState &State) {
|
||||
(GeneratedValue->getType()->isVectorTy() == !GeneratesPerFirstLaneOnly ||
|
||||
State.VF.isScalar()) &&
|
||||
"scalar value but not only first lane defined");
|
||||
State.set(this, GeneratedValue, 0,
|
||||
State.set(this, GeneratedValue,
|
||||
/*IsScalar*/ GeneratesPerFirstLaneOnly);
|
||||
}
|
||||
|
||||
@@ -921,7 +920,7 @@ void VPWidenCallRecipe::execute(VPTransformState &State) {
|
||||
else if (VFTy && !VFTy->getParamType(I.index())->isVectorTy())
|
||||
Arg = State.get(I.value(), VPIteration(0, 0));
|
||||
else
|
||||
Arg = State.get(I.value(), 0);
|
||||
Arg = State.get(I.value());
|
||||
if (UseIntrinsic &&
|
||||
isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, I.index()))
|
||||
TysForDecl.push_back(Arg->getType());
|
||||
@@ -952,7 +951,7 @@ void VPWidenCallRecipe::execute(VPTransformState &State) {
|
||||
V->copyFastMathFlags(CI);
|
||||
|
||||
if (!V->getType()->isVoidTy())
|
||||
State.set(this, V, 0);
|
||||
State.set(this, V);
|
||||
State.addMetadata(V, CI);
|
||||
}
|
||||
|
||||
@@ -1056,11 +1055,11 @@ void VPWidenSelectRecipe::execute(VPTransformState &State) {
|
||||
auto *InvarCond =
|
||||
isInvariantCond() ? State.get(getCond(), VPIteration(0, 0)) : nullptr;
|
||||
|
||||
Value *Cond = InvarCond ? InvarCond : State.get(getCond(), 0);
|
||||
Value *Op0 = State.get(getOperand(1), 0);
|
||||
Value *Op1 = State.get(getOperand(2), 0);
|
||||
Value *Cond = InvarCond ? InvarCond : State.get(getCond());
|
||||
Value *Op0 = State.get(getOperand(1));
|
||||
Value *Op1 = State.get(getOperand(2));
|
||||
Value *Sel = State.Builder.CreateSelect(Cond, Op0, Op1);
|
||||
State.set(this, Sel, 0);
|
||||
State.set(this, Sel);
|
||||
State.addMetadata(Sel, dyn_cast_or_null<Instruction>(getUnderlyingValue()));
|
||||
}
|
||||
|
||||
@@ -1146,7 +1145,7 @@ void VPWidenRecipe::execute(VPTransformState &State) {
|
||||
// Just widen unops and binops.
|
||||
SmallVector<Value *, 2> Ops;
|
||||
for (VPValue *VPOp : operands())
|
||||
Ops.push_back(State.get(VPOp, 0));
|
||||
Ops.push_back(State.get(VPOp));
|
||||
|
||||
Value *V = Builder.CreateNAryOp(Opcode, Ops);
|
||||
|
||||
@@ -1154,23 +1153,23 @@ void VPWidenRecipe::execute(VPTransformState &State) {
|
||||
setFlags(VecOp);
|
||||
|
||||
// Use this vector value for all users of the original instruction.
|
||||
State.set(this, V, 0);
|
||||
State.set(this, V);
|
||||
State.addMetadata(V, dyn_cast_or_null<Instruction>(getUnderlyingValue()));
|
||||
break;
|
||||
}
|
||||
case Instruction::Freeze: {
|
||||
Value *Op = State.get(getOperand(0), 0);
|
||||
Value *Op = State.get(getOperand(0));
|
||||
|
||||
Value *Freeze = Builder.CreateFreeze(Op);
|
||||
State.set(this, Freeze, 0);
|
||||
State.set(this, Freeze);
|
||||
break;
|
||||
}
|
||||
case Instruction::ICmp:
|
||||
case Instruction::FCmp: {
|
||||
// Widen compares. Generate vector compares.
|
||||
bool FCmp = Opcode == Instruction::FCmp;
|
||||
Value *A = State.get(getOperand(0), 0);
|
||||
Value *B = State.get(getOperand(1), 0);
|
||||
Value *A = State.get(getOperand(0));
|
||||
Value *B = State.get(getOperand(1));
|
||||
Value *C = nullptr;
|
||||
if (FCmp) {
|
||||
// Propagate fast math flags.
|
||||
@@ -1181,7 +1180,7 @@ void VPWidenRecipe::execute(VPTransformState &State) {
|
||||
} else {
|
||||
C = Builder.CreateICmp(getPredicate(), A, B);
|
||||
}
|
||||
State.set(this, C, 0);
|
||||
State.set(this, C);
|
||||
State.addMetadata(C, dyn_cast_or_null<Instruction>(getUnderlyingValue()));
|
||||
break;
|
||||
}
|
||||
@@ -1196,7 +1195,7 @@ void VPWidenRecipe::execute(VPTransformState &State) {
|
||||
// Verify that VPlan type inference results agree with the type of the
|
||||
// generated values.
|
||||
assert(VectorType::get(State.TypeAnalysis.inferScalarType(this), State.VF) ==
|
||||
State.get(this, 0)->getType() &&
|
||||
State.get(this)->getType() &&
|
||||
"inferred type and type from generated instructions do not match");
|
||||
#endif
|
||||
}
|
||||
@@ -1283,11 +1282,11 @@ void VPWidenEVLRecipe::execute(VPTransformState &State) {
|
||||
|
||||
State.setDebugLocFrom(getDebugLoc());
|
||||
|
||||
assert(State.get(getOperand(0), 0)->getType()->isVectorTy() &&
|
||||
assert(State.get(getOperand(0))->getType()->isVectorTy() &&
|
||||
"VPWidenEVLRecipe should not be used for scalars");
|
||||
|
||||
VPValue *EVL = getEVL();
|
||||
Value *EVLArg = State.get(EVL, 0, /*NeedsScalar=*/true);
|
||||
Value *EVLArg = State.get(EVL, /*NeedsScalar=*/true);
|
||||
IRBuilderBase &BuilderIR = State.Builder;
|
||||
VectorBuilder Builder(BuilderIR);
|
||||
Value *Mask = BuilderIR.CreateVectorSplat(State.VF, BuilderIR.getTrue());
|
||||
@@ -1295,7 +1294,7 @@ void VPWidenEVLRecipe::execute(VPTransformState &State) {
|
||||
SmallVector<Value *, 4> Ops;
|
||||
for (unsigned I = 0, E = getNumOperands() - 1; I < E; ++I) {
|
||||
VPValue *VPOp = getOperand(I);
|
||||
Ops.push_back(State.get(VPOp, 0));
|
||||
Ops.push_back(State.get(VPOp));
|
||||
}
|
||||
|
||||
Builder.setMask(Mask).setEVL(EVLArg);
|
||||
@@ -1306,7 +1305,7 @@ void VPWidenEVLRecipe::execute(VPTransformState &State) {
|
||||
if (isa<FPMathOperator>(VPInst))
|
||||
setFlags(cast<Instruction>(VPInst));
|
||||
|
||||
State.set(this, VPInst, 0);
|
||||
State.set(this, VPInst);
|
||||
State.addMetadata(VPInst,
|
||||
dyn_cast_or_null<Instruction>(getUnderlyingValue()));
|
||||
}
|
||||
@@ -1338,9 +1337,9 @@ void VPWidenCastRecipe::execute(VPTransformState &State) {
|
||||
assert(State.VF.isVector() && "Not vectorizing?");
|
||||
Type *DestTy = VectorType::get(getResultType(), State.VF);
|
||||
VPValue *Op = getOperand(0);
|
||||
Value *A = State.get(Op, 0);
|
||||
Value *A = State.get(Op);
|
||||
Value *Cast = Builder.CreateCast(Instruction::CastOps(Opcode), A, DestTy);
|
||||
State.set(this, Cast, 0);
|
||||
State.set(this, Cast);
|
||||
State.addMetadata(Cast, cast_or_null<Instruction>(getUnderlyingValue()));
|
||||
}
|
||||
|
||||
@@ -1474,7 +1473,7 @@ void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
|
||||
if (VPValue *SplatVFOperand = getSplatVFValue()) {
|
||||
// The recipe has been unrolled. In that case, fetch the splat value for the
|
||||
// induction increment.
|
||||
SplatVF = State.get(SplatVFOperand, 0);
|
||||
SplatVF = State.get(SplatVFOperand);
|
||||
} else {
|
||||
// Multiply the vectorization factor by the step using integer or
|
||||
// floating-point arithmetic as appropriate.
|
||||
@@ -1503,7 +1502,7 @@ void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
|
||||
PHINode *VecInd = PHINode::Create(SteppedStart->getType(), 2, "vec.ind");
|
||||
VecInd->insertBefore(State.CFG.PrevBB->getFirstInsertionPt());
|
||||
VecInd->setDebugLoc(EntryVal->getDebugLoc());
|
||||
State.set(this, VecInd, 0);
|
||||
State.set(this, VecInd);
|
||||
|
||||
Instruction *LastInduction = cast<Instruction>(Builder.CreateBinOp(
|
||||
AddOp, VecInd, SplatVF, "vec.ind.next", EntryVal->getDebugLoc()));
|
||||
@@ -1628,7 +1627,7 @@ void VPScalarIVStepsRecipe::execute(VPTransformState &State) {
|
||||
InitVec = Builder.CreateSIToFP(InitVec, VecIVTy);
|
||||
auto *Mul = Builder.CreateBinOp(MulOp, InitVec, SplatStep);
|
||||
auto *Add = Builder.CreateBinOp(AddOp, SplatIV, Mul);
|
||||
State.set(this, Add, 0);
|
||||
State.set(this, Add);
|
||||
// It's useful to record the lane values too for the known minimum number
|
||||
// of elements so we do those below. This improves the code quality when
|
||||
// trying to extract the first element, for example.
|
||||
@@ -1691,7 +1690,7 @@ void VPWidenGEPRecipe::execute(VPTransformState &State) {
|
||||
State.Builder.CreateGEP(GEP->getSourceElementType(), Ops[0],
|
||||
ArrayRef(Ops).drop_front(), "", isInBounds());
|
||||
Value *Splat = State.Builder.CreateVectorSplat(State.VF, NewGEP);
|
||||
State.set(this, Splat, 0);
|
||||
State.set(this, Splat);
|
||||
State.addMetadata(Splat, GEP);
|
||||
} else {
|
||||
// If the GEP has at least one loop-varying operand, we are sure to
|
||||
@@ -1700,7 +1699,7 @@ void VPWidenGEPRecipe::execute(VPTransformState &State) {
|
||||
// won't broadcast it.
|
||||
auto *Ptr = isPointerLoopInvariant()
|
||||
? State.get(getOperand(0), VPIteration(0, 0))
|
||||
: State.get(getOperand(0), 0);
|
||||
: State.get(getOperand(0));
|
||||
|
||||
// Collect all the indices for the new GEP. If any index is
|
||||
// loop-invariant, we won't broadcast it.
|
||||
@@ -1710,7 +1709,7 @@ void VPWidenGEPRecipe::execute(VPTransformState &State) {
|
||||
if (isIndexLoopInvariant(I - 1))
|
||||
Indices.push_back(State.get(Operand, VPIteration(0, 0)));
|
||||
else
|
||||
Indices.push_back(State.get(Operand, 0));
|
||||
Indices.push_back(State.get(Operand));
|
||||
}
|
||||
|
||||
// Create the new GEP. Note that this GEP may be a scalar if VF == 1,
|
||||
@@ -1719,7 +1718,7 @@ void VPWidenGEPRecipe::execute(VPTransformState &State) {
|
||||
Indices, "", isInBounds());
|
||||
assert((State.VF.isScalar() || NewGEP->getType()->isVectorTy()) &&
|
||||
"NewGEP is not a pointer vector");
|
||||
State.set(this, NewGEP, 0);
|
||||
State.set(this, NewGEP);
|
||||
State.addMetadata(NewGEP, GEP);
|
||||
}
|
||||
}
|
||||
@@ -1773,7 +1772,7 @@ void VPVectorPointerRecipe ::execute(VPTransformState &State) {
|
||||
ResultPtr = Builder.CreateGEP(IndexedTy, Ptr, Increment, "", InBounds);
|
||||
}
|
||||
|
||||
State.set(this, ResultPtr, 0, /*IsScalar*/ true);
|
||||
State.set(this, ResultPtr, /*IsScalar*/ true);
|
||||
}
|
||||
|
||||
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
||||
@@ -1813,17 +1812,17 @@ void VPBlendRecipe::execute(VPTransformState &State) {
|
||||
for (unsigned In = 0; In < NumIncoming; ++In) {
|
||||
// We might have single edge PHIs (blocks) - use an identity
|
||||
// 'select' for the first PHI operand.
|
||||
Value *In0 = State.get(getIncomingValue(In), 0, OnlyFirstLaneUsed);
|
||||
Value *In0 = State.get(getIncomingValue(In), OnlyFirstLaneUsed);
|
||||
if (In == 0)
|
||||
Result = In0; // Initialize with the first incoming value.
|
||||
else {
|
||||
// Select between the current value and the previous incoming edge
|
||||
// based on the incoming mask.
|
||||
Value *Cond = State.get(getMask(In), 0, OnlyFirstLaneUsed);
|
||||
Value *Cond = State.get(getMask(In), OnlyFirstLaneUsed);
|
||||
Result = State.Builder.CreateSelect(Cond, In0, Result, "predphi");
|
||||
}
|
||||
}
|
||||
State.set(this, Result, 0, OnlyFirstLaneUsed);
|
||||
State.set(this, Result, OnlyFirstLaneUsed);
|
||||
}
|
||||
|
||||
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
||||
@@ -1852,14 +1851,14 @@ void VPBlendRecipe::print(raw_ostream &O, const Twine &Indent,
|
||||
|
||||
void VPReductionRecipe::execute(VPTransformState &State) {
|
||||
assert(!State.Instance && "Reduction being replicated.");
|
||||
Value *PrevInChain = State.get(getChainOp(), 0, /*IsScalar*/ true);
|
||||
Value *PrevInChain = State.get(getChainOp(), /*IsScalar*/ true);
|
||||
RecurKind Kind = RdxDesc.getRecurrenceKind();
|
||||
// Propagate the fast-math flags carried by the underlying instruction.
|
||||
IRBuilderBase::FastMathFlagGuard FMFGuard(State.Builder);
|
||||
State.Builder.setFastMathFlags(RdxDesc.getFastMathFlags());
|
||||
Value *NewVecOp = State.get(getVecOp(), 0);
|
||||
Value *NewVecOp = State.get(getVecOp());
|
||||
if (VPValue *Cond = getCondOp()) {
|
||||
Value *NewCond = State.get(Cond, 0, State.VF.isScalar());
|
||||
Value *NewCond = State.get(Cond, State.VF.isScalar());
|
||||
VectorType *VecTy = dyn_cast<VectorType>(NewVecOp->getType());
|
||||
Type *ElementTy = VecTy ? VecTy->getElementType() : NewVecOp->getType();
|
||||
|
||||
@@ -1888,7 +1887,7 @@ void VPReductionRecipe::execute(VPTransformState &State) {
|
||||
PrevInChain = NewRed;
|
||||
NextInChain = NewRed;
|
||||
} else {
|
||||
PrevInChain = State.get(getChainOp(), 0, /*IsScalar*/ true);
|
||||
PrevInChain = State.get(getChainOp(), /*IsScalar*/ true);
|
||||
NewRed = createReduction(State.Builder, RdxDesc, NewVecOp);
|
||||
if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind))
|
||||
NextInChain = createMinMaxOp(State.Builder, RdxDesc.getRecurrenceKind(),
|
||||
@@ -1897,7 +1896,7 @@ void VPReductionRecipe::execute(VPTransformState &State) {
|
||||
NextInChain = State.Builder.CreateBinOp(
|
||||
(Instruction::BinaryOps)RdxDesc.getOpcode(Kind), NewRed, PrevInChain);
|
||||
}
|
||||
State.set(this, NextInChain, 0, /*IsScalar*/ true);
|
||||
State.set(this, NextInChain, /*IsScalar*/ true);
|
||||
}
|
||||
|
||||
void VPReductionEVLRecipe::execute(VPTransformState &State) {
|
||||
@@ -1910,8 +1909,8 @@ void VPReductionEVLRecipe::execute(VPTransformState &State) {
|
||||
Builder.setFastMathFlags(RdxDesc.getFastMathFlags());
|
||||
|
||||
RecurKind Kind = RdxDesc.getRecurrenceKind();
|
||||
Value *Prev = State.get(getChainOp(), 0, /*IsScalar*/ true);
|
||||
Value *VecOp = State.get(getVecOp(), 0);
|
||||
Value *Prev = State.get(getChainOp(), /*IsScalar*/ true);
|
||||
Value *VecOp = State.get(getVecOp());
|
||||
Value *EVL = State.get(getEVL(), VPIteration(0, 0));
|
||||
|
||||
VectorBuilder VBuilder(Builder);
|
||||
@@ -1919,7 +1918,7 @@ void VPReductionEVLRecipe::execute(VPTransformState &State) {
|
||||
Value *Mask;
|
||||
// TODO: move the all-true mask generation into VectorBuilder.
|
||||
if (VPValue *CondOp = getCondOp())
|
||||
Mask = State.get(CondOp, 0);
|
||||
Mask = State.get(CondOp);
|
||||
else
|
||||
Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
|
||||
VBuilder.setMask(Mask);
|
||||
@@ -1935,7 +1934,7 @@ void VPReductionEVLRecipe::execute(VPTransformState &State) {
|
||||
NewRed = Builder.CreateBinOp(
|
||||
(Instruction::BinaryOps)RdxDesc.getOpcode(Kind), NewRed, Prev);
|
||||
}
|
||||
State.set(this, NewRed, 0, /*IsScalar*/ true);
|
||||
State.set(this, NewRed, /*IsScalar*/ true);
|
||||
}
|
||||
|
||||
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
||||
@@ -2065,7 +2064,7 @@ void VPBranchOnMaskRecipe::execute(VPTransformState &State) {
|
||||
Value *ConditionBit = nullptr;
|
||||
VPValue *BlockInMask = getMask();
|
||||
if (BlockInMask) {
|
||||
ConditionBit = State.get(BlockInMask, 0);
|
||||
ConditionBit = State.get(BlockInMask);
|
||||
if (ConditionBit->getType()->isVectorTy())
|
||||
ConditionBit = State.Builder.CreateExtractElement(
|
||||
ConditionBit, State.Builder.getInt32(Lane));
|
||||
@@ -2098,19 +2097,19 @@ void VPPredInstPHIRecipe::execute(VPTransformState &State) {
|
||||
// needed. In this case the recipe of the predicated instruction is marked to
|
||||
// also do that packing, thereby "hoisting" the insert-element sequence.
|
||||
// Otherwise, a phi node for the scalar value is needed.
|
||||
if (State.hasVectorValue(getOperand(0), 0)) {
|
||||
Value *VectorValue = State.get(getOperand(0), 0);
|
||||
if (State.hasVectorValue(getOperand(0))) {
|
||||
Value *VectorValue = State.get(getOperand(0));
|
||||
InsertElementInst *IEI = cast<InsertElementInst>(VectorValue);
|
||||
PHINode *VPhi = State.Builder.CreatePHI(IEI->getType(), 2);
|
||||
VPhi->addIncoming(IEI->getOperand(0), PredicatingBB); // Unmodified vector.
|
||||
VPhi->addIncoming(IEI, PredicatedBB); // New vector with inserted element.
|
||||
if (State.hasVectorValue(this, 0))
|
||||
State.reset(this, VPhi, 0);
|
||||
if (State.hasVectorValue(this))
|
||||
State.reset(this, VPhi);
|
||||
else
|
||||
State.set(this, VPhi, 0);
|
||||
State.set(this, VPhi);
|
||||
// NOTE: Currently we need to update the value of the operand, so the next
|
||||
// predicated iteration inserts its generated value in the correct vector.
|
||||
State.reset(getOperand(0), VPhi, 0);
|
||||
State.reset(getOperand(0), VPhi);
|
||||
} else {
|
||||
Type *PredInstType = getOperand(0)->getUnderlyingValue()->getType();
|
||||
PHINode *Phi = State.Builder.CreatePHI(PredInstType, 2);
|
||||
@@ -2190,12 +2189,12 @@ void VPWidenLoadRecipe::execute(VPTransformState &State) {
|
||||
if (auto *VPMask = getMask()) {
|
||||
// Mask reversal is only needed for non-all-one (null) masks, as reverse
|
||||
// of a null all-one mask is a null mask.
|
||||
Mask = State.get(VPMask, 0);
|
||||
Mask = State.get(VPMask);
|
||||
if (isReverse())
|
||||
Mask = Builder.CreateVectorReverse(Mask, "reverse");
|
||||
}
|
||||
|
||||
Value *Addr = State.get(getAddr(), 0, /*IsScalar*/ !CreateGather);
|
||||
Value *Addr = State.get(getAddr(), /*IsScalar*/ !CreateGather);
|
||||
Value *NewLI;
|
||||
if (CreateGather) {
|
||||
NewLI = Builder.CreateMaskedGather(DataTy, Addr, Alignment, Mask, nullptr,
|
||||
@@ -2211,7 +2210,7 @@ void VPWidenLoadRecipe::execute(VPTransformState &State) {
|
||||
State.addMetadata(NewLI, LI);
|
||||
if (Reverse)
|
||||
NewLI = Builder.CreateVectorReverse(NewLI, "reverse");
|
||||
State.set(this, NewLI, 0);
|
||||
State.set(this, NewLI);
|
||||
}
|
||||
|
||||
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
||||
@@ -2247,10 +2246,10 @@ void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
|
||||
State.setDebugLocFrom(getDebugLoc());
|
||||
CallInst *NewLI;
|
||||
Value *EVL = State.get(getEVL(), VPIteration(0, 0));
|
||||
Value *Addr = State.get(getAddr(), 0, !CreateGather);
|
||||
Value *Addr = State.get(getAddr(), !CreateGather);
|
||||
Value *Mask = nullptr;
|
||||
if (VPValue *VPMask = getMask()) {
|
||||
Mask = State.get(VPMask, 0);
|
||||
Mask = State.get(VPMask);
|
||||
if (isReverse())
|
||||
Mask = createReverseEVL(Builder, Mask, EVL, "vp.reverse.mask");
|
||||
} else {
|
||||
@@ -2273,7 +2272,7 @@ void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
|
||||
Instruction *Res = NewLI;
|
||||
if (isReverse())
|
||||
Res = createReverseEVL(Builder, Res, EVL, "vp.reverse");
|
||||
State.set(this, Res, 0);
|
||||
State.set(this, Res);
|
||||
}
|
||||
|
||||
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
||||
@@ -2300,12 +2299,12 @@ void VPWidenStoreRecipe::execute(VPTransformState &State) {
|
||||
if (auto *VPMask = getMask()) {
|
||||
// Mask reversal is only needed for non-all-one (null) masks, as reverse
|
||||
// of a null all-one mask is a null mask.
|
||||
Mask = State.get(VPMask, 0);
|
||||
Mask = State.get(VPMask);
|
||||
if (isReverse())
|
||||
Mask = Builder.CreateVectorReverse(Mask, "reverse");
|
||||
}
|
||||
|
||||
Value *StoredVal = State.get(StoredVPValue, 0);
|
||||
Value *StoredVal = State.get(StoredVPValue);
|
||||
if (isReverse()) {
|
||||
// If we store to reverse consecutive memory locations, then we need
|
||||
// to reverse the order of elements in the stored value.
|
||||
@@ -2313,7 +2312,7 @@ void VPWidenStoreRecipe::execute(VPTransformState &State) {
|
||||
// We don't want to update the value in the map as it might be used in
|
||||
// another expression. So don't call resetVectorValue(StoredVal).
|
||||
}
|
||||
Value *Addr = State.get(getAddr(), 0, /*IsScalar*/ !CreateScatter);
|
||||
Value *Addr = State.get(getAddr(), /*IsScalar*/ !CreateScatter);
|
||||
Instruction *NewSI = nullptr;
|
||||
if (CreateScatter)
|
||||
NewSI = Builder.CreateMaskedScatter(StoredVal, Addr, Alignment, Mask);
|
||||
@@ -2343,19 +2342,19 @@ void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
|
||||
State.setDebugLocFrom(getDebugLoc());
|
||||
|
||||
CallInst *NewSI = nullptr;
|
||||
Value *StoredVal = State.get(StoredValue, 0);
|
||||
Value *StoredVal = State.get(StoredValue);
|
||||
Value *EVL = State.get(getEVL(), VPIteration(0, 0));
|
||||
if (isReverse())
|
||||
StoredVal = createReverseEVL(Builder, StoredVal, EVL, "vp.reverse");
|
||||
Value *Mask = nullptr;
|
||||
if (VPValue *VPMask = getMask()) {
|
||||
Mask = State.get(VPMask, 0);
|
||||
Mask = State.get(VPMask);
|
||||
if (isReverse())
|
||||
Mask = createReverseEVL(Builder, Mask, EVL, "vp.reverse.mask");
|
||||
} else {
|
||||
Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
|
||||
}
|
||||
Value *Addr = State.get(getAddr(), 0, !CreateScatter);
|
||||
Value *Addr = State.get(getAddr(), !CreateScatter);
|
||||
if (CreateScatter) {
|
||||
NewSI = Builder.CreateIntrinsic(Type::getVoidTy(EVL->getContext()),
|
||||
Intrinsic::vp_scatter,
|
||||
@@ -2536,7 +2535,7 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
|
||||
assert(!MaskForGaps && "Interleaved groups with gaps are not supported.");
|
||||
assert(InterleaveFactor == 2 &&
|
||||
"Unsupported deinterleave factor for scalable vectors");
|
||||
auto *ResBlockInMask = State.get(BlockInMask, 0);
|
||||
auto *ResBlockInMask = State.get(BlockInMask);
|
||||
SmallVector<Value *, 2> Ops = {ResBlockInMask, ResBlockInMask};
|
||||
auto *MaskTy = VectorType::get(State.Builder.getInt1Ty(),
|
||||
State.VF.getKnownMinValue() * 2, true);
|
||||
@@ -2548,7 +2547,7 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
|
||||
if (!BlockInMask)
|
||||
return MaskForGaps;
|
||||
|
||||
Value *ResBlockInMask = State.get(BlockInMask, 0);
|
||||
Value *ResBlockInMask = State.get(BlockInMask);
|
||||
Value *ShuffledMask = State.Builder.CreateShuffleVector(
|
||||
ResBlockInMask,
|
||||
createReplicatedMask(InterleaveFactor, State.VF.getKnownMinValue()),
|
||||
@@ -2608,7 +2607,7 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
|
||||
if (Group->isReverse())
|
||||
StridedVec = State.Builder.CreateVectorReverse(StridedVec, "reverse");
|
||||
|
||||
State.set(VPDefs[J], StridedVec, 0);
|
||||
State.set(VPDefs[J], StridedVec);
|
||||
++J;
|
||||
}
|
||||
|
||||
@@ -2641,7 +2640,7 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
|
||||
if (Group->isReverse())
|
||||
StridedVec = State.Builder.CreateVectorReverse(StridedVec, "reverse");
|
||||
|
||||
State.set(VPDefs[J], StridedVec, 0);
|
||||
State.set(VPDefs[J], StridedVec);
|
||||
++J;
|
||||
}
|
||||
return;
|
||||
@@ -2671,7 +2670,7 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
|
||||
continue;
|
||||
}
|
||||
|
||||
Value *StoredVec = State.get(StoredValues[StoredIdx], 0);
|
||||
Value *StoredVec = State.get(StoredValues[StoredIdx]);
|
||||
++StoredIdx;
|
||||
|
||||
if (Group->isReverse())
|
||||
@@ -2738,7 +2737,7 @@ void VPCanonicalIVPHIRecipe::execute(VPTransformState &State) {
|
||||
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
|
||||
Phi->addIncoming(Start, VectorPH);
|
||||
Phi->setDebugLoc(getDebugLoc());
|
||||
State.set(this, Phi, 0, /*IsScalar*/ true);
|
||||
State.set(this, Phi, /*IsScalar*/ true);
|
||||
}
|
||||
|
||||
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
||||
@@ -2783,7 +2782,7 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
|
||||
"Recipe should have been replaced");
|
||||
|
||||
auto *IVR = getParent()->getPlan()->getCanonicalIV();
|
||||
PHINode *CanonicalIV = cast<PHINode>(State.get(IVR, 0, /*IsScalar*/ true));
|
||||
PHINode *CanonicalIV = cast<PHINode>(State.get(IVR, /*IsScalar*/ true));
|
||||
unsigned CurrentPart = getUnrollPart(*this);
|
||||
|
||||
// Build a pointer phi
|
||||
@@ -2800,7 +2799,7 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
|
||||
// The recipe has been unrolled. In that case, fetch the single pointer phi
|
||||
// shared among all unrolled parts of the recipe.
|
||||
auto *GEP =
|
||||
cast<GetElementPtrInst>(State.get(getFirstUnrolledPartOperand(), 0));
|
||||
cast<GetElementPtrInst>(State.get(getFirstUnrolledPartOperand()));
|
||||
NewPointerPhi = cast<PHINode>(GEP->getPointerOperand());
|
||||
}
|
||||
|
||||
@@ -2847,7 +2846,7 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
|
||||
State.Builder.CreateMul(StartOffset, State.Builder.CreateVectorSplat(
|
||||
State.VF, ScalarStepValue)),
|
||||
"vector.gep");
|
||||
State.set(this, GEP, 0);
|
||||
State.set(this, GEP);
|
||||
}
|
||||
|
||||
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
||||
@@ -2892,7 +2891,7 @@ void VPExpandSCEVRecipe::print(raw_ostream &O, const Twine &Indent,
|
||||
#endif
|
||||
|
||||
void VPWidenCanonicalIVRecipe::execute(VPTransformState &State) {
|
||||
Value *CanonicalIV = State.get(getOperand(0), 0, /*IsScalar*/ true);
|
||||
Value *CanonicalIV = State.get(getOperand(0), /*IsScalar*/ true);
|
||||
Type *STy = CanonicalIV->getType();
|
||||
IRBuilder<> Builder(State.CFG.PrevBB->getTerminator());
|
||||
ElementCount VF = State.VF;
|
||||
@@ -2906,7 +2905,7 @@ void VPWidenCanonicalIVRecipe::execute(VPTransformState &State) {
|
||||
Builder.CreateAdd(VStep, Builder.CreateStepVector(VStep->getType()));
|
||||
}
|
||||
Value *CanonicalVectorIV = Builder.CreateAdd(VStart, VStep, "vec.iv");
|
||||
State.set(this, CanonicalVectorIV, 0);
|
||||
State.set(this, CanonicalVectorIV);
|
||||
}
|
||||
|
||||
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
||||
@@ -2944,7 +2943,7 @@ void VPFirstOrderRecurrencePHIRecipe::execute(VPTransformState &State) {
|
||||
PHINode *Phi = PHINode::Create(VecTy, 2, "vector.recur");
|
||||
Phi->insertBefore(State.CFG.PrevBB->getFirstInsertionPt());
|
||||
Phi->addIncoming(VectorInit, VectorPH);
|
||||
State.set(this, Phi, 0);
|
||||
State.set(this, Phi);
|
||||
}
|
||||
|
||||
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
||||
@@ -2978,7 +2977,7 @@ void VPReductionPHIRecipe::execute(VPTransformState &State) {
|
||||
"recipe must be in the vector loop header");
|
||||
auto *Phi = PHINode::Create(VecTy, 2, "vec.phi");
|
||||
Phi->insertBefore(HeaderBB->getFirstInsertionPt());
|
||||
State.set(this, Phi, 0, IsInLoop);
|
||||
State.set(this, Phi, IsInLoop);
|
||||
|
||||
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
|
||||
|
||||
@@ -2994,7 +2993,7 @@ void VPReductionPHIRecipe::execute(VPTransformState &State) {
|
||||
} else {
|
||||
IRBuilderBase::InsertPointGuard IPBuilder(Builder);
|
||||
Builder.SetInsertPoint(VectorPH->getTerminator());
|
||||
StartV = Iden = State.get(StartVPV, 0);
|
||||
StartV = Iden = State.get(StartVPV);
|
||||
}
|
||||
} else {
|
||||
Iden = llvm::getRecurrenceIdentity(RK, VecTy->getScalarType(),
|
||||
@@ -3016,7 +3015,7 @@ void VPReductionPHIRecipe::execute(VPTransformState &State) {
|
||||
}
|
||||
}
|
||||
|
||||
Phi = cast<PHINode>(State.get(this, 0, IsInLoop));
|
||||
Phi = cast<PHINode>(State.get(this, IsInLoop));
|
||||
Value *StartVal = (CurrentPart == 0) ? StartV : Iden;
|
||||
Phi->addIncoming(StartVal, VectorPH);
|
||||
}
|
||||
@@ -3036,10 +3035,10 @@ void VPWidenPHIRecipe::execute(VPTransformState &State) {
|
||||
assert(EnableVPlanNativePath &&
|
||||
"Non-native vplans are not expected to have VPWidenPHIRecipes.");
|
||||
|
||||
Value *Op0 = State.get(getOperand(0), 0);
|
||||
Value *Op0 = State.get(getOperand(0));
|
||||
Type *VecTy = Op0->getType();
|
||||
Value *VecPhi = State.Builder.CreatePHI(VecTy, 2, "vec.phi");
|
||||
State.set(this, VecPhi, 0);
|
||||
State.set(this, VecPhi);
|
||||
}
|
||||
|
||||
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
||||
@@ -3067,12 +3066,12 @@ void VPWidenPHIRecipe::print(raw_ostream &O, const Twine &Indent,
|
||||
// remove VPActiveLaneMaskPHIRecipe.
|
||||
void VPActiveLaneMaskPHIRecipe::execute(VPTransformState &State) {
|
||||
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
|
||||
Value *StartMask = State.get(getOperand(0), 0);
|
||||
Value *StartMask = State.get(getOperand(0));
|
||||
PHINode *Phi =
|
||||
State.Builder.CreatePHI(StartMask->getType(), 2, "active.lane.mask");
|
||||
Phi->addIncoming(StartMask, VectorPH);
|
||||
Phi->setDebugLoc(getDebugLoc());
|
||||
State.set(this, Phi, 0);
|
||||
State.set(this, Phi);
|
||||
}
|
||||
|
||||
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
||||
@@ -3092,7 +3091,7 @@ void VPEVLBasedIVPHIRecipe::execute(VPTransformState &State) {
|
||||
PHINode *Phi = State.Builder.CreatePHI(Start->getType(), 2, "evl.based.iv");
|
||||
Phi->addIncoming(Start, VectorPH);
|
||||
Phi->setDebugLoc(getDebugLoc());
|
||||
State.set(this, Phi, 0, /*IsScalar=*/true);
|
||||
State.set(this, Phi, /*IsScalar=*/true);
|
||||
}
|
||||
|
||||
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
||||
|
||||
Reference in new issue
Block a user