diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 667529c5c3296..f2be3748799db 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -979,8 +979,7 @@ class LoopVectorizationCostModel { /// Setup cost-based decisions for user vectorization factor. /// \return true if the UserVF is a feasible VF to be chosen. bool selectUserVectorizationFactor(ElementCount UserVF) { - collectUniformsAndScalars(UserVF); - collectInstsToScalarize(UserVF); + collectNonVectorizedAndSetWideningDecisions(UserVF); return expectedCost(UserVF).isValid(); } @@ -1236,13 +1235,14 @@ class LoopVectorizationCostModel { /// the loop. void collectInstsToScalarize(ElementCount VF); - /// Collect Uniform and Scalar values for the given \p VF. + /// Collect values that will not be widened, including Uniforms, Scalars, and + /// Instructions to Scalarize for the given \p VF. /// The sets depend on CM decision for Load/Store instructions /// that may be vectorized as interleave, gather-scatter or scalarized. /// Also make a decision on what to do about call instructions in the loop /// at that VF -- scalarize, call a known vector routine, or call a /// vector intrinsic. - void collectUniformsAndScalars(ElementCount VF) { + void collectNonVectorizedAndSetWideningDecisions(ElementCount VF) { // Do the analysis once. if (VF.isScalar() || Uniforms.contains(VF)) return; @@ -1250,6 +1250,7 @@ class LoopVectorizationCostModel { collectLoopUniforms(VF); setVectorizedCallDecision(VF); collectLoopScalars(VF); + collectInstsToScalarize(VF); } /// Returns true if the target machine supports masked store operation @@ -5285,7 +5286,7 @@ LoopVectorizationCostModel::calculateRegisterUsage(ArrayRef VFs) { RegUsage[ClassID] += 1; } } else { - collectUniformsAndScalars(VFs[J]); + collectNonVectorizedAndSetWideningDecisions(VFs[J]); for (auto *Inst : OpenIntervals) { // Skip ignored values for VF > 1. if (VecValuesToIgnore.count(Inst)) @@ -5382,11 +5383,14 @@ bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(Instruction *I, } void LoopVectorizationCostModel::collectInstsToScalarize(ElementCount VF) { - // If we aren't vectorizing the loop, or if we've already collected the - // instructions to scalarize, there's nothing to do. Collection may already - // have occurred if we have a user-selected VF and are now computing the - // expected cost for interleaving. - if (VF.isScalar() || VF.isZero() || InstsToScalarize.contains(VF)) + assert(VF.isVector() && "Expected VF >= 2"); + + // If we've already collected the instructions to scalarize or the predicated + // BBs after vectorization, there's nothing to do. Collection may already have + // occurred if we have a user-selected VF and are now computing the expected + // cost for interleaving. + if (InstsToScalarize.contains(VF) || + PredicatedBBsAfterVectorization.contains(VF)) return; // Initialize a mapping for VF in InstsToScalalarize. If we find that it's @@ -5394,8 +5398,6 @@ void LoopVectorizationCostModel::collectInstsToScalarize(ElementCount VF) { // map will indicate that we've analyzed it already. ScalarCostsTy &ScalarCostsVF = InstsToScalarize[VF]; - PredicatedBBsAfterVectorization[VF].clear(); - // Find all the instructions that are scalar with predication in the loop and // determine if it would be better to not if-convert the blocks they are in. // If so, we also record the instructions to scalarize. @@ -7187,12 +7189,7 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) { CM.collectInLoopReductions(); for (const auto &VF : VFCandidates) { // Collect Uniform and Scalar instructions after vectorization with VF. - CM.collectUniformsAndScalars(VF); - - // Collect the instructions (and their associated costs) that will be more - // profitable to scalarize. - if (VF.isVector()) - CM.collectInstsToScalarize(VF); + CM.collectNonVectorizedAndSetWideningDecisions(VF); } buildVPlansWithVPRecipes(ElementCount::getFixed(1), MaxFactors.FixedVF);