8cd86ff284
If a phi has fast math flags, we can propagate it to the widened select. To do this, this patch makes VPPhi and VPBlendRecipe subclasses of VPRecipeWithIRFlags, and propagates it through PlainCFGBuilder and VPPredicator. Alive2 proofs for some of the FMFs (it looks like it can't reason about the full "fast" set yet) nnan: https://alive2.llvm.org/ce/z/f0bRd4 nsz: https://alive2.llvm.org/ce/z/u9P96T The actual motivation for this to eventually be able to move the special casing for tail folding in LoopVectorizationPlanner::addReductionResultComputation into the CFG in #176143, which requires passing through FMFs.
4855 lines
180 KiB
C++
4855 lines
180 KiB
C++
//===- VPlan.h - Represent A Vectorizer Plan --------------------*- C++ -*-===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
/// \file
|
|
/// This file contains the declarations of the Vectorization Plan base classes:
|
|
/// 1. VPBasicBlock and VPRegionBlock that inherit from a common pure virtual
|
|
/// VPBlockBase, together implementing a Hierarchical CFG;
|
|
/// 2. Pure virtual VPRecipeBase serving as the base class for recipes contained
|
|
/// within VPBasicBlocks;
|
|
/// 3. Pure virtual VPSingleDefRecipe serving as a base class for recipes that
|
|
/// also inherit from VPValue.
|
|
/// 4. VPInstruction, a concrete Recipe and VPUser modeling a single planned
|
|
/// instruction;
|
|
/// 5. The VPlan class holding a candidate for vectorization;
|
|
/// These are documented in docs/VectorizationPlan.rst.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
|
|
#define LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
|
|
|
|
#include "VPlanValue.h"
|
|
#include "llvm/ADT/MapVector.h"
|
|
#include "llvm/ADT/SmallPtrSet.h"
|
|
#include "llvm/ADT/SmallVector.h"
|
|
#include "llvm/ADT/Twine.h"
|
|
#include "llvm/ADT/ilist.h"
|
|
#include "llvm/ADT/ilist_node.h"
|
|
#include "llvm/Analysis/IVDescriptors.h"
|
|
#include "llvm/Analysis/MemoryLocation.h"
|
|
#include "llvm/Analysis/VectorUtils.h"
|
|
#include "llvm/IR/DebugLoc.h"
|
|
#include "llvm/IR/FMF.h"
|
|
#include "llvm/IR/Operator.h"
|
|
#include "llvm/Support/Compiler.h"
|
|
#include "llvm/Support/InstructionCost.h"
|
|
#include <cassert>
|
|
#include <cstddef>
|
|
#include <functional>
|
|
#include <string>
|
|
#include <utility>
|
|
#include <variant>
|
|
|
|
namespace llvm {
|
|
|
|
class BasicBlock;
|
|
class DominatorTree;
|
|
class InnerLoopVectorizer;
|
|
class IRBuilderBase;
|
|
struct VPTransformState;
|
|
class raw_ostream;
|
|
class RecurrenceDescriptor;
|
|
class SCEV;
|
|
class Type;
|
|
class VPBasicBlock;
|
|
class VPBuilder;
|
|
class VPDominatorTree;
|
|
class VPRegionBlock;
|
|
class VPlan;
|
|
class VPLane;
|
|
class VPReplicateRecipe;
|
|
class VPlanSlp;
|
|
class Value;
|
|
class LoopVectorizationCostModel;
|
|
|
|
struct VPCostContext;
|
|
|
|
namespace Intrinsic {
|
|
typedef unsigned ID;
|
|
}
|
|
|
|
using VPlanPtr = std::unique_ptr<VPlan>;
|
|
|
|
/// VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
|
|
/// A VPBlockBase can be either a VPBasicBlock or a VPRegionBlock.
|
|
class LLVM_ABI_FOR_TEST VPBlockBase {
|
|
friend class VPBlockUtils;
|
|
|
|
const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
|
|
|
|
/// An optional name for the block.
|
|
std::string Name;
|
|
|
|
/// The immediate VPRegionBlock which this VPBlockBase belongs to, or null if
|
|
/// it is a topmost VPBlockBase.
|
|
VPRegionBlock *Parent = nullptr;
|
|
|
|
/// List of predecessor blocks.
|
|
SmallVector<VPBlockBase *, 1> Predecessors;
|
|
|
|
/// List of successor blocks.
|
|
SmallVector<VPBlockBase *, 1> Successors;
|
|
|
|
/// VPlan containing the block. Can only be set on the entry block of the
|
|
/// plan.
|
|
VPlan *Plan = nullptr;
|
|
|
|
/// Add \p Successor as the last successor to this block.
|
|
void appendSuccessor(VPBlockBase *Successor) {
|
|
assert(Successor && "Cannot add nullptr successor!");
|
|
Successors.push_back(Successor);
|
|
}
|
|
|
|
/// Add \p Predecessor as the last predecessor to this block.
|
|
void appendPredecessor(VPBlockBase *Predecessor) {
|
|
assert(Predecessor && "Cannot add nullptr predecessor!");
|
|
Predecessors.push_back(Predecessor);
|
|
}
|
|
|
|
/// Remove \p Predecessor from the predecessors of this block.
|
|
void removePredecessor(VPBlockBase *Predecessor) {
|
|
auto Pos = find(Predecessors, Predecessor);
|
|
assert(Pos && "Predecessor does not exist");
|
|
Predecessors.erase(Pos);
|
|
}
|
|
|
|
/// Remove \p Successor from the successors of this block.
|
|
void removeSuccessor(VPBlockBase *Successor) {
|
|
auto Pos = find(Successors, Successor);
|
|
assert(Pos && "Successor does not exist");
|
|
Successors.erase(Pos);
|
|
}
|
|
|
|
/// This function replaces one predecessor with another, useful when
|
|
/// trying to replace an old block in the CFG with a new one.
|
|
void replacePredecessor(VPBlockBase *Old, VPBlockBase *New) {
|
|
auto I = find(Predecessors, Old);
|
|
assert(I != Predecessors.end());
|
|
assert(Old->getParent() == New->getParent() &&
|
|
"replaced predecessor must have the same parent");
|
|
*I = New;
|
|
}
|
|
|
|
/// This function replaces one successor with another, useful when
|
|
/// trying to replace an old block in the CFG with a new one.
|
|
void replaceSuccessor(VPBlockBase *Old, VPBlockBase *New) {
|
|
auto I = find(Successors, Old);
|
|
assert(I != Successors.end());
|
|
assert(Old->getParent() == New->getParent() &&
|
|
"replaced successor must have the same parent");
|
|
*I = New;
|
|
}
|
|
|
|
protected:
|
|
VPBlockBase(const unsigned char SC, const std::string &N)
|
|
: SubclassID(SC), Name(N) {}
|
|
|
|
public:
|
|
/// An enumeration for keeping track of the concrete subclass of VPBlockBase
|
|
/// that are actually instantiated. Values of this enumeration are kept in the
|
|
/// SubclassID field of the VPBlockBase objects. They are used for concrete
|
|
/// type identification.
|
|
using VPBlockTy = enum { VPRegionBlockSC, VPBasicBlockSC, VPIRBasicBlockSC };
|
|
|
|
using VPBlocksTy = SmallVectorImpl<VPBlockBase *>;
|
|
|
|
virtual ~VPBlockBase() = default;
|
|
|
|
const std::string &getName() const { return Name; }
|
|
|
|
void setName(const Twine &newName) { Name = newName.str(); }
|
|
|
|
/// \return an ID for the concrete type of this object.
|
|
/// This is used to implement the classof checks. This should not be used
|
|
/// for any other purpose, as the values may change as LLVM evolves.
|
|
unsigned getVPBlockID() const { return SubclassID; }
|
|
|
|
VPRegionBlock *getParent() { return Parent; }
|
|
const VPRegionBlock *getParent() const { return Parent; }
|
|
|
|
/// \return A pointer to the plan containing the current block.
|
|
VPlan *getPlan();
|
|
const VPlan *getPlan() const;
|
|
|
|
/// Sets the pointer of the plan containing the block. The block must be the
|
|
/// entry block into the VPlan.
|
|
void setPlan(VPlan *ParentPlan);
|
|
|
|
void setParent(VPRegionBlock *P) { Parent = P; }
|
|
|
|
/// \return the VPBasicBlock that is the entry of this VPBlockBase,
|
|
/// recursively, if the latter is a VPRegionBlock. Otherwise, if this
|
|
/// VPBlockBase is a VPBasicBlock, it is returned.
|
|
const VPBasicBlock *getEntryBasicBlock() const;
|
|
VPBasicBlock *getEntryBasicBlock();
|
|
|
|
/// \return the VPBasicBlock that is the exiting this VPBlockBase,
|
|
/// recursively, if the latter is a VPRegionBlock. Otherwise, if this
|
|
/// VPBlockBase is a VPBasicBlock, it is returned.
|
|
const VPBasicBlock *getExitingBasicBlock() const;
|
|
VPBasicBlock *getExitingBasicBlock();
|
|
|
|
const VPBlocksTy &getSuccessors() const { return Successors; }
|
|
VPBlocksTy &getSuccessors() { return Successors; }
|
|
|
|
iterator_range<VPBlockBase **> successors() { return Successors; }
|
|
iterator_range<VPBlockBase **> predecessors() { return Predecessors; }
|
|
|
|
const VPBlocksTy &getPredecessors() const { return Predecessors; }
|
|
VPBlocksTy &getPredecessors() { return Predecessors; }
|
|
|
|
/// \return the successor of this VPBlockBase if it has a single successor.
|
|
/// Otherwise return a null pointer.
|
|
VPBlockBase *getSingleSuccessor() const {
|
|
return (Successors.size() == 1 ? *Successors.begin() : nullptr);
|
|
}
|
|
|
|
/// \return the predecessor of this VPBlockBase if it has a single
|
|
/// predecessor. Otherwise return a null pointer.
|
|
VPBlockBase *getSinglePredecessor() const {
|
|
return (Predecessors.size() == 1 ? *Predecessors.begin() : nullptr);
|
|
}
|
|
|
|
size_t getNumSuccessors() const { return Successors.size(); }
|
|
size_t getNumPredecessors() const { return Predecessors.size(); }
|
|
|
|
/// Returns true if this block has any predecessors.
|
|
bool hasPredecessors() const { return !Predecessors.empty(); }
|
|
|
|
/// An Enclosing Block of a block B is any block containing B, including B
|
|
/// itself. \return the closest enclosing block starting from "this", which
|
|
/// has successors. \return the root enclosing block if all enclosing blocks
|
|
/// have no successors.
|
|
VPBlockBase *getEnclosingBlockWithSuccessors();
|
|
|
|
/// \return the closest enclosing block starting from "this", which has
|
|
/// predecessors. \return the root enclosing block if all enclosing blocks
|
|
/// have no predecessors.
|
|
VPBlockBase *getEnclosingBlockWithPredecessors();
|
|
|
|
/// \return the successors either attached directly to this VPBlockBase or, if
|
|
/// this VPBlockBase is the exit block of a VPRegionBlock and has no
|
|
/// successors of its own, search recursively for the first enclosing
|
|
/// VPRegionBlock that has successors and return them. If no such
|
|
/// VPRegionBlock exists, return the (empty) successors of the topmost
|
|
/// VPBlockBase reached.
|
|
const VPBlocksTy &getHierarchicalSuccessors() {
|
|
return getEnclosingBlockWithSuccessors()->getSuccessors();
|
|
}
|
|
|
|
/// \return the hierarchical successor of this VPBlockBase if it has a single
|
|
/// hierarchical successor. Otherwise return a null pointer.
|
|
VPBlockBase *getSingleHierarchicalSuccessor() {
|
|
return getEnclosingBlockWithSuccessors()->getSingleSuccessor();
|
|
}
|
|
|
|
/// \return the predecessors either attached directly to this VPBlockBase or,
|
|
/// if this VPBlockBase is the entry block of a VPRegionBlock and has no
|
|
/// predecessors of its own, search recursively for the first enclosing
|
|
/// VPRegionBlock that has predecessors and return them. If no such
|
|
/// VPRegionBlock exists, return the (empty) predecessors of the topmost
|
|
/// VPBlockBase reached.
|
|
const VPBlocksTy &getHierarchicalPredecessors() {
|
|
return getEnclosingBlockWithPredecessors()->getPredecessors();
|
|
}
|
|
|
|
/// \return the hierarchical predecessor of this VPBlockBase if it has a
|
|
/// single hierarchical predecessor. Otherwise return a null pointer.
|
|
VPBlockBase *getSingleHierarchicalPredecessor() {
|
|
return getEnclosingBlockWithPredecessors()->getSinglePredecessor();
|
|
}
|
|
|
|
/// Set a given VPBlockBase \p Successor as the single successor of this
|
|
/// VPBlockBase. This VPBlockBase is not added as predecessor of \p Successor.
|
|
/// This VPBlockBase must have no successors.
|
|
void setOneSuccessor(VPBlockBase *Successor) {
|
|
assert(Successors.empty() && "Setting one successor when others exist.");
|
|
assert(Successor->getParent() == getParent() &&
|
|
"connected blocks must have the same parent");
|
|
appendSuccessor(Successor);
|
|
}
|
|
|
|
/// Set two given VPBlockBases \p IfTrue and \p IfFalse to be the two
|
|
/// successors of this VPBlockBase. This VPBlockBase is not added as
|
|
/// predecessor of \p IfTrue or \p IfFalse. This VPBlockBase must have no
|
|
/// successors.
|
|
void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse) {
|
|
assert(Successors.empty() && "Setting two successors when others exist.");
|
|
appendSuccessor(IfTrue);
|
|
appendSuccessor(IfFalse);
|
|
}
|
|
|
|
/// Set each VPBasicBlock in \p NewPreds as predecessor of this VPBlockBase.
|
|
/// This VPBlockBase must have no predecessors. This VPBlockBase is not added
|
|
/// as successor of any VPBasicBlock in \p NewPreds.
|
|
void setPredecessors(ArrayRef<VPBlockBase *> NewPreds) {
|
|
assert(Predecessors.empty() && "Block predecessors already set.");
|
|
for (auto *Pred : NewPreds)
|
|
appendPredecessor(Pred);
|
|
}
|
|
|
|
/// Set each VPBasicBlock in \p NewSuccss as successor of this VPBlockBase.
|
|
/// This VPBlockBase must have no successors. This VPBlockBase is not added
|
|
/// as predecessor of any VPBasicBlock in \p NewSuccs.
|
|
void setSuccessors(ArrayRef<VPBlockBase *> NewSuccs) {
|
|
assert(Successors.empty() && "Block successors already set.");
|
|
for (auto *Succ : NewSuccs)
|
|
appendSuccessor(Succ);
|
|
}
|
|
|
|
/// Remove all the predecessor of this block.
|
|
void clearPredecessors() { Predecessors.clear(); }
|
|
|
|
/// Remove all the successors of this block.
|
|
void clearSuccessors() { Successors.clear(); }
|
|
|
|
/// Swap predecessors of the block. The block must have exactly 2
|
|
/// predecessors.
|
|
void swapPredecessors() {
|
|
assert(Predecessors.size() == 2 && "must have 2 predecessors to swap");
|
|
std::swap(Predecessors[0], Predecessors[1]);
|
|
}
|
|
|
|
/// Swap successors of the block. The block must have exactly 2 successors.
|
|
// TODO: This should be part of introducing conditional branch recipes rather
|
|
// than being independent.
|
|
void swapSuccessors() {
|
|
assert(Successors.size() == 2 && "must have 2 successors to swap");
|
|
std::swap(Successors[0], Successors[1]);
|
|
}
|
|
|
|
/// Returns the index for \p Pred in the blocks predecessors list.
|
|
unsigned getIndexForPredecessor(const VPBlockBase *Pred) const {
|
|
assert(count(Predecessors, Pred) == 1 &&
|
|
"must have Pred exactly once in Predecessors");
|
|
return std::distance(Predecessors.begin(), find(Predecessors, Pred));
|
|
}
|
|
|
|
/// Returns the index for \p Succ in the blocks successor list.
|
|
unsigned getIndexForSuccessor(const VPBlockBase *Succ) const {
|
|
assert(count(Successors, Succ) == 1 &&
|
|
"must have Succ exactly once in Successors");
|
|
return std::distance(Successors.begin(), find(Successors, Succ));
|
|
}
|
|
|
|
/// The method which generates the output IR that correspond to this
|
|
/// VPBlockBase, thereby "executing" the VPlan.
|
|
virtual void execute(VPTransformState *State) = 0;
|
|
|
|
/// Return the cost of the block.
|
|
virtual InstructionCost cost(ElementCount VF, VPCostContext &Ctx) = 0;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
void printAsOperand(raw_ostream &OS, bool PrintType = false) const {
|
|
OS << getName();
|
|
}
|
|
|
|
/// Print plain-text dump of this VPBlockBase to \p O, prefixing all lines
|
|
/// with \p Indent. \p SlotTracker is used to print unnamed VPValue's using
|
|
/// consequtive numbers.
|
|
///
|
|
/// Note that the numbering is applied to the whole VPlan, so printing
|
|
/// individual blocks is consistent with the whole VPlan printing.
|
|
virtual void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const = 0;
|
|
|
|
/// Print plain-text dump of this VPlan to \p O.
|
|
void print(raw_ostream &O) const;
|
|
|
|
/// Print the successors of this block to \p O, prefixing all lines with \p
|
|
/// Indent.
|
|
void printSuccessors(raw_ostream &O, const Twine &Indent) const;
|
|
|
|
/// Dump this VPBlockBase to dbgs().
|
|
LLVM_DUMP_METHOD void dump() const { print(dbgs()); }
|
|
#endif
|
|
|
|
/// Clone the current block and it's recipes without updating the operands of
|
|
/// the cloned recipes, including all blocks in the single-entry single-exit
|
|
/// region for VPRegionBlocks.
|
|
virtual VPBlockBase *clone() = 0;
|
|
};
|
|
|
|
/// VPRecipeBase is a base class modeling a sequence of one or more output IR
|
|
/// instructions. VPRecipeBase owns the VPValues it defines through VPDef
|
|
/// and is responsible for deleting its defined values. Single-value
|
|
/// recipes must inherit from VPSingleDef instead of inheriting from both
|
|
/// VPRecipeBase and VPValue separately.
|
|
class LLVM_ABI_FOR_TEST VPRecipeBase
|
|
: public ilist_node_with_parent<VPRecipeBase, VPBasicBlock>,
|
|
public VPDef,
|
|
public VPUser {
|
|
friend VPBasicBlock;
|
|
friend class VPBlockUtils;
|
|
|
|
/// Subclass identifier (for isa/dyn_cast).
|
|
const unsigned char SubclassID;
|
|
|
|
/// Each VPRecipe belongs to a single VPBasicBlock.
|
|
VPBasicBlock *Parent = nullptr;
|
|
|
|
/// The debug location for the recipe.
|
|
DebugLoc DL;
|
|
|
|
public:
|
|
/// An enumeration for keeping track of the concrete subclass of VPRecipeBase
|
|
/// that is actually instantiated. Values of this enumeration are kept in the
|
|
/// SubclassID field of the VPRecipeBase objects. They are used for concrete
|
|
/// type identification.
|
|
using VPRecipeTy = enum {
|
|
VPBranchOnMaskSC,
|
|
VPDerivedIVSC,
|
|
VPExpandSCEVSC,
|
|
VPExpressionSC,
|
|
VPIRInstructionSC,
|
|
VPInstructionSC,
|
|
VPInterleaveEVLSC,
|
|
VPInterleaveSC,
|
|
VPReductionEVLSC,
|
|
VPReductionSC,
|
|
VPReplicateSC,
|
|
VPScalarIVStepsSC,
|
|
VPVectorPointerSC,
|
|
VPVectorEndPointerSC,
|
|
VPWidenCallSC,
|
|
VPWidenCanonicalIVSC,
|
|
VPWidenCastSC,
|
|
VPWidenGEPSC,
|
|
VPWidenIntrinsicSC,
|
|
VPWidenLoadEVLSC,
|
|
VPWidenLoadSC,
|
|
VPWidenStoreEVLSC,
|
|
VPWidenStoreSC,
|
|
VPWidenSC,
|
|
VPBlendSC,
|
|
VPHistogramSC,
|
|
// START: Phi-like recipes. Need to be kept together.
|
|
VPWidenPHISC,
|
|
VPPredInstPHISC,
|
|
// START: SubclassID for recipes that inherit VPHeaderPHIRecipe.
|
|
// VPHeaderPHIRecipe need to be kept together.
|
|
VPCanonicalIVPHISC,
|
|
VPActiveLaneMaskPHISC,
|
|
VPEVLBasedIVPHISC,
|
|
VPFirstOrderRecurrencePHISC,
|
|
VPWidenIntOrFpInductionSC,
|
|
VPWidenPointerInductionSC,
|
|
VPReductionPHISC,
|
|
// END: SubclassID for recipes that inherit VPHeaderPHIRecipe
|
|
// END: Phi-like recipes
|
|
VPFirstPHISC = VPWidenPHISC,
|
|
VPFirstHeaderPHISC = VPCanonicalIVPHISC,
|
|
VPLastHeaderPHISC = VPReductionPHISC,
|
|
VPLastPHISC = VPReductionPHISC,
|
|
};
|
|
|
|
VPRecipeBase(const unsigned char SC, ArrayRef<VPValue *> Operands,
|
|
DebugLoc DL = DebugLoc::getUnknown())
|
|
: VPDef(), VPUser(Operands), SubclassID(SC), DL(DL) {}
|
|
|
|
~VPRecipeBase() override = default;
|
|
|
|
/// Clone the current recipe.
|
|
virtual VPRecipeBase *clone() = 0;
|
|
|
|
/// \return the VPBasicBlock which this VPRecipe belongs to.
|
|
VPBasicBlock *getParent() { return Parent; }
|
|
const VPBasicBlock *getParent() const { return Parent; }
|
|
|
|
/// \return the VPRegionBlock which the recipe belongs to.
|
|
VPRegionBlock *getRegion();
|
|
const VPRegionBlock *getRegion() const;
|
|
|
|
/// The method which generates the output IR instructions that correspond to
|
|
/// this VPRecipe, thereby "executing" the VPlan.
|
|
virtual void execute(VPTransformState &State) = 0;
|
|
|
|
/// Return the cost of this recipe, taking into account if the cost
|
|
/// computation should be skipped and the ForceTargetInstructionCost flag.
|
|
/// Also takes care of printing the cost for debugging.
|
|
InstructionCost cost(ElementCount VF, VPCostContext &Ctx);
|
|
|
|
/// Insert an unlinked recipe into a basic block immediately before
|
|
/// the specified recipe.
|
|
void insertBefore(VPRecipeBase *InsertPos);
|
|
/// Insert an unlinked recipe into \p BB immediately before the insertion
|
|
/// point \p IP;
|
|
void insertBefore(VPBasicBlock &BB, iplist<VPRecipeBase>::iterator IP);
|
|
|
|
/// Insert an unlinked Recipe into a basic block immediately after
|
|
/// the specified Recipe.
|
|
void insertAfter(VPRecipeBase *InsertPos);
|
|
|
|
/// Unlink this recipe from its current VPBasicBlock and insert it into
|
|
/// the VPBasicBlock that MovePos lives in, right after MovePos.
|
|
void moveAfter(VPRecipeBase *MovePos);
|
|
|
|
/// Unlink this recipe and insert into BB before I.
|
|
///
|
|
/// \pre I is a valid iterator into BB.
|
|
void moveBefore(VPBasicBlock &BB, iplist<VPRecipeBase>::iterator I);
|
|
|
|
/// This method unlinks 'this' from the containing basic block, but does not
|
|
/// delete it.
|
|
void removeFromParent();
|
|
|
|
/// This method unlinks 'this' from the containing basic block and deletes it.
|
|
///
|
|
/// \returns an iterator pointing to the element after the erased one
|
|
iplist<VPRecipeBase>::iterator eraseFromParent();
|
|
|
|
/// \return an ID for the concrete type of this object.
|
|
unsigned getVPRecipeID() const { return SubclassID; }
|
|
|
|
/// Method to support type inquiry through isa, cast, and dyn_cast.
|
|
static inline bool classof(const VPDef *D) {
|
|
// All VPDefs are also VPRecipeBases.
|
|
return true;
|
|
}
|
|
|
|
static inline bool classof(const VPUser *U) { return true; }
|
|
|
|
/// Returns true if the recipe may have side-effects.
|
|
bool mayHaveSideEffects() const;
|
|
|
|
/// Returns true for PHI-like recipes.
|
|
bool isPhi() const;
|
|
|
|
/// Returns true if the recipe may read from memory.
|
|
bool mayReadFromMemory() const;
|
|
|
|
/// Returns true if the recipe may write to memory.
|
|
bool mayWriteToMemory() const;
|
|
|
|
/// Returns true if the recipe may read from or write to memory.
|
|
bool mayReadOrWriteMemory() const {
|
|
return mayReadFromMemory() || mayWriteToMemory();
|
|
}
|
|
|
|
/// Returns the debug location of the recipe.
|
|
DebugLoc getDebugLoc() const { return DL; }
|
|
|
|
/// Return true if the recipe is a scalar cast.
|
|
bool isScalarCast() const;
|
|
|
|
/// Set the recipe's debug location to \p NewDL.
|
|
void setDebugLoc(DebugLoc NewDL) { DL = NewDL; }
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Dump the recipe to stderr (for debugging).
|
|
LLVM_ABI_FOR_TEST void dump() const;
|
|
|
|
/// Print the recipe, delegating to printRecipe().
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const;
|
|
#endif
|
|
|
|
protected:
|
|
/// Compute the cost of this recipe either using a recipe's specialized
|
|
/// implementation or using the legacy cost model and the underlying
|
|
/// instructions.
|
|
virtual InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Each concrete VPRecipe prints itself, without printing common information,
|
|
/// like debug info or metadata.
|
|
virtual void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const = 0;
|
|
#endif
|
|
};
|
|
|
|
// Helper macro to define common classof implementations for recipes.
|
|
#define VP_CLASSOF_IMPL(VPRecipeID) \
|
|
static inline bool classof(const VPRecipeBase *R) { \
|
|
return R->getVPRecipeID() == VPRecipeID; \
|
|
} \
|
|
static inline bool classof(const VPValue *V) { \
|
|
auto *R = V->getDefiningRecipe(); \
|
|
return R && R->getVPRecipeID() == VPRecipeID; \
|
|
} \
|
|
static inline bool classof(const VPUser *U) { \
|
|
auto *R = dyn_cast<VPRecipeBase>(U); \
|
|
return R && R->getVPRecipeID() == VPRecipeID; \
|
|
} \
|
|
static inline bool classof(const VPSingleDefRecipe *R) { \
|
|
return R->getVPRecipeID() == VPRecipeID; \
|
|
}
|
|
|
|
/// VPSingleDef is a base class for recipes for modeling a sequence of one or
|
|
/// more output IR that define a single result VPValue.
|
|
/// Note that VPRecipeBase must be inherited from before VPValue.
|
|
class VPSingleDefRecipe : public VPRecipeBase, public VPRecipeValue {
|
|
public:
|
|
VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
|
|
DebugLoc DL = DebugLoc::getUnknown())
|
|
: VPRecipeBase(SC, Operands, DL), VPRecipeValue(this) {}
|
|
|
|
VPSingleDefRecipe(const unsigned char SC, ArrayRef<VPValue *> Operands,
|
|
Value *UV, DebugLoc DL = DebugLoc::getUnknown())
|
|
: VPRecipeBase(SC, Operands, DL), VPRecipeValue(this, UV) {}
|
|
|
|
static inline bool classof(const VPRecipeBase *R) {
|
|
switch (R->getVPRecipeID()) {
|
|
case VPRecipeBase::VPDerivedIVSC:
|
|
case VPRecipeBase::VPEVLBasedIVPHISC:
|
|
case VPRecipeBase::VPExpandSCEVSC:
|
|
case VPRecipeBase::VPExpressionSC:
|
|
case VPRecipeBase::VPInstructionSC:
|
|
case VPRecipeBase::VPReductionEVLSC:
|
|
case VPRecipeBase::VPReductionSC:
|
|
case VPRecipeBase::VPReplicateSC:
|
|
case VPRecipeBase::VPScalarIVStepsSC:
|
|
case VPRecipeBase::VPVectorPointerSC:
|
|
case VPRecipeBase::VPVectorEndPointerSC:
|
|
case VPRecipeBase::VPWidenCallSC:
|
|
case VPRecipeBase::VPWidenCanonicalIVSC:
|
|
case VPRecipeBase::VPWidenCastSC:
|
|
case VPRecipeBase::VPWidenGEPSC:
|
|
case VPRecipeBase::VPWidenIntrinsicSC:
|
|
case VPRecipeBase::VPWidenSC:
|
|
case VPRecipeBase::VPBlendSC:
|
|
case VPRecipeBase::VPPredInstPHISC:
|
|
case VPRecipeBase::VPCanonicalIVPHISC:
|
|
case VPRecipeBase::VPActiveLaneMaskPHISC:
|
|
case VPRecipeBase::VPFirstOrderRecurrencePHISC:
|
|
case VPRecipeBase::VPWidenPHISC:
|
|
case VPRecipeBase::VPWidenIntOrFpInductionSC:
|
|
case VPRecipeBase::VPWidenPointerInductionSC:
|
|
case VPRecipeBase::VPReductionPHISC:
|
|
return true;
|
|
case VPRecipeBase::VPBranchOnMaskSC:
|
|
case VPRecipeBase::VPInterleaveEVLSC:
|
|
case VPRecipeBase::VPInterleaveSC:
|
|
case VPRecipeBase::VPIRInstructionSC:
|
|
case VPRecipeBase::VPWidenLoadEVLSC:
|
|
case VPRecipeBase::VPWidenLoadSC:
|
|
case VPRecipeBase::VPWidenStoreEVLSC:
|
|
case VPRecipeBase::VPWidenStoreSC:
|
|
case VPRecipeBase::VPHistogramSC:
|
|
// TODO: Widened stores don't define a value, but widened loads do. Split
|
|
// the recipes to be able to make widened loads VPSingleDefRecipes.
|
|
return false;
|
|
}
|
|
llvm_unreachable("Unhandled VPRecipeID");
|
|
}
|
|
|
|
static inline bool classof(const VPUser *U) {
|
|
auto *R = dyn_cast<VPRecipeBase>(U);
|
|
return R && classof(R);
|
|
}
|
|
|
|
VPSingleDefRecipe *clone() override = 0;
|
|
|
|
/// Returns the underlying instruction.
|
|
Instruction *getUnderlyingInstr() {
|
|
return cast<Instruction>(getUnderlyingValue());
|
|
}
|
|
const Instruction *getUnderlyingInstr() const {
|
|
return cast<Instruction>(getUnderlyingValue());
|
|
}
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print this VPSingleDefRecipe to dbgs() (for debugging).
|
|
LLVM_ABI_FOR_TEST LLVM_DUMP_METHOD void dump() const;
|
|
#endif
|
|
};
|
|
|
|
/// Class to record and manage LLVM IR flags.
|
|
class VPIRFlags {
|
|
enum class OperationType : unsigned char {
|
|
Cmp,
|
|
FCmp,
|
|
OverflowingBinOp,
|
|
Trunc,
|
|
DisjointOp,
|
|
PossiblyExactOp,
|
|
GEPOp,
|
|
FPMathOp,
|
|
NonNegOp,
|
|
ReductionOp,
|
|
Other
|
|
};
|
|
|
|
public:
|
|
struct WrapFlagsTy {
|
|
char HasNUW : 1;
|
|
char HasNSW : 1;
|
|
|
|
WrapFlagsTy(bool HasNUW, bool HasNSW) : HasNUW(HasNUW), HasNSW(HasNSW) {}
|
|
};
|
|
|
|
struct TruncFlagsTy {
|
|
char HasNUW : 1;
|
|
char HasNSW : 1;
|
|
|
|
TruncFlagsTy(bool HasNUW, bool HasNSW) : HasNUW(HasNUW), HasNSW(HasNSW) {}
|
|
};
|
|
|
|
struct DisjointFlagsTy {
|
|
char IsDisjoint : 1;
|
|
DisjointFlagsTy(bool IsDisjoint) : IsDisjoint(IsDisjoint) {}
|
|
};
|
|
|
|
struct NonNegFlagsTy {
|
|
char NonNeg : 1;
|
|
NonNegFlagsTy(bool IsNonNeg) : NonNeg(IsNonNeg) {}
|
|
};
|
|
|
|
private:
|
|
struct ExactFlagsTy {
|
|
char IsExact : 1;
|
|
ExactFlagsTy(bool Exact) : IsExact(Exact) {}
|
|
};
|
|
struct FastMathFlagsTy {
|
|
char AllowReassoc : 1;
|
|
char NoNaNs : 1;
|
|
char NoInfs : 1;
|
|
char NoSignedZeros : 1;
|
|
char AllowReciprocal : 1;
|
|
char AllowContract : 1;
|
|
char ApproxFunc : 1;
|
|
|
|
LLVM_ABI_FOR_TEST FastMathFlagsTy(const FastMathFlags &FMF);
|
|
};
|
|
/// Holds both the predicate and fast-math flags for floating-point
|
|
/// comparisons.
|
|
struct FCmpFlagsTy {
|
|
CmpInst::Predicate Pred;
|
|
FastMathFlagsTy FMFs;
|
|
};
|
|
/// Holds reduction-specific flags: RecurKind, IsOrdered, IsInLoop, and FMFs.
|
|
struct ReductionFlagsTy {
|
|
// RecurKind has ~26 values, needs 5 bits but uses 6 bits to account for
|
|
// additional kinds.
|
|
unsigned char Kind : 6;
|
|
// TODO: Derive order/in-loop from plan and remove here.
|
|
unsigned char IsOrdered : 1;
|
|
unsigned char IsInLoop : 1;
|
|
FastMathFlagsTy FMFs;
|
|
|
|
ReductionFlagsTy(RecurKind Kind, bool IsOrdered, bool IsInLoop,
|
|
FastMathFlags FMFs)
|
|
: Kind(static_cast<unsigned char>(Kind)), IsOrdered(IsOrdered),
|
|
IsInLoop(IsInLoop), FMFs(FMFs) {}
|
|
};
|
|
|
|
OperationType OpType;
|
|
|
|
union {
|
|
CmpInst::Predicate CmpPredicate;
|
|
WrapFlagsTy WrapFlags;
|
|
TruncFlagsTy TruncFlags;
|
|
DisjointFlagsTy DisjointFlags;
|
|
ExactFlagsTy ExactFlags;
|
|
GEPNoWrapFlags GEPFlags;
|
|
NonNegFlagsTy NonNegFlags;
|
|
FastMathFlagsTy FMFs;
|
|
FCmpFlagsTy FCmpFlags;
|
|
ReductionFlagsTy ReductionFlags;
|
|
unsigned AllFlags;
|
|
};
|
|
|
|
public:
|
|
VPIRFlags() : OpType(OperationType::Other), AllFlags(0) {}
|
|
|
|
VPIRFlags(Instruction &I) {
|
|
if (auto *FCmp = dyn_cast<FCmpInst>(&I)) {
|
|
OpType = OperationType::FCmp;
|
|
FCmpFlags.Pred = FCmp->getPredicate();
|
|
FCmpFlags.FMFs = FCmp->getFastMathFlags();
|
|
} else if (auto *Op = dyn_cast<CmpInst>(&I)) {
|
|
OpType = OperationType::Cmp;
|
|
CmpPredicate = Op->getPredicate();
|
|
} else if (auto *Op = dyn_cast<PossiblyDisjointInst>(&I)) {
|
|
OpType = OperationType::DisjointOp;
|
|
DisjointFlags.IsDisjoint = Op->isDisjoint();
|
|
} else if (auto *Op = dyn_cast<OverflowingBinaryOperator>(&I)) {
|
|
OpType = OperationType::OverflowingBinOp;
|
|
WrapFlags = {Op->hasNoUnsignedWrap(), Op->hasNoSignedWrap()};
|
|
} else if (auto *Op = dyn_cast<TruncInst>(&I)) {
|
|
OpType = OperationType::Trunc;
|
|
TruncFlags = {Op->hasNoUnsignedWrap(), Op->hasNoSignedWrap()};
|
|
} else if (auto *Op = dyn_cast<PossiblyExactOperator>(&I)) {
|
|
OpType = OperationType::PossiblyExactOp;
|
|
ExactFlags.IsExact = Op->isExact();
|
|
} else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
|
|
OpType = OperationType::GEPOp;
|
|
GEPFlags = GEP->getNoWrapFlags();
|
|
} else if (auto *PNNI = dyn_cast<PossiblyNonNegInst>(&I)) {
|
|
OpType = OperationType::NonNegOp;
|
|
NonNegFlags.NonNeg = PNNI->hasNonNeg();
|
|
} else if (auto *Op = dyn_cast<FPMathOperator>(&I)) {
|
|
OpType = OperationType::FPMathOp;
|
|
FMFs = Op->getFastMathFlags();
|
|
} else {
|
|
OpType = OperationType::Other;
|
|
AllFlags = 0;
|
|
}
|
|
}
|
|
|
|
VPIRFlags(CmpInst::Predicate Pred)
|
|
: OpType(OperationType::Cmp), CmpPredicate(Pred) {}
|
|
|
|
VPIRFlags(CmpInst::Predicate Pred, FastMathFlags FMFs)
|
|
: OpType(OperationType::FCmp) {
|
|
FCmpFlags.Pred = Pred;
|
|
FCmpFlags.FMFs = FMFs;
|
|
}
|
|
|
|
VPIRFlags(WrapFlagsTy WrapFlags)
|
|
: OpType(OperationType::OverflowingBinOp), WrapFlags(WrapFlags) {}
|
|
|
|
VPIRFlags(TruncFlagsTy TruncFlags)
|
|
: OpType(OperationType::Trunc), TruncFlags(TruncFlags) {}
|
|
|
|
VPIRFlags(FastMathFlags FMFs) : OpType(OperationType::FPMathOp), FMFs(FMFs) {}
|
|
|
|
VPIRFlags(DisjointFlagsTy DisjointFlags)
|
|
: OpType(OperationType::DisjointOp), DisjointFlags(DisjointFlags) {}
|
|
|
|
VPIRFlags(NonNegFlagsTy NonNegFlags)
|
|
: OpType(OperationType::NonNegOp), NonNegFlags(NonNegFlags) {}
|
|
|
|
VPIRFlags(ExactFlagsTy ExactFlags)
|
|
: OpType(OperationType::PossiblyExactOp), ExactFlags(ExactFlags) {}
|
|
|
|
VPIRFlags(GEPNoWrapFlags GEPFlags)
|
|
: OpType(OperationType::GEPOp), GEPFlags(GEPFlags) {}
|
|
|
|
VPIRFlags(RecurKind Kind, bool IsOrdered, bool IsInLoop, FastMathFlags FMFs)
|
|
: OpType(OperationType::ReductionOp),
|
|
ReductionFlags(Kind, IsOrdered, IsInLoop, FMFs) {}
|
|
|
|
void transferFlags(VPIRFlags &Other) {
|
|
OpType = Other.OpType;
|
|
AllFlags = Other.AllFlags;
|
|
}
|
|
|
|
/// Only keep flags also present in \p Other. \p Other must have the same
|
|
/// OpType as the current object.
|
|
void intersectFlags(const VPIRFlags &Other);
|
|
|
|
/// Drop all poison-generating flags.
|
|
void dropPoisonGeneratingFlags() {
|
|
// NOTE: This needs to be kept in-sync with
|
|
// Instruction::dropPoisonGeneratingFlags.
|
|
switch (OpType) {
|
|
case OperationType::OverflowingBinOp:
|
|
WrapFlags.HasNUW = false;
|
|
WrapFlags.HasNSW = false;
|
|
break;
|
|
case OperationType::Trunc:
|
|
TruncFlags.HasNUW = false;
|
|
TruncFlags.HasNSW = false;
|
|
break;
|
|
case OperationType::DisjointOp:
|
|
DisjointFlags.IsDisjoint = false;
|
|
break;
|
|
case OperationType::PossiblyExactOp:
|
|
ExactFlags.IsExact = false;
|
|
break;
|
|
case OperationType::GEPOp:
|
|
GEPFlags = GEPNoWrapFlags::none();
|
|
break;
|
|
case OperationType::FPMathOp:
|
|
case OperationType::FCmp:
|
|
case OperationType::ReductionOp:
|
|
getFMFsRef().NoNaNs = false;
|
|
getFMFsRef().NoInfs = false;
|
|
break;
|
|
case OperationType::NonNegOp:
|
|
NonNegFlags.NonNeg = false;
|
|
break;
|
|
case OperationType::Cmp:
|
|
case OperationType::Other:
|
|
break;
|
|
}
|
|
}
|
|
|
|
/// Apply the IR flags to \p I.
|
|
void applyFlags(Instruction &I) const {
|
|
switch (OpType) {
|
|
case OperationType::OverflowingBinOp:
|
|
I.setHasNoUnsignedWrap(WrapFlags.HasNUW);
|
|
I.setHasNoSignedWrap(WrapFlags.HasNSW);
|
|
break;
|
|
case OperationType::Trunc:
|
|
I.setHasNoUnsignedWrap(TruncFlags.HasNUW);
|
|
I.setHasNoSignedWrap(TruncFlags.HasNSW);
|
|
break;
|
|
case OperationType::DisjointOp:
|
|
cast<PossiblyDisjointInst>(&I)->setIsDisjoint(DisjointFlags.IsDisjoint);
|
|
break;
|
|
case OperationType::PossiblyExactOp:
|
|
I.setIsExact(ExactFlags.IsExact);
|
|
break;
|
|
case OperationType::GEPOp:
|
|
cast<GetElementPtrInst>(&I)->setNoWrapFlags(GEPFlags);
|
|
break;
|
|
case OperationType::FPMathOp:
|
|
case OperationType::FCmp: {
|
|
const FastMathFlagsTy &F = getFMFsRef();
|
|
I.setHasAllowReassoc(F.AllowReassoc);
|
|
I.setHasNoNaNs(F.NoNaNs);
|
|
I.setHasNoInfs(F.NoInfs);
|
|
I.setHasNoSignedZeros(F.NoSignedZeros);
|
|
I.setHasAllowReciprocal(F.AllowReciprocal);
|
|
I.setHasAllowContract(F.AllowContract);
|
|
I.setHasApproxFunc(F.ApproxFunc);
|
|
break;
|
|
}
|
|
case OperationType::NonNegOp:
|
|
I.setNonNeg(NonNegFlags.NonNeg);
|
|
break;
|
|
case OperationType::ReductionOp:
|
|
llvm_unreachable("reduction ops should not use applyFlags");
|
|
case OperationType::Cmp:
|
|
case OperationType::Other:
|
|
break;
|
|
}
|
|
}
|
|
|
|
CmpInst::Predicate getPredicate() const {
|
|
assert((OpType == OperationType::Cmp || OpType == OperationType::FCmp) &&
|
|
"recipe doesn't have a compare predicate");
|
|
return OpType == OperationType::FCmp ? FCmpFlags.Pred : CmpPredicate;
|
|
}
|
|
|
|
void setPredicate(CmpInst::Predicate Pred) {
|
|
assert((OpType == OperationType::Cmp || OpType == OperationType::FCmp) &&
|
|
"recipe doesn't have a compare predicate");
|
|
if (OpType == OperationType::FCmp)
|
|
FCmpFlags.Pred = Pred;
|
|
else
|
|
CmpPredicate = Pred;
|
|
}
|
|
|
|
GEPNoWrapFlags getGEPNoWrapFlags() const { return GEPFlags; }
|
|
|
|
/// Returns true if the recipe has a comparison predicate.
|
|
bool hasPredicate() const {
|
|
return OpType == OperationType::Cmp || OpType == OperationType::FCmp;
|
|
}
|
|
|
|
/// Returns true if the recipe has fast-math flags.
|
|
bool hasFastMathFlags() const {
|
|
return OpType == OperationType::FPMathOp || OpType == OperationType::FCmp ||
|
|
OpType == OperationType::ReductionOp;
|
|
}
|
|
|
|
LLVM_ABI_FOR_TEST FastMathFlags getFastMathFlags() const;
|
|
|
|
/// Returns true if the recipe has non-negative flag.
|
|
bool hasNonNegFlag() const { return OpType == OperationType::NonNegOp; }
|
|
|
|
bool isNonNeg() const {
|
|
assert(OpType == OperationType::NonNegOp &&
|
|
"recipe doesn't have a NNEG flag");
|
|
return NonNegFlags.NonNeg;
|
|
}
|
|
|
|
bool hasNoUnsignedWrap() const {
|
|
switch (OpType) {
|
|
case OperationType::OverflowingBinOp:
|
|
return WrapFlags.HasNUW;
|
|
case OperationType::Trunc:
|
|
return TruncFlags.HasNUW;
|
|
default:
|
|
llvm_unreachable("recipe doesn't have a NUW flag");
|
|
}
|
|
}
|
|
|
|
bool hasNoSignedWrap() const {
|
|
switch (OpType) {
|
|
case OperationType::OverflowingBinOp:
|
|
return WrapFlags.HasNSW;
|
|
case OperationType::Trunc:
|
|
return TruncFlags.HasNSW;
|
|
default:
|
|
llvm_unreachable("recipe doesn't have a NSW flag");
|
|
}
|
|
}
|
|
|
|
bool isDisjoint() const {
|
|
assert(OpType == OperationType::DisjointOp &&
|
|
"recipe cannot have a disjoing flag");
|
|
return DisjointFlags.IsDisjoint;
|
|
}
|
|
|
|
RecurKind getRecurKind() const {
|
|
assert(OpType == OperationType::ReductionOp &&
|
|
"recipe doesn't have reduction flags");
|
|
return static_cast<RecurKind>(ReductionFlags.Kind);
|
|
}
|
|
|
|
bool isReductionOrdered() const {
|
|
assert(OpType == OperationType::ReductionOp &&
|
|
"recipe doesn't have reduction flags");
|
|
return ReductionFlags.IsOrdered;
|
|
}
|
|
|
|
bool isReductionInLoop() const {
|
|
assert(OpType == OperationType::ReductionOp &&
|
|
"recipe doesn't have reduction flags");
|
|
return ReductionFlags.IsInLoop;
|
|
}
|
|
|
|
private:
|
|
/// Get a reference to the fast-math flags for FPMathOp, FCmp or ReductionOp.
|
|
FastMathFlagsTy &getFMFsRef() {
|
|
if (OpType == OperationType::FCmp)
|
|
return FCmpFlags.FMFs;
|
|
if (OpType == OperationType::ReductionOp)
|
|
return ReductionFlags.FMFs;
|
|
return FMFs;
|
|
}
|
|
const FastMathFlagsTy &getFMFsRef() const {
|
|
if (OpType == OperationType::FCmp)
|
|
return FCmpFlags.FMFs;
|
|
if (OpType == OperationType::ReductionOp)
|
|
return ReductionFlags.FMFs;
|
|
return FMFs;
|
|
}
|
|
|
|
public:
|
|
/// Returns default flags for \p Opcode for opcodes that support it, asserts
|
|
/// otherwise. Opcodes not supporting default flags include compares and
|
|
/// ComputeReductionResult.
|
|
static VPIRFlags getDefaultFlags(unsigned Opcode);
|
|
|
|
#if !defined(NDEBUG)
|
|
/// Returns true if the set flags are valid for \p Opcode.
|
|
LLVM_ABI_FOR_TEST bool flagsValidForOpcode(unsigned Opcode) const;
|
|
|
|
/// Returns true if \p Opcode has its required flags set.
|
|
LLVM_ABI_FOR_TEST bool hasRequiredFlagsForOpcode(unsigned Opcode) const;
|
|
#endif
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
void printFlags(raw_ostream &O) const;
|
|
#endif
|
|
};
|
|
|
|
/// A pure-virtual common base class for recipes defining a single VPValue and
|
|
/// using IR flags.
|
|
struct VPRecipeWithIRFlags : public VPSingleDefRecipe, public VPIRFlags {
|
|
VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
|
|
const VPIRFlags &Flags,
|
|
DebugLoc DL = DebugLoc::getUnknown())
|
|
: VPSingleDefRecipe(SC, Operands, DL), VPIRFlags(Flags) {}
|
|
|
|
static inline bool classof(const VPRecipeBase *R) {
|
|
return R->getVPRecipeID() == VPRecipeBase::VPBlendSC ||
|
|
R->getVPRecipeID() == VPRecipeBase::VPInstructionSC ||
|
|
R->getVPRecipeID() == VPRecipeBase::VPWidenSC ||
|
|
R->getVPRecipeID() == VPRecipeBase::VPWidenGEPSC ||
|
|
R->getVPRecipeID() == VPRecipeBase::VPWidenCallSC ||
|
|
R->getVPRecipeID() == VPRecipeBase::VPWidenCastSC ||
|
|
R->getVPRecipeID() == VPRecipeBase::VPWidenIntrinsicSC ||
|
|
R->getVPRecipeID() == VPRecipeBase::VPReductionSC ||
|
|
R->getVPRecipeID() == VPRecipeBase::VPReductionEVLSC ||
|
|
R->getVPRecipeID() == VPRecipeBase::VPReplicateSC ||
|
|
R->getVPRecipeID() == VPRecipeBase::VPVectorEndPointerSC ||
|
|
R->getVPRecipeID() == VPRecipeBase::VPVectorPointerSC;
|
|
}
|
|
|
|
static inline bool classof(const VPUser *U) {
|
|
auto *R = dyn_cast<VPRecipeBase>(U);
|
|
return R && classof(R);
|
|
}
|
|
|
|
static inline bool classof(const VPValue *V) {
|
|
auto *R = V->getDefiningRecipe();
|
|
return R && classof(R);
|
|
}
|
|
|
|
VPRecipeWithIRFlags *clone() override = 0;
|
|
|
|
static inline bool classof(const VPSingleDefRecipe *R) {
|
|
return classof(static_cast<const VPRecipeBase *>(R));
|
|
}
|
|
|
|
void execute(VPTransformState &State) override = 0;
|
|
|
|
/// Compute the cost for this recipe for \p VF, using \p Opcode and \p Ctx.
|
|
InstructionCost getCostForRecipeWithOpcode(unsigned Opcode, ElementCount VF,
|
|
VPCostContext &Ctx) const;
|
|
};
|
|
|
|
/// Helper to access the operand that contains the unroll part for this recipe
|
|
/// after unrolling.
|
|
template <unsigned PartOpIdx> class LLVM_ABI_FOR_TEST VPUnrollPartAccessor {
|
|
protected:
|
|
/// Return the VPValue operand containing the unroll part or null if there is
|
|
/// no such operand.
|
|
VPValue *getUnrollPartOperand(const VPUser &U) const;
|
|
|
|
/// Return the unroll part.
|
|
unsigned getUnrollPart(const VPUser &U) const;
|
|
};
|
|
|
|
/// Helper to manage IR metadata for recipes. It filters out metadata that
|
|
/// cannot be propagated.
|
|
class VPIRMetadata {
|
|
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
|
|
|
|
public:
|
|
VPIRMetadata() = default;
|
|
|
|
/// Adds metatadata that can be preserved from the original instruction
|
|
/// \p I.
|
|
VPIRMetadata(Instruction &I) { getMetadataToPropagate(&I, Metadata); }
|
|
|
|
/// Copy constructor for cloning.
|
|
VPIRMetadata(const VPIRMetadata &Other) = default;
|
|
|
|
VPIRMetadata &operator=(const VPIRMetadata &Other) = default;
|
|
|
|
/// Add all metadata to \p I.
|
|
void applyMetadata(Instruction &I) const;
|
|
|
|
/// Set metadata with kind \p Kind to \p Node. If metadata with \p Kind
|
|
/// already exists, it will be replaced. Otherwise, it will be added.
|
|
void setMetadata(unsigned Kind, MDNode *Node) {
|
|
auto It =
|
|
llvm::find_if(Metadata, [Kind](const std::pair<unsigned, MDNode *> &P) {
|
|
return P.first == Kind;
|
|
});
|
|
if (It != Metadata.end())
|
|
It->second = Node;
|
|
else
|
|
Metadata.emplace_back(Kind, Node);
|
|
}
|
|
|
|
/// Intersect this VPIRMetadata object with \p MD, keeping only metadata
|
|
/// nodes that are common to both.
|
|
void intersect(const VPIRMetadata &MD);
|
|
|
|
/// Get metadata of kind \p Kind. Returns nullptr if not found.
|
|
MDNode *getMetadata(unsigned Kind) const {
|
|
auto It =
|
|
find_if(Metadata, [Kind](const auto &P) { return P.first == Kind; });
|
|
return It != Metadata.end() ? It->second : nullptr;
|
|
}
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print metadata with node IDs.
|
|
void print(raw_ostream &O, VPSlotTracker &SlotTracker) const;
|
|
#endif
|
|
};
|
|
|
|
/// This is a concrete Recipe that models a single VPlan-level instruction.
|
|
/// While as any Recipe it may generate a sequence of IR instructions when
|
|
/// executed, these instructions would always form a single-def expression as
|
|
/// the VPInstruction is also a single def-use vertex. Most VPInstruction
|
|
/// opcodes can take an optional mask. Masks may be assigned during
|
|
/// predication.
|
|
class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
|
|
public VPIRMetadata,
|
|
public VPUnrollPartAccessor<1> {
|
|
friend class VPlanSlp;
|
|
|
|
public:
|
|
/// VPlan opcodes, extending LLVM IR with idiomatics instructions.
|
|
enum {
|
|
FirstOrderRecurrenceSplice =
|
|
Instruction::OtherOpsEnd + 1, // Combines the incoming and previous
|
|
// values of a first-order recurrence.
|
|
Not,
|
|
SLPLoad,
|
|
SLPStore,
|
|
// Creates a mask where each lane is active (true) whilst the current
|
|
// counter (first operand + index) is less than the second operand. i.e.
|
|
// mask[i] = icmpt ult (op0 + i), op1
|
|
// The size of the mask returned is VF * Multiplier (UF, third op).
|
|
ActiveLaneMask,
|
|
ExplicitVectorLength,
|
|
CalculateTripCountMinusVF,
|
|
// Increment the canonical IV separately for each unrolled part.
|
|
CanonicalIVIncrementForPart,
|
|
// Abstract instruction that compares two values and branches. This is
|
|
// lowered to ICmp + BranchOnCond during VPlan to VPlan transformation.
|
|
BranchOnCount,
|
|
BranchOnCond,
|
|
// Branch with 2 boolean condition operands and 3 successors. If condition
|
|
// 0 is true, branches to successor 0; if condition 1 is true, branches to
|
|
// successor 1; otherwise branches to successor 2. Expanded after region
|
|
// dissolution into: (1) an OR of the two conditions branching to
|
|
// middle.split or successor 2, and (2) middle.split branching to successor
|
|
// 0 or successor 1 based on condition 0.
|
|
BranchOnTwoConds,
|
|
Broadcast,
|
|
/// Given operands of (the same) struct type, creates a struct of fixed-
|
|
/// width vectors each containing a struct field of all operands. The
|
|
/// number of operands matches the element count of every vector.
|
|
BuildStructVector,
|
|
/// Creates a fixed-width vector containing all operands. The number of
|
|
/// operands matches the vector element count.
|
|
BuildVector,
|
|
/// Extracts all lanes from its (non-scalable) vector operand. This is an
|
|
/// abstract VPInstruction whose single defined VPValue represents VF
|
|
/// scalars extracted from a vector, to be replaced by VF ExtractElement
|
|
/// VPInstructions.
|
|
Unpack,
|
|
/// Compute the final result of a AnyOf reduction with select(cmp(),x,y),
|
|
/// where one of (x,y) is loop invariant, and both x and y are integer type.
|
|
ComputeAnyOfResult,
|
|
ComputeReductionResult,
|
|
// Extracts the last part of its operand. Removed during unrolling.
|
|
ExtractLastPart,
|
|
// Extracts the last lane of its vector operand, per part.
|
|
ExtractLastLane,
|
|
// Extracts the second-to-last lane from its operand or the second-to-last
|
|
// part if it is scalar. In the latter case, the recipe will be removed
|
|
// during unrolling.
|
|
ExtractPenultimateElement,
|
|
LogicalAnd, // Non-poison propagating logical And.
|
|
// Add an offset in bytes (second operand) to a base pointer (first
|
|
// operand). Only generates scalar values (either for the first lane only or
|
|
// for all lanes, depending on its uses).
|
|
PtrAdd,
|
|
// Add a vector offset in bytes (second operand) to a scalar base pointer
|
|
// (first operand).
|
|
WidePtrAdd,
|
|
// Returns a scalar boolean value, which is true if any lane of its
|
|
// (boolean) vector operands is true. It produces the reduced value across
|
|
// all unrolled iterations. Unrolling will add all copies of its original
|
|
// operand as additional operands. AnyOf is poison-safe as all operands
|
|
// will be frozen.
|
|
AnyOf,
|
|
// Calculates the first active lane index of the vector predicate operands.
|
|
// It produces the lane index across all unrolled iterations. Unrolling will
|
|
// add all copies of its original operand as additional operands.
|
|
// Implemented with @llvm.experimental.cttz.elts, but returns the expected
|
|
// result even with operands that are all zeroes.
|
|
FirstActiveLane,
|
|
// Calculates the last active lane index of the vector predicate operands.
|
|
// The predicates must be prefix-masks (all 1s before all 0s). Used when
|
|
// tail-folding to extract the correct live-out value from the last active
|
|
// iteration. It produces the lane index across all unrolled iterations.
|
|
// Unrolling will add all copies of its original operand as additional
|
|
// operands.
|
|
LastActiveLane,
|
|
// Returns a reversed vector for the operand.
|
|
Reverse,
|
|
|
|
// The opcodes below are used for VPInstructionWithType.
|
|
//
|
|
/// Scale the first operand (vector step) by the second operand
|
|
/// (scalar-step). Casts both operands to the result type if needed.
|
|
WideIVStep,
|
|
/// Start vector for reductions with 3 operands: the original start value,
|
|
/// the identity value for the reduction and an integer indicating the
|
|
/// scaling factor.
|
|
ReductionStartVector,
|
|
// Creates a step vector starting from 0 to VF with a step of 1.
|
|
StepVector,
|
|
/// Extracts a single lane (first operand) from a set of vector operands.
|
|
/// The lane specifies an index into a vector formed by combining all vector
|
|
/// operands (all operands after the first one).
|
|
ExtractLane,
|
|
/// Explicit user for the resume phi of the canonical induction in the main
|
|
/// VPlan, used by the epilogue vector loop.
|
|
ResumeForEpilogue,
|
|
/// Extracts the lane from the first operand corresponding to the last
|
|
/// active (non-zero) lane in the mask (second operand), or if no lanes
|
|
/// were active in the mask, returns the default value (third operand).
|
|
ExtractLastActive,
|
|
|
|
/// Returns the value for vscale.
|
|
VScale,
|
|
/// Compute the exiting value of a wide induction after vectorization, that
|
|
/// is the value of the last lane of the induction increment (i.e. its
|
|
/// backedge value). Takes the wide induction recipe and the original
|
|
/// backedge value as operands.
|
|
ExitingIVValue,
|
|
OpsEnd = ExitingIVValue,
|
|
};
|
|
|
|
/// Returns true if this VPInstruction generates scalar values for all lanes.
|
|
/// Most VPInstructions generate a single value per part, either vector or
|
|
/// scalar. VPReplicateRecipe takes care of generating multiple (scalar)
|
|
/// values per all lanes, stemming from an original ingredient. This method
|
|
/// identifies the (rare) cases of VPInstructions that do so as well, w/o an
|
|
/// underlying ingredient.
|
|
bool doesGeneratePerAllLanes() const;
|
|
|
|
/// Return the number of operands determined by the opcode of the
|
|
/// VPInstruction, excluding mask. Returns -1u if the number of operands
|
|
/// cannot be determined directly by the opcode.
|
|
unsigned getNumOperandsForOpcode() const;
|
|
|
|
private:
|
|
typedef unsigned char OpcodeTy;
|
|
OpcodeTy Opcode;
|
|
|
|
/// An optional name that can be used for the generated IR instruction.
|
|
std::string Name;
|
|
|
|
/// Returns true if we can generate a scalar for the first lane only if
|
|
/// needed.
|
|
bool canGenerateScalarForFirstLane() const;
|
|
|
|
/// Utility methods serving execute(): generates a single vector instance of
|
|
/// the modeled instruction. \returns the generated value. . In some cases an
|
|
/// existing value is returned rather than a generated one.
|
|
Value *generate(VPTransformState &State);
|
|
|
|
/// Returns true if the VPInstruction does not need masking.
|
|
bool alwaysUnmasked() const {
|
|
// For now only VPInstructions with underlying values use masks.
|
|
// TODO: provide masks to VPInstructions w/o underlying values.
|
|
if (!getUnderlyingValue())
|
|
return true;
|
|
|
|
return Opcode == Instruction::PHI || Opcode == Instruction::GetElementPtr;
|
|
}
|
|
|
|
public:
|
|
VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
|
|
const VPIRFlags &Flags = {}, const VPIRMetadata &MD = {},
|
|
DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "");
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPInstructionSC)
|
|
|
|
VPInstruction *clone() override {
|
|
auto *New = new VPInstruction(Opcode, operands(), *this, *this,
|
|
getDebugLoc(), Name);
|
|
if (getUnderlyingValue())
|
|
New->setUnderlyingValue(getUnderlyingInstr());
|
|
return New;
|
|
}
|
|
|
|
unsigned getOpcode() const { return Opcode; }
|
|
|
|
/// Generate the instruction.
|
|
/// TODO: We currently execute only per-part unless a specific instance is
|
|
/// provided.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPInstruction.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the VPInstruction to dbgs() (for debugging).
|
|
LLVM_DUMP_METHOD void dump() const;
|
|
#endif
|
|
|
|
bool hasResult() const {
|
|
// CallInst may or may not have a result, depending on the called function.
|
|
// Conservatively return calls have results for now.
|
|
switch (getOpcode()) {
|
|
case Instruction::Ret:
|
|
case Instruction::Br:
|
|
case Instruction::Store:
|
|
case Instruction::Switch:
|
|
case Instruction::IndirectBr:
|
|
case Instruction::Resume:
|
|
case Instruction::CatchRet:
|
|
case Instruction::Unreachable:
|
|
case Instruction::Fence:
|
|
case Instruction::AtomicRMW:
|
|
case VPInstruction::BranchOnCond:
|
|
case VPInstruction::BranchOnTwoConds:
|
|
case VPInstruction::BranchOnCount:
|
|
return false;
|
|
default:
|
|
return true;
|
|
}
|
|
}
|
|
|
|
/// Returns true if the VPInstruction has a mask operand.
|
|
bool isMasked() const {
|
|
unsigned NumOpsForOpcode = getNumOperandsForOpcode();
|
|
// VPInstructions without a fixed number of operands cannot be masked.
|
|
if (NumOpsForOpcode == -1u)
|
|
return false;
|
|
return NumOpsForOpcode + 1 == getNumOperands();
|
|
}
|
|
|
|
/// Returns the number of operands, excluding the mask if the VPInstruction is
|
|
/// masked.
|
|
unsigned getNumOperandsWithoutMask() const {
|
|
return getNumOperands() - isMasked();
|
|
}
|
|
|
|
/// Add mask \p Mask to an unmasked VPInstruction, if it needs masking.
|
|
void addMask(VPValue *Mask) {
|
|
assert(!isMasked() && "recipe is already masked");
|
|
if (alwaysUnmasked())
|
|
return;
|
|
addOperand(Mask);
|
|
}
|
|
|
|
/// Returns the mask for the VPInstruction. Returns nullptr for unmasked
|
|
/// VPInstructions.
|
|
VPValue *getMask() const {
|
|
return isMasked() ? getOperand(getNumOperands() - 1) : nullptr;
|
|
}
|
|
|
|
/// Returns an iterator range over the operands excluding the mask operand
|
|
/// if present.
|
|
iterator_range<operand_iterator> operandsWithoutMask() {
|
|
return make_range(op_begin(), op_begin() + getNumOperandsWithoutMask());
|
|
}
|
|
iterator_range<const_operand_iterator> operandsWithoutMask() const {
|
|
return make_range(op_begin(), op_begin() + getNumOperandsWithoutMask());
|
|
}
|
|
|
|
/// Returns true if the underlying opcode may read from or write to memory.
|
|
bool opcodeMayReadOrWriteFromMemory() const;
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool usesFirstLaneOnly(const VPValue *Op) const override;
|
|
|
|
/// Returns true if the recipe only uses the first part of operand \p Op.
|
|
bool usesFirstPartOnly(const VPValue *Op) const override;
|
|
|
|
/// Returns true if this VPInstruction produces a scalar value from a vector,
|
|
/// e.g. by performing a reduction or extracting a lane.
|
|
bool isVectorToScalar() const;
|
|
|
|
/// Returns true if this VPInstruction's operands are single scalars and the
|
|
/// result is also a single scalar.
|
|
bool isSingleScalar() const;
|
|
|
|
/// Returns the symbolic name assigned to the VPInstruction.
|
|
StringRef getName() const { return Name; }
|
|
|
|
/// Set the symbolic name for the VPInstruction.
|
|
void setName(StringRef NewName) { Name = NewName.str(); }
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the VPInstruction to \p O.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A specialization of VPInstruction augmenting it with a dedicated result
|
|
/// type, to be used when the opcode and operands of the VPInstruction don't
|
|
/// directly determine the result type. Note that there is no separate recipe ID
|
|
/// for VPInstructionWithType; it shares the same ID as VPInstruction and is
|
|
/// distinguished purely by the opcode.
|
|
class VPInstructionWithType : public VPInstruction {
|
|
/// Scalar result type produced by the recipe.
|
|
Type *ResultTy;
|
|
|
|
public:
|
|
VPInstructionWithType(unsigned Opcode, ArrayRef<VPValue *> Operands,
|
|
Type *ResultTy, const VPIRFlags &Flags = {},
|
|
const VPIRMetadata &Metadata = {},
|
|
DebugLoc DL = DebugLoc::getUnknown(),
|
|
const Twine &Name = "")
|
|
: VPInstruction(Opcode, Operands, Flags, Metadata, DL, Name),
|
|
ResultTy(ResultTy) {}
|
|
|
|
static inline bool classof(const VPRecipeBase *R) {
|
|
// VPInstructionWithType are VPInstructions with specific opcodes requiring
|
|
// type information.
|
|
if (R->isScalarCast())
|
|
return true;
|
|
auto *VPI = dyn_cast<VPInstruction>(R);
|
|
if (!VPI)
|
|
return false;
|
|
switch (VPI->getOpcode()) {
|
|
case VPInstruction::WideIVStep:
|
|
case VPInstruction::StepVector:
|
|
case VPInstruction::VScale:
|
|
return true;
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
static inline bool classof(const VPUser *R) {
|
|
return isa<VPInstructionWithType>(cast<VPRecipeBase>(R));
|
|
}
|
|
|
|
VPInstruction *clone() override {
|
|
auto *New =
|
|
new VPInstructionWithType(getOpcode(), operands(), getResultType(),
|
|
*this, *this, getDebugLoc(), getName());
|
|
New->setUnderlyingValue(getUnderlyingValue());
|
|
return New;
|
|
}
|
|
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPInstruction.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override {
|
|
// TODO: Compute accurate cost after retiring the legacy cost model.
|
|
return 0;
|
|
}
|
|
|
|
Type *getResultType() const { return ResultTy; }
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// Helper type to provide functions to access incoming values and blocks for
|
|
/// phi-like recipes.
|
|
class VPPhiAccessors {
|
|
protected:
|
|
/// Return a VPRecipeBase* to the current object.
|
|
virtual const VPRecipeBase *getAsRecipe() const = 0;
|
|
|
|
public:
|
|
virtual ~VPPhiAccessors() = default;
|
|
|
|
/// Returns the incoming VPValue with index \p Idx.
|
|
VPValue *getIncomingValue(unsigned Idx) const {
|
|
return getAsRecipe()->getOperand(Idx);
|
|
}
|
|
|
|
/// Returns the incoming block with index \p Idx.
|
|
const VPBasicBlock *getIncomingBlock(unsigned Idx) const;
|
|
|
|
/// Returns the number of incoming values, also number of incoming blocks.
|
|
virtual unsigned getNumIncoming() const {
|
|
return getAsRecipe()->getNumOperands();
|
|
}
|
|
|
|
/// Returns an interator range over the incoming values.
|
|
VPUser::const_operand_range incoming_values() const {
|
|
return make_range(getAsRecipe()->op_begin(),
|
|
getAsRecipe()->op_begin() + getNumIncoming());
|
|
}
|
|
|
|
using const_incoming_blocks_range = iterator_range<mapped_iterator<
|
|
detail::index_iterator, std::function<const VPBasicBlock *(size_t)>>>;
|
|
|
|
/// Returns an iterator range over the incoming blocks.
|
|
const_incoming_blocks_range incoming_blocks() const {
|
|
std::function<const VPBasicBlock *(size_t)> GetBlock = [this](size_t Idx) {
|
|
return getIncomingBlock(Idx);
|
|
};
|
|
return map_range(index_range(0, getNumIncoming()), GetBlock);
|
|
}
|
|
|
|
/// Returns an iterator range over pairs of incoming values and corresponding
|
|
/// incoming blocks.
|
|
detail::zippy<llvm::detail::zip_first, VPUser::const_operand_range,
|
|
const_incoming_blocks_range>
|
|
incoming_values_and_blocks() const {
|
|
return zip_equal(incoming_values(), incoming_blocks());
|
|
}
|
|
|
|
/// Removes the incoming value for \p IncomingBlock, which must be a
|
|
/// predecessor.
|
|
void removeIncomingValueFor(VPBlockBase *IncomingBlock) const;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printPhiOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const;
|
|
#endif
|
|
};
|
|
|
|
struct LLVM_ABI_FOR_TEST VPPhi : public VPInstruction, public VPPhiAccessors {
|
|
VPPhi(ArrayRef<VPValue *> Operands, const VPIRFlags &Flags, DebugLoc DL,
|
|
const Twine &Name = "")
|
|
: VPInstruction(Instruction::PHI, Operands, Flags, {}, DL, Name) {}
|
|
|
|
static inline bool classof(const VPUser *U) {
|
|
auto *VPI = dyn_cast<VPInstruction>(U);
|
|
return VPI && VPI->getOpcode() == Instruction::PHI;
|
|
}
|
|
|
|
static inline bool classof(const VPValue *V) {
|
|
auto *VPI = dyn_cast<VPInstruction>(V);
|
|
return VPI && VPI->getOpcode() == Instruction::PHI;
|
|
}
|
|
|
|
static inline bool classof(const VPSingleDefRecipe *SDR) {
|
|
auto *VPI = dyn_cast<VPInstruction>(SDR);
|
|
return VPI && VPI->getOpcode() == Instruction::PHI;
|
|
}
|
|
|
|
VPPhi *clone() override {
|
|
auto *PhiR = new VPPhi(operands(), *this, getDebugLoc(), getName());
|
|
PhiR->setUnderlyingValue(getUnderlyingValue());
|
|
return PhiR;
|
|
}
|
|
|
|
void execute(VPTransformState &State) override;
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
const VPRecipeBase *getAsRecipe() const override { return this; }
|
|
};
|
|
|
|
/// A recipe to wrap on original IR instruction not to be modified during
|
|
/// execution, except for PHIs. PHIs are modeled via the VPIRPhi subclass.
|
|
/// Expect PHIs, VPIRInstructions cannot have any operands.
|
|
class VPIRInstruction : public VPRecipeBase {
|
|
Instruction &I;
|
|
|
|
protected:
|
|
/// VPIRInstruction::create() should be used to create VPIRInstructions, as
|
|
/// subclasses may need to be created, e.g. VPIRPhi.
|
|
VPIRInstruction(Instruction &I)
|
|
: VPRecipeBase(VPRecipeBase::VPIRInstructionSC, {}), I(I) {}
|
|
|
|
public:
|
|
~VPIRInstruction() override = default;
|
|
|
|
/// Create a new VPIRPhi for \p \I, if it is a PHINode, otherwise create a
|
|
/// VPIRInstruction.
|
|
LLVM_ABI_FOR_TEST static VPIRInstruction *create(Instruction &I);
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPIRInstructionSC)
|
|
|
|
VPIRInstruction *clone() override {
|
|
auto *R = create(I);
|
|
for (auto *Op : operands())
|
|
R->addOperand(Op);
|
|
return R;
|
|
}
|
|
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPIRInstruction.
|
|
LLVM_ABI_FOR_TEST InstructionCost
|
|
computeCost(ElementCount VF, VPCostContext &Ctx) const override;
|
|
|
|
Instruction &getInstruction() const { return I; }
|
|
|
|
bool usesScalars(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
|
|
bool usesFirstPartOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
|
|
bool usesFirstLaneOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
|
|
/// Update the recipe's first operand to the last lane of the last part of the
|
|
/// operand using \p Builder. Must only be used for VPIRInstructions with at
|
|
/// least one operand wrapping a PHINode.
|
|
void extractLastLaneOfLastPartOfFirstOperand(VPBuilder &Builder);
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// An overlay for VPIRInstructions wrapping PHI nodes enabling convenient use
|
|
/// cast/dyn_cast/isa and execute() implementation. A single VPValue operand is
|
|
/// allowed, and it is used to add a new incoming value for the single
|
|
/// predecessor VPBB.
|
|
struct LLVM_ABI_FOR_TEST VPIRPhi : public VPIRInstruction,
|
|
public VPPhiAccessors {
|
|
VPIRPhi(PHINode &PN) : VPIRInstruction(PN) {}
|
|
|
|
static inline bool classof(const VPRecipeBase *U) {
|
|
auto *R = dyn_cast<VPIRInstruction>(U);
|
|
return R && isa<PHINode>(R->getInstruction());
|
|
}
|
|
|
|
PHINode &getIRPhi() { return cast<PHINode>(getInstruction()); }
|
|
|
|
void execute(VPTransformState &State) override;
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
const VPRecipeBase *getAsRecipe() const override { return this; }
|
|
};
|
|
|
|
/// VPWidenRecipe is a recipe for producing a widened instruction using the
|
|
/// opcode and operands of the recipe. This recipe covers most of the
|
|
/// traditional vectorization cases where each recipe transforms into a
|
|
/// vectorized version of itself.
|
|
class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags,
|
|
public VPIRMetadata {
|
|
unsigned Opcode;
|
|
|
|
public:
|
|
VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands,
|
|
const VPIRFlags &Flags = {}, const VPIRMetadata &Metadata = {},
|
|
DebugLoc DL = {})
|
|
: VPRecipeWithIRFlags(VPRecipeBase::VPWidenSC, Operands, Flags, DL),
|
|
VPIRMetadata(Metadata), Opcode(I.getOpcode()) {
|
|
setUnderlyingValue(&I);
|
|
}
|
|
|
|
VPWidenRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
|
|
const VPIRFlags &Flags = {}, const VPIRMetadata &Metadata = {},
|
|
DebugLoc DL = {})
|
|
: VPRecipeWithIRFlags(VPRecipeBase::VPWidenSC, Operands, Flags, DL),
|
|
VPIRMetadata(Metadata), Opcode(Opcode) {}
|
|
|
|
~VPWidenRecipe() override = default;
|
|
|
|
VPWidenRecipe *clone() override {
|
|
if (auto *UV = getUnderlyingValue())
|
|
return new VPWidenRecipe(*cast<Instruction>(UV), operands(), *this, *this,
|
|
getDebugLoc());
|
|
return new VPWidenRecipe(Opcode, operands(), *this, *this, getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenSC)
|
|
|
|
/// Produce a widened instruction using the opcode and operands of the recipe,
|
|
/// processing State.VF elements.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPWidenRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
unsigned getOpcode() const { return Opcode; }
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool usesFirstLaneOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return Opcode == Instruction::Select && Op == getOperand(0) &&
|
|
Op->isDefinedOutsideLoopRegions();
|
|
}
|
|
};
|
|
|
|
/// VPWidenCastRecipe is a recipe to create vector cast instructions.
|
|
class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
|
|
/// Cast instruction opcode.
|
|
Instruction::CastOps Opcode;
|
|
|
|
/// Result type for the cast.
|
|
Type *ResultTy;
|
|
|
|
public:
|
|
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
|
|
CastInst *CI = nullptr, const VPIRFlags &Flags = {},
|
|
const VPIRMetadata &Metadata = {},
|
|
DebugLoc DL = DebugLoc::getUnknown())
|
|
: VPRecipeWithIRFlags(VPRecipeBase::VPWidenCastSC, Op, Flags, DL),
|
|
VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) {
|
|
assert(flagsValidForOpcode(Opcode) &&
|
|
"Set flags not supported for the provided opcode");
|
|
assert(hasRequiredFlagsForOpcode(Opcode) &&
|
|
"Opcode requires specific flags to be set");
|
|
setUnderlyingValue(CI);
|
|
}
|
|
|
|
~VPWidenCastRecipe() override = default;
|
|
|
|
VPWidenCastRecipe *clone() override {
|
|
return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy,
|
|
cast_or_null<CastInst>(getUnderlyingValue()),
|
|
*this, *this, getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenCastSC)
|
|
|
|
/// Produce widened copies of the cast.
|
|
LLVM_ABI_FOR_TEST void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPWidenCastRecipe.
|
|
LLVM_ABI_FOR_TEST InstructionCost
|
|
computeCost(ElementCount VF, VPCostContext &Ctx) const override;
|
|
|
|
Instruction::CastOps getOpcode() const { return Opcode; }
|
|
|
|
/// Returns the result type of the cast.
|
|
Type *getResultType() const { return ResultTy; }
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
LLVM_ABI_FOR_TEST void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe for widening vector intrinsics.
|
|
class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
|
|
/// ID of the vector intrinsic to widen.
|
|
Intrinsic::ID VectorIntrinsicID;
|
|
|
|
/// Scalar return type of the intrinsic.
|
|
Type *ResultTy;
|
|
|
|
/// True if the intrinsic may read from memory.
|
|
bool MayReadFromMemory;
|
|
|
|
/// True if the intrinsic may read write to memory.
|
|
bool MayWriteToMemory;
|
|
|
|
/// True if the intrinsic may have side-effects.
|
|
bool MayHaveSideEffects;
|
|
|
|
public:
|
|
VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID,
|
|
ArrayRef<VPValue *> CallArguments, Type *Ty,
|
|
const VPIRFlags &Flags = {},
|
|
const VPIRMetadata &MD = {},
|
|
DebugLoc DL = DebugLoc::getUnknown())
|
|
: VPRecipeWithIRFlags(VPRecipeBase::VPWidenIntrinsicSC, CallArguments,
|
|
Flags, DL),
|
|
VPIRMetadata(MD), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
|
|
MayReadFromMemory(CI.mayReadFromMemory()),
|
|
MayWriteToMemory(CI.mayWriteToMemory()),
|
|
MayHaveSideEffects(CI.mayHaveSideEffects()) {
|
|
setUnderlyingValue(&CI);
|
|
}
|
|
|
|
VPWidenIntrinsicRecipe(Intrinsic::ID VectorIntrinsicID,
|
|
ArrayRef<VPValue *> CallArguments, Type *Ty,
|
|
const VPIRFlags &Flags = {},
|
|
const VPIRMetadata &Metadata = {},
|
|
DebugLoc DL = DebugLoc::getUnknown())
|
|
: VPRecipeWithIRFlags(VPRecipeBase::VPWidenIntrinsicSC, CallArguments,
|
|
Flags, DL),
|
|
VPIRMetadata(Metadata), VectorIntrinsicID(VectorIntrinsicID),
|
|
ResultTy(Ty) {
|
|
LLVMContext &Ctx = Ty->getContext();
|
|
AttributeSet Attrs = Intrinsic::getFnAttributes(Ctx, VectorIntrinsicID);
|
|
MemoryEffects ME = Attrs.getMemoryEffects();
|
|
MayReadFromMemory = !ME.onlyWritesMemory();
|
|
MayWriteToMemory = !ME.onlyReadsMemory();
|
|
MayHaveSideEffects = MayWriteToMemory ||
|
|
!Attrs.hasAttribute(Attribute::NoUnwind) ||
|
|
!Attrs.hasAttribute(Attribute::WillReturn);
|
|
}
|
|
|
|
~VPWidenIntrinsicRecipe() override = default;
|
|
|
|
VPWidenIntrinsicRecipe *clone() override {
|
|
if (Value *CI = getUnderlyingValue())
|
|
return new VPWidenIntrinsicRecipe(*cast<CallInst>(CI), VectorIntrinsicID,
|
|
operands(), ResultTy, *this, *this,
|
|
getDebugLoc());
|
|
return new VPWidenIntrinsicRecipe(VectorIntrinsicID, operands(), ResultTy,
|
|
*this, *this, getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenIntrinsicSC)
|
|
|
|
/// Produce a widened version of the vector intrinsic.
|
|
LLVM_ABI_FOR_TEST void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this vector intrinsic.
|
|
LLVM_ABI_FOR_TEST InstructionCost
|
|
computeCost(ElementCount VF, VPCostContext &Ctx) const override;
|
|
|
|
/// Return the ID of the intrinsic.
|
|
Intrinsic::ID getVectorIntrinsicID() const { return VectorIntrinsicID; }
|
|
|
|
/// Return the scalar return type of the intrinsic.
|
|
Type *getResultType() const { return ResultTy; }
|
|
|
|
/// Return to name of the intrinsic as string.
|
|
StringRef getIntrinsicName() const;
|
|
|
|
/// Returns true if the intrinsic may read from memory.
|
|
bool mayReadFromMemory() const { return MayReadFromMemory; }
|
|
|
|
/// Returns true if the intrinsic may write to memory.
|
|
bool mayWriteToMemory() const { return MayWriteToMemory; }
|
|
|
|
/// Returns true if the intrinsic may have side-effects.
|
|
bool mayHaveSideEffects() const { return MayHaveSideEffects; }
|
|
|
|
LLVM_ABI_FOR_TEST bool usesFirstLaneOnly(const VPValue *Op) const override;
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
LLVM_ABI_FOR_TEST void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe for widening Call instructions using library calls.
|
|
class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,
|
|
public VPIRMetadata {
|
|
/// Variant stores a pointer to the chosen function. There is a 1:1 mapping
|
|
/// between a given VF and the chosen vectorized variant, so there will be a
|
|
/// different VPlan for each VF with a valid variant.
|
|
Function *Variant;
|
|
|
|
public:
|
|
VPWidenCallRecipe(Value *UV, Function *Variant,
|
|
ArrayRef<VPValue *> CallArguments,
|
|
const VPIRFlags &Flags = {},
|
|
const VPIRMetadata &Metadata = {}, DebugLoc DL = {})
|
|
: VPRecipeWithIRFlags(VPRecipeBase::VPWidenCallSC, CallArguments, Flags,
|
|
DL),
|
|
VPIRMetadata(Metadata), Variant(Variant) {
|
|
setUnderlyingValue(UV);
|
|
assert(
|
|
isa<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue()) &&
|
|
"last operand must be the called function");
|
|
}
|
|
|
|
~VPWidenCallRecipe() override = default;
|
|
|
|
VPWidenCallRecipe *clone() override {
|
|
return new VPWidenCallRecipe(getUnderlyingValue(), Variant, operands(),
|
|
*this, *this, getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenCallSC)
|
|
|
|
/// Produce a widened version of the call instruction.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPWidenCallRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
Function *getCalledScalarFunction() const {
|
|
return cast<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue());
|
|
}
|
|
|
|
operand_range args() { return drop_end(operands()); }
|
|
const_operand_range args() const { return drop_end(operands()); }
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe representing a sequence of load -> update -> store as part of
|
|
/// a histogram operation. This means there may be aliasing between vector
|
|
/// lanes, which is handled by the llvm.experimental.vector.histogram family
|
|
/// of intrinsics. The only update operations currently supported are
|
|
/// 'add' and 'sub' where the other term is loop-invariant.
|
|
class VPHistogramRecipe : public VPRecipeBase {
|
|
/// Opcode of the update operation, currently either add or sub.
|
|
unsigned Opcode;
|
|
|
|
public:
|
|
VPHistogramRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
|
|
DebugLoc DL = DebugLoc::getUnknown())
|
|
: VPRecipeBase(VPRecipeBase::VPHistogramSC, Operands, DL),
|
|
Opcode(Opcode) {}
|
|
|
|
~VPHistogramRecipe() override = default;
|
|
|
|
VPHistogramRecipe *clone() override {
|
|
return new VPHistogramRecipe(Opcode, operands(), getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPHistogramSC);
|
|
|
|
/// Produce a vectorized histogram operation.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPHistogramRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
unsigned getOpcode() const { return Opcode; }
|
|
|
|
/// Return the mask operand if one was provided, or a null pointer if all
|
|
/// lanes should be executed unconditionally.
|
|
VPValue *getMask() const {
|
|
return getNumOperands() == 3 ? getOperand(2) : nullptr;
|
|
}
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe for handling GEP instructions.
|
|
class LLVM_ABI_FOR_TEST VPWidenGEPRecipe : public VPRecipeWithIRFlags {
|
|
Type *SourceElementTy;
|
|
|
|
bool isPointerLoopInvariant() const {
|
|
return getOperand(0)->isDefinedOutsideLoopRegions();
|
|
}
|
|
|
|
bool isIndexLoopInvariant(unsigned I) const {
|
|
return getOperand(I + 1)->isDefinedOutsideLoopRegions();
|
|
}
|
|
|
|
public:
|
|
VPWidenGEPRecipe(GetElementPtrInst *GEP, ArrayRef<VPValue *> Operands,
|
|
const VPIRFlags &Flags = {},
|
|
DebugLoc DL = DebugLoc::getUnknown())
|
|
: VPRecipeWithIRFlags(VPRecipeBase::VPWidenGEPSC, Operands, Flags, DL),
|
|
SourceElementTy(GEP->getSourceElementType()) {
|
|
setUnderlyingValue(GEP);
|
|
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
|
|
(void)Metadata;
|
|
getMetadataToPropagate(GEP, Metadata);
|
|
assert(Metadata.empty() && "unexpected metadata on GEP");
|
|
}
|
|
|
|
~VPWidenGEPRecipe() override = default;
|
|
|
|
VPWidenGEPRecipe *clone() override {
|
|
return new VPWidenGEPRecipe(cast<GetElementPtrInst>(getUnderlyingInstr()),
|
|
operands(), *this, getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenGEPSC)
|
|
|
|
/// This recipe generates a GEP instruction.
|
|
unsigned getOpcode() const { return Instruction::GetElementPtr; }
|
|
|
|
/// Generate the gep nodes.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
Type *getSourceElementType() const { return SourceElementTy; }
|
|
|
|
/// Return the cost of this VPWidenGEPRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override {
|
|
// TODO: Compute accurate cost after retiring the legacy cost model.
|
|
return 0;
|
|
}
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool usesFirstLaneOnly(const VPValue *Op) const override;
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe to compute a pointer to the last element of each part of a widened
|
|
/// memory access for widened memory accesses of SourceElementTy. Used for
|
|
/// VPWidenMemoryRecipes or VPInterleaveRecipes that are reversed.
|
|
class VPVectorEndPointerRecipe : public VPRecipeWithIRFlags,
|
|
public VPUnrollPartAccessor<2> {
|
|
Type *SourceElementTy;
|
|
|
|
/// The constant stride of the pointer computed by this recipe, expressed in
|
|
/// units of SourceElementTy.
|
|
int64_t Stride;
|
|
|
|
public:
|
|
VPVectorEndPointerRecipe(VPValue *Ptr, VPValue *VF, Type *SourceElementTy,
|
|
int64_t Stride, GEPNoWrapFlags GEPFlags, DebugLoc DL)
|
|
: VPRecipeWithIRFlags(VPRecipeBase::VPVectorEndPointerSC, {Ptr, VF},
|
|
GEPFlags, DL),
|
|
SourceElementTy(SourceElementTy), Stride(Stride) {
|
|
assert(Stride < 0 && "Stride must be negative");
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPVectorEndPointerSC)
|
|
|
|
Type *getSourceElementType() const { return SourceElementTy; }
|
|
VPValue *getVFValue() { return getOperand(1); }
|
|
const VPValue *getVFValue() const { return getOperand(1); }
|
|
|
|
void execute(VPTransformState &State) override;
|
|
|
|
bool usesFirstLaneOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
|
|
/// Return the cost of this VPVectorPointerRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override {
|
|
// TODO: Compute accurate cost after retiring the legacy cost model.
|
|
return 0;
|
|
}
|
|
|
|
/// Returns true if the recipe only uses the first part of operand \p Op.
|
|
bool usesFirstPartOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
assert(getNumOperands() <= 2 && "must have at most two operands");
|
|
return true;
|
|
}
|
|
|
|
VPVectorEndPointerRecipe *clone() override {
|
|
return new VPVectorEndPointerRecipe(getOperand(0), getVFValue(),
|
|
getSourceElementType(), Stride,
|
|
getGEPNoWrapFlags(), getDebugLoc());
|
|
}
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe to compute the pointers for widened memory accesses of \p
|
|
/// SourceElementTy. Unrolling adds an extra offset operand for unrolled parts >
|
|
/// 0 and it produces `GEP Ptr, Offset`. The offset for unrolled part 0 is 0.
|
|
class VPVectorPointerRecipe : public VPRecipeWithIRFlags {
|
|
Type *SourceElementTy;
|
|
|
|
public:
|
|
VPVectorPointerRecipe(VPValue *Ptr, Type *SourceElementTy,
|
|
GEPNoWrapFlags GEPFlags, DebugLoc DL)
|
|
: VPRecipeWithIRFlags(VPRecipeBase::VPVectorPointerSC, Ptr, GEPFlags, DL),
|
|
SourceElementTy(SourceElementTy) {}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPVectorPointerSC)
|
|
|
|
VPValue *getOffset() {
|
|
return getNumOperands() == 2 ? getOperand(1) : nullptr;
|
|
}
|
|
|
|
void execute(VPTransformState &State) override;
|
|
|
|
Type *getSourceElementType() const { return SourceElementTy; }
|
|
|
|
bool usesFirstLaneOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
|
|
/// Returns true if the recipe only uses the first part of operand \p Op.
|
|
bool usesFirstPartOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
assert(getNumOperands() <= 2 && "must have at most two operands");
|
|
return true;
|
|
}
|
|
|
|
VPVectorPointerRecipe *clone() override {
|
|
auto *Clone = new VPVectorPointerRecipe(getOperand(0), SourceElementTy,
|
|
getGEPNoWrapFlags(), getDebugLoc());
|
|
if (auto *Off = getOffset())
|
|
Clone->addOperand(Off);
|
|
return Clone;
|
|
}
|
|
|
|
/// Return the cost of this VPHeaderPHIRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override {
|
|
// TODO: Compute accurate cost after retiring the legacy cost model.
|
|
return 0;
|
|
}
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A pure virtual base class for all recipes modeling header phis, including
|
|
/// phis for first order recurrences, pointer inductions and reductions. The
|
|
/// start value is the first operand of the recipe and the incoming value from
|
|
/// the backedge is the second operand.
|
|
///
|
|
/// Inductions are modeled using the following sub-classes:
|
|
/// * VPCanonicalIVPHIRecipe: Canonical scalar induction of the vector loop,
|
|
/// starting at a specified value (zero for the main vector loop, the resume
|
|
/// value for the epilogue vector loop) and stepping by 1. The induction
|
|
/// controls exiting of the vector loop by comparing against the vector trip
|
|
/// count. Produces a single scalar PHI for the induction value per
|
|
/// iteration.
|
|
/// * VPWidenIntOrFpInductionRecipe: Generates vector values for integer and
|
|
/// floating point inductions with arbitrary start and step values. Produces
|
|
/// a vector PHI per-part.
|
|
/// * VPDerivedIVRecipe: Converts the canonical IV value to the corresponding
|
|
/// value of an IV with different start and step values. Produces a single
|
|
/// scalar value per iteration
|
|
/// * VPScalarIVStepsRecipe: Generates scalar values per-lane based on a
|
|
/// canonical or derived induction.
|
|
/// * VPWidenPointerInductionRecipe: Generate vector and scalar values for a
|
|
/// pointer induction. Produces either a vector PHI per-part or scalar values
|
|
/// per-lane based on the canonical induction.
|
|
class LLVM_ABI_FOR_TEST VPHeaderPHIRecipe : public VPSingleDefRecipe,
|
|
public VPPhiAccessors {
|
|
protected:
|
|
VPHeaderPHIRecipe(unsigned char VPRecipeID, Instruction *UnderlyingInstr,
|
|
VPValue *Start, DebugLoc DL = DebugLoc::getUnknown())
|
|
: VPSingleDefRecipe(VPRecipeID, Start, UnderlyingInstr, DL) {}
|
|
|
|
const VPRecipeBase *getAsRecipe() const override { return this; }
|
|
|
|
public:
|
|
~VPHeaderPHIRecipe() override = default;
|
|
|
|
/// Method to support type inquiry through isa, cast, and dyn_cast.
|
|
static inline bool classof(const VPRecipeBase *R) {
|
|
return R->getVPRecipeID() >= VPRecipeBase::VPFirstHeaderPHISC &&
|
|
R->getVPRecipeID() <= VPRecipeBase::VPLastHeaderPHISC;
|
|
}
|
|
static inline bool classof(const VPValue *V) {
|
|
return isa<VPHeaderPHIRecipe>(V->getDefiningRecipe());
|
|
}
|
|
static inline bool classof(const VPSingleDefRecipe *R) {
|
|
return isa<VPHeaderPHIRecipe>(static_cast<const VPRecipeBase *>(R));
|
|
}
|
|
|
|
/// Generate the phi nodes.
|
|
void execute(VPTransformState &State) override = 0;
|
|
|
|
/// Return the cost of this header phi recipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
/// Returns the start value of the phi, if one is set.
|
|
VPValue *getStartValue() {
|
|
return getNumOperands() == 0 ? nullptr : getOperand(0);
|
|
}
|
|
VPValue *getStartValue() const {
|
|
return getNumOperands() == 0 ? nullptr : getOperand(0);
|
|
}
|
|
|
|
/// Update the start value of the recipe.
|
|
void setStartValue(VPValue *V) { setOperand(0, V); }
|
|
|
|
/// Returns the incoming value from the loop backedge.
|
|
virtual VPValue *getBackedgeValue() {
|
|
return getOperand(1);
|
|
}
|
|
|
|
/// Update the incoming value from the loop backedge.
|
|
void setBackedgeValue(VPValue *V) { setOperand(1, V); }
|
|
|
|
/// Returns the backedge value as a recipe. The backedge value is guaranteed
|
|
/// to be a recipe.
|
|
virtual VPRecipeBase &getBackedgeRecipe() {
|
|
return *getBackedgeValue()->getDefiningRecipe();
|
|
}
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override = 0;
|
|
#endif
|
|
};
|
|
|
|
/// Base class for widened induction (VPWidenIntOrFpInductionRecipe and
|
|
/// VPWidenPointerInductionRecipe), providing shared functionality, including
|
|
/// retrieving the step value, induction descriptor and original phi node.
|
|
class VPWidenInductionRecipe : public VPHeaderPHIRecipe {
|
|
const InductionDescriptor &IndDesc;
|
|
|
|
public:
|
|
VPWidenInductionRecipe(unsigned char Kind, PHINode *IV, VPValue *Start,
|
|
VPValue *Step, const InductionDescriptor &IndDesc,
|
|
DebugLoc DL)
|
|
: VPHeaderPHIRecipe(Kind, IV, Start, DL), IndDesc(IndDesc) {
|
|
addOperand(Step);
|
|
}
|
|
|
|
static inline bool classof(const VPRecipeBase *R) {
|
|
return R->getVPRecipeID() == VPRecipeBase::VPWidenIntOrFpInductionSC ||
|
|
R->getVPRecipeID() == VPRecipeBase::VPWidenPointerInductionSC;
|
|
}
|
|
|
|
static inline bool classof(const VPValue *V) {
|
|
auto *R = V->getDefiningRecipe();
|
|
return R && classof(R);
|
|
}
|
|
|
|
static inline bool classof(const VPSingleDefRecipe *R) {
|
|
return classof(static_cast<const VPRecipeBase *>(R));
|
|
}
|
|
|
|
void execute(VPTransformState &State) override = 0;
|
|
|
|
/// Returns the start value of the induction.
|
|
VPIRValue *getStartValue() const { return cast<VPIRValue>(getOperand(0)); }
|
|
|
|
/// Returns the step value of the induction.
|
|
VPValue *getStepValue() { return getOperand(1); }
|
|
const VPValue *getStepValue() const { return getOperand(1); }
|
|
|
|
/// Update the step value of the recipe.
|
|
void setStepValue(VPValue *V) { setOperand(1, V); }
|
|
|
|
VPValue *getVFValue() { return getOperand(2); }
|
|
const VPValue *getVFValue() const { return getOperand(2); }
|
|
|
|
/// Returns the number of incoming values, also number of incoming blocks.
|
|
/// Note that at the moment, VPWidenPointerInductionRecipe only has a single
|
|
/// incoming value, its start value.
|
|
unsigned getNumIncoming() const override { return 1; }
|
|
|
|
PHINode *getPHINode() const { return cast<PHINode>(getUnderlyingValue()); }
|
|
|
|
/// Returns the induction descriptor for the recipe.
|
|
const InductionDescriptor &getInductionDescriptor() const { return IndDesc; }
|
|
|
|
VPValue *getBackedgeValue() override {
|
|
// TODO: All operands of base recipe must exist and be at same index in
|
|
// derived recipe.
|
|
llvm_unreachable(
|
|
"VPWidenIntOrFpInductionRecipe generates its own backedge value");
|
|
}
|
|
|
|
VPRecipeBase &getBackedgeRecipe() override {
|
|
// TODO: All operands of base recipe must exist and be at same index in
|
|
// derived recipe.
|
|
llvm_unreachable(
|
|
"VPWidenIntOrFpInductionRecipe generates its own backedge value");
|
|
}
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool usesFirstLaneOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
// The recipe creates its own wide start value, so it only requests the
|
|
// first lane of the operand.
|
|
// TODO: Remove once creating the start value is modeled separately.
|
|
return Op == getStartValue() || Op == getStepValue();
|
|
}
|
|
};
|
|
|
|
/// A recipe for handling phi nodes of integer and floating-point inductions,
|
|
/// producing their vector values. This is an abstract recipe and must be
|
|
/// converted to concrete recipes before executing.
|
|
class VPWidenIntOrFpInductionRecipe : public VPWidenInductionRecipe,
|
|
public VPIRFlags {
|
|
TruncInst *Trunc;
|
|
|
|
// If this recipe is unrolled it will have 2 additional operands.
|
|
bool isUnrolled() const { return getNumOperands() == 5; }
|
|
|
|
public:
|
|
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPIRValue *Start, VPValue *Step,
|
|
VPValue *VF, const InductionDescriptor &IndDesc,
|
|
const VPIRFlags &Flags, DebugLoc DL)
|
|
: VPWidenInductionRecipe(VPRecipeBase::VPWidenIntOrFpInductionSC, IV,
|
|
Start, Step, IndDesc, DL),
|
|
VPIRFlags(Flags), Trunc(nullptr) {
|
|
addOperand(VF);
|
|
}
|
|
|
|
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPIRValue *Start, VPValue *Step,
|
|
VPValue *VF, const InductionDescriptor &IndDesc,
|
|
TruncInst *Trunc, const VPIRFlags &Flags,
|
|
DebugLoc DL)
|
|
: VPWidenInductionRecipe(VPRecipeBase::VPWidenIntOrFpInductionSC, IV,
|
|
Start, Step, IndDesc, DL),
|
|
VPIRFlags(Flags), Trunc(Trunc) {
|
|
addOperand(VF);
|
|
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
|
|
(void)Metadata;
|
|
if (Trunc)
|
|
getMetadataToPropagate(Trunc, Metadata);
|
|
assert(Metadata.empty() && "unexpected metadata on Trunc");
|
|
}
|
|
|
|
~VPWidenIntOrFpInductionRecipe() override = default;
|
|
|
|
VPWidenIntOrFpInductionRecipe *clone() override {
|
|
return new VPWidenIntOrFpInductionRecipe(
|
|
getPHINode(), getStartValue(), getStepValue(), getVFValue(),
|
|
getInductionDescriptor(), Trunc, *this, getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenIntOrFpInductionSC)
|
|
|
|
void execute(VPTransformState &State) override {
|
|
llvm_unreachable("cannot execute this recipe, should be expanded via "
|
|
"expandVPWidenIntOrFpInductionRecipe");
|
|
}
|
|
|
|
/// Returns the start value of the induction.
|
|
VPIRValue *getStartValue() const { return cast<VPIRValue>(getOperand(0)); }
|
|
|
|
/// If the recipe has been unrolled, return the VPValue for the induction
|
|
/// increment, otherwise return null.
|
|
VPValue *getSplatVFValue() const {
|
|
return isUnrolled() ? getOperand(getNumOperands() - 2) : nullptr;
|
|
}
|
|
|
|
/// Returns the number of incoming values, also number of incoming blocks.
|
|
/// Note that at the moment, VPWidenIntOrFpInductionRecipes only have a single
|
|
/// incoming value, its start value.
|
|
unsigned getNumIncoming() const override { return 1; }
|
|
|
|
/// Returns the first defined value as TruncInst, if it is one or nullptr
|
|
/// otherwise.
|
|
TruncInst *getTruncInst() { return Trunc; }
|
|
const TruncInst *getTruncInst() const { return Trunc; }
|
|
|
|
/// Returns true if the induction is canonical, i.e. starting at 0 and
|
|
/// incremented by UF * VF (= the original IV is incremented by 1) and has the
|
|
/// same type as the canonical induction.
|
|
bool isCanonical() const;
|
|
|
|
/// Returns the scalar type of the induction.
|
|
Type *getScalarType() const {
|
|
return Trunc ? Trunc->getType() : getStartValue()->getType();
|
|
}
|
|
|
|
/// Returns the VPValue representing the value of this induction at
|
|
/// the last unrolled part, if it exists. Returns itself if unrolling did not
|
|
/// take place.
|
|
VPValue *getLastUnrolledPartOperand() {
|
|
return isUnrolled() ? getOperand(getNumOperands() - 1) : this;
|
|
}
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
class VPWidenPointerInductionRecipe : public VPWidenInductionRecipe {
|
|
public:
|
|
/// Create a new VPWidenPointerInductionRecipe for \p Phi with start value \p
|
|
/// Start and the number of elements unrolled \p NumUnrolledElems, typically
|
|
/// VF*UF.
|
|
VPWidenPointerInductionRecipe(PHINode *Phi, VPValue *Start, VPValue *Step,
|
|
VPValue *NumUnrolledElems,
|
|
const InductionDescriptor &IndDesc, DebugLoc DL)
|
|
: VPWidenInductionRecipe(VPRecipeBase::VPWidenPointerInductionSC, Phi,
|
|
Start, Step, IndDesc, DL) {
|
|
addOperand(NumUnrolledElems);
|
|
}
|
|
|
|
~VPWidenPointerInductionRecipe() override = default;
|
|
|
|
VPWidenPointerInductionRecipe *clone() override {
|
|
return new VPWidenPointerInductionRecipe(
|
|
cast<PHINode>(getUnderlyingInstr()), getOperand(0), getOperand(1),
|
|
getOperand(2), getInductionDescriptor(), getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenPointerInductionSC)
|
|
|
|
/// Generate vector values for the pointer induction.
|
|
void execute(VPTransformState &State) override {
|
|
llvm_unreachable("cannot execute this recipe, should be expanded via "
|
|
"expandVPWidenPointerInduction");
|
|
};
|
|
|
|
/// Returns true if only scalar values will be generated.
|
|
bool onlyScalarsGenerated(bool IsScalable);
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe for widened phis. Incoming values are operands of the recipe and
|
|
/// their operand index corresponds to the incoming predecessor block. If the
|
|
/// recipe is placed in an entry block to a (non-replicate) region, it must have
|
|
/// exactly 2 incoming values, the first from the predecessor of the region and
|
|
/// the second from the exiting block of the region.
|
|
class LLVM_ABI_FOR_TEST VPWidenPHIRecipe : public VPSingleDefRecipe,
|
|
public VPPhiAccessors {
|
|
/// Name to use for the generated IR instruction for the widened phi.
|
|
std::string Name;
|
|
|
|
public:
|
|
/// Create a new VPWidenPHIRecipe for \p Phi with start value \p Start and
|
|
/// debug location \p DL.
|
|
VPWidenPHIRecipe(PHINode *Phi, VPValue *Start = nullptr,
|
|
DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "")
|
|
: VPSingleDefRecipe(VPRecipeBase::VPWidenPHISC, {}, Phi, DL),
|
|
Name(Name.str()) {
|
|
if (Start)
|
|
addOperand(Start);
|
|
}
|
|
|
|
VPWidenPHIRecipe *clone() override {
|
|
auto *C = new VPWidenPHIRecipe(cast<PHINode>(getUnderlyingValue()),
|
|
getOperand(0), getDebugLoc(), Name);
|
|
for (VPValue *Op : llvm::drop_begin(operands()))
|
|
C->addOperand(Op);
|
|
return C;
|
|
}
|
|
|
|
~VPWidenPHIRecipe() override = default;
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenPHISC)
|
|
|
|
/// Generate the phi/select nodes.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPWidenPHIRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
|
|
const VPRecipeBase *getAsRecipe() const override { return this; }
|
|
};
|
|
|
|
/// A recipe for handling first-order recurrence phis. The start value is the
|
|
/// first operand of the recipe and the incoming value from the backedge is the
|
|
/// second operand.
|
|
struct VPFirstOrderRecurrencePHIRecipe : public VPHeaderPHIRecipe {
|
|
VPFirstOrderRecurrencePHIRecipe(PHINode *Phi, VPValue &Start,
|
|
VPValue &BackedgeValue)
|
|
: VPHeaderPHIRecipe(VPRecipeBase::VPFirstOrderRecurrencePHISC, Phi,
|
|
&Start) {
|
|
addOperand(&BackedgeValue);
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPFirstOrderRecurrencePHISC)
|
|
|
|
VPFirstOrderRecurrencePHIRecipe *clone() override {
|
|
return new VPFirstOrderRecurrencePHIRecipe(
|
|
cast<PHINode>(getUnderlyingInstr()), *getOperand(0), *getOperand(1));
|
|
}
|
|
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this first-order recurrence phi recipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool usesFirstLaneOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return Op == getStartValue();
|
|
}
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// Possible variants of a reduction.
|
|
|
|
/// This reduction is ordered and in-loop.
|
|
struct RdxOrdered {};
|
|
/// This reduction is in-loop.
|
|
struct RdxInLoop {};
|
|
/// This reduction is unordered with the partial result scaled down by some
|
|
/// factor.
|
|
struct RdxUnordered {
|
|
unsigned VFScaleFactor;
|
|
};
|
|
using ReductionStyle = std::variant<RdxOrdered, RdxInLoop, RdxUnordered>;
|
|
|
|
inline ReductionStyle getReductionStyle(bool InLoop, bool Ordered,
|
|
unsigned ScaleFactor) {
|
|
assert((!Ordered || InLoop) && "Ordered implies in-loop");
|
|
if (Ordered)
|
|
return RdxOrdered{};
|
|
if (InLoop)
|
|
return RdxInLoop{};
|
|
return RdxUnordered{/*VFScaleFactor=*/ScaleFactor};
|
|
}
|
|
|
|
/// A recipe for handling reduction phis. The start value is the first operand
|
|
/// of the recipe and the incoming value from the backedge is the second
|
|
/// operand.
|
|
class VPReductionPHIRecipe : public VPHeaderPHIRecipe,
|
|
public VPUnrollPartAccessor<2> {
|
|
/// The recurrence kind of the reduction.
|
|
const RecurKind Kind;
|
|
|
|
ReductionStyle Style;
|
|
|
|
/// The phi is part of a multi-use reduction (e.g., used in FindIV
|
|
/// patterns for argmin/argmax).
|
|
/// TODO: Also support cases where the phi itself has a single use, but its
|
|
/// compare has multiple uses.
|
|
bool HasUsesOutsideReductionChain;
|
|
|
|
public:
|
|
/// Create a new VPReductionPHIRecipe for the reduction \p Phi.
|
|
VPReductionPHIRecipe(PHINode *Phi, RecurKind Kind, VPValue &Start,
|
|
VPValue &BackedgeValue, ReductionStyle Style,
|
|
bool HasUsesOutsideReductionChain = false)
|
|
: VPHeaderPHIRecipe(VPRecipeBase::VPReductionPHISC, Phi, &Start),
|
|
Kind(Kind), Style(Style),
|
|
HasUsesOutsideReductionChain(HasUsesOutsideReductionChain) {
|
|
addOperand(&BackedgeValue);
|
|
}
|
|
|
|
~VPReductionPHIRecipe() override = default;
|
|
|
|
VPReductionPHIRecipe *clone() override {
|
|
return new VPReductionPHIRecipe(
|
|
dyn_cast_or_null<PHINode>(getUnderlyingValue()), getRecurrenceKind(),
|
|
*getOperand(0), *getBackedgeValue(), Style,
|
|
HasUsesOutsideReductionChain);
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPReductionPHISC)
|
|
|
|
/// Generate the phi/select nodes.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Get the factor that the VF of this recipe's output should be scaled by, or
|
|
/// 1 if it isn't scaled.
|
|
unsigned getVFScaleFactor() const {
|
|
auto *Partial = std::get_if<RdxUnordered>(&Style);
|
|
return Partial ? Partial->VFScaleFactor : 1;
|
|
}
|
|
|
|
/// Set the VFScaleFactor for this reduction phi. Can only be set to a factor
|
|
/// > 1.
|
|
void setVFScaleFactor(unsigned ScaleFactor) {
|
|
assert(ScaleFactor > 1 && "must set to scale factor > 1");
|
|
Style = RdxUnordered{ScaleFactor};
|
|
}
|
|
|
|
/// Returns the number of incoming values, also number of incoming blocks.
|
|
/// Note that at the moment, VPWidenPointerInductionRecipe only has a single
|
|
/// incoming value, its start value.
|
|
unsigned getNumIncoming() const override { return 2; }
|
|
|
|
/// Returns the recurrence kind of the reduction.
|
|
RecurKind getRecurrenceKind() const { return Kind; }
|
|
|
|
/// Returns true, if the phi is part of an ordered reduction.
|
|
bool isOrdered() const { return std::holds_alternative<RdxOrdered>(Style); }
|
|
|
|
/// Returns true if the phi is part of an in-loop reduction.
|
|
bool isInLoop() const {
|
|
return std::holds_alternative<RdxInLoop>(Style) ||
|
|
std::holds_alternative<RdxOrdered>(Style);
|
|
}
|
|
|
|
/// Returns true if the reduction outputs a vector with a scaled down VF.
|
|
bool isPartialReduction() const { return getVFScaleFactor() > 1; }
|
|
|
|
/// Returns true, if the phi is part of a multi-use reduction.
|
|
bool hasUsesOutsideReductionChain() const {
|
|
return HasUsesOutsideReductionChain;
|
|
}
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool usesFirstLaneOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return isOrdered() || isInLoop();
|
|
}
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe for vectorizing a phi-node as a sequence of mask-based select
|
|
/// instructions.
|
|
class LLVM_ABI_FOR_TEST VPBlendRecipe : public VPRecipeWithIRFlags {
|
|
public:
|
|
/// The blend operation is a User of the incoming values and of their
|
|
/// respective masks, ordered [I0, M0, I1, M1, I2, M2, ...]. Note that M0 can
|
|
/// be omitted (implied by passing an odd number of operands) in which case
|
|
/// all other incoming values are merged into it.
|
|
VPBlendRecipe(PHINode *Phi, ArrayRef<VPValue *> Operands,
|
|
const VPIRFlags &Flags, DebugLoc DL)
|
|
: VPRecipeWithIRFlags(VPRecipeBase::VPBlendSC, Operands, Flags, DL) {
|
|
assert(Operands.size() >= 2 && "Expected at least two operands!");
|
|
setUnderlyingValue(Phi);
|
|
}
|
|
|
|
VPBlendRecipe *clone() override {
|
|
return new VPBlendRecipe(cast_or_null<PHINode>(getUnderlyingValue()),
|
|
operands(), *this, getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPBlendSC)
|
|
|
|
/// A normalized blend is one that has an odd number of operands, whereby the
|
|
/// first operand does not have an associated mask.
|
|
bool isNormalized() const { return getNumOperands() % 2; }
|
|
|
|
/// Return the number of incoming values, taking into account when normalized
|
|
/// the first incoming value will have no mask.
|
|
unsigned getNumIncomingValues() const {
|
|
return (getNumOperands() + isNormalized()) / 2;
|
|
}
|
|
|
|
/// Return incoming value number \p Idx.
|
|
VPValue *getIncomingValue(unsigned Idx) const {
|
|
return Idx == 0 ? getOperand(0) : getOperand(Idx * 2 - isNormalized());
|
|
}
|
|
|
|
/// Return mask number \p Idx.
|
|
VPValue *getMask(unsigned Idx) const {
|
|
assert((Idx > 0 || !isNormalized()) && "First index has no mask!");
|
|
return Idx == 0 ? getOperand(1) : getOperand(Idx * 2 + !isNormalized());
|
|
}
|
|
|
|
/// Set mask number \p Idx to \p V.
|
|
void setMask(unsigned Idx, VPValue *V) {
|
|
assert((Idx > 0 || !isNormalized()) && "First index has no mask!");
|
|
Idx == 0 ? setOperand(1, V) : setOperand(Idx * 2 + !isNormalized(), V);
|
|
}
|
|
|
|
void execute(VPTransformState &State) override {
|
|
llvm_unreachable("VPBlendRecipe should be expanded by simplifyBlends");
|
|
}
|
|
|
|
/// Return the cost of this VPWidenMemoryRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool usesFirstLaneOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
// Recursing through Blend recipes only, must terminate at header phi's the
|
|
// latest.
|
|
return all_of(users(),
|
|
[this](VPUser *U) { return U->usesFirstLaneOnly(this); });
|
|
}
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A common base class for interleaved memory operations.
|
|
/// An Interleaved memory operation is a memory access method that combines
|
|
/// multiple strided loads/stores into a single wide load/store with shuffles.
|
|
/// The first operand is the start address. The optional operands are, in order,
|
|
/// the stored values and the mask.
|
|
class LLVM_ABI_FOR_TEST VPInterleaveBase : public VPRecipeBase,
|
|
public VPIRMetadata {
|
|
const InterleaveGroup<Instruction> *IG;
|
|
|
|
/// Indicates if the interleave group is in a conditional block and requires a
|
|
/// mask.
|
|
bool HasMask = false;
|
|
|
|
/// Indicates if gaps between members of the group need to be masked out or if
|
|
/// unusued gaps can be loaded speculatively.
|
|
bool NeedsMaskForGaps = false;
|
|
|
|
protected:
|
|
VPInterleaveBase(const unsigned char SC,
|
|
const InterleaveGroup<Instruction> *IG,
|
|
ArrayRef<VPValue *> Operands,
|
|
ArrayRef<VPValue *> StoredValues, VPValue *Mask,
|
|
bool NeedsMaskForGaps, const VPIRMetadata &MD, DebugLoc DL)
|
|
: VPRecipeBase(SC, Operands, DL), VPIRMetadata(MD), IG(IG),
|
|
NeedsMaskForGaps(NeedsMaskForGaps) {
|
|
// TODO: extend the masked interleaved-group support to reversed access.
|
|
assert((!Mask || !IG->isReverse()) &&
|
|
"Reversed masked interleave-group not supported.");
|
|
if (StoredValues.empty()) {
|
|
for (unsigned I = 0; I < IG->getFactor(); ++I)
|
|
if (Instruction *Inst = IG->getMember(I)) {
|
|
assert(!Inst->getType()->isVoidTy() && "must have result");
|
|
new VPRecipeValue(this, Inst);
|
|
}
|
|
} else {
|
|
for (auto *SV : StoredValues)
|
|
addOperand(SV);
|
|
}
|
|
if (Mask) {
|
|
HasMask = true;
|
|
addOperand(Mask);
|
|
}
|
|
}
|
|
|
|
public:
|
|
VPInterleaveBase *clone() override = 0;
|
|
|
|
static inline bool classof(const VPRecipeBase *R) {
|
|
return R->getVPRecipeID() == VPRecipeBase::VPInterleaveSC ||
|
|
R->getVPRecipeID() == VPRecipeBase::VPInterleaveEVLSC;
|
|
}
|
|
|
|
static inline bool classof(const VPUser *U) {
|
|
auto *R = dyn_cast<VPRecipeBase>(U);
|
|
return R && classof(R);
|
|
}
|
|
|
|
/// Return the address accessed by this recipe.
|
|
VPValue *getAddr() const {
|
|
return getOperand(0); // Address is the 1st, mandatory operand.
|
|
}
|
|
|
|
/// Return the mask used by this recipe. Note that a full mask is represented
|
|
/// by a nullptr.
|
|
VPValue *getMask() const {
|
|
// Mask is optional and the last operand.
|
|
return HasMask ? getOperand(getNumOperands() - 1) : nullptr;
|
|
}
|
|
|
|
/// Return true if the access needs a mask because of the gaps.
|
|
bool needsMaskForGaps() const { return NeedsMaskForGaps; }
|
|
|
|
const InterleaveGroup<Instruction> *getInterleaveGroup() const { return IG; }
|
|
|
|
Instruction *getInsertPos() const { return IG->getInsertPos(); }
|
|
|
|
void execute(VPTransformState &State) override {
|
|
llvm_unreachable("VPInterleaveBase should not be instantiated.");
|
|
}
|
|
|
|
/// Return the cost of this recipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool usesFirstLaneOnly(const VPValue *Op) const override = 0;
|
|
|
|
/// Returns the number of stored operands of this interleave group. Returns 0
|
|
/// for load interleave groups.
|
|
virtual unsigned getNumStoreOperands() const = 0;
|
|
|
|
/// Return the VPValues stored by this interleave group. If it is a load
|
|
/// interleave group, return an empty ArrayRef.
|
|
ArrayRef<VPValue *> getStoredValues() const {
|
|
return {op_end() - (getNumStoreOperands() + (HasMask ? 1 : 0)),
|
|
getNumStoreOperands()};
|
|
}
|
|
};
|
|
|
|
/// VPInterleaveRecipe is a recipe for transforming an interleave group of load
|
|
/// or stores into one wide load/store and shuffles. The first operand of a
|
|
/// VPInterleave recipe is the address, followed by the stored values, followed
|
|
/// by an optional mask.
|
|
class LLVM_ABI_FOR_TEST VPInterleaveRecipe final : public VPInterleaveBase {
|
|
public:
|
|
VPInterleaveRecipe(const InterleaveGroup<Instruction> *IG, VPValue *Addr,
|
|
ArrayRef<VPValue *> StoredValues, VPValue *Mask,
|
|
bool NeedsMaskForGaps, const VPIRMetadata &MD, DebugLoc DL)
|
|
: VPInterleaveBase(VPRecipeBase::VPInterleaveSC, IG, Addr, StoredValues,
|
|
Mask, NeedsMaskForGaps, MD, DL) {}
|
|
|
|
~VPInterleaveRecipe() override = default;
|
|
|
|
VPInterleaveRecipe *clone() override {
|
|
return new VPInterleaveRecipe(getInterleaveGroup(), getAddr(),
|
|
getStoredValues(), getMask(),
|
|
needsMaskForGaps(), *this, getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPInterleaveSC)
|
|
|
|
/// Generate the wide load or store, and shuffles.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
bool usesFirstLaneOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return Op == getAddr() && !llvm::is_contained(getStoredValues(), Op);
|
|
}
|
|
|
|
unsigned getNumStoreOperands() const override {
|
|
return getNumOperands() - (getMask() ? 2 : 1);
|
|
}
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe for interleaved memory operations with vector-predication
|
|
/// intrinsics. The first operand is the address, the second operand is the
|
|
/// explicit vector length. Stored values and mask are optional operands.
|
|
class LLVM_ABI_FOR_TEST VPInterleaveEVLRecipe final : public VPInterleaveBase {
|
|
public:
|
|
VPInterleaveEVLRecipe(VPInterleaveRecipe &R, VPValue &EVL, VPValue *Mask)
|
|
: VPInterleaveBase(VPRecipeBase::VPInterleaveEVLSC,
|
|
R.getInterleaveGroup(), {R.getAddr(), &EVL},
|
|
R.getStoredValues(), Mask, R.needsMaskForGaps(), R,
|
|
R.getDebugLoc()) {
|
|
assert(!getInterleaveGroup()->isReverse() &&
|
|
"Reversed interleave-group with tail folding is not supported.");
|
|
assert(!needsMaskForGaps() && "Interleaved access with gap mask is not "
|
|
"supported for scalable vector.");
|
|
}
|
|
|
|
~VPInterleaveEVLRecipe() override = default;
|
|
|
|
VPInterleaveEVLRecipe *clone() override {
|
|
llvm_unreachable("cloning not implemented yet");
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPInterleaveEVLSC)
|
|
|
|
/// The VPValue of the explicit vector length.
|
|
VPValue *getEVL() const { return getOperand(1); }
|
|
|
|
/// Generate the wide load or store, and shuffles.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// The recipe only uses the first lane of the address, and EVL operand.
|
|
bool usesFirstLaneOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return (Op == getAddr() && !llvm::is_contained(getStoredValues(), Op)) ||
|
|
Op == getEVL();
|
|
}
|
|
|
|
unsigned getNumStoreOperands() const override {
|
|
return getNumOperands() - (getMask() ? 3 : 2);
|
|
}
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe to represent inloop, ordered or partial reduction operations. It
|
|
/// performs a reduction on a vector operand into a scalar (vector in the case
|
|
/// of a partial reduction) value, and adds the result to a chain. The Operands
|
|
/// are {ChainOp, VecOp, [Condition]}.
|
|
class LLVM_ABI_FOR_TEST VPReductionRecipe : public VPRecipeWithIRFlags {
|
|
|
|
/// The recurrence kind for the reduction in question.
|
|
RecurKind RdxKind;
|
|
/// Whether the reduction is conditional.
|
|
bool IsConditional = false;
|
|
ReductionStyle Style;
|
|
|
|
protected:
|
|
VPReductionRecipe(const unsigned char SC, RecurKind RdxKind,
|
|
FastMathFlags FMFs, Instruction *I,
|
|
ArrayRef<VPValue *> Operands, VPValue *CondOp,
|
|
ReductionStyle Style, DebugLoc DL)
|
|
: VPRecipeWithIRFlags(SC, Operands, FMFs, DL), RdxKind(RdxKind),
|
|
Style(Style) {
|
|
if (CondOp) {
|
|
IsConditional = true;
|
|
addOperand(CondOp);
|
|
}
|
|
setUnderlyingValue(I);
|
|
}
|
|
|
|
public:
|
|
VPReductionRecipe(RecurKind RdxKind, FastMathFlags FMFs, Instruction *I,
|
|
VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp,
|
|
ReductionStyle Style, DebugLoc DL = DebugLoc::getUnknown())
|
|
: VPReductionRecipe(VPRecipeBase::VPReductionSC, RdxKind, FMFs, I,
|
|
{ChainOp, VecOp}, CondOp, Style, DL) {}
|
|
|
|
VPReductionRecipe(const RecurKind RdxKind, FastMathFlags FMFs,
|
|
VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp,
|
|
ReductionStyle Style, DebugLoc DL = DebugLoc::getUnknown())
|
|
: VPReductionRecipe(VPRecipeBase::VPReductionSC, RdxKind, FMFs, nullptr,
|
|
{ChainOp, VecOp}, CondOp, Style, DL) {}
|
|
|
|
~VPReductionRecipe() override = default;
|
|
|
|
VPReductionRecipe *clone() override {
|
|
return new VPReductionRecipe(RdxKind, getFastMathFlags(),
|
|
getUnderlyingInstr(), getChainOp(), getVecOp(),
|
|
getCondOp(), Style, getDebugLoc());
|
|
}
|
|
|
|
static inline bool classof(const VPRecipeBase *R) {
|
|
return R->getVPRecipeID() == VPRecipeBase::VPReductionSC ||
|
|
R->getVPRecipeID() == VPRecipeBase::VPReductionEVLSC;
|
|
}
|
|
|
|
static inline bool classof(const VPUser *U) {
|
|
auto *R = dyn_cast<VPRecipeBase>(U);
|
|
return R && classof(R);
|
|
}
|
|
|
|
static inline bool classof(const VPValue *VPV) {
|
|
const VPRecipeBase *R = VPV->getDefiningRecipe();
|
|
return R && classof(R);
|
|
}
|
|
|
|
static inline bool classof(const VPSingleDefRecipe *R) {
|
|
return classof(static_cast<const VPRecipeBase *>(R));
|
|
}
|
|
|
|
/// Generate the reduction in the loop.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of VPReductionRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
/// Return the recurrence kind for the in-loop reduction.
|
|
RecurKind getRecurrenceKind() const { return RdxKind; }
|
|
/// Return true if the in-loop reduction is ordered.
|
|
bool isOrdered() const { return std::holds_alternative<RdxOrdered>(Style); };
|
|
/// Return true if the in-loop reduction is conditional.
|
|
bool isConditional() const { return IsConditional; };
|
|
/// Returns true if the reduction outputs a vector with a scaled down VF.
|
|
bool isPartialReduction() const { return getVFScaleFactor() > 1; }
|
|
/// Returns true if the reduction is in-loop.
|
|
bool isInLoop() const {
|
|
return std::holds_alternative<RdxInLoop>(Style) ||
|
|
std::holds_alternative<RdxOrdered>(Style);
|
|
}
|
|
/// The VPValue of the scalar Chain being accumulated.
|
|
VPValue *getChainOp() const { return getOperand(0); }
|
|
/// The VPValue of the vector value to be reduced.
|
|
VPValue *getVecOp() const { return getOperand(1); }
|
|
/// The VPValue of the condition for the block.
|
|
VPValue *getCondOp() const {
|
|
return isConditional() ? getOperand(getNumOperands() - 1) : nullptr;
|
|
}
|
|
/// Get the factor that the VF of this recipe's output should be scaled by, or
|
|
/// 1 if it isn't scaled.
|
|
unsigned getVFScaleFactor() const {
|
|
auto *Partial = std::get_if<RdxUnordered>(&Style);
|
|
return Partial ? Partial->VFScaleFactor : 1;
|
|
}
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe to represent inloop reduction operations with vector-predication
|
|
/// intrinsics, performing a reduction on a vector operand with the explicit
|
|
/// vector length (EVL) into a scalar value, and adding the result to a chain.
|
|
/// The Operands are {ChainOp, VecOp, EVL, [Condition]}.
|
|
class LLVM_ABI_FOR_TEST VPReductionEVLRecipe : public VPReductionRecipe {
|
|
public:
|
|
VPReductionEVLRecipe(VPReductionRecipe &R, VPValue &EVL, VPValue *CondOp,
|
|
DebugLoc DL = DebugLoc::getUnknown())
|
|
: VPReductionRecipe(VPRecipeBase::VPReductionEVLSC, R.getRecurrenceKind(),
|
|
R.getFastMathFlags(),
|
|
cast_or_null<Instruction>(R.getUnderlyingValue()),
|
|
{R.getChainOp(), R.getVecOp(), &EVL}, CondOp,
|
|
getReductionStyle(/*InLoop=*/true, R.isOrdered(), 1),
|
|
DL) {}
|
|
|
|
~VPReductionEVLRecipe() override = default;
|
|
|
|
VPReductionEVLRecipe *clone() override {
|
|
llvm_unreachable("cloning not implemented yet");
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPReductionEVLSC)
|
|
|
|
/// Generate the reduction in the loop
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// The VPValue of the explicit vector length.
|
|
VPValue *getEVL() const { return getOperand(2); }
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool usesFirstLaneOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return Op == getEVL();
|
|
}
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// VPReplicateRecipe replicates a given instruction producing multiple scalar
|
|
/// copies of the original scalar type, one per lane, instead of producing a
|
|
/// single copy of widened type for all lanes. If the instruction is known to be
|
|
/// a single scalar, only one copy, per lane zero, will be generated.
|
|
class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
|
|
public VPIRMetadata {
|
|
/// Indicator if only a single replica per lane is needed.
|
|
bool IsSingleScalar;
|
|
|
|
/// Indicator if the replicas are also predicated.
|
|
bool IsPredicated;
|
|
|
|
public:
|
|
VPReplicateRecipe(Instruction *I, ArrayRef<VPValue *> Operands,
|
|
bool IsSingleScalar, VPValue *Mask = nullptr,
|
|
const VPIRFlags &Flags = {}, VPIRMetadata Metadata = {},
|
|
DebugLoc DL = DebugLoc::getUnknown())
|
|
: VPRecipeWithIRFlags(VPRecipeBase::VPReplicateSC, Operands, Flags, DL),
|
|
VPIRMetadata(Metadata), IsSingleScalar(IsSingleScalar),
|
|
IsPredicated(Mask) {
|
|
setUnderlyingValue(I);
|
|
if (Mask)
|
|
addOperand(Mask);
|
|
}
|
|
|
|
~VPReplicateRecipe() override = default;
|
|
|
|
VPReplicateRecipe *clone() override {
|
|
auto *Copy = new VPReplicateRecipe(
|
|
getUnderlyingInstr(), operands(), IsSingleScalar,
|
|
isPredicated() ? getMask() : nullptr, *this, *this, getDebugLoc());
|
|
Copy->transferFlags(*this);
|
|
return Copy;
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPReplicateSC)
|
|
|
|
/// Generate replicas of the desired Ingredient. Replicas will be generated
|
|
/// for all parts and lanes unless a specific part and lane are specified in
|
|
/// the \p State.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPReplicateRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
bool isSingleScalar() const { return IsSingleScalar; }
|
|
|
|
bool isPredicated() const { return IsPredicated; }
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool usesFirstLaneOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return isSingleScalar();
|
|
}
|
|
|
|
/// Returns true if the recipe uses scalars of operand \p Op.
|
|
bool usesScalars(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
|
|
/// Returns true if the recipe is used by a widened recipe via an intervening
|
|
/// VPPredInstPHIRecipe. In this case, the scalar values should also be packed
|
|
/// in a vector.
|
|
bool shouldPack() const;
|
|
|
|
/// Return the mask of a predicated VPReplicateRecipe.
|
|
VPValue *getMask() {
|
|
assert(isPredicated() && "Trying to get the mask of a unpredicated recipe");
|
|
return getOperand(getNumOperands() - 1);
|
|
}
|
|
|
|
unsigned getOpcode() const { return getUnderlyingInstr()->getOpcode(); }
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe for generating conditional branches on the bits of a mask.
|
|
class LLVM_ABI_FOR_TEST VPBranchOnMaskRecipe : public VPRecipeBase {
|
|
public:
|
|
VPBranchOnMaskRecipe(VPValue *BlockInMask, DebugLoc DL)
|
|
: VPRecipeBase(VPRecipeBase::VPBranchOnMaskSC, {BlockInMask}, DL) {}
|
|
|
|
VPBranchOnMaskRecipe *clone() override {
|
|
return new VPBranchOnMaskRecipe(getOperand(0), getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPBranchOnMaskSC)
|
|
|
|
/// Generate the extraction of the appropriate bit from the block mask and the
|
|
/// conditional branch.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPBranchOnMaskRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override {
|
|
O << Indent << "BRANCH-ON-MASK ";
|
|
printOperands(O, SlotTracker);
|
|
}
|
|
#endif
|
|
|
|
/// Returns true if the recipe uses scalars of operand \p Op.
|
|
bool usesScalars(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
};
|
|
|
|
/// A recipe to combine multiple recipes into a single 'expression' recipe,
|
|
/// which should be considered a single entity for cost-modeling and transforms.
|
|
/// The recipe needs to be 'decomposed', i.e. replaced by its individual
|
|
/// expression recipes, before execute. The individual expression recipes are
|
|
/// completely disconnected from the def-use graph of other recipes not part of
|
|
/// the expression. Def-use edges between pairs of expression recipes remain
|
|
/// intact, whereas every edge between an expression recipe and a recipe outside
|
|
/// the expression is elevated to connect the non-expression recipe with the
|
|
/// VPExpressionRecipe itself.
|
|
class VPExpressionRecipe : public VPSingleDefRecipe {
|
|
/// Recipes included in this VPExpressionRecipe. This could contain
|
|
/// duplicates.
|
|
SmallVector<VPSingleDefRecipe *> ExpressionRecipes;
|
|
|
|
/// Temporary VPValues used for external operands of the expression, i.e.
|
|
/// operands not defined by recipes in the expression.
|
|
SmallVector<VPValue *> LiveInPlaceholders;
|
|
|
|
enum class ExpressionTypes {
|
|
/// Represents an inloop extended reduction operation, performing a
|
|
/// reduction on an extended vector operand into a scalar value, and adding
|
|
/// the result to a chain.
|
|
ExtendedReduction,
|
|
/// Represent an inloop multiply-accumulate reduction, multiplying the
|
|
/// extended vector operands, performing a reduction.add on the result, and
|
|
/// adding the scalar result to a chain.
|
|
ExtMulAccReduction,
|
|
/// Represent an inloop multiply-accumulate reduction, multiplying the
|
|
/// vector operands, performing a reduction.add on the result, and adding
|
|
/// the scalar result to a chain.
|
|
MulAccReduction,
|
|
/// Represent an inloop multiply-accumulate reduction, multiplying the
|
|
/// extended vector operands, negating the multiplication, performing a
|
|
/// reduction.add on the result, and adding the scalar result to a chain.
|
|
ExtNegatedMulAccReduction,
|
|
};
|
|
|
|
/// Type of the expression.
|
|
ExpressionTypes ExpressionType;
|
|
|
|
/// Construct a new VPExpressionRecipe by internalizing recipes in \p
|
|
/// ExpressionRecipes. External operands (i.e. not defined by another recipe
|
|
/// in the expression) are replaced by temporary VPValues and the original
|
|
/// operands are transferred to the VPExpressionRecipe itself. Clone recipes
|
|
/// as needed (excluding last) to ensure they are only used by other recipes
|
|
/// in the expression.
|
|
VPExpressionRecipe(ExpressionTypes ExpressionType,
|
|
ArrayRef<VPSingleDefRecipe *> ExpressionRecipes);
|
|
|
|
public:
|
|
VPExpressionRecipe(VPWidenCastRecipe *Ext, VPReductionRecipe *Red)
|
|
: VPExpressionRecipe(ExpressionTypes::ExtendedReduction, {Ext, Red}) {}
|
|
VPExpressionRecipe(VPWidenRecipe *Mul, VPReductionRecipe *Red)
|
|
: VPExpressionRecipe(ExpressionTypes::MulAccReduction, {Mul, Red}) {}
|
|
VPExpressionRecipe(VPWidenCastRecipe *Ext0, VPWidenCastRecipe *Ext1,
|
|
VPWidenRecipe *Mul, VPReductionRecipe *Red)
|
|
: VPExpressionRecipe(ExpressionTypes::ExtMulAccReduction,
|
|
{Ext0, Ext1, Mul, Red}) {}
|
|
VPExpressionRecipe(VPWidenCastRecipe *Ext0, VPWidenCastRecipe *Ext1,
|
|
VPWidenRecipe *Mul, VPWidenRecipe *Sub,
|
|
VPReductionRecipe *Red)
|
|
: VPExpressionRecipe(ExpressionTypes::ExtNegatedMulAccReduction,
|
|
{Ext0, Ext1, Mul, Sub, Red}) {
|
|
assert(Mul->getOpcode() == Instruction::Mul && "Expected a mul");
|
|
assert(Red->getRecurrenceKind() == RecurKind::Add &&
|
|
"Expected an add reduction");
|
|
assert(getNumOperands() >= 3 && "Expected at least three operands");
|
|
[[maybe_unused]] auto *SubConst = dyn_cast<VPConstantInt>(getOperand(2));
|
|
assert(SubConst && SubConst->isZero() &&
|
|
Sub->getOpcode() == Instruction::Sub && "Expected a negating sub");
|
|
}
|
|
|
|
~VPExpressionRecipe() override {
|
|
SmallPtrSet<VPSingleDefRecipe *, 4> ExpressionRecipesSeen;
|
|
for (auto *R : reverse(ExpressionRecipes)) {
|
|
if (ExpressionRecipesSeen.insert(R).second)
|
|
delete R;
|
|
}
|
|
for (VPValue *T : LiveInPlaceholders)
|
|
delete T;
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPExpressionSC)
|
|
|
|
VPExpressionRecipe *clone() override {
|
|
assert(!ExpressionRecipes.empty() && "empty expressions should be removed");
|
|
SmallVector<VPSingleDefRecipe *> NewExpressiondRecipes;
|
|
for (auto *R : ExpressionRecipes)
|
|
NewExpressiondRecipes.push_back(R->clone());
|
|
for (auto *New : NewExpressiondRecipes) {
|
|
for (const auto &[Idx, Old] : enumerate(ExpressionRecipes))
|
|
New->replaceUsesOfWith(Old, NewExpressiondRecipes[Idx]);
|
|
// Update placeholder operands in the cloned recipe to use the external
|
|
// operands, to be internalized when the cloned expression is constructed.
|
|
for (const auto &[Placeholder, OutsideOp] :
|
|
zip(LiveInPlaceholders, operands()))
|
|
New->replaceUsesOfWith(Placeholder, OutsideOp);
|
|
}
|
|
return new VPExpressionRecipe(ExpressionType, NewExpressiondRecipes);
|
|
}
|
|
|
|
/// Return the VPValue to use to infer the result type of the recipe.
|
|
VPValue *getOperandOfResultType() const {
|
|
unsigned OpIdx =
|
|
cast<VPReductionRecipe>(ExpressionRecipes.back())->isConditional() ? 2
|
|
: 1;
|
|
return getOperand(getNumOperands() - OpIdx);
|
|
}
|
|
|
|
/// Insert the recipes of the expression back into the VPlan, directly before
|
|
/// the current recipe. Leaves the expression recipe empty, which must be
|
|
/// removed before codegen.
|
|
void decompose();
|
|
|
|
unsigned getVFScaleFactor() const {
|
|
auto *PR = dyn_cast<VPReductionRecipe>(ExpressionRecipes.back());
|
|
return PR ? PR->getVFScaleFactor() : 1;
|
|
}
|
|
|
|
/// Method for generating code, must not be called as this recipe is abstract.
|
|
void execute(VPTransformState &State) override {
|
|
llvm_unreachable("recipe must be removed before execute");
|
|
}
|
|
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
/// Returns true if this expression contains recipes that may read from or
|
|
/// write to memory.
|
|
bool mayReadOrWriteMemory() const;
|
|
|
|
/// Returns true if this expression contains recipes that may have side
|
|
/// effects.
|
|
bool mayHaveSideEffects() const;
|
|
|
|
/// Returns true if the result of this VPExpressionRecipe is a single-scalar.
|
|
bool isSingleScalar() const;
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// VPPredInstPHIRecipe is a recipe for generating the phi nodes needed when
|
|
/// control converges back from a Branch-on-Mask. The phi nodes are needed in
|
|
/// order to merge values that are set under such a branch and feed their uses.
|
|
/// The phi nodes can be scalar or vector depending on the users of the value.
|
|
/// This recipe works in concert with VPBranchOnMaskRecipe.
|
|
class LLVM_ABI_FOR_TEST VPPredInstPHIRecipe : public VPSingleDefRecipe {
|
|
public:
|
|
/// Construct a VPPredInstPHIRecipe given \p PredInst whose value needs a phi
|
|
/// nodes after merging back from a Branch-on-Mask.
|
|
VPPredInstPHIRecipe(VPValue *PredV, DebugLoc DL)
|
|
: VPSingleDefRecipe(VPRecipeBase::VPPredInstPHISC, PredV, DL) {}
|
|
~VPPredInstPHIRecipe() override = default;
|
|
|
|
VPPredInstPHIRecipe *clone() override {
|
|
return new VPPredInstPHIRecipe(getOperand(0), getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPPredInstPHISC)
|
|
|
|
/// Generates phi nodes for live-outs (from a replicate region) as needed to
|
|
/// retain SSA form.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPPredInstPHIRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override {
|
|
// TODO: Compute accurate cost after retiring the legacy cost model.
|
|
return 0;
|
|
}
|
|
|
|
/// Returns true if the recipe uses scalars of operand \p Op.
|
|
bool usesScalars(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A common base class for widening memory operations. An optional mask can be
|
|
/// provided as the last operand.
|
|
class LLVM_ABI_FOR_TEST VPWidenMemoryRecipe : public VPRecipeBase,
|
|
public VPIRMetadata {
|
|
protected:
|
|
Instruction &Ingredient;
|
|
|
|
/// Alignment information for this memory access.
|
|
Align Alignment;
|
|
|
|
/// Whether the accessed addresses are consecutive.
|
|
bool Consecutive;
|
|
|
|
/// Whether the consecutive accessed addresses are in reverse order.
|
|
bool Reverse;
|
|
|
|
/// Whether the memory access is masked.
|
|
bool IsMasked = false;
|
|
|
|
void setMask(VPValue *Mask) {
|
|
assert(!IsMasked && "cannot re-set mask");
|
|
if (!Mask)
|
|
return;
|
|
addOperand(Mask);
|
|
IsMasked = true;
|
|
}
|
|
|
|
VPWidenMemoryRecipe(const char unsigned SC, Instruction &I,
|
|
std::initializer_list<VPValue *> Operands,
|
|
bool Consecutive, bool Reverse,
|
|
const VPIRMetadata &Metadata, DebugLoc DL)
|
|
: VPRecipeBase(SC, Operands, DL), VPIRMetadata(Metadata), Ingredient(I),
|
|
Alignment(getLoadStoreAlignment(&I)), Consecutive(Consecutive),
|
|
Reverse(Reverse) {
|
|
assert((Consecutive || !Reverse) && "Reverse implies consecutive");
|
|
assert((isa<VPVectorEndPointerRecipe>(getAddr()) || !Reverse) &&
|
|
"Reversed acccess without VPVectorEndPointerRecipe address?");
|
|
}
|
|
|
|
public:
|
|
VPWidenMemoryRecipe *clone() override {
|
|
llvm_unreachable("cloning not supported");
|
|
}
|
|
|
|
static inline bool classof(const VPRecipeBase *R) {
|
|
return R->getVPRecipeID() == VPRecipeBase::VPWidenLoadSC ||
|
|
R->getVPRecipeID() == VPRecipeBase::VPWidenStoreSC ||
|
|
R->getVPRecipeID() == VPRecipeBase::VPWidenLoadEVLSC ||
|
|
R->getVPRecipeID() == VPRecipeBase::VPWidenStoreEVLSC;
|
|
}
|
|
|
|
static inline bool classof(const VPUser *U) {
|
|
auto *R = dyn_cast<VPRecipeBase>(U);
|
|
return R && classof(R);
|
|
}
|
|
|
|
/// Return whether the loaded-from / stored-to addresses are consecutive.
|
|
bool isConsecutive() const { return Consecutive; }
|
|
|
|
/// Return whether the consecutive loaded/stored addresses are in reverse
|
|
/// order.
|
|
bool isReverse() const { return Reverse; }
|
|
|
|
/// Return the address accessed by this recipe.
|
|
VPValue *getAddr() const { return getOperand(0); }
|
|
|
|
/// Returns true if the recipe is masked.
|
|
bool isMasked() const { return IsMasked; }
|
|
|
|
/// Return the mask used by this recipe. Note that a full mask is represented
|
|
/// by a nullptr.
|
|
VPValue *getMask() const {
|
|
// Mask is optional and therefore the last operand.
|
|
return isMasked() ? getOperand(getNumOperands() - 1) : nullptr;
|
|
}
|
|
|
|
/// Returns the alignment of the memory access.
|
|
Align getAlign() const { return Alignment; }
|
|
|
|
/// Generate the wide load/store.
|
|
void execute(VPTransformState &State) override {
|
|
llvm_unreachable("VPWidenMemoryRecipe should not be instantiated.");
|
|
}
|
|
|
|
/// Return the cost of this VPWidenMemoryRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override;
|
|
|
|
Instruction &getIngredient() const { return Ingredient; }
|
|
};
|
|
|
|
/// A recipe for widening load operations, using the address to load from and an
|
|
/// optional mask.
|
|
struct LLVM_ABI_FOR_TEST VPWidenLoadRecipe final : public VPWidenMemoryRecipe,
|
|
public VPRecipeValue {
|
|
VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask,
|
|
bool Consecutive, bool Reverse,
|
|
const VPIRMetadata &Metadata, DebugLoc DL)
|
|
: VPWidenMemoryRecipe(VPRecipeBase::VPWidenLoadSC, Load, {Addr},
|
|
Consecutive, Reverse, Metadata, DL),
|
|
VPRecipeValue(this, &Load) {
|
|
setMask(Mask);
|
|
}
|
|
|
|
VPWidenLoadRecipe *clone() override {
|
|
return new VPWidenLoadRecipe(cast<LoadInst>(Ingredient), getAddr(),
|
|
getMask(), Consecutive, Reverse, *this,
|
|
getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenLoadSC);
|
|
|
|
/// Generate a wide load or gather.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool usesFirstLaneOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
// Widened, consecutive loads operations only demand the first lane of
|
|
// their address.
|
|
return Op == getAddr() && isConsecutive();
|
|
}
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe for widening load operations with vector-predication intrinsics,
|
|
/// using the address to load from, the explicit vector length and an optional
|
|
/// mask.
|
|
struct VPWidenLoadEVLRecipe final : public VPWidenMemoryRecipe,
|
|
public VPRecipeValue {
|
|
VPWidenLoadEVLRecipe(VPWidenLoadRecipe &L, VPValue *Addr, VPValue &EVL,
|
|
VPValue *Mask)
|
|
: VPWidenMemoryRecipe(VPRecipeBase::VPWidenLoadEVLSC, L.getIngredient(),
|
|
{Addr, &EVL}, L.isConsecutive(), L.isReverse(), L,
|
|
L.getDebugLoc()),
|
|
VPRecipeValue(this, &getIngredient()) {
|
|
setMask(Mask);
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenLoadEVLSC)
|
|
|
|
/// Return the EVL operand.
|
|
VPValue *getEVL() const { return getOperand(1); }
|
|
|
|
/// Generate the wide load or gather.
|
|
LLVM_ABI_FOR_TEST void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPWidenLoadEVLRecipe.
|
|
LLVM_ABI_FOR_TEST InstructionCost
|
|
computeCost(ElementCount VF, VPCostContext &Ctx) const override;
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool usesFirstLaneOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
// Widened loads only demand the first lane of EVL and consecutive loads
|
|
// only demand the first lane of their address.
|
|
return Op == getEVL() || (Op == getAddr() && isConsecutive());
|
|
}
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
LLVM_ABI_FOR_TEST void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe for widening store operations, using the stored value, the address
|
|
/// to store to and an optional mask.
|
|
struct LLVM_ABI_FOR_TEST VPWidenStoreRecipe final : public VPWidenMemoryRecipe {
|
|
VPWidenStoreRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredVal,
|
|
VPValue *Mask, bool Consecutive, bool Reverse,
|
|
const VPIRMetadata &Metadata, DebugLoc DL)
|
|
: VPWidenMemoryRecipe(VPRecipeBase::VPWidenStoreSC, Store,
|
|
{Addr, StoredVal}, Consecutive, Reverse, Metadata,
|
|
DL) {
|
|
setMask(Mask);
|
|
}
|
|
|
|
VPWidenStoreRecipe *clone() override {
|
|
return new VPWidenStoreRecipe(cast<StoreInst>(Ingredient), getAddr(),
|
|
getStoredValue(), getMask(), Consecutive,
|
|
Reverse, *this, getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenStoreSC);
|
|
|
|
/// Return the value stored by this recipe.
|
|
VPValue *getStoredValue() const { return getOperand(1); }
|
|
|
|
/// Generate a wide store or scatter.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool usesFirstLaneOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
// Widened, consecutive stores only demand the first lane of their address,
|
|
// unless the same operand is also stored.
|
|
return Op == getAddr() && isConsecutive() && Op != getStoredValue();
|
|
}
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe for widening store operations with vector-predication intrinsics,
|
|
/// using the value to store, the address to store to, the explicit vector
|
|
/// length and an optional mask.
|
|
struct VPWidenStoreEVLRecipe final : public VPWidenMemoryRecipe {
|
|
VPWidenStoreEVLRecipe(VPWidenStoreRecipe &S, VPValue *Addr,
|
|
VPValue *StoredVal, VPValue &EVL, VPValue *Mask)
|
|
: VPWidenMemoryRecipe(VPRecipeBase::VPWidenStoreEVLSC, S.getIngredient(),
|
|
{Addr, StoredVal, &EVL}, S.isConsecutive(),
|
|
S.isReverse(), S, S.getDebugLoc()) {
|
|
setMask(Mask);
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenStoreEVLSC)
|
|
|
|
/// Return the address accessed by this recipe.
|
|
VPValue *getStoredValue() const { return getOperand(1); }
|
|
|
|
/// Return the EVL operand.
|
|
VPValue *getEVL() const { return getOperand(2); }
|
|
|
|
/// Generate the wide store or scatter.
|
|
LLVM_ABI_FOR_TEST void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPWidenStoreEVLRecipe.
|
|
LLVM_ABI_FOR_TEST InstructionCost
|
|
computeCost(ElementCount VF, VPCostContext &Ctx) const override;
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool usesFirstLaneOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
if (Op == getEVL()) {
|
|
assert(getStoredValue() != Op && "unexpected store of EVL");
|
|
return true;
|
|
}
|
|
// Widened, consecutive memory operations only demand the first lane of
|
|
// their address, unless the same operand is also stored. That latter can
|
|
// happen with opaque pointers.
|
|
return Op == getAddr() && isConsecutive() && Op != getStoredValue();
|
|
}
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
LLVM_ABI_FOR_TEST void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// Recipe to expand a SCEV expression.
|
|
class VPExpandSCEVRecipe : public VPSingleDefRecipe {
|
|
const SCEV *Expr;
|
|
|
|
public:
|
|
VPExpandSCEVRecipe(const SCEV *Expr)
|
|
: VPSingleDefRecipe(VPRecipeBase::VPExpandSCEVSC, {}), Expr(Expr) {}
|
|
|
|
~VPExpandSCEVRecipe() override = default;
|
|
|
|
VPExpandSCEVRecipe *clone() override { return new VPExpandSCEVRecipe(Expr); }
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPExpandSCEVSC)
|
|
|
|
void execute(VPTransformState &State) override {
|
|
llvm_unreachable("SCEV expressions must be expanded before final execute");
|
|
}
|
|
|
|
/// Return the cost of this VPExpandSCEVRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override {
|
|
// TODO: Compute accurate cost after retiring the legacy cost model.
|
|
return 0;
|
|
}
|
|
|
|
const SCEV *getSCEV() const { return Expr; }
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// Canonical scalar induction phi of the vector loop. Starting at the specified
|
|
/// start value (either 0 or the resume value when vectorizing the epilogue
|
|
/// loop). VPWidenCanonicalIVRecipe represents the vector version of the
|
|
/// canonical induction variable.
|
|
class VPCanonicalIVPHIRecipe : public VPHeaderPHIRecipe {
|
|
public:
|
|
VPCanonicalIVPHIRecipe(VPIRValue *StartV, DebugLoc DL)
|
|
: VPHeaderPHIRecipe(VPRecipeBase::VPCanonicalIVPHISC, nullptr, StartV,
|
|
DL) {}
|
|
|
|
~VPCanonicalIVPHIRecipe() override = default;
|
|
|
|
VPCanonicalIVPHIRecipe *clone() override {
|
|
auto *R = new VPCanonicalIVPHIRecipe(getStartValue(), getDebugLoc());
|
|
R->addOperand(getBackedgeValue());
|
|
return R;
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPCanonicalIVPHISC)
|
|
|
|
void execute(VPTransformState &State) override {
|
|
llvm_unreachable("cannot execute this recipe, should be replaced by a "
|
|
"scalar phi recipe");
|
|
}
|
|
|
|
/// Returns the start value of the canonical induction.
|
|
VPIRValue *getStartValue() const { return cast<VPIRValue>(getOperand(0)); }
|
|
|
|
/// Returns the scalar type of the induction.
|
|
Type *getScalarType() const { return getStartValue()->getType(); }
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool usesFirstLaneOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
|
|
/// Returns true if the recipe only uses the first part of operand \p Op.
|
|
bool usesFirstPartOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
|
|
/// Return the cost of this VPCanonicalIVPHIRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override {
|
|
// For now, match the behavior of the legacy cost model.
|
|
return 0;
|
|
}
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
LLVM_ABI_FOR_TEST void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe for generating the active lane mask for the vector loop that is
|
|
/// used to predicate the vector operations.
|
|
class VPActiveLaneMaskPHIRecipe : public VPHeaderPHIRecipe {
|
|
public:
|
|
VPActiveLaneMaskPHIRecipe(VPValue *StartMask, DebugLoc DL)
|
|
: VPHeaderPHIRecipe(VPRecipeBase::VPActiveLaneMaskPHISC, nullptr,
|
|
StartMask, DL) {}
|
|
|
|
~VPActiveLaneMaskPHIRecipe() override = default;
|
|
|
|
VPActiveLaneMaskPHIRecipe *clone() override {
|
|
auto *R = new VPActiveLaneMaskPHIRecipe(getOperand(0), getDebugLoc());
|
|
if (getNumOperands() == 2)
|
|
R->addOperand(getOperand(1));
|
|
return R;
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPActiveLaneMaskPHISC)
|
|
|
|
/// Generate the active lane mask phi of the vector loop.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe for generating the phi node for the current index of elements,
|
|
/// adjusted in accordance with EVL value. It starts at the start value of the
|
|
/// canonical induction and gets incremented by EVL in each iteration of the
|
|
/// vector loop.
|
|
class VPEVLBasedIVPHIRecipe : public VPHeaderPHIRecipe {
|
|
public:
|
|
VPEVLBasedIVPHIRecipe(VPValue *StartIV, DebugLoc DL)
|
|
: VPHeaderPHIRecipe(VPRecipeBase::VPEVLBasedIVPHISC, nullptr, StartIV,
|
|
DL) {}
|
|
|
|
~VPEVLBasedIVPHIRecipe() override = default;
|
|
|
|
VPEVLBasedIVPHIRecipe *clone() override {
|
|
llvm_unreachable("cloning not implemented yet");
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPEVLBasedIVPHISC)
|
|
|
|
void execute(VPTransformState &State) override {
|
|
llvm_unreachable("cannot execute this recipe, should be replaced by a "
|
|
"scalar phi recipe");
|
|
}
|
|
|
|
/// Return the cost of this VPEVLBasedIVPHIRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override {
|
|
// For now, match the behavior of the legacy cost model.
|
|
return 0;
|
|
}
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool usesFirstLaneOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
LLVM_ABI_FOR_TEST void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A Recipe for widening the canonical induction variable of the vector loop.
|
|
class VPWidenCanonicalIVRecipe : public VPSingleDefRecipe,
|
|
public VPUnrollPartAccessor<1> {
|
|
public:
|
|
VPWidenCanonicalIVRecipe(VPCanonicalIVPHIRecipe *CanonicalIV)
|
|
: VPSingleDefRecipe(VPRecipeBase::VPWidenCanonicalIVSC, {CanonicalIV}) {}
|
|
|
|
~VPWidenCanonicalIVRecipe() override = default;
|
|
|
|
VPWidenCanonicalIVRecipe *clone() override {
|
|
return new VPWidenCanonicalIVRecipe(
|
|
cast<VPCanonicalIVPHIRecipe>(getOperand(0)));
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPWidenCanonicalIVSC)
|
|
|
|
/// Generate a canonical vector induction variable of the vector loop, with
|
|
/// start = {<Part*VF, Part*VF+1, ..., Part*VF+VF-1> for 0 <= Part < UF}, and
|
|
/// step = <VF*UF, VF*UF, ..., VF*UF>.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPWidenCanonicalIVPHIRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override {
|
|
// TODO: Compute accurate cost after retiring the legacy cost model.
|
|
return 0;
|
|
}
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe for converting the input value \p IV value to the corresponding
|
|
/// value of an IV with different start and step values, using Start + IV *
|
|
/// Step.
|
|
class VPDerivedIVRecipe : public VPSingleDefRecipe {
|
|
/// Kind of the induction.
|
|
const InductionDescriptor::InductionKind Kind;
|
|
/// If not nullptr, the floating point induction binary operator. Must be set
|
|
/// for floating point inductions.
|
|
const FPMathOperator *FPBinOp;
|
|
|
|
/// Name to use for the generated IR instruction for the derived IV.
|
|
std::string Name;
|
|
|
|
public:
|
|
VPDerivedIVRecipe(const InductionDescriptor &IndDesc, VPIRValue *Start,
|
|
VPCanonicalIVPHIRecipe *CanonicalIV, VPValue *Step,
|
|
const Twine &Name = "")
|
|
: VPDerivedIVRecipe(
|
|
IndDesc.getKind(),
|
|
dyn_cast_or_null<FPMathOperator>(IndDesc.getInductionBinOp()),
|
|
Start, CanonicalIV, Step, Name) {}
|
|
|
|
VPDerivedIVRecipe(InductionDescriptor::InductionKind Kind,
|
|
const FPMathOperator *FPBinOp, VPIRValue *Start,
|
|
VPValue *IV, VPValue *Step, const Twine &Name = "")
|
|
: VPSingleDefRecipe(VPRecipeBase::VPDerivedIVSC, {Start, IV, Step}),
|
|
Kind(Kind), FPBinOp(FPBinOp), Name(Name.str()) {}
|
|
|
|
~VPDerivedIVRecipe() override = default;
|
|
|
|
VPDerivedIVRecipe *clone() override {
|
|
return new VPDerivedIVRecipe(Kind, FPBinOp, getStartValue(), getOperand(1),
|
|
getStepValue());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPDerivedIVSC)
|
|
|
|
/// Generate the transformed value of the induction at offset StartValue (1.
|
|
/// operand) + IV (2. operand) * StepValue (3, operand).
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPDerivedIVRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override {
|
|
// TODO: Compute accurate cost after retiring the legacy cost model.
|
|
return 0;
|
|
}
|
|
|
|
Type *getScalarType() const { return getStartValue()->getType(); }
|
|
|
|
VPIRValue *getStartValue() const { return cast<VPIRValue>(getOperand(0)); }
|
|
VPValue *getStepValue() const { return getOperand(2); }
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool usesFirstLaneOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// A recipe for handling phi nodes of integer and floating-point inductions,
|
|
/// producing their scalar values. Before unrolling by UF the recipe represents
|
|
/// the VF*UF scalar values to be produced, or UF scalar values if only first
|
|
/// lane is used, and has 3 operands: IV, step and VF. Unrolling adds one extra
|
|
/// operand StartIndex to all unroll parts except part 0, as the recipe
|
|
/// represents the VF scalar values (this number of values is taken from
|
|
/// State.VF rather than from the VF operand) starting at IV + StartIndex.
|
|
class LLVM_ABI_FOR_TEST VPScalarIVStepsRecipe : public VPRecipeWithIRFlags {
|
|
Instruction::BinaryOps InductionOpcode;
|
|
|
|
public:
|
|
VPScalarIVStepsRecipe(VPValue *IV, VPValue *Step, VPValue *VF,
|
|
Instruction::BinaryOps Opcode, FastMathFlags FMFs,
|
|
DebugLoc DL)
|
|
: VPRecipeWithIRFlags(VPRecipeBase::VPScalarIVStepsSC, {IV, Step, VF},
|
|
FMFs, DL),
|
|
InductionOpcode(Opcode) {}
|
|
|
|
VPScalarIVStepsRecipe(const InductionDescriptor &IndDesc, VPValue *IV,
|
|
VPValue *Step, VPValue *VF,
|
|
DebugLoc DL = DebugLoc::getUnknown())
|
|
: VPScalarIVStepsRecipe(
|
|
IV, Step, VF, IndDesc.getInductionOpcode(),
|
|
dyn_cast_or_null<FPMathOperator>(IndDesc.getInductionBinOp())
|
|
? IndDesc.getInductionBinOp()->getFastMathFlags()
|
|
: FastMathFlags(),
|
|
DL) {}
|
|
|
|
~VPScalarIVStepsRecipe() override = default;
|
|
|
|
VPScalarIVStepsRecipe *clone() override {
|
|
return new VPScalarIVStepsRecipe(
|
|
getOperand(0), getOperand(1), getOperand(2), InductionOpcode,
|
|
hasFastMathFlags() ? getFastMathFlags() : FastMathFlags(),
|
|
getDebugLoc());
|
|
}
|
|
|
|
VP_CLASSOF_IMPL(VPRecipeBase::VPScalarIVStepsSC)
|
|
|
|
/// Generate the scalarized versions of the phi node as needed by their users.
|
|
void execute(VPTransformState &State) override;
|
|
|
|
/// Return the cost of this VPScalarIVStepsRecipe.
|
|
InstructionCost computeCost(ElementCount VF,
|
|
VPCostContext &Ctx) const override {
|
|
// TODO: Compute accurate cost after retiring the legacy cost model.
|
|
return 0;
|
|
}
|
|
|
|
VPValue *getStepValue() const { return getOperand(1); }
|
|
|
|
/// Return the number of scalars to produce per unroll part, used to compute
|
|
/// StartIndex during unrolling.
|
|
VPValue *getVFValue() const { return getOperand(2); }
|
|
|
|
/// Return the StartIndex, or null if known to be zero, valid only after
|
|
/// unrolling.
|
|
VPValue *getStartIndex() const {
|
|
return getNumOperands() == 4 ? getOperand(3) : nullptr;
|
|
}
|
|
|
|
/// Returns true if the recipe only uses the first lane of operand \p Op.
|
|
bool usesFirstLaneOnly(const VPValue *Op) const override {
|
|
assert(is_contained(operands(), Op) &&
|
|
"Op must be an operand of the recipe");
|
|
return true;
|
|
}
|
|
|
|
protected:
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the recipe.
|
|
void printRecipe(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
#endif
|
|
};
|
|
|
|
/// Casting from VPRecipeBase -> VPPhiAccessors is supported for all recipe
|
|
/// types implementing VPPhiAccessors. Used by isa<> & co.
|
|
template <> struct CastIsPossible<VPPhiAccessors, const VPRecipeBase *> {
|
|
static inline bool isPossible(const VPRecipeBase *f) {
|
|
// TODO: include VPPredInstPHIRecipe too, once it implements VPPhiAccessors.
|
|
return isa<VPIRPhi, VPHeaderPHIRecipe, VPWidenPHIRecipe, VPPhi>(f);
|
|
}
|
|
};
|
|
/// Support casting from VPRecipeBase -> VPPhiAccessors, by down-casting to the
|
|
/// recipe types implementing VPPhiAccessors. Used by cast<>, dyn_cast<> & co.
|
|
template <typename SrcTy>
|
|
struct CastInfoVPPhiAccessors : public CastIsPossible<VPPhiAccessors, SrcTy> {
|
|
|
|
using Self = CastInfo<VPPhiAccessors, SrcTy>;
|
|
|
|
/// doCast is used by cast<>.
|
|
static inline VPPhiAccessors *doCast(SrcTy R) {
|
|
return const_cast<VPPhiAccessors *>([R]() -> const VPPhiAccessors * {
|
|
switch (R->getVPRecipeID()) {
|
|
case VPRecipeBase::VPInstructionSC:
|
|
return cast<VPPhi>(R);
|
|
case VPRecipeBase::VPIRInstructionSC:
|
|
return cast<VPIRPhi>(R);
|
|
case VPRecipeBase::VPWidenPHISC:
|
|
return cast<VPWidenPHIRecipe>(R);
|
|
default:
|
|
return cast<VPHeaderPHIRecipe>(R);
|
|
}
|
|
}());
|
|
}
|
|
|
|
/// doCastIfPossible is used by dyn_cast<>.
|
|
static inline VPPhiAccessors *doCastIfPossible(SrcTy f) {
|
|
if (!Self::isPossible(f))
|
|
return nullptr;
|
|
return doCast(f);
|
|
}
|
|
};
|
|
template <>
|
|
struct CastInfo<VPPhiAccessors, VPRecipeBase *>
|
|
: CastInfoVPPhiAccessors<VPRecipeBase *> {};
|
|
template <>
|
|
struct CastInfo<VPPhiAccessors, const VPRecipeBase *>
|
|
: CastInfoVPPhiAccessors<const VPRecipeBase *> {};
|
|
|
|
/// Casting from (const) VPRecipeBase -> (const) VPIRMetadata is supported for
|
|
/// all recipe types implementing VPIRMetadata. Used by isa<> & co.
|
|
namespace detail {
|
|
template <typename DstTy, typename RecipeBasePtrTy>
|
|
static inline auto castToVPIRMetadata(RecipeBasePtrTy R) -> DstTy {
|
|
switch (R->getVPRecipeID()) {
|
|
case VPRecipeBase::VPInstructionSC:
|
|
return cast<VPInstruction>(R);
|
|
case VPRecipeBase::VPWidenSC:
|
|
return cast<VPWidenRecipe>(R);
|
|
case VPRecipeBase::VPWidenCastSC:
|
|
return cast<VPWidenCastRecipe>(R);
|
|
case VPRecipeBase::VPWidenIntrinsicSC:
|
|
return cast<VPWidenIntrinsicRecipe>(R);
|
|
case VPRecipeBase::VPWidenCallSC:
|
|
return cast<VPWidenCallRecipe>(R);
|
|
case VPRecipeBase::VPReplicateSC:
|
|
return cast<VPReplicateRecipe>(R);
|
|
case VPRecipeBase::VPInterleaveSC:
|
|
case VPRecipeBase::VPInterleaveEVLSC:
|
|
return cast<VPInterleaveBase>(R);
|
|
case VPRecipeBase::VPWidenLoadSC:
|
|
case VPRecipeBase::VPWidenLoadEVLSC:
|
|
case VPRecipeBase::VPWidenStoreSC:
|
|
case VPRecipeBase::VPWidenStoreEVLSC:
|
|
return cast<VPWidenMemoryRecipe>(R);
|
|
default:
|
|
llvm_unreachable("invalid recipe for VPIRMetadata cast");
|
|
}
|
|
}
|
|
} // namespace detail
|
|
|
|
/// Support casting from VPRecipeBase -> VPIRMetadata, by down-casting to the
|
|
/// recipe types implementing VPIRMetadata. Used by cast<>, dyn_cast<> & co.
|
|
template <typename DstTy, typename SrcTy>
|
|
struct CastInfoVPIRMetadata : public CastIsPossible<DstTy, SrcTy> {
|
|
static inline bool isPossible(SrcTy R) {
|
|
// NOTE: Each recipe inheriting from VPIRMetadata must be listed here and
|
|
// also handled in castToVPIRMetadata.
|
|
return isa<VPInstruction, VPWidenRecipe, VPWidenCastRecipe,
|
|
VPWidenIntrinsicRecipe, VPWidenCallRecipe, VPReplicateRecipe,
|
|
VPInterleaveRecipe, VPInterleaveEVLRecipe, VPWidenLoadRecipe,
|
|
VPWidenLoadEVLRecipe, VPWidenStoreRecipe, VPWidenStoreEVLRecipe>(
|
|
R);
|
|
}
|
|
|
|
using RetTy = DstTy *;
|
|
|
|
/// doCast is used by cast<>.
|
|
static inline RetTy doCast(SrcTy R) {
|
|
return detail::castToVPIRMetadata<RetTy, SrcTy>(R);
|
|
}
|
|
|
|
/// doCastIfPossible is used by dyn_cast<>.
|
|
static inline RetTy doCastIfPossible(SrcTy R) {
|
|
if (!isPossible(R))
|
|
return nullptr;
|
|
return doCast(R);
|
|
}
|
|
};
|
|
template <>
|
|
struct CastInfo<VPIRMetadata, VPRecipeBase *>
|
|
: CastInfoVPIRMetadata<VPIRMetadata, VPRecipeBase *> {};
|
|
template <>
|
|
struct CastInfo<VPIRMetadata, const VPRecipeBase *>
|
|
: CastInfoVPIRMetadata<const VPIRMetadata, const VPRecipeBase *> {};
|
|
|
|
/// VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph. It
|
|
/// holds a sequence of zero or more VPRecipe's each representing a sequence of
|
|
/// output IR instructions. All PHI-like recipes must come before any non-PHI recipes.
|
|
class LLVM_ABI_FOR_TEST VPBasicBlock : public VPBlockBase {
|
|
friend class VPlan;
|
|
|
|
/// Use VPlan::createVPBasicBlock to create VPBasicBlocks.
|
|
VPBasicBlock(const Twine &Name = "", VPRecipeBase *Recipe = nullptr)
|
|
: VPBlockBase(VPBasicBlockSC, Name.str()) {
|
|
if (Recipe)
|
|
appendRecipe(Recipe);
|
|
}
|
|
|
|
public:
|
|
using RecipeListTy = iplist<VPRecipeBase>;
|
|
|
|
protected:
|
|
/// The VPRecipes held in the order of output instructions to generate.
|
|
RecipeListTy Recipes;
|
|
|
|
VPBasicBlock(const unsigned char BlockSC, const Twine &Name = "")
|
|
: VPBlockBase(BlockSC, Name.str()) {}
|
|
|
|
public:
|
|
~VPBasicBlock() override {
|
|
while (!Recipes.empty())
|
|
Recipes.pop_back();
|
|
}
|
|
|
|
/// Instruction iterators...
|
|
using iterator = RecipeListTy::iterator;
|
|
using const_iterator = RecipeListTy::const_iterator;
|
|
using reverse_iterator = RecipeListTy::reverse_iterator;
|
|
using const_reverse_iterator = RecipeListTy::const_reverse_iterator;
|
|
|
|
//===--------------------------------------------------------------------===//
|
|
/// Recipe iterator methods
|
|
///
|
|
inline iterator begin() { return Recipes.begin(); }
|
|
inline const_iterator begin() const { return Recipes.begin(); }
|
|
inline iterator end() { return Recipes.end(); }
|
|
inline const_iterator end() const { return Recipes.end(); }
|
|
|
|
inline reverse_iterator rbegin() { return Recipes.rbegin(); }
|
|
inline const_reverse_iterator rbegin() const { return Recipes.rbegin(); }
|
|
inline reverse_iterator rend() { return Recipes.rend(); }
|
|
inline const_reverse_iterator rend() const { return Recipes.rend(); }
|
|
|
|
inline size_t size() const { return Recipes.size(); }
|
|
inline bool empty() const { return Recipes.empty(); }
|
|
inline const VPRecipeBase &front() const { return Recipes.front(); }
|
|
inline VPRecipeBase &front() { return Recipes.front(); }
|
|
inline const VPRecipeBase &back() const { return Recipes.back(); }
|
|
inline VPRecipeBase &back() { return Recipes.back(); }
|
|
|
|
/// Returns a reference to the list of recipes.
|
|
RecipeListTy &getRecipeList() { return Recipes; }
|
|
|
|
/// Returns a pointer to a member of the recipe list.
|
|
static RecipeListTy VPBasicBlock::*getSublistAccess(VPRecipeBase *) {
|
|
return &VPBasicBlock::Recipes;
|
|
}
|
|
|
|
/// Method to support type inquiry through isa, cast, and dyn_cast.
|
|
static inline bool classof(const VPBlockBase *V) {
|
|
return V->getVPBlockID() == VPBlockBase::VPBasicBlockSC ||
|
|
V->getVPBlockID() == VPBlockBase::VPIRBasicBlockSC;
|
|
}
|
|
|
|
void insert(VPRecipeBase *Recipe, iterator InsertPt) {
|
|
assert(Recipe && "No recipe to append.");
|
|
assert(!Recipe->Parent && "Recipe already in VPlan");
|
|
Recipe->Parent = this;
|
|
Recipes.insert(InsertPt, Recipe);
|
|
}
|
|
|
|
/// Augment the existing recipes of a VPBasicBlock with an additional
|
|
/// \p Recipe as the last recipe.
|
|
void appendRecipe(VPRecipeBase *Recipe) { insert(Recipe, end()); }
|
|
|
|
/// The method which generates the output IR instructions that correspond to
|
|
/// this VPBasicBlock, thereby "executing" the VPlan.
|
|
void execute(VPTransformState *State) override;
|
|
|
|
/// Return the cost of this VPBasicBlock.
|
|
InstructionCost cost(ElementCount VF, VPCostContext &Ctx) override;
|
|
|
|
/// Return the position of the first non-phi node recipe in the block.
|
|
iterator getFirstNonPhi();
|
|
|
|
/// Returns an iterator range over the PHI-like recipes in the block.
|
|
iterator_range<iterator> phis() {
|
|
return make_range(begin(), getFirstNonPhi());
|
|
}
|
|
|
|
/// Split current block at \p SplitAt by inserting a new block between the
|
|
/// current block and its successors and moving all recipes starting at
|
|
/// SplitAt to the new block. Returns the new block.
|
|
VPBasicBlock *splitAt(iterator SplitAt);
|
|
|
|
VPRegionBlock *getEnclosingLoopRegion();
|
|
const VPRegionBlock *getEnclosingLoopRegion() const;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print this VPBsicBlock to \p O, prefixing all lines with \p Indent. \p
|
|
/// SlotTracker is used to print unnamed VPValue's using consequtive numbers.
|
|
///
|
|
/// Note that the numbering is applied to the whole VPlan, so printing
|
|
/// individual blocks is consistent with the whole VPlan printing.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
using VPBlockBase::print; // Get the print(raw_stream &O) version.
|
|
#endif
|
|
|
|
/// If the block has multiple successors, return the branch recipe terminating
|
|
/// the block. If there are no or only a single successor, return nullptr;
|
|
VPRecipeBase *getTerminator();
|
|
const VPRecipeBase *getTerminator() const;
|
|
|
|
/// Returns true if the block is exiting it's parent region.
|
|
bool isExiting() const;
|
|
|
|
/// Clone the current block and it's recipes, without updating the operands of
|
|
/// the cloned recipes.
|
|
VPBasicBlock *clone() override;
|
|
|
|
/// Returns the predecessor block at index \p Idx with the predecessors as per
|
|
/// the corresponding plain CFG. If the block is an entry block to a region,
|
|
/// the first predecessor is the single predecessor of a region, and the
|
|
/// second predecessor is the exiting block of the region.
|
|
const VPBasicBlock *getCFGPredecessor(unsigned Idx) const;
|
|
|
|
protected:
|
|
/// Execute the recipes in the IR basic block \p BB.
|
|
void executeRecipes(VPTransformState *State, BasicBlock *BB);
|
|
|
|
/// Connect the VPBBs predecessors' in the VPlan CFG to the IR basic block
|
|
/// generated for this VPBB.
|
|
void connectToPredecessors(VPTransformState &State);
|
|
|
|
private:
|
|
/// Create an IR BasicBlock to hold the output instructions generated by this
|
|
/// VPBasicBlock, and return it. Update the CFGState accordingly.
|
|
BasicBlock *createEmptyBasicBlock(VPTransformState &State);
|
|
};
|
|
|
|
inline const VPBasicBlock *
|
|
VPPhiAccessors::getIncomingBlock(unsigned Idx) const {
|
|
return getAsRecipe()->getParent()->getCFGPredecessor(Idx);
|
|
}
|
|
|
|
/// A special type of VPBasicBlock that wraps an existing IR basic block.
|
|
/// Recipes of the block get added before the first non-phi instruction in the
|
|
/// wrapped block.
|
|
/// Note: At the moment, VPIRBasicBlock can only be used to wrap VPlan's
|
|
/// preheader block.
|
|
class VPIRBasicBlock : public VPBasicBlock {
|
|
friend class VPlan;
|
|
|
|
BasicBlock *IRBB;
|
|
|
|
/// Use VPlan::createVPIRBasicBlock to create VPIRBasicBlocks.
|
|
VPIRBasicBlock(BasicBlock *IRBB)
|
|
: VPBasicBlock(VPIRBasicBlockSC,
|
|
(Twine("ir-bb<") + IRBB->getName() + Twine(">")).str()),
|
|
IRBB(IRBB) {}
|
|
|
|
public:
|
|
~VPIRBasicBlock() override = default;
|
|
|
|
static inline bool classof(const VPBlockBase *V) {
|
|
return V->getVPBlockID() == VPBlockBase::VPIRBasicBlockSC;
|
|
}
|
|
|
|
/// The method which generates the output IR instructions that correspond to
|
|
/// this VPBasicBlock, thereby "executing" the VPlan.
|
|
void execute(VPTransformState *State) override;
|
|
|
|
VPIRBasicBlock *clone() override;
|
|
|
|
BasicBlock *getIRBasicBlock() const { return IRBB; }
|
|
};
|
|
|
|
/// VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks
|
|
/// which form a Single-Entry-Single-Exiting subgraph of the output IR CFG.
|
|
/// A VPRegionBlock may indicate that its contents are to be replicated several
|
|
/// times. This is designed to support predicated scalarization, in which a
|
|
/// scalar if-then code structure needs to be generated VF * UF times. Having
|
|
/// this replication indicator helps to keep a single model for multiple
|
|
/// candidate VF's. The actual replication takes place only once the desired VF
|
|
/// and UF have been determined.
|
|
class LLVM_ABI_FOR_TEST VPRegionBlock : public VPBlockBase {
|
|
friend class VPlan;
|
|
|
|
/// Hold the Single Entry of the SESE region modelled by the VPRegionBlock.
|
|
VPBlockBase *Entry;
|
|
|
|
/// Hold the Single Exiting block of the SESE region modelled by the
|
|
/// VPRegionBlock.
|
|
VPBlockBase *Exiting;
|
|
|
|
/// An indicator whether this region is to generate multiple replicated
|
|
/// instances of output IR corresponding to its VPBlockBases.
|
|
bool IsReplicator;
|
|
|
|
/// Use VPlan::createVPRegionBlock to create VPRegionBlocks.
|
|
VPRegionBlock(VPBlockBase *Entry, VPBlockBase *Exiting,
|
|
const std::string &Name = "", bool IsReplicator = false)
|
|
: VPBlockBase(VPRegionBlockSC, Name), Entry(Entry), Exiting(Exiting),
|
|
IsReplicator(IsReplicator) {
|
|
assert(Entry->getPredecessors().empty() && "Entry block has predecessors.");
|
|
assert(Exiting->getSuccessors().empty() && "Exit block has successors.");
|
|
Entry->setParent(this);
|
|
Exiting->setParent(this);
|
|
}
|
|
VPRegionBlock(const std::string &Name = "", bool IsReplicator = false)
|
|
: VPBlockBase(VPRegionBlockSC, Name), Entry(nullptr), Exiting(nullptr),
|
|
IsReplicator(IsReplicator) {}
|
|
|
|
public:
|
|
~VPRegionBlock() override = default;
|
|
|
|
/// Method to support type inquiry through isa, cast, and dyn_cast.
|
|
static inline bool classof(const VPBlockBase *V) {
|
|
return V->getVPBlockID() == VPBlockBase::VPRegionBlockSC;
|
|
}
|
|
|
|
const VPBlockBase *getEntry() const { return Entry; }
|
|
VPBlockBase *getEntry() { return Entry; }
|
|
|
|
/// Set \p EntryBlock as the entry VPBlockBase of this VPRegionBlock. \p
|
|
/// EntryBlock must have no predecessors.
|
|
void setEntry(VPBlockBase *EntryBlock) {
|
|
assert(EntryBlock->getPredecessors().empty() &&
|
|
"Entry block cannot have predecessors.");
|
|
Entry = EntryBlock;
|
|
EntryBlock->setParent(this);
|
|
}
|
|
|
|
const VPBlockBase *getExiting() const { return Exiting; }
|
|
VPBlockBase *getExiting() { return Exiting; }
|
|
|
|
/// Set \p ExitingBlock as the exiting VPBlockBase of this VPRegionBlock. \p
|
|
/// ExitingBlock must have no successors.
|
|
void setExiting(VPBlockBase *ExitingBlock) {
|
|
assert(ExitingBlock->getSuccessors().empty() &&
|
|
"Exit block cannot have successors.");
|
|
Exiting = ExitingBlock;
|
|
ExitingBlock->setParent(this);
|
|
}
|
|
|
|
/// Returns the pre-header VPBasicBlock of the loop region.
|
|
VPBasicBlock *getPreheaderVPBB() {
|
|
assert(!isReplicator() && "should only get pre-header of loop regions");
|
|
return getSinglePredecessor()->getExitingBasicBlock();
|
|
}
|
|
|
|
/// An indicator whether this region is to generate multiple replicated
|
|
/// instances of output IR corresponding to its VPBlockBases.
|
|
bool isReplicator() const { return IsReplicator; }
|
|
|
|
/// The method which generates the output IR instructions that correspond to
|
|
/// this VPRegionBlock, thereby "executing" the VPlan.
|
|
void execute(VPTransformState *State) override;
|
|
|
|
// Return the cost of this region.
|
|
InstructionCost cost(ElementCount VF, VPCostContext &Ctx) override;
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print this VPRegionBlock to \p O (recursively), prefixing all lines with
|
|
/// \p Indent. \p SlotTracker is used to print unnamed VPValue's using
|
|
/// consequtive numbers.
|
|
///
|
|
/// Note that the numbering is applied to the whole VPlan, so printing
|
|
/// individual regions is consistent with the whole VPlan printing.
|
|
void print(raw_ostream &O, const Twine &Indent,
|
|
VPSlotTracker &SlotTracker) const override;
|
|
using VPBlockBase::print; // Get the print(raw_stream &O) version.
|
|
#endif
|
|
|
|
/// Clone all blocks in the single-entry single-exit region of the block and
|
|
/// their recipes without updating the operands of the cloned recipes.
|
|
VPRegionBlock *clone() override;
|
|
|
|
/// Remove the current region from its VPlan, connecting its predecessor to
|
|
/// its entry, and its exiting block to its successor.
|
|
void dissolveToCFGLoop();
|
|
|
|
/// Returns the canonical induction recipe of the region.
|
|
VPCanonicalIVPHIRecipe *getCanonicalIV() {
|
|
VPBasicBlock *EntryVPBB = getEntryBasicBlock();
|
|
if (EntryVPBB->empty()) {
|
|
// VPlan native path. TODO: Unify both code paths.
|
|
EntryVPBB = cast<VPBasicBlock>(EntryVPBB->getSingleSuccessor());
|
|
}
|
|
return cast<VPCanonicalIVPHIRecipe>(&*EntryVPBB->begin());
|
|
}
|
|
const VPCanonicalIVPHIRecipe *getCanonicalIV() const {
|
|
return const_cast<VPRegionBlock *>(this)->getCanonicalIV();
|
|
}
|
|
|
|
/// Return the type of the canonical IV for loop regions.
|
|
Type *getCanonicalIVType() { return getCanonicalIV()->getScalarType(); }
|
|
const Type *getCanonicalIVType() const {
|
|
return getCanonicalIV()->getScalarType();
|
|
}
|
|
};
|
|
|
|
inline VPRegionBlock *VPRecipeBase::getRegion() {
|
|
return getParent()->getParent();
|
|
}
|
|
|
|
inline const VPRegionBlock *VPRecipeBase::getRegion() const {
|
|
return getParent()->getParent();
|
|
}
|
|
|
|
/// VPlan models a candidate for vectorization, encoding various decisions take
|
|
/// to produce efficient output IR, including which branches, basic-blocks and
|
|
/// output IR instructions to generate, and their cost. VPlan holds a
|
|
/// Hierarchical-CFG of VPBasicBlocks and VPRegionBlocks rooted at an Entry
|
|
/// VPBasicBlock.
|
|
class VPlan {
|
|
friend class VPlanPrinter;
|
|
friend class VPSlotTracker;
|
|
|
|
/// VPBasicBlock corresponding to the original preheader. Used to place
|
|
/// VPExpandSCEV recipes for expressions used during skeleton creation and the
|
|
/// rest of VPlan execution.
|
|
/// When this VPlan is used for the epilogue vector loop, the entry will be
|
|
/// replaced by a new entry block created during skeleton creation.
|
|
VPBasicBlock *Entry;
|
|
|
|
/// VPIRBasicBlock wrapping the header of the original scalar loop.
|
|
VPIRBasicBlock *ScalarHeader;
|
|
|
|
/// Immutable list of VPIRBasicBlocks wrapping the exit blocks of the original
|
|
/// scalar loop. Note that some exit blocks may be unreachable at the moment,
|
|
/// e.g. if the scalar epilogue always executes.
|
|
SmallVector<VPIRBasicBlock *, 2> ExitBlocks;
|
|
|
|
/// Holds the VFs applicable to this VPlan.
|
|
SmallSetVector<ElementCount, 2> VFs;
|
|
|
|
/// Holds the UFs applicable to this VPlan. If empty, the VPlan is valid for
|
|
/// any UF.
|
|
SmallSetVector<unsigned, 2> UFs;
|
|
|
|
/// Holds the name of the VPlan, for printing.
|
|
std::string Name;
|
|
|
|
/// Represents the trip count of the original loop, for folding
|
|
/// the tail.
|
|
VPValue *TripCount = nullptr;
|
|
|
|
/// Represents the backedge taken count of the original loop, for folding
|
|
/// the tail. It equals TripCount - 1.
|
|
VPSymbolicValue *BackedgeTakenCount = nullptr;
|
|
|
|
/// Represents the vector trip count.
|
|
VPSymbolicValue VectorTripCount;
|
|
|
|
/// Represents the vectorization factor of the loop.
|
|
VPSymbolicValue VF;
|
|
|
|
/// Represents the loop-invariant VF * UF of the vector loop region.
|
|
VPSymbolicValue VFxUF;
|
|
|
|
/// Contains all the external definitions created for this VPlan, as a mapping
|
|
/// from IR Values to VPIRValues.
|
|
SmallMapVector<Value *, VPIRValue *, 16> LiveIns;
|
|
|
|
/// Blocks allocated and owned by the VPlan. They will be deleted once the
|
|
/// VPlan is destroyed.
|
|
SmallVector<VPBlockBase *> CreatedBlocks;
|
|
|
|
/// Construct a VPlan with \p Entry to the plan and with \p ScalarHeader
|
|
/// wrapping the original header of the scalar loop.
|
|
VPlan(VPBasicBlock *Entry, VPIRBasicBlock *ScalarHeader)
|
|
: Entry(Entry), ScalarHeader(ScalarHeader) {
|
|
Entry->setPlan(this);
|
|
assert(ScalarHeader->getNumSuccessors() == 0 &&
|
|
"scalar header must be a leaf node");
|
|
}
|
|
|
|
public:
|
|
/// Construct a VPlan for \p L. This will create VPIRBasicBlocks wrapping the
|
|
/// original preheader and scalar header of \p L, to be used as entry and
|
|
/// scalar header blocks of the new VPlan.
|
|
VPlan(Loop *L);
|
|
|
|
/// Construct a VPlan with a new VPBasicBlock as entry, a VPIRBasicBlock
|
|
/// wrapping \p ScalarHeaderBB and a trip count of \p TC.
|
|
VPlan(BasicBlock *ScalarHeaderBB) {
|
|
setEntry(createVPBasicBlock("preheader"));
|
|
ScalarHeader = createVPIRBasicBlock(ScalarHeaderBB);
|
|
}
|
|
|
|
LLVM_ABI_FOR_TEST ~VPlan();
|
|
|
|
void setEntry(VPBasicBlock *VPBB) {
|
|
Entry = VPBB;
|
|
VPBB->setPlan(this);
|
|
}
|
|
|
|
/// Generate the IR code for this VPlan.
|
|
void execute(VPTransformState *State);
|
|
|
|
/// Return the cost of this plan.
|
|
InstructionCost cost(ElementCount VF, VPCostContext &Ctx);
|
|
|
|
VPBasicBlock *getEntry() { return Entry; }
|
|
const VPBasicBlock *getEntry() const { return Entry; }
|
|
|
|
/// Returns the preheader of the vector loop region, if one exists, or null
|
|
/// otherwise.
|
|
VPBasicBlock *getVectorPreheader() {
|
|
VPRegionBlock *VectorRegion = getVectorLoopRegion();
|
|
return VectorRegion
|
|
? cast<VPBasicBlock>(VectorRegion->getSinglePredecessor())
|
|
: nullptr;
|
|
}
|
|
|
|
/// Returns the VPRegionBlock of the vector loop.
|
|
LLVM_ABI_FOR_TEST VPRegionBlock *getVectorLoopRegion();
|
|
LLVM_ABI_FOR_TEST const VPRegionBlock *getVectorLoopRegion() const;
|
|
|
|
/// Returns the 'middle' block of the plan, that is the block that selects
|
|
/// whether to execute the scalar tail loop or the exit block from the loop
|
|
/// latch. If there is an early exit from the vector loop, the middle block
|
|
/// conceptully has the early exit block as third successor, split accross 2
|
|
/// VPBBs. In that case, the second VPBB selects whether to execute the scalar
|
|
/// tail loop or the exit block. If the scalar tail loop or exit block are
|
|
/// known to always execute, the middle block may branch directly to that
|
|
/// block. This function cannot be called once the vector loop region has been
|
|
/// removed.
|
|
VPBasicBlock *getMiddleBlock() {
|
|
VPRegionBlock *LoopRegion = getVectorLoopRegion();
|
|
assert(
|
|
LoopRegion &&
|
|
"cannot call the function after vector loop region has been removed");
|
|
// The middle block is always the last successor of the region.
|
|
return cast<VPBasicBlock>(LoopRegion->getSuccessors().back());
|
|
}
|
|
|
|
const VPBasicBlock *getMiddleBlock() const {
|
|
return const_cast<VPlan *>(this)->getMiddleBlock();
|
|
}
|
|
|
|
/// Return the VPBasicBlock for the preheader of the scalar loop.
|
|
VPBasicBlock *getScalarPreheader() const {
|
|
return cast<VPBasicBlock>(getScalarHeader()->getSinglePredecessor());
|
|
}
|
|
|
|
/// Return the VPIRBasicBlock wrapping the header of the scalar loop.
|
|
VPIRBasicBlock *getScalarHeader() const { return ScalarHeader; }
|
|
|
|
/// Return an ArrayRef containing VPIRBasicBlocks wrapping the exit blocks of
|
|
/// the original scalar loop.
|
|
ArrayRef<VPIRBasicBlock *> getExitBlocks() const { return ExitBlocks; }
|
|
|
|
/// Return the VPIRBasicBlock corresponding to \p IRBB. \p IRBB must be an
|
|
/// exit block.
|
|
VPIRBasicBlock *getExitBlock(BasicBlock *IRBB) const;
|
|
|
|
/// Returns true if \p VPBB is an exit block.
|
|
bool isExitBlock(VPBlockBase *VPBB);
|
|
|
|
/// The trip count of the original loop.
|
|
VPValue *getTripCount() const {
|
|
assert(TripCount && "trip count needs to be set before accessing it");
|
|
return TripCount;
|
|
}
|
|
|
|
/// Set the trip count assuming it is currently null; if it is not - use
|
|
/// resetTripCount().
|
|
void setTripCount(VPValue *NewTripCount) {
|
|
assert(!TripCount && NewTripCount && "TripCount should not be set yet.");
|
|
TripCount = NewTripCount;
|
|
}
|
|
|
|
/// Resets the trip count for the VPlan. The caller must make sure all uses of
|
|
/// the original trip count have been replaced.
|
|
void resetTripCount(VPValue *NewTripCount) {
|
|
assert(TripCount && NewTripCount && TripCount->getNumUsers() == 0 &&
|
|
"TripCount must be set when resetting");
|
|
TripCount = NewTripCount;
|
|
}
|
|
|
|
/// The backedge taken count of the original loop.
|
|
VPValue *getOrCreateBackedgeTakenCount() {
|
|
if (!BackedgeTakenCount)
|
|
BackedgeTakenCount = new VPSymbolicValue();
|
|
return BackedgeTakenCount;
|
|
}
|
|
VPValue *getBackedgeTakenCount() const { return BackedgeTakenCount; }
|
|
|
|
/// The vector trip count.
|
|
VPSymbolicValue &getVectorTripCount() { return VectorTripCount; }
|
|
|
|
/// Returns the VF of the vector loop region.
|
|
VPValue &getVF() { return VF; };
|
|
const VPValue &getVF() const { return VF; };
|
|
|
|
/// Returns VF * UF of the vector loop region.
|
|
VPValue &getVFxUF() { return VFxUF; }
|
|
|
|
LLVMContext &getContext() const {
|
|
return getScalarHeader()->getIRBasicBlock()->getContext();
|
|
}
|
|
|
|
void addVF(ElementCount VF) { VFs.insert(VF); }
|
|
|
|
void setVF(ElementCount VF) {
|
|
assert(hasVF(VF) && "Cannot set VF not already in plan");
|
|
VFs.clear();
|
|
VFs.insert(VF);
|
|
}
|
|
|
|
bool hasVF(ElementCount VF) const { return VFs.count(VF); }
|
|
bool hasScalableVF() const {
|
|
return any_of(VFs, [](ElementCount VF) { return VF.isScalable(); });
|
|
}
|
|
|
|
/// Returns an iterator range over all VFs of the plan.
|
|
iterator_range<SmallSetVector<ElementCount, 2>::iterator>
|
|
vectorFactors() const {
|
|
return VFs;
|
|
}
|
|
|
|
bool hasScalarVFOnly() const {
|
|
bool HasScalarVFOnly = VFs.size() == 1 && VFs[0].isScalar();
|
|
assert(HasScalarVFOnly == hasVF(ElementCount::getFixed(1)) &&
|
|
"Plan with scalar VF should only have a single VF");
|
|
return HasScalarVFOnly;
|
|
}
|
|
|
|
bool hasUF(unsigned UF) const { return UFs.empty() || UFs.contains(UF); }
|
|
|
|
unsigned getUF() const {
|
|
assert(UFs.size() == 1 && "Expected a single UF");
|
|
return UFs[0];
|
|
}
|
|
|
|
void setUF(unsigned UF) {
|
|
assert(hasUF(UF) && "Cannot set the UF not already in plan");
|
|
UFs.clear();
|
|
UFs.insert(UF);
|
|
}
|
|
|
|
/// Returns true if the VPlan already has been unrolled, i.e. it has a single
|
|
/// concrete UF.
|
|
bool isUnrolled() const { return UFs.size() == 1; }
|
|
|
|
/// Return a string with the name of the plan and the applicable VFs and UFs.
|
|
std::string getName() const;
|
|
|
|
void setName(const Twine &newName) { Name = newName.str(); }
|
|
|
|
/// Gets the live-in VPIRValue for \p V or adds a new live-in (if none exists
|
|
/// yet) for \p V.
|
|
VPIRValue *getOrAddLiveIn(Value *V) {
|
|
assert(V && "Trying to get or add the VPIRValue of a null Value");
|
|
auto [It, Inserted] = LiveIns.try_emplace(V);
|
|
if (Inserted) {
|
|
if (auto *CI = dyn_cast<ConstantInt>(V))
|
|
It->second = new VPConstantInt(CI);
|
|
else
|
|
It->second = new VPIRValue(V);
|
|
}
|
|
|
|
assert(isa<VPIRValue>(It->second) &&
|
|
"Only VPIRValues should be in mapping");
|
|
return It->second;
|
|
}
|
|
VPIRValue *getOrAddLiveIn(VPIRValue *V) {
|
|
assert(V && "Trying to get or add the VPIRValue of a null VPIRValue");
|
|
return getOrAddLiveIn(V->getValue());
|
|
}
|
|
|
|
/// Return a VPIRValue wrapping i1 true.
|
|
VPIRValue *getTrue() { return getConstantInt(1, 1); }
|
|
|
|
/// Return a VPIRValue wrapping i1 false.
|
|
VPIRValue *getFalse() { return getConstantInt(1, 0); }
|
|
|
|
/// Return a VPIRValue wrapping a ConstantInt with the given type and value.
|
|
VPIRValue *getConstantInt(Type *Ty, uint64_t Val, bool IsSigned = false) {
|
|
return getOrAddLiveIn(ConstantInt::get(Ty, Val, IsSigned));
|
|
}
|
|
|
|
/// Return a VPIRValue wrapping a ConstantInt with the given bitwidth and
|
|
/// value.
|
|
VPIRValue *getConstantInt(unsigned BitWidth, uint64_t Val,
|
|
bool IsSigned = false) {
|
|
return getConstantInt(APInt(BitWidth, Val, IsSigned));
|
|
}
|
|
|
|
/// Return a VPIRValue wrapping a ConstantInt with the given APInt value.
|
|
VPIRValue *getConstantInt(const APInt &Val) {
|
|
return getOrAddLiveIn(ConstantInt::get(getContext(), Val));
|
|
}
|
|
|
|
/// Return the live-in VPIRValue for \p V, if there is one or nullptr
|
|
/// otherwise.
|
|
VPIRValue *getLiveIn(Value *V) const { return LiveIns.lookup(V); }
|
|
|
|
/// Return the list of live-in VPValues available in the VPlan.
|
|
auto getLiveIns() const { return LiveIns.values(); }
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
/// Print the live-ins of this VPlan to \p O.
|
|
void printLiveIns(raw_ostream &O) const;
|
|
|
|
/// Print this VPlan to \p O.
|
|
LLVM_ABI_FOR_TEST void print(raw_ostream &O) const;
|
|
|
|
/// Print this VPlan in DOT format to \p O.
|
|
LLVM_ABI_FOR_TEST void printDOT(raw_ostream &O) const;
|
|
|
|
/// Dump the plan to stderr (for debugging).
|
|
LLVM_DUMP_METHOD void dump() const;
|
|
#endif
|
|
|
|
/// Clone the current VPlan, update all VPValues of the new VPlan and cloned
|
|
/// recipes to refer to the clones, and return it.
|
|
LLVM_ABI_FOR_TEST VPlan *duplicate();
|
|
|
|
/// Create a new VPBasicBlock with \p Name and containing \p Recipe if
|
|
/// present. The returned block is owned by the VPlan and deleted once the
|
|
/// VPlan is destroyed.
|
|
VPBasicBlock *createVPBasicBlock(const Twine &Name,
|
|
VPRecipeBase *Recipe = nullptr) {
|
|
auto *VPB = new VPBasicBlock(Name, Recipe);
|
|
CreatedBlocks.push_back(VPB);
|
|
return VPB;
|
|
}
|
|
|
|
/// Create a new loop region with \p Name and entry and exiting blocks set
|
|
/// to \p Entry and \p Exiting respectively, if set. The returned block is
|
|
/// owned by the VPlan and deleted once the VPlan is destroyed.
|
|
VPRegionBlock *createLoopRegion(const std::string &Name = "",
|
|
VPBlockBase *Entry = nullptr,
|
|
VPBlockBase *Exiting = nullptr) {
|
|
auto *VPB = Entry ? new VPRegionBlock(Entry, Exiting, Name)
|
|
: new VPRegionBlock(Name);
|
|
CreatedBlocks.push_back(VPB);
|
|
return VPB;
|
|
}
|
|
|
|
/// Create a new replicate region with \p Entry, \p Exiting and \p Name. The
|
|
/// returned block is owned by the VPlan and deleted once the VPlan is
|
|
/// destroyed.
|
|
VPRegionBlock *createReplicateRegion(VPBlockBase *Entry, VPBlockBase *Exiting,
|
|
const std::string &Name = "") {
|
|
auto *VPB = new VPRegionBlock(Entry, Exiting, Name, true);
|
|
CreatedBlocks.push_back(VPB);
|
|
return VPB;
|
|
}
|
|
|
|
/// Create a VPIRBasicBlock wrapping \p IRBB, but do not create
|
|
/// VPIRInstructions wrapping the instructions in t\p IRBB. The returned
|
|
/// block is owned by the VPlan and deleted once the VPlan is destroyed.
|
|
VPIRBasicBlock *createEmptyVPIRBasicBlock(BasicBlock *IRBB);
|
|
|
|
/// Create a VPIRBasicBlock from \p IRBB containing VPIRInstructions for all
|
|
/// instructions in \p IRBB, except its terminator which is managed by the
|
|
/// successors of the block in VPlan. The returned block is owned by the VPlan
|
|
/// and deleted once the VPlan is destroyed.
|
|
LLVM_ABI_FOR_TEST VPIRBasicBlock *createVPIRBasicBlock(BasicBlock *IRBB);
|
|
|
|
/// Returns true if the VPlan is based on a loop with an early exit. That is
|
|
/// the case if the VPlan has either more than one exit block or a single exit
|
|
/// block with multiple predecessors (one for the exit via the latch and one
|
|
/// via the other early exit).
|
|
bool hasEarlyExit() const {
|
|
return count_if(ExitBlocks,
|
|
[](VPIRBasicBlock *EB) { return EB->hasPredecessors(); }) >
|
|
1 ||
|
|
(ExitBlocks.size() == 1 && ExitBlocks[0]->getNumPredecessors() > 1);
|
|
}
|
|
|
|
/// Returns true if the scalar tail may execute after the vector loop. Note
|
|
/// that this relies on unneeded branches to the scalar tail loop being
|
|
/// removed.
|
|
bool hasScalarTail() const {
|
|
return !(!getScalarPreheader()->hasPredecessors() ||
|
|
getScalarPreheader()->getSinglePredecessor() == getEntry());
|
|
}
|
|
};
|
|
|
|
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
|
|
inline raw_ostream &operator<<(raw_ostream &OS, const VPlan &Plan) {
|
|
Plan.print(OS);
|
|
return OS;
|
|
}
|
|
#endif
|
|
|
|
} // end namespace llvm
|
|
|
|
#endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
|