5ea5b9eb8e
This code calls `getInitializer` without checking if the global has an initializer, which causes an assert if there isn't one. These globals may have no initializer after optimization from `globalopt`. Check for an initializer first. This is already locked down with `CodeGen/SPIRV/legalize-zero-size-arrays-appending.ll` so we don't need a new test. This issue was exposed by https://github.com/llvm/llvm-project/pull/192730. Signed-off-by: Nick Sarnie <nick.sarnie@intel.com>
272 lines
10 KiB
C++
272 lines
10 KiB
C++
//===- ProfileVerify.cpp - Verify profile info for testing ----------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "llvm/Transforms/Utils/ProfileVerify.h"
|
|
#include "llvm/ADT/DynamicAPInt.h"
|
|
#include "llvm/ADT/STLExtras.h"
|
|
#include "llvm/ADT/SmallString.h"
|
|
#include "llvm/Analysis/BranchProbabilityInfo.h"
|
|
#include "llvm/IR/Analysis.h"
|
|
#include "llvm/IR/Constants.h"
|
|
#include "llvm/IR/Dominators.h"
|
|
#include "llvm/IR/Function.h"
|
|
#include "llvm/IR/GlobalValue.h"
|
|
#include "llvm/IR/GlobalVariable.h"
|
|
#include "llvm/IR/Instructions.h"
|
|
#include "llvm/IR/LLVMContext.h"
|
|
#include "llvm/IR/MDBuilder.h"
|
|
#include "llvm/IR/Module.h"
|
|
#include "llvm/IR/PassManager.h"
|
|
#include "llvm/IR/ProfDataUtils.h"
|
|
#include "llvm/Support/BranchProbability.h"
|
|
#include "llvm/Support/Casting.h"
|
|
#include "llvm/Support/CommandLine.h"
|
|
|
|
using namespace llvm;
|
|
static cl::opt<int64_t>
|
|
DefaultFunctionEntryCount("profcheck-default-function-entry-count",
|
|
cl::init(1000));
|
|
static cl::opt<bool>
|
|
AnnotateSelect("profcheck-annotate-select", cl::init(true),
|
|
cl::desc("Also inject (if missing) and verify MD_prof for "
|
|
"`select` instructions"));
|
|
static cl::opt<bool>
|
|
WeightsForTest("profcheck-weights-for-test", cl::init(false),
|
|
cl::desc("Generate weights with small values for tests."));
|
|
|
|
static cl::opt<uint32_t> SelectTrueWeight(
|
|
"profcheck-default-select-true-weight", cl::init(2U),
|
|
cl::desc("When annotating `select` instructions, this value will be used "
|
|
"for the first ('true') case."));
|
|
static cl::opt<uint32_t> SelectFalseWeight(
|
|
"profcheck-default-select-false-weight", cl::init(3U),
|
|
cl::desc("When annotating `select` instructions, this value will be used "
|
|
"for the second ('false') case."));
|
|
namespace {
|
|
class ProfileInjector {
|
|
Function &F;
|
|
FunctionAnalysisManager &FAM;
|
|
|
|
public:
|
|
static const Instruction *
|
|
getTerminatorBenefitingFromMDProf(const BasicBlock &BB) {
|
|
if (succ_size(&BB) < 2)
|
|
return nullptr;
|
|
auto *Term = BB.getTerminator();
|
|
return (isa<CondBrInst>(Term) || isa<SwitchInst>(Term) ||
|
|
isa<IndirectBrInst>(Term) || isa<CallBrInst>(Term))
|
|
? Term
|
|
: nullptr;
|
|
}
|
|
|
|
static Instruction *getTerminatorBenefitingFromMDProf(BasicBlock &BB) {
|
|
return const_cast<Instruction *>(
|
|
getTerminatorBenefitingFromMDProf(const_cast<const BasicBlock &>(BB)));
|
|
}
|
|
|
|
ProfileInjector(Function &F, FunctionAnalysisManager &FAM) : F(F), FAM(FAM) {}
|
|
bool inject();
|
|
};
|
|
|
|
bool isAsmOnly(const Function &F) {
|
|
if (!F.hasFnAttribute(Attribute::AttrKind::Naked))
|
|
return false;
|
|
for (const auto &BB : F)
|
|
for (const auto &I : drop_end(BB)) {
|
|
const auto *CB = dyn_cast<CallBase>(&I);
|
|
if (!CB || !CB->isInlineAsm())
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
void emitProfileError(StringRef Msg, Function &F) {
|
|
F.getContext().emitError("Profile verification failed for function '" +
|
|
F.getName() + "': " + Msg);
|
|
}
|
|
|
|
} // namespace
|
|
|
|
// FIXME: currently this injects only for terminators. Select isn't yet
|
|
// supported.
|
|
bool ProfileInjector::inject() {
|
|
// skip purely asm functions
|
|
if (isAsmOnly(F))
|
|
return false;
|
|
// Get whatever branch probability info can be derived from the given IR -
|
|
// whether it has or not metadata. The main intention for this pass is to
|
|
// ensure that other passes don't drop or "forget" to update MD_prof. We do
|
|
// this as a mode in which lit tests would run. We want to avoid changing the
|
|
// behavior of those tests. A pass may use BPI (or BFI, which is computed from
|
|
// BPI). If no metadata is present, BPI is guesstimated by
|
|
// BranchProbabilityAnalysis. The injector (this pass) only persists whatever
|
|
// information the analysis provides, in other words, the pass being tested
|
|
// will get the same BPI it does if the injector wasn't running.
|
|
auto &BPI = FAM.getResult<BranchProbabilityAnalysis>(F);
|
|
|
|
// Inject a function count if there's none. It's reasonable for a pass to
|
|
// want to clear the MD_prof of a function with zero entry count. If the
|
|
// original profile (iFDO or AFDO) is empty for a function, it's simpler to
|
|
// require assigning it the 0-entry count explicitly than to mark every branch
|
|
// as cold (we do want some explicit information in the spirit of what this
|
|
// verifier wants to achieve - make dropping / corrupting MD_prof
|
|
// unit-testable)
|
|
if (!F.getEntryCount(/*AllowSynthetic=*/true))
|
|
F.setEntryCount(DefaultFunctionEntryCount);
|
|
// If there is an entry count that's 0, then don't bother injecting. We won't
|
|
// verify these either.
|
|
if (F.getEntryCount(/*AllowSynthetic=*/true)->getCount() == 0)
|
|
return false;
|
|
bool Changed = false;
|
|
// Cycle through the weights list. If we didn't, tests with more than (say)
|
|
// one conditional branch would have the same !prof metadata on all of them,
|
|
// and numerically that may make for a poor unit test.
|
|
uint32_t WeightsForTestOffset = 0;
|
|
for (auto &BB : F) {
|
|
if (AnnotateSelect) {
|
|
for (auto &I : BB) {
|
|
if (auto *SI = dyn_cast<SelectInst>(&I)) {
|
|
if (SI->getCondition()->getType()->isVectorTy())
|
|
continue;
|
|
if (I.getMetadata(LLVMContext::MD_prof))
|
|
continue;
|
|
setBranchWeights(I, {SelectTrueWeight, SelectFalseWeight},
|
|
/*IsExpected=*/false);
|
|
}
|
|
}
|
|
}
|
|
auto *Term = getTerminatorBenefitingFromMDProf(BB);
|
|
if (!Term || Term->getMetadata(LLVMContext::MD_prof))
|
|
continue;
|
|
SmallVector<BranchProbability> Probs;
|
|
|
|
SmallVector<uint32_t> Weights;
|
|
Weights.reserve(Term->getNumSuccessors());
|
|
if (WeightsForTest) {
|
|
static const std::array Primes{3, 5, 7, 11, 13, 17, 19, 23, 29, 31,
|
|
37, 41, 43, 47, 53, 59, 61, 67, 71};
|
|
for (uint32_t I = 0, E = Term->getNumSuccessors(); I < E; ++I)
|
|
Weights.emplace_back(
|
|
Primes[(WeightsForTestOffset + I) % Primes.size()]);
|
|
++WeightsForTestOffset;
|
|
} else {
|
|
Probs.reserve(Term->getNumSuccessors());
|
|
for (auto I = 0U, E = Term->getNumSuccessors(); I < E; ++I)
|
|
Probs.emplace_back(BPI.getEdgeProbability(&BB, Term->getSuccessor(I)));
|
|
|
|
assert(llvm::find_if(Probs,
|
|
[](const BranchProbability &P) {
|
|
return P.isUnknown();
|
|
}) == Probs.end() &&
|
|
"All branch probabilities should be valid");
|
|
const auto *FirstZeroDenominator =
|
|
find_if(Probs, [](const BranchProbability &P) {
|
|
return P.getDenominator() == 0;
|
|
});
|
|
(void)FirstZeroDenominator;
|
|
assert(FirstZeroDenominator == Probs.end());
|
|
const auto *FirstNonZeroNumerator = find_if(
|
|
Probs, [](const BranchProbability &P) { return !P.isZero(); });
|
|
assert(FirstNonZeroNumerator != Probs.end());
|
|
DynamicAPInt LCM(Probs[0].getDenominator());
|
|
DynamicAPInt GCD(FirstNonZeroNumerator->getNumerator());
|
|
for (const auto &Prob : drop_begin(Probs)) {
|
|
if (!Prob.getNumerator())
|
|
continue;
|
|
LCM = llvm::lcm(LCM, DynamicAPInt(Prob.getDenominator()));
|
|
GCD = llvm::gcd(GCD, DynamicAPInt(Prob.getNumerator()));
|
|
}
|
|
for (const auto &Prob : Probs) {
|
|
DynamicAPInt W =
|
|
(Prob.getNumerator() * LCM / GCD) / Prob.getDenominator();
|
|
Weights.emplace_back(static_cast<uint32_t>((int64_t)W));
|
|
}
|
|
}
|
|
setBranchWeights(*Term, Weights, /*IsExpected=*/false);
|
|
Changed = true;
|
|
}
|
|
return Changed;
|
|
}
|
|
|
|
PreservedAnalyses ProfileInjectorPass::run(Function &F,
|
|
FunctionAnalysisManager &FAM) {
|
|
ProfileInjector PI(F, FAM);
|
|
if (!PI.inject())
|
|
return PreservedAnalyses::all();
|
|
|
|
return PreservedAnalyses::none();
|
|
}
|
|
|
|
PreservedAnalyses ProfileVerifierPass::run(Module &M,
|
|
ModuleAnalysisManager &MAM) {
|
|
auto PopulateIgnoreList = [&](StringRef GVName) {
|
|
if (const auto *CT = M.getGlobalVariable(GVName))
|
|
if (CT->hasInitializer())
|
|
if (const auto *CA =
|
|
dyn_cast_if_present<ConstantArray>(CT->getInitializer()))
|
|
for (const auto &Elt : CA->operands())
|
|
if (const auto *CS = dyn_cast<ConstantStruct>(Elt))
|
|
if (CS->getNumOperands() >= 2 && CS->getOperand(1))
|
|
if (const auto *F = dyn_cast<Function>(
|
|
CS->getOperand(1)->stripPointerCasts()))
|
|
IgnoreList.insert(F);
|
|
};
|
|
PopulateIgnoreList("llvm.global_ctors");
|
|
PopulateIgnoreList("llvm.global_dtors");
|
|
|
|
// expose the function-level run as public through a wrapper, so we can use
|
|
// pass manager mechanisms dealing with declarations and with composing the
|
|
// returned PreservedAnalyses values.
|
|
struct Wrapper : PassInfoMixin<Wrapper> {
|
|
ProfileVerifierPass &PVP;
|
|
PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM) {
|
|
return PVP.run(F, FAM);
|
|
}
|
|
explicit Wrapper(ProfileVerifierPass &PVP) : PVP(PVP) {}
|
|
};
|
|
|
|
return createModuleToFunctionPassAdaptor(Wrapper(*this)).run(M, MAM);
|
|
}
|
|
|
|
PreservedAnalyses ProfileVerifierPass::run(Function &F,
|
|
FunctionAnalysisManager &FAM) {
|
|
// skip purely asm functions
|
|
if (isAsmOnly(F))
|
|
return PreservedAnalyses::all();
|
|
if (IgnoreList.contains(&F))
|
|
return PreservedAnalyses::all();
|
|
|
|
const auto EntryCount = F.getEntryCount(/*AllowSynthetic=*/true);
|
|
if (!EntryCount) {
|
|
auto *MD = F.getMetadata(LLVMContext::MD_prof);
|
|
if (!MD || !isExplicitlyUnknownProfileMetadata(*MD)) {
|
|
emitProfileError("function entry count missing (set to 0 if cold)", F);
|
|
return PreservedAnalyses::all();
|
|
}
|
|
} else if (EntryCount->getCount() == 0) {
|
|
return PreservedAnalyses::all();
|
|
}
|
|
for (const auto &BB : F) {
|
|
if (AnnotateSelect) {
|
|
for (const auto &I : BB)
|
|
if (auto *SI = dyn_cast<SelectInst>(&I)) {
|
|
if (SI->getCondition()->getType()->isVectorTy())
|
|
continue;
|
|
if (I.getMetadata(LLVMContext::MD_prof))
|
|
continue;
|
|
emitProfileError("select annotation missing", F);
|
|
}
|
|
}
|
|
if (const auto *Term =
|
|
ProfileInjector::getTerminatorBenefitingFromMDProf(BB))
|
|
if (!Term->getMetadata(LLVMContext::MD_prof))
|
|
emitProfileError("branch annotation missing", F);
|
|
}
|
|
return PreservedAnalyses::all();
|
|
}
|