2f422a52fd
This PR implements the prefetch insertion in the InsertCodePrefetch pass based on the [RFC](https://discourse.llvm.org/t/rfc-code-prefetch-insertion/88668). If the prefetch target is not defined in the same module (i.e, prefetch target function is not defined in the same module), we emit a fallback weak symbol after the prefetch instruction so that if the symbol is not ever defined, we don't get undefined symbol error and the prefetch instruction prefetches the next address: ``` prefetchit1 __llvm_prefetch_target_foo(%rip) .weak __llvm_prefetch_target_foo __llvm_prefetch_target_foo: ``` The weak symbol semantic is tied to ELF, so this makes this PR target-dependent.
677 lines
28 KiB
C++
677 lines
28 KiB
C++
//===-- BasicBlockSectionsProfileReader.cpp -------------------------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// Implementation of the basic block sections profile reader pass. It parses
|
|
// and stores the basic block sections profile file (which is specified via the
|
|
// `-basic-block-sections` flag).
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
|
|
#include "llvm/ADT/DenseSet.h"
|
|
#include "llvm/ADT/SmallSet.h"
|
|
#include "llvm/ADT/SmallString.h"
|
|
#include "llvm/ADT/SmallVector.h"
|
|
#include "llvm/ADT/StringMap.h"
|
|
#include "llvm/ADT/StringRef.h"
|
|
#include "llvm/IR/DebugInfoMetadata.h"
|
|
#include "llvm/Pass.h"
|
|
#include "llvm/Support/Error.h"
|
|
#include "llvm/Support/ErrorHandling.h"
|
|
#include "llvm/Support/LineIterator.h"
|
|
#include "llvm/Support/MemoryBuffer.h"
|
|
#include "llvm/Support/Path.h"
|
|
#include "llvm/Support/UniqueBBID.h"
|
|
#include <llvm/ADT/STLExtras.h>
|
|
|
|
using namespace llvm;
|
|
|
|
char BasicBlockSectionsProfileReaderWrapperPass::ID = 0;
|
|
INITIALIZE_PASS(BasicBlockSectionsProfileReaderWrapperPass,
|
|
"bbsections-profile-reader",
|
|
"Reads and parses a basic block sections profile.", false,
|
|
false)
|
|
|
|
Expected<UniqueBBID>
|
|
BasicBlockSectionsProfileReader::parseUniqueBBID(StringRef S) const {
|
|
SmallVector<StringRef, 2> Parts;
|
|
S.split(Parts, '.');
|
|
if (Parts.size() > 2)
|
|
return createProfileParseError(Twine("unable to parse basic block id: '") +
|
|
S + "'");
|
|
unsigned long long BaseBBID;
|
|
if (getAsUnsignedInteger(Parts[0], 10, BaseBBID))
|
|
return createProfileParseError(
|
|
Twine("unable to parse BB id: '" + Parts[0]) +
|
|
"': unsigned integer expected");
|
|
unsigned long long CloneID = 0;
|
|
if (Parts.size() > 1 && getAsUnsignedInteger(Parts[1], 10, CloneID))
|
|
return createProfileParseError(Twine("unable to parse clone id: '") +
|
|
Parts[1] + "': unsigned integer expected");
|
|
return UniqueBBID{static_cast<unsigned>(BaseBBID),
|
|
static_cast<unsigned>(CloneID)};
|
|
}
|
|
|
|
bool BasicBlockSectionsProfileReader::isFunctionHot(StringRef FuncName) const {
|
|
return !getClusterInfoForFunction(FuncName).empty();
|
|
}
|
|
|
|
SmallVector<BBClusterInfo>
|
|
BasicBlockSectionsProfileReader::getClusterInfoForFunction(
|
|
StringRef FuncName) const {
|
|
auto R = ProgramOptimizationProfile.find(getAliasName(FuncName));
|
|
return R != ProgramOptimizationProfile.end() ? R->second.ClusterInfo
|
|
: SmallVector<BBClusterInfo>();
|
|
}
|
|
|
|
SmallVector<SmallVector<unsigned>>
|
|
BasicBlockSectionsProfileReader::getClonePathsForFunction(
|
|
StringRef FuncName) const {
|
|
auto R = ProgramOptimizationProfile.find(getAliasName(FuncName));
|
|
return R != ProgramOptimizationProfile.end()
|
|
? R->second.ClonePaths
|
|
: SmallVector<SmallVector<unsigned>>();
|
|
}
|
|
|
|
uint64_t BasicBlockSectionsProfileReader::getEdgeCount(
|
|
StringRef FuncName, const UniqueBBID &SrcBBID,
|
|
const UniqueBBID &SinkBBID) const {
|
|
const CFGProfile *CFG = getFunctionCFGProfile(FuncName);
|
|
if (CFG == nullptr)
|
|
return 0;
|
|
auto NodeIt = CFG->EdgeCounts.find(SrcBBID);
|
|
if (NodeIt == CFG->EdgeCounts.end())
|
|
return 0;
|
|
auto EdgeIt = NodeIt->second.find(SinkBBID);
|
|
if (EdgeIt == NodeIt->second.end())
|
|
return 0;
|
|
return EdgeIt->second;
|
|
}
|
|
|
|
SmallVector<CallsiteID>
|
|
BasicBlockSectionsProfileReader::getPrefetchTargetsForFunction(
|
|
StringRef FuncName) const {
|
|
auto R = ProgramOptimizationProfile.find(getAliasName(FuncName));
|
|
return R != ProgramOptimizationProfile.end() ? R->second.PrefetchTargets
|
|
: SmallVector<CallsiteID>();
|
|
}
|
|
|
|
SmallVector<PrefetchHint>
|
|
BasicBlockSectionsProfileReader::getPrefetchHintsForFunction(
|
|
StringRef FuncName) const {
|
|
StringMap<FunctionOptimizationProfile>::const_iterator It =
|
|
ProgramOptimizationProfile.find(getAliasName(FuncName));
|
|
return It != ProgramOptimizationProfile.end() ? It->second.PrefetchHints
|
|
: SmallVector<PrefetchHint>();
|
|
}
|
|
|
|
// Reads the version 1 basic block sections profile. Profile for each function
|
|
// is encoded as follows:
|
|
// m <module_name>
|
|
// f <function_name_1> <function_name_2> ...
|
|
// c <bb_id_1> <bb_id_2> <bb_id_3>
|
|
// c <bb_id_4> <bb_id_5>
|
|
// ...
|
|
// Module name specifier (starting with 'm') is optional and allows
|
|
// distinguishing profile for internal-linkage functions with the same name. If
|
|
// not specified, it will apply to any function with the same name. Function
|
|
// name specifier (starting with 'f') can specify multiple function name
|
|
// aliases. Basic block clusters are specified by 'c' and specify the cluster of
|
|
// basic blocks, and the internal order in which they must be placed in the same
|
|
// section.
|
|
// This profile can also specify cloning paths which instruct the compiler to
|
|
// clone basic blocks along a path. The cloned blocks are then specified in the
|
|
// cluster information.
|
|
// The following profile lists two cloning paths (starting with 'p') for
|
|
// function bar and places the total 9 blocks within two clusters. The first two
|
|
// blocks of a cloning path specify the edge along which the path is cloned. For
|
|
// instance, path 1 (1 -> 3 -> 4) instructs that 3 and 4 must be cloned along
|
|
// the edge 1->3. Within the given clusters, each cloned block is identified by
|
|
// "<original block id>.<clone id>". For instance, 3.1 represents the first
|
|
// clone of block 3. Original blocks are specified just with their block ids. A
|
|
// block cloned multiple times appears with distinct clone ids. The CFG for bar
|
|
// is shown below before and after cloning with its final clusters labeled.
|
|
//
|
|
// f main
|
|
// f bar
|
|
// p 1 3 4 # cloning path 1
|
|
// p 4 2 # cloning path 2
|
|
// c 1 3.1 4.1 6 # basic block cluster 1
|
|
// c 0 2 3 4 2.1 5 # basic block cluster 2
|
|
// ****************************************************************************
|
|
// function bar before and after cloning with basic block clusters shown.
|
|
// ****************************************************************************
|
|
// .... ..............
|
|
// 0 -------+ : 0 :---->: 1 ---> 3.1 :
|
|
// | | : | : :........ | :
|
|
// v v : v : : v :
|
|
// +--> 2 --> 5 1 ~~~~~~> +---: 2 : : 4.1: clsuter 1
|
|
// | | | | : | : : | :
|
|
// | v | | : v ....... : v :
|
|
// | 3 <------+ | : 3 <--+ : : 6 :
|
|
// | | | : | | : :....:
|
|
// | v | : v | :
|
|
// +--- 4 ---> 6 | : 4 | :
|
|
// | : | | :
|
|
// | : v | :
|
|
// | :2.1---+ : cluster 2
|
|
// | : | ......:
|
|
// | : v :
|
|
// +-->: 5 :
|
|
// ....
|
|
// ****************************************************************************
|
|
// This profile can also specify prefetch targets (starting with 't') which
|
|
// instruct the compiler to emit a prefetch symbol for the given target and
|
|
// prefetch hints (starting with 'i') which instruct the compiler to insert a
|
|
// prefetch hint instruction at the given site for the given target.
|
|
//
|
|
// A prefetch target is specified by a pair "<bbid>,<subblock_index>" where
|
|
// bbid specifies the target basic block and subblock_index is a zero-based
|
|
// index. Subblock 0 refers to the region at the beginning of the block up to
|
|
// the first callsite. Subblock `i > 0` refers to the region immediately after
|
|
// the `i`-th callsite up to the `i+1`-th callsite (or the end of the block).
|
|
// The prefetch target is always emitted at the beginning of the subblock.
|
|
// This is the beginning of the basic block for `i = 0` and immediately after
|
|
// the `i`-th call for every `i > 0`.
|
|
//
|
|
// A prefetch hint is specified by a pair "site target", where site is
|
|
// specified as a pair "<bbid>,<callsite_index>" similar to prefetch
|
|
// targets, and target is specified as a triple
|
|
// "<function_name>,<bbid>,<callsite_index>".
|
|
//
|
|
// Example: A basic block in function "foo" with BBID 10 and two call
|
|
// instructions (call_A, call_B). This block is conceptually split into
|
|
// subblocks, with the prefetch target symbol emitted at the beginning of
|
|
// each subblock.
|
|
//
|
|
// +----------------------------------+
|
|
// | __llvm_prefetch_target_foo_10_0: | <--- Subblock 0 (before call_A)
|
|
// | Instruction 1 |
|
|
// | Instruction 2 |
|
|
// | call_A (Callsite 0) |
|
|
// | __llvm_prefetch_target_foo_10_1: | <--- Subblock 1 (after call_A,
|
|
// | | before call_B)
|
|
// | Instruction 3 |
|
|
// | call_B (Callsite 1) |
|
|
// | __llvm_prefetch_target_foo_10_2: | <--- Subblock 2 (after call_B,
|
|
// | | before call_C)
|
|
// | Instruction 4 |
|
|
// +----------------------------------+
|
|
//
|
|
// A prefetch hint specified in function "bar" as "120,1 foo,10,2" results
|
|
// in a hint inserted after the first call in block #120 of bar targeting the
|
|
// address immediately after the second call in block #10 of function foo.
|
|
//
|
|
// B
|
|
// +----------------------------------------------------+
|
|
// | Instruction 1 |
|
|
// | call_C (Callsite 1) |
|
|
// | code_prefetch __llvm_prefetch_target_foo_10 |
|
|
// | Instruction 2 |
|
|
// +----------------------------------------------------+
|
|
//
|
|
Error BasicBlockSectionsProfileReader::ReadV1Profile() {
|
|
auto FI = ProgramOptimizationProfile.end();
|
|
|
|
// Current cluster ID corresponding to this function.
|
|
unsigned CurrentCluster = 0;
|
|
// Current position in the current cluster.
|
|
unsigned CurrentPosition = 0;
|
|
|
|
// Temporary set to ensure every basic block ID appears once in the clusters
|
|
// of a function.
|
|
DenseSet<UniqueBBID> FuncBBIDs;
|
|
|
|
// Debug-info-based module filename for the current function. Empty string
|
|
// means no filename.
|
|
StringRef DIFilename;
|
|
|
|
for (; !LineIt.is_at_eof(); ++LineIt) {
|
|
StringRef S(*LineIt);
|
|
char Specifier = S[0];
|
|
S = S.drop_front().trim();
|
|
SmallVector<StringRef, 4> Values;
|
|
S.split(Values, ' ');
|
|
switch (Specifier) {
|
|
case '@':
|
|
continue;
|
|
case 'm': // Module name speicifer.
|
|
if (Values.size() != 1) {
|
|
return createProfileParseError(Twine("invalid module name value: '") +
|
|
S + "'");
|
|
}
|
|
DIFilename = sys::path::remove_leading_dotslash(Values[0]);
|
|
continue;
|
|
case 'f': { // Function names specifier.
|
|
bool FunctionFound = any_of(Values, [&](StringRef Alias) {
|
|
auto It = FunctionNameToDIFilename.find(Alias);
|
|
// No match if this function name is not found in this module.
|
|
if (It == FunctionNameToDIFilename.end())
|
|
return false;
|
|
// Return a match if debug-info-filename is not specified. Otherwise,
|
|
// check for equality.
|
|
return DIFilename.empty() || It->second == DIFilename;
|
|
});
|
|
if (!FunctionFound) {
|
|
// Skip the following profile by setting the profile iterator (FI) to
|
|
// the past-the-end element.
|
|
FI = ProgramOptimizationProfile.end();
|
|
DIFilename = "";
|
|
continue;
|
|
}
|
|
for (size_t i = 1; i < Values.size(); ++i)
|
|
FuncAliasMap.try_emplace(Values[i], Values.front());
|
|
|
|
// Prepare for parsing clusters of this function name.
|
|
// Start a new cluster map for this function name.
|
|
auto R = ProgramOptimizationProfile.try_emplace(Values.front());
|
|
// Report error when multiple profiles have been specified for the same
|
|
// function.
|
|
if (!R.second)
|
|
return createProfileParseError("duplicate profile for function '" +
|
|
Values.front() + "'");
|
|
FI = R.first;
|
|
CurrentCluster = 0;
|
|
FuncBBIDs.clear();
|
|
// We won't need DIFilename anymore. Clean it up to avoid its application
|
|
// on the next function.
|
|
DIFilename = "";
|
|
continue;
|
|
}
|
|
case 'c': // Basic block cluster specifier.
|
|
// Skip the profile when we the profile iterator (FI) refers to the
|
|
// past-the-end element.
|
|
if (FI == ProgramOptimizationProfile.end())
|
|
continue;
|
|
// Reset current cluster position.
|
|
CurrentPosition = 0;
|
|
for (auto BasicBlockIDStr : Values) {
|
|
auto BasicBlockID = parseUniqueBBID(BasicBlockIDStr);
|
|
if (!BasicBlockID)
|
|
return BasicBlockID.takeError();
|
|
if (!FuncBBIDs.insert(*BasicBlockID).second)
|
|
return createProfileParseError(
|
|
Twine("duplicate basic block id found '") + BasicBlockIDStr +
|
|
"'");
|
|
|
|
FI->second.ClusterInfo.emplace_back(BBClusterInfo{
|
|
*std::move(BasicBlockID), CurrentCluster, CurrentPosition++});
|
|
}
|
|
CurrentCluster++;
|
|
continue;
|
|
case 'p': { // Basic block cloning path specifier.
|
|
// Skip the profile when we the profile iterator (FI) refers to the
|
|
// past-the-end element.
|
|
if (FI == ProgramOptimizationProfile.end())
|
|
continue;
|
|
SmallSet<unsigned, 5> BBsInPath;
|
|
FI->second.ClonePaths.push_back({});
|
|
for (size_t I = 0; I < Values.size(); ++I) {
|
|
auto BaseBBIDStr = Values[I];
|
|
unsigned long long BaseBBID = 0;
|
|
if (getAsUnsignedInteger(BaseBBIDStr, 10, BaseBBID))
|
|
return createProfileParseError(Twine("unsigned integer expected: '") +
|
|
BaseBBIDStr + "'");
|
|
if (I != 0 && !BBsInPath.insert(BaseBBID).second)
|
|
return createProfileParseError(
|
|
Twine("duplicate cloned block in path: '") + BaseBBIDStr + "'");
|
|
FI->second.ClonePaths.back().push_back(BaseBBID);
|
|
}
|
|
continue;
|
|
}
|
|
case 'g': { // CFG profile specifier.
|
|
// Skip the profile when we the profile iterator (FI) refers to the
|
|
// past-the-end element.
|
|
if (FI == ProgramOptimizationProfile.end())
|
|
continue;
|
|
// For each node, its CFG profile is encoded as
|
|
// <src>:<count>,<sink_1>:<count_1>,<sink_2>:<count_2>,...
|
|
for (auto BasicBlockEdgeProfile : Values) {
|
|
if (BasicBlockEdgeProfile.empty())
|
|
continue;
|
|
SmallVector<StringRef, 4> NodeEdgeCounts;
|
|
BasicBlockEdgeProfile.split(NodeEdgeCounts, ',');
|
|
UniqueBBID SrcBBID;
|
|
for (size_t i = 0; i < NodeEdgeCounts.size(); ++i) {
|
|
auto [BBIDStr, CountStr] = NodeEdgeCounts[i].split(':');
|
|
auto BBID = parseUniqueBBID(BBIDStr);
|
|
if (!BBID)
|
|
return BBID.takeError();
|
|
unsigned long long Count = 0;
|
|
if (getAsUnsignedInteger(CountStr, 10, Count))
|
|
return createProfileParseError(
|
|
Twine("unsigned integer expected: '") + CountStr + "'");
|
|
if (i == 0) {
|
|
// The first element represents the source and its total count.
|
|
FI->second.CFG.NodeCounts[SrcBBID = *BBID] = Count;
|
|
continue;
|
|
}
|
|
FI->second.CFG.EdgeCounts[SrcBBID][*BBID] = Count;
|
|
}
|
|
}
|
|
continue;
|
|
}
|
|
case 'h': { // Basic block hash secifier.
|
|
// Skip the profile when the profile iterator (FI) refers to the
|
|
// past-the-end element.
|
|
if (FI == ProgramOptimizationProfile.end())
|
|
continue;
|
|
for (auto BBIDHashStr : Values) {
|
|
auto [BBIDStr, HashStr] = BBIDHashStr.split(':');
|
|
unsigned long long BBID = 0, Hash = 0;
|
|
if (getAsUnsignedInteger(BBIDStr, 10, BBID))
|
|
return createProfileParseError(Twine("unsigned integer expected: '") +
|
|
BBIDStr + "'");
|
|
if (getAsUnsignedInteger(HashStr, 16, Hash))
|
|
return createProfileParseError(
|
|
Twine("unsigned integer expected in hex format: '") + HashStr +
|
|
"'");
|
|
FI->second.CFG.BBHashes[BBID] = Hash;
|
|
}
|
|
continue;
|
|
}
|
|
case 't': { // Callsite target specifier.
|
|
// Skip the profile when we the profile iterator (FI) refers to the
|
|
// past-the-end element.
|
|
if (FI == ProgramOptimizationProfile.end())
|
|
continue;
|
|
SmallVector<StringRef, 2> PrefetchTargetStr;
|
|
Values[0].split(PrefetchTargetStr, ',');
|
|
if (PrefetchTargetStr.size() != 2)
|
|
return createProfileParseError(Twine("Callsite target expected: ") +
|
|
Values[0]);
|
|
auto TargetBBID = parseUniqueBBID(PrefetchTargetStr[0]);
|
|
if (!TargetBBID)
|
|
return TargetBBID.takeError();
|
|
unsigned long long CallsiteIndex;
|
|
if (getAsUnsignedInteger(PrefetchTargetStr[1], 10, CallsiteIndex))
|
|
return createProfileParseError(Twine("signed integer expected: '") +
|
|
PrefetchTargetStr[1]);
|
|
FI->second.PrefetchTargets.push_back(
|
|
CallsiteID{*TargetBBID, static_cast<unsigned>(CallsiteIndex)});
|
|
continue;
|
|
}
|
|
|
|
case 'i': { // Prefetch hint specifier.
|
|
// Skip the profile when the profile iterator (FI) refers to the
|
|
// past-the-end element.
|
|
if (FI == ProgramOptimizationProfile.end())
|
|
continue;
|
|
if (Values.size() != 2)
|
|
return createProfileParseError(
|
|
Twine("Prefetch hint expected of format '<prefetch-site> "
|
|
"<prefetch-target>': " +
|
|
S));
|
|
SmallVector<StringRef, 2> PrefetchSiteStr;
|
|
Values[0].split(PrefetchSiteStr, ',');
|
|
if (PrefetchSiteStr.size() != 2)
|
|
return createProfileParseError(Twine("Prefetch site expected of format "
|
|
"'<block-id>,<callsite-id>': ") +
|
|
Values[0]);
|
|
auto SiteBBID = parseUniqueBBID(PrefetchSiteStr[0]);
|
|
if (!SiteBBID)
|
|
return SiteBBID.takeError();
|
|
unsigned long long SiteCallsiteIndex;
|
|
if (getAsUnsignedInteger(PrefetchSiteStr[1], 10, SiteCallsiteIndex))
|
|
return createProfileParseError(Twine("unsigned integer expected: '") +
|
|
PrefetchSiteStr[1]);
|
|
|
|
SmallVector<StringRef, 3> PrefetchTargetStr;
|
|
Values[1].split(PrefetchTargetStr, ',');
|
|
if (PrefetchTargetStr.size() != 3)
|
|
return createProfileParseError(
|
|
Twine("Prefetch target expected of format "
|
|
"'<function-name>,<block-id>,<callsite-id>': ") +
|
|
Values[1]);
|
|
auto TargetBBID = parseUniqueBBID(PrefetchTargetStr[1]);
|
|
if (!TargetBBID)
|
|
return TargetBBID.takeError();
|
|
unsigned long long TargetCallsiteIndex;
|
|
if (getAsUnsignedInteger(PrefetchTargetStr[2], 10, TargetCallsiteIndex))
|
|
return createProfileParseError(Twine("unsigned integer expected: '") +
|
|
PrefetchTargetStr[2]);
|
|
FI->second.PrefetchHints.push_back(PrefetchHint{
|
|
CallsiteID{*SiteBBID, static_cast<unsigned>(SiteCallsiteIndex)},
|
|
PrefetchTargetStr[0],
|
|
CallsiteID{*TargetBBID, static_cast<unsigned>(TargetCallsiteIndex)}});
|
|
continue;
|
|
}
|
|
default:
|
|
return createProfileParseError(Twine("invalid specifier: '") +
|
|
Twine(Specifier) + "'");
|
|
}
|
|
llvm_unreachable("should not break from this switch statement");
|
|
}
|
|
return Error::success();
|
|
}
|
|
|
|
Error BasicBlockSectionsProfileReader::ReadV0Profile() {
|
|
auto FI = ProgramOptimizationProfile.end();
|
|
// Current cluster ID corresponding to this function.
|
|
unsigned CurrentCluster = 0;
|
|
// Current position in the current cluster.
|
|
unsigned CurrentPosition = 0;
|
|
|
|
// Temporary set to ensure every basic block ID appears once in the clusters
|
|
// of a function.
|
|
SmallSet<unsigned, 4> FuncBBIDs;
|
|
|
|
for (; !LineIt.is_at_eof(); ++LineIt) {
|
|
StringRef S(*LineIt);
|
|
if (S[0] == '@')
|
|
continue;
|
|
// Check for the leading "!"
|
|
if (!S.consume_front("!") || S.empty())
|
|
break;
|
|
// Check for second "!" which indicates a cluster of basic blocks.
|
|
if (S.consume_front("!")) {
|
|
// Skip the profile when we the profile iterator (FI) refers to the
|
|
// past-the-end element.
|
|
if (FI == ProgramOptimizationProfile.end())
|
|
continue;
|
|
SmallVector<StringRef, 4> BBIDs;
|
|
S.split(BBIDs, ' ');
|
|
// Reset current cluster position.
|
|
CurrentPosition = 0;
|
|
for (auto BBIDStr : BBIDs) {
|
|
unsigned long long BBID;
|
|
if (getAsUnsignedInteger(BBIDStr, 10, BBID))
|
|
return createProfileParseError(Twine("unsigned integer expected: '") +
|
|
BBIDStr + "'");
|
|
if (!FuncBBIDs.insert(BBID).second)
|
|
return createProfileParseError(
|
|
Twine("duplicate basic block id found '") + BBIDStr + "'");
|
|
|
|
FI->second.ClusterInfo.emplace_back(
|
|
BBClusterInfo({{static_cast<unsigned>(BBID), 0},
|
|
CurrentCluster,
|
|
CurrentPosition++}));
|
|
}
|
|
CurrentCluster++;
|
|
} else {
|
|
// This is a function name specifier. It may include a debug info filename
|
|
// specifier starting with `M=`.
|
|
auto [AliasesStr, DIFilenameStr] = S.split(' ');
|
|
SmallString<128> DIFilename;
|
|
if (DIFilenameStr.starts_with("M=")) {
|
|
DIFilename =
|
|
sys::path::remove_leading_dotslash(DIFilenameStr.substr(2));
|
|
if (DIFilename.empty())
|
|
return createProfileParseError("empty module name specifier");
|
|
} else if (!DIFilenameStr.empty()) {
|
|
return createProfileParseError("unknown string found: '" +
|
|
DIFilenameStr + "'");
|
|
}
|
|
// Function aliases are separated using '/'. We use the first function
|
|
// name for the cluster info mapping and delegate all other aliases to
|
|
// this one.
|
|
SmallVector<StringRef, 4> Aliases;
|
|
AliasesStr.split(Aliases, '/');
|
|
bool FunctionFound = any_of(Aliases, [&](StringRef Alias) {
|
|
auto It = FunctionNameToDIFilename.find(Alias);
|
|
// No match if this function name is not found in this module.
|
|
if (It == FunctionNameToDIFilename.end())
|
|
return false;
|
|
// Return a match if debug-info-filename is not specified. Otherwise,
|
|
// check for equality.
|
|
return DIFilename.empty() || It->second == DIFilename;
|
|
});
|
|
if (!FunctionFound) {
|
|
// Skip the following profile by setting the profile iterator (FI) to
|
|
// the past-the-end element.
|
|
FI = ProgramOptimizationProfile.end();
|
|
continue;
|
|
}
|
|
for (size_t i = 1; i < Aliases.size(); ++i)
|
|
FuncAliasMap.try_emplace(Aliases[i], Aliases.front());
|
|
|
|
// Prepare for parsing clusters of this function name.
|
|
// Start a new cluster map for this function name.
|
|
auto R = ProgramOptimizationProfile.try_emplace(Aliases.front());
|
|
// Report error when multiple profiles have been specified for the same
|
|
// function.
|
|
if (!R.second)
|
|
return createProfileParseError("duplicate profile for function '" +
|
|
Aliases.front() + "'");
|
|
FI = R.first;
|
|
CurrentCluster = 0;
|
|
FuncBBIDs.clear();
|
|
}
|
|
}
|
|
return Error::success();
|
|
}
|
|
|
|
// Basic Block Sections can be enabled for a subset of machine basic blocks.
|
|
// This is done by passing a file containing names of functions for which basic
|
|
// block sections are desired. Additionally, machine basic block ids of the
|
|
// functions can also be specified for a finer granularity. Moreover, a cluster
|
|
// of basic blocks could be assigned to the same section.
|
|
// Optionally, a debug-info filename can be specified for each function to allow
|
|
// distinguishing internal-linkage functions of the same name.
|
|
// A file with basic block sections for all of function main and three blocks
|
|
// for function foo (of which 1 and 2 are placed in a cluster) looks like this:
|
|
// (Profile for function foo is only loaded when its debug-info filename
|
|
// matches 'path/to/foo_file.cc').
|
|
// ----------------------------
|
|
// list.txt:
|
|
// !main
|
|
// !foo M=path/to/foo_file.cc
|
|
// !!1 2
|
|
// !!4
|
|
Error BasicBlockSectionsProfileReader::ReadProfile() {
|
|
assert(MBuf);
|
|
|
|
unsigned long long Version = 0;
|
|
StringRef FirstLine(*LineIt);
|
|
if (FirstLine.consume_front("v")) {
|
|
if (getAsUnsignedInteger(FirstLine, 10, Version)) {
|
|
return createProfileParseError(Twine("version number expected: '") +
|
|
FirstLine + "'");
|
|
}
|
|
if (Version > 1) {
|
|
return createProfileParseError(Twine("invalid profile version: ") +
|
|
Twine(Version));
|
|
}
|
|
++LineIt;
|
|
}
|
|
|
|
switch (Version) {
|
|
case 0:
|
|
// TODO: Deprecate V0 once V1 is fully integrated downstream.
|
|
return ReadV0Profile();
|
|
case 1:
|
|
return ReadV1Profile();
|
|
default:
|
|
llvm_unreachable("Invalid profile version.");
|
|
}
|
|
}
|
|
|
|
bool BasicBlockSectionsProfileReaderWrapperPass::doInitialization(Module &M) {
|
|
if (!BBSPR.MBuf)
|
|
return false;
|
|
// Get the function name to debug info filename mapping.
|
|
BBSPR.FunctionNameToDIFilename.clear();
|
|
for (const Function &F : M) {
|
|
SmallString<128> DIFilename;
|
|
if (F.isDeclaration())
|
|
continue;
|
|
DISubprogram *Subprogram = F.getSubprogram();
|
|
if (Subprogram) {
|
|
llvm::DICompileUnit *CU = Subprogram->getUnit();
|
|
if (CU)
|
|
DIFilename = sys::path::remove_leading_dotslash(CU->getFilename());
|
|
}
|
|
[[maybe_unused]] bool inserted =
|
|
BBSPR.FunctionNameToDIFilename.try_emplace(F.getName(), DIFilename)
|
|
.second;
|
|
assert(inserted);
|
|
}
|
|
if (auto Err = BBSPR.ReadProfile())
|
|
report_fatal_error(std::move(Err));
|
|
return false;
|
|
}
|
|
|
|
AnalysisKey BasicBlockSectionsProfileReaderAnalysis::Key;
|
|
|
|
BasicBlockSectionsProfileReader
|
|
BasicBlockSectionsProfileReaderAnalysis::run(Function &F,
|
|
FunctionAnalysisManager &AM) {
|
|
return BasicBlockSectionsProfileReader(TM->getBBSectionsFuncListBuf());
|
|
}
|
|
|
|
bool BasicBlockSectionsProfileReaderWrapperPass::isFunctionHot(
|
|
StringRef FuncName) const {
|
|
return BBSPR.isFunctionHot(FuncName);
|
|
}
|
|
|
|
SmallVector<BBClusterInfo>
|
|
BasicBlockSectionsProfileReaderWrapperPass::getClusterInfoForFunction(
|
|
StringRef FuncName) const {
|
|
return BBSPR.getClusterInfoForFunction(FuncName);
|
|
}
|
|
|
|
SmallVector<SmallVector<unsigned>>
|
|
BasicBlockSectionsProfileReaderWrapperPass::getClonePathsForFunction(
|
|
StringRef FuncName) const {
|
|
return BBSPR.getClonePathsForFunction(FuncName);
|
|
}
|
|
|
|
const CFGProfile *
|
|
BasicBlockSectionsProfileReaderWrapperPass::getFunctionCFGProfile(
|
|
StringRef FuncName) const {
|
|
return BBSPR.getFunctionCFGProfile(FuncName);
|
|
}
|
|
|
|
uint64_t BasicBlockSectionsProfileReaderWrapperPass::getEdgeCount(
|
|
StringRef FuncName, const UniqueBBID &SrcBBID,
|
|
const UniqueBBID &SinkBBID) const {
|
|
return BBSPR.getEdgeCount(FuncName, SrcBBID, SinkBBID);
|
|
}
|
|
|
|
SmallVector<CallsiteID>
|
|
BasicBlockSectionsProfileReaderWrapperPass::getPrefetchTargetsForFunction(
|
|
StringRef FuncName) const {
|
|
return BBSPR.getPrefetchTargetsForFunction(FuncName);
|
|
}
|
|
|
|
SmallVector<PrefetchHint>
|
|
BasicBlockSectionsProfileReaderWrapperPass::getPrefetchHintsForFunction(
|
|
StringRef FuncName) const {
|
|
return BBSPR.getPrefetchHintsForFunction(FuncName);
|
|
}
|
|
|
|
BasicBlockSectionsProfileReader &
|
|
BasicBlockSectionsProfileReaderWrapperPass::getBBSPR() {
|
|
return BBSPR;
|
|
}
|
|
|
|
ImmutablePass *llvm::createBasicBlockSectionsProfileReaderWrapperPass(
|
|
const MemoryBuffer *Buf) {
|
|
return new BasicBlockSectionsProfileReaderWrapperPass(Buf);
|
|
}
|