Files
llvm-project/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp
Florian Hahn cac7fe50e0 [VPlan] Make canonical IV part of the region (#156262)
The canonical IV is directly tied to a loop region. To directly ensure
there's a single, unique canonical IV, directly define it by the region.

Depends on https://github.com/llvm/llvm-project/pull/161589.

PR: https://github.com/llvm/llvm-project/pull/156262
2026-04-19 22:21:04 +01:00

490 lines
17 KiB
C++

//===- llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp ----------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "../lib/Transforms/Vectorize/VPlanVerifier.h"
#include "../lib/Transforms/Vectorize/VPlan.h"
#include "../lib/Transforms/Vectorize/VPlanCFG.h"
#include "VPlanTestBase.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "gtest/gtest.h"
using namespace llvm;
namespace llvm {
LLVM_ABI extern cl::opt<bool> VerifyEachVPlan;
} // namespace llvm
using VPVerifierTest = VPlanTestBase;
namespace {
TEST_F(VPVerifierTest, VPInstructionUseBeforeDefSameBB) {
VPlan &Plan = getPlan();
VPIRValue *Zero = Plan.getConstantInt(32, 0);
VPInstruction *DefI =
new VPInstruction(Instruction::Add, {Zero, Zero},
VPIRFlags::getDefaultFlags(Instruction::Add));
VPInstruction *UseI =
new VPInstruction(Instruction::Sub, {DefI, Zero},
VPIRFlags::getDefaultFlags(Instruction::Sub));
VPBasicBlock *VPBB1 = Plan.getEntry();
VPBB1->appendRecipe(UseI);
VPBB1->appendRecipe(DefI);
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
// Add a dummy instruction to VPBB2 to make it non-empty
VPInstruction *Dummy =
new VPInstruction(Instruction::Add, {Zero, Zero},
VPIRFlags::getDefaultFlags(Instruction::Add));
VPBB2->appendRecipe(Dummy);
VPRegionBlock *R1 = Plan.createLoopRegion(Type::getInt32Ty(C), DebugLoc(),
"R1", VPBB2, VPBB2);
VPBlockUtils::connectBlocks(VPBB1, R1);
VPBlockUtils::connectBlocks(R1, Plan.getScalarHeader());
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
#endif
EXPECT_FALSE(verifyVPlanIsValid(Plan));
#if GTEST_HAS_STREAM_REDIRECTION
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
EXPECT_STREQ("Use before def!\n"
" EMIT vp<%1> = sub vp<%2>, ir<0>\n"
" before\n"
" EMIT vp<%2> = add ir<0>, ir<0>\n",
::testing::internal::GetCapturedStderr().c_str());
#else
EXPECT_STREQ("Use before def!\n",
::testing::internal::GetCapturedStderr().c_str());
#endif
#endif
}
TEST_F(VPVerifierTest, VPInstructionUseBeforeDefDifferentBB) {
VPlan &Plan = getPlan();
VPIRValue *Zero = Plan.getConstantInt(32, 0);
VPInstruction *DefI =
new VPInstruction(Instruction::Add, {Zero, Zero},
VPIRFlags::getDefaultFlags(Instruction::Add));
VPInstruction *UseI =
new VPInstruction(Instruction::Sub, {DefI, Zero},
VPIRFlags::getDefaultFlags(Instruction::Sub));
VPInstruction *BranchOnCond =
new VPInstruction(VPInstruction::BranchOnCond, {UseI});
VPBasicBlock *VPBB1 = Plan.getEntry();
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
VPBB1->appendRecipe(UseI);
VPBB2->appendRecipe(DefI);
VPBB2->appendRecipe(BranchOnCond);
VPRegionBlock *R1 = Plan.createLoopRegion(Type::getInt32Ty(C), DebugLoc(),
"R1", VPBB2, VPBB2);
VPBlockUtils::connectBlocks(VPBB1, R1);
VPBlockUtils::connectBlocks(R1, Plan.getScalarHeader());
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
#endif
EXPECT_FALSE(verifyVPlanIsValid(Plan));
#if GTEST_HAS_STREAM_REDIRECTION
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
EXPECT_STREQ("Use before def!\n"
" EMIT vp<%1> = sub vp<%3>, ir<0>\n"
" before\n"
" EMIT vp<%3> = add ir<0>, ir<0>\n",
::testing::internal::GetCapturedStderr().c_str());
#else
EXPECT_STREQ("Use before def!\n",
::testing::internal::GetCapturedStderr().c_str());
#endif
#endif
}
TEST_F(VPVerifierTest, VPBlendUseBeforeDefDifferentBB) {
VPlan &Plan = getPlan();
IntegerType *Int32 = IntegerType::get(C, 32);
auto *Phi = PHINode::Create(Int32, 1);
VPIRValue *Zero = Plan.getConstantInt(Int32, 0);
VPInstruction *DefI =
new VPInstruction(Instruction::Add, {Zero, Zero},
VPIRFlags::getDefaultFlags(Instruction::Add));
VPInstruction *BranchOnCond =
new VPInstruction(VPInstruction::BranchOnCond, {DefI});
auto *Blend = new VPBlendRecipe(Phi, {DefI, Plan.getTrue()}, {}, {});
VPBasicBlock *VPBB1 = Plan.getEntry();
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
VPBasicBlock *VPBB3 = Plan.createVPBasicBlock("");
VPBasicBlock *VPBB4 = Plan.createVPBasicBlock("");
VPBB3->appendRecipe(Blend);
VPBB4->appendRecipe(DefI);
VPBB4->appendRecipe(BranchOnCond);
VPBlockUtils::connectBlocks(VPBB2, VPBB3);
VPBlockUtils::connectBlocks(VPBB3, VPBB4);
VPRegionBlock *R1 = Plan.createLoopRegion(Type::getInt32Ty(C), DebugLoc(),
"R1", VPBB2, VPBB4);
VPBlockUtils::connectBlocks(VPBB1, R1);
VPBB3->setParent(R1);
VPBlockUtils::connectBlocks(R1, Plan.getScalarHeader());
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
#endif
EXPECT_FALSE(verifyVPlanIsValid(Plan));
#if GTEST_HAS_STREAM_REDIRECTION
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
EXPECT_STREQ("Use before def!\n"
" BLEND ir<<badref>> = vp<%2>\n"
" before\n"
" EMIT vp<%2> = add ir<0>, ir<0>\n",
::testing::internal::GetCapturedStderr().c_str());
#else
EXPECT_STREQ("Use before def!\n",
::testing::internal::GetCapturedStderr().c_str());
#endif
#endif
delete Phi;
}
TEST_F(VPVerifierTest, VPPhiIncomingValueDoesntDominateIncomingBlock) {
VPlan &Plan = getPlan();
IntegerType *Int32 = IntegerType::get(C, 32);
VPIRValue *Zero = Plan.getConstantInt(Int32, 0);
VPBasicBlock *VPBB1 = Plan.getEntry();
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
VPBasicBlock *VPBB3 = Plan.createVPBasicBlock("");
VPBasicBlock *VPBB4 = Plan.createVPBasicBlock("");
VPInstruction *DefI =
new VPInstruction(Instruction::Add, {Zero, Zero},
VPIRFlags::getDefaultFlags(Instruction::Add));
VPPhi *Phi = new VPPhi({DefI}, {}, {});
VPBB2->appendRecipe(Phi);
VPBB2->appendRecipe(DefI);
VPRegionBlock *R1 = Plan.createLoopRegion(Type::getInt32Ty(C), DebugLoc(),
"R1", VPBB3, VPBB3);
VPBlockUtils::connectBlocks(VPBB1, VPBB2);
VPBlockUtils::connectBlocks(VPBB2, R1);
VPBlockUtils::connectBlocks(VPBB4, Plan.getScalarHeader());
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
#endif
EXPECT_FALSE(verifyVPlanIsValid(Plan));
#if GTEST_HAS_STREAM_REDIRECTION
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
EXPECT_STREQ("Incoming def does not dominate incoming block!\n"
" EMIT vp<%2> = add ir<0>, ir<0>\n"
" does not dominate preheader for\n"
" EMIT-SCALAR vp<%1> = phi [ vp<%2>, preheader ]",
::testing::internal::GetCapturedStderr().c_str());
#else
EXPECT_STREQ("Incoming def does not dominate incoming block!\n",
::testing::internal::GetCapturedStderr().c_str());
#endif
#endif
}
TEST_F(VPVerifierTest, DuplicateSuccessorsOutsideRegion) {
VPlan &Plan = getPlan();
VPIRValue *Zero = Plan.getConstantInt(32, 0);
VPInstruction *I1 =
new VPInstruction(Instruction::Add, {Zero, Zero},
VPIRFlags::getDefaultFlags(Instruction::Add));
VPInstruction *BranchOnCond =
new VPInstruction(VPInstruction::BranchOnCond, {I1});
VPInstruction *BranchOnCond2 =
new VPInstruction(VPInstruction::BranchOnCond, {I1});
VPBasicBlock *VPBB1 = Plan.getEntry();
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
VPBB1->appendRecipe(I1);
VPBB1->appendRecipe(BranchOnCond2);
VPBB2->appendRecipe(BranchOnCond);
VPRegionBlock *R1 = Plan.createLoopRegion(Type::getInt32Ty(C), DebugLoc(),
"R1", VPBB2, VPBB2);
VPBlockUtils::connectBlocks(VPBB1, R1);
VPBlockUtils::connectBlocks(VPBB1, R1);
VPBlockUtils::connectBlocks(R1, Plan.getScalarHeader());
// Duplicate successors are allowed for blocks with conditional terminators.
EXPECT_TRUE(verifyVPlanIsValid(Plan));
}
TEST_F(VPVerifierTest, DuplicateSuccessorsInsideRegion) {
VPlan &Plan = getPlan();
VPIRValue *Zero = Plan.getConstantInt(32, 0);
VPInstruction *I1 =
new VPInstruction(Instruction::Add, {Zero, Zero},
VPIRFlags::getDefaultFlags(Instruction::Add));
VPInstruction *BranchOnCond =
new VPInstruction(VPInstruction::BranchOnCond, {I1});
VPInstruction *BranchOnCond2 =
new VPInstruction(VPInstruction::BranchOnCond, {I1});
VPBasicBlock *VPBB1 = Plan.getEntry();
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
VPBasicBlock *VPBB3 = Plan.createVPBasicBlock("");
VPBB1->appendRecipe(I1);
VPBB2->appendRecipe(BranchOnCond2);
VPBB3->appendRecipe(BranchOnCond);
VPBlockUtils::connectBlocks(VPBB2, VPBB3);
VPBlockUtils::connectBlocks(VPBB2, VPBB3);
VPRegionBlock *R1 = Plan.createLoopRegion(Type::getInt32Ty(C), DebugLoc(),
"R1", VPBB2, VPBB3);
VPBlockUtils::connectBlocks(VPBB1, R1);
VPBB3->setParent(R1);
VPBlockUtils::connectBlocks(R1, Plan.getScalarHeader());
// Duplicate successors are allowed for blocks with conditional terminators.
EXPECT_TRUE(verifyVPlanIsValid(Plan));
}
TEST_F(VPVerifierTest, BlockOutsideRegionWithParent) {
VPlan &Plan = getPlan();
VPBasicBlock *VPBB1 = Plan.getEntry();
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("");
VPValue *Zero = Plan.getConstantInt(32, 0);
VPInstruction *DefI =
new VPInstruction(Instruction::Add, {Zero, Zero},
VPIRFlags::getDefaultFlags(Instruction::Add));
VPInstruction *BranchOnCond =
new VPInstruction(VPInstruction::BranchOnCond, {DefI});
VPBB1->appendRecipe(DefI);
VPBB2->appendRecipe(BranchOnCond);
VPRegionBlock *R1 = Plan.createLoopRegion(Type::getInt32Ty(C), DebugLoc(),
"R1", VPBB2, VPBB2);
VPBlockUtils::connectBlocks(VPBB1, R1);
VPBlockUtils::connectBlocks(R1, Plan.getScalarHeader());
VPBB1->setParent(R1);
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
#endif
EXPECT_FALSE(verifyVPlanIsValid(Plan));
#if GTEST_HAS_STREAM_REDIRECTION
EXPECT_STREQ("Predecessor is not in the same region.\n",
::testing::internal::GetCapturedStderr().c_str());
#endif
}
TEST_F(VPVerifierTest, NonHeaderPHIInHeader) {
VPlan &Plan = getPlan();
VPValue *Zero = Plan.getConstantInt(32, 0);
auto *BranchOnCond = new VPInstruction(VPInstruction::BranchOnCond, {Zero});
VPBasicBlock *VPBB1 = Plan.getEntry();
VPBasicBlock *VPBB2 = Plan.createVPBasicBlock("header");
PHINode *PHINode = PHINode::Create(Type::getInt32Ty(C), 2);
auto *IRPhi = new VPIRPhi(*PHINode);
VPBB2->appendRecipe(IRPhi);
VPBB2->appendRecipe(BranchOnCond);
VPRegionBlock *R1 = Plan.createLoopRegion(Type::getInt32Ty(C), DebugLoc(),
"R1", VPBB2, VPBB2);
VPBlockUtils::connectBlocks(VPBB1, R1);
VPBlockUtils::connectBlocks(R1, Plan.getScalarHeader());
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
#endif
EXPECT_FALSE(verifyVPlanIsValid(Plan));
#if GTEST_HAS_STREAM_REDIRECTION
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
EXPECT_STREQ(
"Found non-header PHI recipe in header VPBB: IR <badref> = phi i32 \n",
::testing::internal::GetCapturedStderr().c_str());
#else
EXPECT_STREQ("Found non-header PHI recipe in header VPBB",
::testing::internal::GetCapturedStderr().c_str());
#endif
#endif
delete PHINode;
}
TEST_F(VPVerifierTest, testRUN_VPLAN_PASS) {
VPlan &Plan = getPlan();
VPIRValue *Zero = Plan.getConstantInt(32, 0);
VPInstruction *DefI =
new VPInstruction(Instruction::Add, {Zero, Zero},
VPIRFlags::getDefaultFlags(Instruction::Add));
VPInstruction *UseI =
new VPInstruction(Instruction::Sub, {DefI, Zero},
VPIRFlags::getDefaultFlags(Instruction::Sub));
VPBasicBlock *VPBB1 = Plan.getEntry();
VPBB1->appendRecipe(UseI);
VPBB1->appendRecipe(DefI);
bool OrigVerifyEachVPlan = VerifyEachVPlan;
VerifyEachVPlan = true;
llvm::scope_exit _([&]() { VerifyEachVPlan = OrigVerifyEachVPlan; });
auto NopPass = [](VPlan &Plan) {};
EXPECT_DEATH(
{ VPlanTransforms::runPass("simplifyRecipes", NopPass, Plan); },
"Broken VPlan found, compilation aborted!");
}
class VPIRVerifierTest : public VPlanTestIRBase {};
TEST_F(VPIRVerifierTest, testVerifyIRPhiInScalarHeaderVPIRBB) {
const char *ModuleString =
"define void @f(ptr %A, i64 %N) {\n"
"entry:\n"
" br label %loop\n"
"loop:\n"
" %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]\n"
" %iv.next = add i64 %iv, 1\n"
" %exitcond = icmp ne i64 %iv.next, %N\n"
" br i1 %exitcond, label %loop, label %for.end\n"
"for.end:\n"
" %p = phi i64 [ %iv, %loop ]\n"
" ret void\n"
"}\n";
Module &M = parseModule(ModuleString);
Function *F = M.getFunction("f");
BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
auto Plan = buildVPlan(LoopHeader);
VPValue *Zero = Plan->getConstantInt(32, 0);
Plan->getScalarHeader()->front().addOperand(Zero);
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
#endif
EXPECT_FALSE(verifyVPlanIsValid(*Plan));
#if GTEST_HAS_STREAM_REDIRECTION
EXPECT_STREQ(
"Phi-like recipe with different number of operands and predecessors.\n",
::testing::internal::GetCapturedStderr().c_str());
#endif
}
TEST_F(VPIRVerifierTest, testVerifyIRPhiInExitVPIRBB) {
const char *ModuleString =
"define void @f(ptr %A, i64 %N) {\n"
"entry:\n"
" br label %loop\n"
"loop:\n"
" %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]\n"
" %iv.next = add i64 %iv, 1\n"
" %exitcond = icmp ne i64 %iv.next, %N\n"
" br i1 %exitcond, label %loop, label %for.end\n"
"for.end:\n"
" %p = phi i64 [ %iv, %loop ]\n"
" ret void\n"
"}\n";
Module &M = parseModule(ModuleString);
Function *F = M.getFunction("f");
BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
auto Plan = buildVPlan(LoopHeader);
// Create a definition in the vector loop header that will be used by the phi.
auto *HeaderBlock =
cast<VPBasicBlock>(Plan->getVectorLoopRegion()->getEntry());
VPInstruction *DefI =
new VPInstruction(VPInstruction::ExtractLastLane,
{HeaderBlock->front().getVPSingleValue()});
DefI->insertBefore(Plan->getMiddleBlock()->getTerminator());
Plan->getExitBlocks()[0]->front().addOperand(DefI);
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
#endif
EXPECT_FALSE(verifyVPlanIsValid(*Plan));
#if GTEST_HAS_STREAM_REDIRECTION
EXPECT_STREQ(
"Phi-like recipe with different number of operands and predecessors.\n",
::testing::internal::GetCapturedStderr().c_str());
#endif
}
TEST_F(VPIRVerifierTest, BranchOnTwoCondsLatchHeaderVerification) {
const char *ModuleString =
"define void @f(ptr dereferenceable(40) align 2 %pred) {\n"
"entry:\n"
" br label %for.body\n"
"for.body:\n"
" %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]\n"
" %uncountable.addr = getelementptr inbounds nuw i16, ptr %pred, i64 "
"%iv\n"
" %uncountable.val = load i16, ptr %uncountable.addr, align 2\n"
" %uncountable.cond = icmp sgt i16 %uncountable.val, 500\n"
" br i1 %uncountable.cond, label %exit, label %for.inc\n"
"for.inc:\n"
" %iv.next = add nuw nsw i64 %iv, 1\n"
" %countable.cond = icmp eq i64 %iv.next, 20\n"
" br i1 %countable.cond, label %exit, label %for.body\n"
"exit:\n"
" ret void\n"
"}\n";
Module &M = parseModule(ModuleString);
Function *F = M.getFunction("f");
BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
// Build a plain CFG VPlan with BranchOnTwoConds as the latch terminator
// (3 successors), without wrapping blocks in loop regions.
auto Plan = buildVPlan(LoopHeader, UncountableExitStyle::ReadOnly,
/*CreateLoopRegions=*/false);
auto *MiddleVPBB =
cast<VPBasicBlock>(Plan->getScalarPreheader()->getPredecessors()[0]);
auto *Latch = cast<VPBasicBlock>(MiddleVPBB->getSinglePredecessor());
auto *Header = Latch->getSuccessors().back();
ASSERT_EQ(Header->getPredecessors()[1], Latch);
ASSERT_EQ(Latch->getNumSuccessors(), 3u);
auto *Term = cast<VPInstruction>(&Latch->back());
EXPECT_EQ(Term->getOpcode(), VPInstruction::BranchOnTwoConds);
// Verify the plan is valid; this exercises isLatch with a 3-successor latch.
EXPECT_TRUE(verifyVPlanIsValid(*Plan));
// Swap the latch's first and last successors, placing the header at index 0
// instead of the last position. isLatch checks the last successor, so the
// latch is no longer recognized, triggering the header predecessor check.
auto &Succs = Latch->getSuccessors();
std::swap(Succs[0], Succs[2]);
#if GTEST_HAS_STREAM_REDIRECTION
::testing::internal::CaptureStderr();
#endif
EXPECT_FALSE(verifyVPlanIsValid(*Plan));
#if GTEST_HAS_STREAM_REDIRECTION
EXPECT_STREQ("Header's second predecessor must be the latch!\n",
::testing::internal::GetCapturedStderr().c_str());
#endif
}
} // namespace