Doomhowl-Interactive/llvm-project
0
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity
Files
main
llvm-project/llvm/test/Transforms/LoopVectorize/tail-folding-div.ll
Florian Hahn ec14a1f40c [VPlan] Add transform to replace VPWidenCanonicalIV with wide IV. (#194267) 
Add a new cost-based transform that replaces VPWidenCanonicalIVRecipe
with a canonical VPWidenIntOrFpInductionPHIRecipe, if it does not
increase spills.

The main benefit of VPWidenCanonicalIVRecipe is that it has shorter
live-ranges than wide IV phis. The new transform introduces wide IV
unless VPWidenCanonicalIVRecipe is cheaper or the wide IV introduces
additional spills.

This introduces wide IVs in a number of cases, where previously had
VPWidenCanonicalIVRecipe, because there was no existing wide canonical
IV we could re-use. It should also help avoid somewhat unrelated changes
in https://github.com/llvm/llvm-project/pull/190191.

PR: https://github.com/llvm/llvm-project/pull/194267
2026-04-29 19:54:35 +00:00

341 lines
17 KiB
LLVM
Raw Permalink Blame History

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
; RUN: opt -S --passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-vector-width=2 -force-target-supports-masked-memory-ops < %s | FileCheck %s
; No predicate for sdiv because RHS has no masked-off lanes.
define void @test_sdiv_variant_divisor_induction(ptr noalias %a, ptr noalias %c) {
; CHECK-LABEL: define void @test_sdiv_variant_divisor_induction(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[C:%.*]]) {
; CHECK-NEXT: [[LOOP_PREHEADER:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND1:%.*]] = phi <2 x i64> [ <i64 1, i64 2>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND2:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <2 x i16> [[VEC_IND2]], splat (i16 1024)
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[A]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[TMP12:%.*]] = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr align 4 [[TMP4]], <2 x i1> [[TMP2]], <2 x i64> poison)
; CHECK-NEXT: [[VEC_IND:%.*]] = select <2 x i1> [[TMP2]], <2 x i64> [[VEC_IND1]], <2 x i64> splat (i64 1)
; CHECK-NEXT: [[TMP13:%.*]] = sdiv <2 x i64> [[TMP12]], [[VEC_IND]]
; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[C]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: call void @llvm.masked.store.v2i64.p0(<2 x i64> [[TMP13]], ptr align 4 [[TMP15]], <2 x i1> [[TMP2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND1]], splat (i64 2)
; CHECK-NEXT: [[VEC_IND_NEXT2]] = add nuw <2 x i16> [[VEC_IND2]], splat (i16 2)
; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1026
; CHECK-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
loop.preheader:
br label %loop
loop:
%iv = phi i64 [ %iv.next, %loop ], [ 1, %loop.preheader ]
%a.gep = getelementptr i64, ptr %a, i64 %iv
%0 = load i64, ptr %a.gep
%2 = sdiv i64 %0, %iv
%c.gep = getelementptr i64, ptr %c, i64 %iv
store i64 %2, ptr %c.gep
%iv.next = add i64 %iv, 1
%done = icmp eq i64 %iv.next, 1026
br i1 %done, label %exit, label %loop
exit:
ret void
}
; Predicate for sdiv because RHS have masked-off lanes.
define void @test_sdiv_variant_divisor_load(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
; CHECK-LABEL: define void @test_sdiv_variant_divisor_load(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) {
; CHECK-NEXT: [[VECTOR_PH:.*:]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE:.*]]
; CHECK: [[PRED_LOAD_CONTINUE]]:
; CHECK-NEXT: [[TMP1:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, %[[VECTOR_BODY]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE]] ]
; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <2 x i16> [[VEC_IND]], splat (i16 1024)
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[A]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP19:%.*]] = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr align 4 [[TMP13]], <2 x i1> [[TMP2]], <2 x i64> poison)
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[B]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP20:%.*]] = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr align 4 [[TMP16]], <2 x i1> [[TMP2]], <2 x i64> poison)
; CHECK-NEXT: [[TMP21:%.*]] = select <2 x i1> [[TMP2]], <2 x i64> [[TMP20]], <2 x i64> splat (i64 1)
; CHECK-NEXT: [[TMP22:%.*]] = sdiv <2 x i64> [[TMP19]], [[TMP21]]
; CHECK-NEXT: [[TMP27:%.*]] = getelementptr i64, ptr [[C]], i64 [[TMP1]]
; CHECK-NEXT: call void @llvm.masked.store.v2i64.p0(<2 x i64> [[TMP22]], ptr align 4 [[TMP27]], <2 x i1> [[TMP2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP1]], 2
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <2 x i16> [[VEC_IND]], splat (i16 2)
; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1026
; CHECK-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_LOAD_CONTINUE]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
loop.preheader:
br label %loop
loop:
%iv = phi i64 [ %iv.next, %loop ], [ 0, %loop.preheader ]
%a.gep = getelementptr i64, ptr %a, i64 %iv
%0 = load i64, ptr %a.gep
%b.gep = getelementptr i64, ptr %b, i64 %iv
%1 = load i64, ptr %b.gep
%2 = sdiv i64 %0, %1
%c.gep = getelementptr i64, ptr %c, i64 %iv
store i64 %2, ptr %c.gep
%iv.next = add i64 %iv, 1
%done = icmp eq i64 %iv.next, 1025
br i1 %done, label %exit, label %loop
exit:
ret void
}
; Predicate for sdiv because RHS may be -1.
define void @test_sdiv_invariant_divisor_nonconst(ptr noalias %a, i64 %b, ptr noalias %c) {
; CHECK-LABEL: define void @test_sdiv_invariant_divisor_nonconst(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[B:%.*]], ptr noalias [[C:%.*]]) {
; CHECK-NEXT: [[LOOP_PREHEADER:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[B]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE:.*]]
; CHECK: [[PRED_LOAD_CONTINUE]]:
; CHECK-NEXT: [[TMP1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE]] ]
; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <2 x i16> [[VEC_IND]], splat (i16 1024)
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[A]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP12:%.*]] = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr align 4 [[TMP9]], <2 x i1> [[TMP2]], <2 x i64> poison)
; CHECK-NEXT: [[TMP13:%.*]] = select <2 x i1> [[TMP2]], <2 x i64> [[BROADCAST_SPLAT]], <2 x i64> splat (i64 1)
; CHECK-NEXT: [[TMP14:%.*]] = sdiv <2 x i64> [[TMP12]], [[TMP13]]
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[C]], i64 [[TMP1]]
; CHECK-NEXT: call void @llvm.masked.store.v2i64.p0(<2 x i64> [[TMP14]], ptr align 4 [[TMP19]], <2 x i1> [[TMP2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP1]], 2
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <2 x i16> [[VEC_IND]], splat (i16 2)
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1026
; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_LOAD_CONTINUE]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
loop.preheader:
br label %loop
loop:
%iv = phi i64 [ %iv.next, %loop ], [ 0, %loop.preheader ]
%a.gep = getelementptr i64, ptr %a, i64 %iv
%0 = load i64, ptr %a.gep
%2 = sdiv i64 %0, %b
%c.gep = getelementptr i64, ptr %c, i64 %iv
store i64 %2, ptr %c.gep
%iv.next = add i64 %iv, 1
%done = icmp eq i64 %iv.next, 1025
br i1 %done, label %exit, label %loop
exit:
ret void
}
; Predicate for sdiv because RHS is -1.
define void @test_sdiv_invariant_divisor_minusone(ptr noalias %a, ptr noalias %c) {
; CHECK-LABEL: define void @test_sdiv_invariant_divisor_minusone(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[C:%.*]]) {
; CHECK-NEXT: [[VECTOR_PH:.*:]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE:.*]]
; CHECK: [[PRED_LOAD_CONTINUE]]:
; CHECK-NEXT: [[TMP1:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, %[[VECTOR_BODY]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE]] ]
; CHECK-NEXT: [[TMP2:%.*]] = icmp ule <2 x i16> [[VEC_IND]], splat (i16 1024)
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[A]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP12:%.*]] = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr align 4 [[TMP9]], <2 x i1> [[TMP2]], <2 x i64> poison)
; CHECK-NEXT: [[TMP13:%.*]] = select <2 x i1> [[TMP2]], <2 x i64> splat (i64 -1), <2 x i64> splat (i64 1)
; CHECK-NEXT: [[TMP14:%.*]] = sdiv <2 x i64> [[TMP12]], [[TMP13]]
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[C]], i64 [[TMP1]]
; CHECK-NEXT: call void @llvm.masked.store.v2i64.p0(<2 x i64> [[TMP14]], ptr align 4 [[TMP19]], <2 x i1> [[TMP2]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP1]], 2
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <2 x i16> [[VEC_IND]], splat (i16 2)
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1026
; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_LOAD_CONTINUE]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
loop.preheader:
br label %loop
loop:
%iv = phi i64 [ %iv.next, %loop ], [ 0, %loop.preheader ]
%a.gep = getelementptr i64, ptr %a, i64 %iv
%0 = load i64, ptr %a.gep
%2 = sdiv i64 %0, -1
%c.gep = getelementptr i64, ptr %c, i64 %iv
store i64 %2, ptr %c.gep
%iv.next = add i64 %iv, 1
%done = icmp eq i64 %iv.next, 1025
br i1 %done, label %exit, label %loop
exit:
ret void
}
; No predicate for sdiv because RHS is 3.
define void @test_sdiv_invariant_divisor_safeimm(ptr noalias %a, ptr noalias %c) {
; CHECK-LABEL: define void @test_sdiv_invariant_divisor_safeimm(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[C:%.*]]) {
; CHECK-NEXT: [[VECTOR_PH:.*:]]
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE:.*]]
; CHECK: [[PRED_LOAD_CONTINUE]]:
; CHECK-NEXT: [[TMP1:%.*]] = phi i64 [ 0, %[[VECTOR_BODY]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, %[[VECTOR_BODY]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE]] ]
; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <2 x i16> [[VEC_IND]], splat (i16 1024)
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[A]], i64 [[TMP1]]
; CHECK-NEXT: [[TMP12:%.*]] = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr align 4 [[TMP9]], <2 x i1> [[TMP0]], <2 x i64> poison)
; CHECK-NEXT: [[TMP13:%.*]] = sdiv <2 x i64> [[TMP12]], splat (i64 3)
; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[C]], i64 [[TMP1]]
; CHECK-NEXT: call void @llvm.masked.store.v2i64.p0(<2 x i64> [[TMP13]], ptr align 4 [[TMP18]], <2 x i1> [[TMP0]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP1]], 2
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <2 x i16> [[VEC_IND]], splat (i16 2)
; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1026
; CHECK-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_LOAD_CONTINUE]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
loop.preheader:
br label %loop
loop:
%iv = phi i64 [ %iv.next, %loop ], [ 0, %loop.preheader ]
%a.gep = getelementptr i64, ptr %a, i64 %iv
%0 = load i64, ptr %a.gep
%2 = sdiv i64 %0, 3
%c.gep = getelementptr i64, ptr %c, i64 %iv
store i64 %2, ptr %c.gep
%iv.next = add i64 %iv, 1
%done = icmp eq i64 %iv.next, 1025
br i1 %done, label %exit, label %loop
exit:
ret void
}
; No predicate for sdiv because LHS & RHS has no masked-off lanes.
define void @test_sdiv_variant_dividend_induction(i64 %a, ptr noalias %c) {
; CHECK-LABEL: define void @test_sdiv_variant_dividend_induction(
; CHECK-SAME: i64 [[A:%.*]], ptr noalias [[C:%.*]]) {
; CHECK-NEXT: [[LOOP_PREHEADER:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[A]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[TMP7:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = icmp ule <2 x i64> [[VEC_IND]], splat (i64 1024)
; CHECK-NEXT: [[TMP11:%.*]] = select <2 x i1> [[TMP0]], <2 x i64> [[BROADCAST_SPLAT]], <2 x i64> splat (i64 1)
; CHECK-NEXT: [[TMP1:%.*]] = sdiv <2 x i64> [[VEC_IND]], [[TMP11]]
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[C]], i64 [[TMP7]]
; CHECK-NEXT: call void @llvm.masked.store.v2i64.p0(<2 x i64> [[TMP1]], ptr align 4 [[TMP8]], <2 x i1> [[TMP0]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP7]], 2
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1026
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
loop.preheader:
br label %loop
loop:
%iv = phi i64 [ %iv.next, %loop ], [ 0, %loop.preheader ]
%2 = sdiv i64 %iv, %a
%c.gep = getelementptr i64, ptr %c, i64 %iv
store i64 %2, ptr %c.gep
%iv.next = add i64 %iv, 1
%done = icmp eq i64 %iv.next, 1025
br i1 %done, label %exit, label %loop
exit:
ret void
}
; No predicate for sdiv because both LHS & RHS have no masked-off lanes.
define void @test_sdiv_both_invariant_nonconst(ptr noalias %a, i64 %b, i64 %b2, ptr noalias %c) {
; CHECK-LABEL: define void @test_sdiv_both_invariant_nonconst(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[B:%.*]], i64 [[B2:%.*]], ptr noalias [[C:%.*]]) {
; CHECK-NEXT: [[LOOP_PREHEADER:.*:]]
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[B]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <2 x i64> poison, i64 [[B2]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT1]], <2 x i64> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE:.*]]
; CHECK: [[PRED_LOAD_CONTINUE]]:
; CHECK-NEXT: [[TMP2:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <2 x i16> [ <i16 0, i16 1>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE]] ]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ule <2 x i16> [[VEC_IND]], splat (i16 1024)
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[A]], i64 [[TMP2]]
; CHECK-NEXT: [[TMP22:%.*]] = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr align 4 [[TMP10]], <2 x i1> [[TMP3]], <2 x i64> poison)
; CHECK-NEXT: [[TMP13:%.*]] = select <2 x i1> [[TMP3]], <2 x i64> [[BROADCAST_SPLAT2]], <2 x i64> splat (i64 1)
; CHECK-NEXT: [[TMP23:%.*]] = sdiv <2 x i64> [[BROADCAST_SPLAT]], [[TMP13]]
; CHECK-NEXT: [[TMP14:%.*]] = add <2 x i64> [[TMP22]], [[TMP23]]
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr i64, ptr [[C]], i64 [[TMP2]]
; CHECK-NEXT: call void @llvm.masked.store.v2i64.p0(<2 x i64> [[TMP14]], ptr align 4 [[TMP19]], <2 x i1> [[TMP3]])
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP2]], 2
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw <2 x i16> [[VEC_IND]], splat (i16 2)
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1026
; CHECK-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_LOAD_CONTINUE]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret void
;
loop.preheader:
br label %loop
loop:
%iv = phi i64 [ %iv.next, %loop ], [ 0, %loop.preheader ]
%a.gep = getelementptr i64, ptr %a, i64 %iv
%0 = load i64, ptr %a.gep
%2 = sdiv i64 %b, %b2
%3 = add i64 %0, %2
%c.gep = getelementptr i64, ptr %c, i64 %iv
store i64 %3, ptr %c.gep
%iv.next = add i64 %iv, 1
%done = icmp eq i64 %iv.next, 1025
br i1 %done, label %exit, label %loop
exit:
ret void
}
;.
; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]}
; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]}
; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]}
; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]}
;.
Reference in New Issue View Git Blame Copy Permalink