1d9775f684
The canonical form preferred by instcombine is to use 64-bit values for the index when it is a constant. We should try to do the same where possible in the loop vectoriser as this reduces churn in the compiler. It also makes other work easier, such as removing extra unnecessary passes on the RUN line in the test directory which I plan to do afterwards.
2899 lines
181 KiB
LLVM
2899 lines
181 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
|
|
; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -prefer-inloop-reductions -S | FileCheck %s
|
|
; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=4 -prefer-inloop-reductions -S | FileCheck %s --check-prefix=CHECK-INTERLEAVED
|
|
|
|
target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
|
|
|
|
define i32 @reduction_sum_single(ptr noalias nocapture %A) {
|
|
; CHECK-LABEL: define i32 @reduction_sum_single(
|
|
; CHECK-SAME: ptr noalias captures(none) [[A:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP2:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD]])
|
|
; CHECK-NEXT: [[TMP2]] = add i32 [[VEC_PHI]], [[TMP1]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: br [[DOT_CRIT_EDGE:label %.*]]
|
|
; CHECK: [[__CRIT_EDGE:.*:]]
|
|
; CHECK-NEXT: ret i32 [[TMP2]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define i32 @reduction_sum_single(
|
|
; CHECK-INTERLEAVED-SAME: ptr noalias captures(none) [[A:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3]] = add i32 [[VEC_PHI]], [[TMP2]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD2]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5]] = add i32 [[VEC_PHI1]], [[TMP4]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[SUM_0_LCSSA:%.*]] = add i32 [[TMP5]], [[TMP3]]
|
|
; CHECK-INTERLEAVED-NEXT: br [[DOT_CRIT_EDGE:label %.*]]
|
|
; CHECK-INTERLEAVED: [[__CRIT_EDGE:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_0_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %.lr.ph
|
|
|
|
.lr.ph:
|
|
%indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %entry ]
|
|
%sum.02 = phi i32 [ %l7, %.lr.ph ], [ 0, %entry ]
|
|
%l2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
|
|
%l3 = load i32, ptr %l2, align 4
|
|
%l7 = add i32 %sum.02, %l3
|
|
%indvars.iv.next = add i64 %indvars.iv, 1
|
|
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
|
|
%exitcond = icmp eq i32 %lftr.wideiv, 256
|
|
br i1 %exitcond, label %._crit_edge, label %.lr.ph
|
|
|
|
._crit_edge:
|
|
%sum.0.lcssa = phi i32 [ %l7, %.lr.ph ]
|
|
ret i32 %sum.0.lcssa
|
|
}
|
|
|
|
define i32 @reduction_sum(ptr noalias nocapture %A, ptr noalias nocapture %B) {
|
|
; CHECK-LABEL: define i32 @reduction_sum(
|
|
; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[VEC_IND]])
|
|
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[VEC_PHI]], [[TMP2]]
|
|
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD]])
|
|
; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[TMP3]], [[TMP4]]
|
|
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD1]])
|
|
; CHECK-NEXT: [[TMP7]] = add i32 [[TMP5]], [[TMP6]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
|
|
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: br [[DOT_CRIT_EDGE:label %.*]]
|
|
; CHECK: [[__CRIT_EDGE:.*:]]
|
|
; CHECK-NEXT: ret i32 [[TMP7]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define i32 @reduction_sum(
|
|
; CHECK-INTERLEAVED-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[VEC_IND]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = add i32 [[VEC_PHI]], [[TMP4]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[STEP_ADD]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = add i32 [[VEC_PHI1]], [[TMP6]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = add i32 [[TMP5]], [[TMP8]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD2]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = add i32 [[TMP7]], [[TMP10]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD3]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP13]] = add i32 [[TMP9]], [[TMP12]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD4]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP15]] = add i32 [[TMP11]], [[TMP14]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], splat (i32 4)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[SUM_0_LCSSA:%.*]] = add i32 [[TMP15]], [[TMP13]]
|
|
; CHECK-INTERLEAVED-NEXT: br [[DOT_CRIT_EDGE:label %.*]]
|
|
; CHECK-INTERLEAVED: [[__CRIT_EDGE:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_0_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %.lr.ph
|
|
|
|
.lr.ph:
|
|
%indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %entry ]
|
|
%sum.02 = phi i32 [ %l9, %.lr.ph ], [ 0, %entry ]
|
|
%l2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
|
|
%l3 = load i32, ptr %l2, align 4
|
|
%l4 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv
|
|
%l5 = load i32, ptr %l4, align 4
|
|
%l6 = trunc i64 %indvars.iv to i32
|
|
%l7 = add i32 %sum.02, %l6
|
|
%l8 = add i32 %l7, %l3
|
|
%l9 = add i32 %l8, %l5
|
|
%indvars.iv.next = add i64 %indvars.iv, 1
|
|
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
|
|
%exitcond = icmp eq i32 %lftr.wideiv, 256
|
|
br i1 %exitcond, label %._crit_edge, label %.lr.ph
|
|
|
|
._crit_edge:
|
|
%sum.0.lcssa = phi i32 [ %l9, %.lr.ph ]
|
|
ret i32 %sum.0.lcssa
|
|
}
|
|
|
|
define i32 @reduction_sum_const(ptr noalias nocapture %A) {
|
|
; CHECK-LABEL: define i32 @reduction_sum_const(
|
|
; CHECK-SAME: ptr noalias captures(none) [[A:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD]])
|
|
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[VEC_PHI]], [[TMP1]]
|
|
; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> splat (i32 3))
|
|
; CHECK-NEXT: [[TMP3]] = add i32 [[TMP2]], [[TMP5]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: br [[DOT_CRIT_EDGE:label %.*]]
|
|
; CHECK: [[__CRIT_EDGE:.*:]]
|
|
; CHECK-NEXT: ret i32 [[TMP3]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define i32 @reduction_sum_const(
|
|
; CHECK-INTERLEAVED-SAME: ptr noalias captures(none) [[A:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = add i32 [[VEC_PHI]], [[TMP2]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD2]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = add i32 [[VEC_PHI1]], [[TMP4]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> splat (i32 3))
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6]] = add i32 [[TMP3]], [[TMP9]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> splat (i32 3))
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7]] = add i32 [[TMP5]], [[TMP10]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[SUM_0_LCSSA:%.*]] = add i32 [[TMP7]], [[TMP6]]
|
|
; CHECK-INTERLEAVED-NEXT: br [[DOT_CRIT_EDGE:label %.*]]
|
|
; CHECK-INTERLEAVED: [[__CRIT_EDGE:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_0_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %.lr.ph
|
|
|
|
.lr.ph:
|
|
%indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %entry ]
|
|
%sum.02 = phi i32 [ %l9, %.lr.ph ], [ 0, %entry ]
|
|
%l2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
|
|
%l3 = load i32, ptr %l2, align 4
|
|
%l7 = add i32 %sum.02, %l3
|
|
%l9 = add i32 %l7, 3
|
|
%indvars.iv.next = add i64 %indvars.iv, 1
|
|
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
|
|
%exitcond = icmp eq i32 %lftr.wideiv, 256
|
|
br i1 %exitcond, label %._crit_edge, label %.lr.ph
|
|
|
|
._crit_edge:
|
|
%sum.0.lcssa = phi i32 [ %l9, %.lr.ph ]
|
|
ret i32 %sum.0.lcssa
|
|
}
|
|
|
|
define i32 @reduction_prod(ptr noalias nocapture %A, ptr noalias nocapture %B) {
|
|
; CHECK-LABEL: define i32 @reduction_prod(
|
|
; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 1, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[VEC_IND]])
|
|
; CHECK-NEXT: [[TMP3:%.*]] = mul i32 [[VEC_PHI]], [[TMP2]]
|
|
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[WIDE_LOAD]])
|
|
; CHECK-NEXT: [[TMP5:%.*]] = mul i32 [[TMP3]], [[TMP4]]
|
|
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[WIDE_LOAD1]])
|
|
; CHECK-NEXT: [[TMP7]] = mul i32 [[TMP5]], [[TMP6]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
|
|
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: br [[DOT_CRIT_EDGE:label %.*]]
|
|
; CHECK: [[__CRIT_EDGE:.*:]]
|
|
; CHECK-NEXT: ret i32 [[TMP7]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define i32 @reduction_prod(
|
|
; CHECK-INTERLEAVED-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 1, %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 1, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[VEC_IND]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = mul i32 [[VEC_PHI]], [[TMP4]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[STEP_ADD]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = mul i32 [[VEC_PHI1]], [[TMP6]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[WIDE_LOAD]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = mul i32 [[TMP5]], [[TMP8]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[WIDE_LOAD2]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = mul i32 [[TMP7]], [[TMP10]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[WIDE_LOAD3]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP13]] = mul i32 [[TMP9]], [[TMP12]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[WIDE_LOAD4]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP15]] = mul i32 [[TMP11]], [[TMP14]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], splat (i32 4)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[PROD_0_LCSSA:%.*]] = mul i32 [[TMP15]], [[TMP13]]
|
|
; CHECK-INTERLEAVED-NEXT: br [[DOT_CRIT_EDGE:label %.*]]
|
|
; CHECK-INTERLEAVED: [[__CRIT_EDGE:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: ret i32 [[PROD_0_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %.lr.ph
|
|
|
|
.lr.ph:
|
|
%indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %entry ]
|
|
%prod.02 = phi i32 [ %l9, %.lr.ph ], [ 1, %entry ]
|
|
%l2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
|
|
%l3 = load i32, ptr %l2, align 4
|
|
%l4 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv
|
|
%l5 = load i32, ptr %l4, align 4
|
|
%l6 = trunc i64 %indvars.iv to i32
|
|
%l7 = mul i32 %prod.02, %l6
|
|
%l8 = mul i32 %l7, %l3
|
|
%l9 = mul i32 %l8, %l5
|
|
%indvars.iv.next = add i64 %indvars.iv, 1
|
|
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
|
|
%exitcond = icmp eq i32 %lftr.wideiv, 256
|
|
br i1 %exitcond, label %._crit_edge, label %.lr.ph
|
|
|
|
._crit_edge:
|
|
%prod.0.lcssa = phi i32 [ %l9, %.lr.ph ]
|
|
ret i32 %prod.0.lcssa
|
|
}
|
|
|
|
define i32 @reduction_mix(ptr noalias nocapture %A, ptr noalias nocapture %B) {
|
|
; CHECK-LABEL: define i32 @reduction_mix(
|
|
; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4
|
|
; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[VEC_IND]])
|
|
; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[VEC_PHI]], [[TMP3]]
|
|
; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD2]], [[WIDE_LOAD1]]
|
|
; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]])
|
|
; CHECK-NEXT: [[TMP6]] = add i32 [[TMP4]], [[TMP5]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
|
|
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: br [[DOT_CRIT_EDGE:label %.*]]
|
|
; CHECK: [[__CRIT_EDGE:.*:]]
|
|
; CHECK-NEXT: ret i32 [[TMP6]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define i32 @reduction_mix(
|
|
; CHECK-INTERLEAVED-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[VEC_IND]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = add i32 [[VEC_PHI]], [[TMP4]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[STEP_ADD]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = add i32 [[VEC_PHI1]], [[TMP6]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD3]], [[WIDE_LOAD]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP8]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10]] = add i32 [[TMP5]], [[TMP9]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD4]], [[WIDE_LOAD2]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP11]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP13]] = add i32 [[TMP7]], [[TMP12]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], splat (i32 4)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[SUM_0_LCSSA:%.*]] = add i32 [[TMP13]], [[TMP10]]
|
|
; CHECK-INTERLEAVED-NEXT: br [[DOT_CRIT_EDGE:label %.*]]
|
|
; CHECK-INTERLEAVED: [[__CRIT_EDGE:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_0_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %.lr.ph
|
|
|
|
.lr.ph:
|
|
%indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %entry ]
|
|
%sum.02 = phi i32 [ %l9, %.lr.ph ], [ 0, %entry ]
|
|
%l2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
|
|
%l3 = load i32, ptr %l2, align 4
|
|
%l4 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv
|
|
%l5 = load i32, ptr %l4, align 4
|
|
%l6 = mul nsw i32 %l5, %l3
|
|
%l7 = trunc i64 %indvars.iv to i32
|
|
%l8 = add i32 %sum.02, %l7
|
|
%l9 = add i32 %l8, %l6
|
|
%indvars.iv.next = add i64 %indvars.iv, 1
|
|
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
|
|
%exitcond = icmp eq i32 %lftr.wideiv, 256
|
|
br i1 %exitcond, label %._crit_edge, label %.lr.ph
|
|
|
|
._crit_edge:
|
|
%sum.0.lcssa = phi i32 [ %l9, %.lr.ph ]
|
|
ret i32 %sum.0.lcssa
|
|
}
|
|
|
|
define i32 @reduction_mul(ptr noalias nocapture %A, ptr noalias nocapture %B) {
|
|
; CHECK-LABEL: define i32 @reduction_mul(
|
|
; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 19, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[WIDE_LOAD]])
|
|
; CHECK-NEXT: [[TMP3:%.*]] = mul i32 [[VEC_PHI]], [[TMP2]]
|
|
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[WIDE_LOAD1]])
|
|
; CHECK-NEXT: [[TMP5]] = mul i32 [[TMP3]], [[TMP4]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: br [[DOT_CRIT_EDGE:label %.*]]
|
|
; CHECK: [[__CRIT_EDGE:.*:]]
|
|
; CHECK-NEXT: ret i32 [[TMP5]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define i32 @reduction_mul(
|
|
; CHECK-INTERLEAVED-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 19, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 1, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[WIDE_LOAD]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = mul i32 [[VEC_PHI]], [[TMP4]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[WIDE_LOAD2]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = mul i32 [[VEC_PHI1]], [[TMP6]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[WIDE_LOAD3]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9]] = mul i32 [[TMP5]], [[TMP8]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> [[WIDE_LOAD4]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP11]] = mul i32 [[TMP7]], [[TMP10]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[SUM_0_LCSSA:%.*]] = mul i32 [[TMP11]], [[TMP9]]
|
|
; CHECK-INTERLEAVED-NEXT: br [[DOT_CRIT_EDGE:label %.*]]
|
|
; CHECK-INTERLEAVED: [[__CRIT_EDGE:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_0_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %.lr.ph
|
|
|
|
.lr.ph:
|
|
%indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %entry ]
|
|
%sum.02 = phi i32 [ %l7, %.lr.ph ], [ 19, %entry ]
|
|
%l2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
|
|
%l3 = load i32, ptr %l2, align 4
|
|
%l4 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv
|
|
%l5 = load i32, ptr %l4, align 4
|
|
%l6 = mul i32 %sum.02, %l3
|
|
%l7 = mul i32 %l6, %l5
|
|
%indvars.iv.next = add i64 %indvars.iv, 1
|
|
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
|
|
%exitcond = icmp eq i32 %lftr.wideiv, 256
|
|
br i1 %exitcond, label %._crit_edge, label %.lr.ph
|
|
|
|
._crit_edge:
|
|
%sum.0.lcssa = phi i32 [ %l7, %.lr.ph ]
|
|
ret i32 %sum.0.lcssa
|
|
}
|
|
|
|
define i32 @start_at_non_zero(ptr nocapture %in, ptr nocapture %coeff, ptr nocapture %out) {
|
|
; CHECK-LABEL: define i32 @start_at_non_zero(
|
|
; CHECK-SAME: ptr captures(none) [[IN:%.*]], ptr captures(none) [[COEFF:%.*]], ptr captures(none) [[OUT:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 120, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[IN]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[COEFF]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP6]], align 4
|
|
; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD2]], [[WIDE_LOAD1]]
|
|
; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]])
|
|
; CHECK-NEXT: [[TMP4]] = add i32 [[VEC_PHI]], [[TMP3]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: br label %[[FOR_END:.*]]
|
|
; CHECK: [[FOR_END]]:
|
|
; CHECK-NEXT: ret i32 [[TMP4]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define i32 @start_at_non_zero(
|
|
; CHECK-INTERLEAVED-SAME: ptr captures(none) [[IN:%.*]], ptr captures(none) [[COEFF:%.*]], ptr captures(none) [[OUT:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 120, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[IN]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[COEFF]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD3]], [[WIDE_LOAD]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6]] = add i32 [[VEC_PHI]], [[TMP5]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD4]], [[WIDE_LOAD2]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9]] = add i32 [[VEC_PHI1]], [[TMP8]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add i32 [[TMP9]], [[TMP6]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_END:.*]]
|
|
; CHECK-INTERLEAVED: [[FOR_END]]:
|
|
; CHECK-INTERLEAVED-NEXT: ret i32 [[BIN_RDX]]
|
|
;
|
|
entry:
|
|
br label %for.body
|
|
|
|
for.body:
|
|
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
|
|
%sum.09 = phi i32 [ %add, %for.body ], [ 120, %entry ]
|
|
%arrayidx = getelementptr inbounds i32, ptr %in, i64 %indvars.iv
|
|
%l0 = load i32, ptr %arrayidx, align 4
|
|
%arrayidx2 = getelementptr inbounds i32, ptr %coeff, i64 %indvars.iv
|
|
%l1 = load i32, ptr %arrayidx2, align 4
|
|
%mul = mul nsw i32 %l1, %l0
|
|
%add = add nsw i32 %mul, %sum.09
|
|
%indvars.iv.next = add i64 %indvars.iv, 1
|
|
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
|
|
%exitcond = icmp eq i32 %lftr.wideiv, 256
|
|
br i1 %exitcond, label %for.end, label %for.body
|
|
|
|
for.end:
|
|
%sum.0.lcssa = phi i32 [ %add, %for.body ]
|
|
ret i32 %sum.0.lcssa
|
|
}
|
|
|
|
define i32 @reduction_and(ptr nocapture %A, ptr nocapture %B) {
|
|
; CHECK-LABEL: define i32 @reduction_and(
|
|
; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ -1, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[WIDE_LOAD]])
|
|
; CHECK-NEXT: [[TMP3:%.*]] = and i32 [[VEC_PHI]], [[TMP2]]
|
|
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[WIDE_LOAD1]])
|
|
; CHECK-NEXT: [[TMP5]] = and i32 [[TMP3]], [[TMP4]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: br label %[[FOR_END:.*]]
|
|
; CHECK: [[FOR_END]]:
|
|
; CHECK-NEXT: ret i32 [[TMP5]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define i32 @reduction_and(
|
|
; CHECK-INTERLEAVED-SAME: ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi i32 [ -1, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ -1, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[WIDE_LOAD]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = and i32 [[VEC_PHI]], [[TMP4]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[WIDE_LOAD2]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = and i32 [[VEC_PHI1]], [[TMP6]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[WIDE_LOAD3]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9]] = and i32 [[TMP5]], [[TMP8]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[WIDE_LOAD4]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP11]] = and i32 [[TMP7]], [[TMP10]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[RESULT_0_LCSSA:%.*]] = and i32 [[TMP11]], [[TMP9]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_END:.*]]
|
|
; CHECK-INTERLEAVED: [[FOR_END]]:
|
|
; CHECK-INTERLEAVED-NEXT: ret i32 [[RESULT_0_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %for.body
|
|
|
|
for.body:
|
|
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
|
|
%result.08 = phi i32 [ %and, %for.body ], [ -1, %entry ]
|
|
%arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
|
|
%l0 = load i32, ptr %arrayidx, align 4
|
|
%arrayidx2 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv
|
|
%l1 = load i32, ptr %arrayidx2, align 4
|
|
%add = and i32 %result.08, %l0
|
|
%and = and i32 %add, %l1
|
|
%indvars.iv.next = add i64 %indvars.iv, 1
|
|
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
|
|
%exitcond = icmp eq i32 %lftr.wideiv, 256
|
|
br i1 %exitcond, label %for.end, label %for.body
|
|
|
|
for.end:
|
|
%result.0.lcssa = phi i32 [ %and, %for.body ]
|
|
ret i32 %result.0.lcssa
|
|
}
|
|
|
|
define i32 @reduction_or(ptr nocapture %A, ptr nocapture %B) {
|
|
; CHECK-LABEL: define i32 @reduction_or(
|
|
; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]
|
|
; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP2]])
|
|
; CHECK-NEXT: [[TMP4]] = or i32 [[VEC_PHI]], [[TMP3]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: br label %[[FOR_END:.*]]
|
|
; CHECK: [[FOR_END]]:
|
|
; CHECK-NEXT: ret i32 [[TMP4]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define i32 @reduction_or(
|
|
; CHECK-INTERLEAVED-SAME: ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> [[WIDE_LOAD3]], [[WIDE_LOAD]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD4]], [[WIDE_LOAD2]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP4]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7]] = or i32 [[VEC_PHI]], [[TMP6]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[TMP5]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9]] = or i32 [[VEC_PHI1]], [[TMP8]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[RESULT_0_LCSSA:%.*]] = or i32 [[TMP9]], [[TMP7]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_END:.*]]
|
|
; CHECK-INTERLEAVED: [[FOR_END]]:
|
|
; CHECK-INTERLEAVED-NEXT: ret i32 [[RESULT_0_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %for.body
|
|
|
|
for.body:
|
|
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
|
|
%result.08 = phi i32 [ %or, %for.body ], [ 0, %entry ]
|
|
%arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
|
|
%l0 = load i32, ptr %arrayidx, align 4
|
|
%arrayidx2 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv
|
|
%l1 = load i32, ptr %arrayidx2, align 4
|
|
%add = add nsw i32 %l1, %l0
|
|
%or = or i32 %add, %result.08
|
|
%indvars.iv.next = add i64 %indvars.iv, 1
|
|
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
|
|
%exitcond = icmp eq i32 %lftr.wideiv, 256
|
|
br i1 %exitcond, label %for.end, label %for.body
|
|
|
|
for.end:
|
|
%result.0.lcssa = phi i32 [ %or, %for.body ]
|
|
ret i32 %result.0.lcssa
|
|
}
|
|
|
|
define i32 @reduction_xor(ptr nocapture %A, ptr nocapture %B) {
|
|
; CHECK-LABEL: define i32 @reduction_xor(
|
|
; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]
|
|
; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> [[TMP2]])
|
|
; CHECK-NEXT: [[TMP4]] = xor i32 [[VEC_PHI]], [[TMP3]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: br label %[[FOR_END:.*]]
|
|
; CHECK: [[FOR_END]]:
|
|
; CHECK-NEXT: ret i32 [[TMP4]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define i32 @reduction_xor(
|
|
; CHECK-INTERLEAVED-SAME: ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = add nsw <4 x i32> [[WIDE_LOAD3]], [[WIDE_LOAD]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = add nsw <4 x i32> [[WIDE_LOAD4]], [[WIDE_LOAD2]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> [[TMP4]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7]] = xor i32 [[VEC_PHI]], [[TMP6]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> [[TMP5]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9]] = xor i32 [[VEC_PHI1]], [[TMP8]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[RESULT_0_LCSSA:%.*]] = xor i32 [[TMP9]], [[TMP7]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_END:.*]]
|
|
; CHECK-INTERLEAVED: [[FOR_END]]:
|
|
; CHECK-INTERLEAVED-NEXT: ret i32 [[RESULT_0_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %for.body
|
|
|
|
for.body:
|
|
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
|
|
%result.08 = phi i32 [ %xor, %for.body ], [ 0, %entry ]
|
|
%arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
|
|
%l0 = load i32, ptr %arrayidx, align 4
|
|
%arrayidx2 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv
|
|
%l1 = load i32, ptr %arrayidx2, align 4
|
|
%add = add nsw i32 %l1, %l0
|
|
%xor = xor i32 %add, %result.08
|
|
%indvars.iv.next = add i64 %indvars.iv, 1
|
|
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
|
|
%exitcond = icmp eq i32 %lftr.wideiv, 256
|
|
br i1 %exitcond, label %for.end, label %for.body
|
|
|
|
for.end:
|
|
%result.0.lcssa = phi i32 [ %xor, %for.body ]
|
|
ret i32 %result.0.lcssa
|
|
}
|
|
|
|
define float @reduction_fadd(ptr nocapture %A, ptr nocapture %B) {
|
|
; CHECK-LABEL: define float @reduction_fadd(
|
|
; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP3:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
|
|
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
|
|
; CHECK-NEXT: [[TMP2:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[WIDE_LOAD]])
|
|
; CHECK-NEXT: [[TMP5:%.*]] = fadd fast float [[VEC_PHI]], [[TMP2]]
|
|
; CHECK-NEXT: [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[WIDE_LOAD1]])
|
|
; CHECK-NEXT: [[TMP3]] = fadd fast float [[TMP5]], [[TMP6]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: br label %[[FOR_END:.*]]
|
|
; CHECK: [[FOR_END]]:
|
|
; CHECK-NEXT: ret float [[TMP3]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define float @reduction_fadd(
|
|
; CHECK-INTERLEAVED-SAME: ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[TMP0]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[TMP2]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[WIDE_LOAD]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = fadd fast float [[VEC_PHI]], [[TMP4]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[WIDE_LOAD2]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = fadd fast float [[VEC_PHI1]], [[TMP9]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[WIDE_LOAD3]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6]] = fadd fast float [[TMP5]], [[TMP12]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[WIDE_LOAD4]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7]] = fadd fast float [[TMP11]], [[TMP10]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[RESULT_0_LCSSA:%.*]] = fadd fast float [[TMP7]], [[TMP6]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_END:.*]]
|
|
; CHECK-INTERLEAVED: [[FOR_END]]:
|
|
; CHECK-INTERLEAVED-NEXT: ret float [[RESULT_0_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %for.body
|
|
|
|
for.body:
|
|
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
|
|
%result.08 = phi float [ %fadd, %for.body ], [ 0.0, %entry ]
|
|
%arrayidx = getelementptr inbounds float, ptr %A, i64 %indvars.iv
|
|
%l0 = load float, ptr %arrayidx, align 4
|
|
%arrayidx2 = getelementptr inbounds float, ptr %B, i64 %indvars.iv
|
|
%l1 = load float, ptr %arrayidx2, align 4
|
|
%add = fadd fast float %result.08, %l0
|
|
%fadd = fadd fast float %add, %l1
|
|
%indvars.iv.next = add i64 %indvars.iv, 1
|
|
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
|
|
%exitcond = icmp eq i32 %lftr.wideiv, 256
|
|
br i1 %exitcond, label %for.end, label %for.body
|
|
|
|
for.end:
|
|
%result.0.lcssa = phi float [ %fadd, %for.body ]
|
|
ret float %result.0.lcssa
|
|
}
|
|
|
|
define float @reduction_fmul(ptr nocapture %A, ptr nocapture %B) {
|
|
; CHECK-LABEL: define float @reduction_fmul(
|
|
; CHECK-SAME: ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
|
|
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
|
|
; CHECK-NEXT: [[TMP2:%.*]] = call fast float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[WIDE_LOAD]])
|
|
; CHECK-NEXT: [[TMP3:%.*]] = fmul fast float [[VEC_PHI]], [[TMP2]]
|
|
; CHECK-NEXT: [[TMP4:%.*]] = call fast float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[WIDE_LOAD1]])
|
|
; CHECK-NEXT: [[TMP5]] = fmul fast float [[TMP3]], [[TMP4]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: br label %[[FOR_END:.*]]
|
|
; CHECK: [[FOR_END]]:
|
|
; CHECK-NEXT: ret float [[TMP5]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define float @reduction_fmul(
|
|
; CHECK-INTERLEAVED-SAME: ptr captures(none) [[A:%.*]], ptr captures(none) [[B:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi float [ 1.000000e+00, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[TMP0]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[TMP2]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = call fast float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[WIDE_LOAD]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = fmul fast float [[VEC_PHI]], [[TMP4]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[WIDE_LOAD2]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = fmul fast float [[VEC_PHI1]], [[TMP6]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = call fast float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[WIDE_LOAD3]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9]] = fmul fast float [[TMP5]], [[TMP8]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = call fast float @llvm.vector.reduce.fmul.v4f32(float 1.000000e+00, <4 x float> [[WIDE_LOAD4]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP11]] = fmul fast float [[TMP7]], [[TMP10]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[RESULT_0_LCSSA:%.*]] = fmul fast float [[TMP11]], [[TMP9]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_END:.*]]
|
|
; CHECK-INTERLEAVED: [[FOR_END]]:
|
|
; CHECK-INTERLEAVED-NEXT: ret float [[RESULT_0_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %for.body
|
|
|
|
for.body:
|
|
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
|
|
%result.08 = phi float [ %fmul, %for.body ], [ 0.0, %entry ]
|
|
%arrayidx = getelementptr inbounds float, ptr %A, i64 %indvars.iv
|
|
%l0 = load float, ptr %arrayidx, align 4
|
|
%arrayidx2 = getelementptr inbounds float, ptr %B, i64 %indvars.iv
|
|
%l1 = load float, ptr %arrayidx2, align 4
|
|
%add = fmul fast float %result.08, %l0
|
|
%fmul = fmul fast float %add, %l1
|
|
%indvars.iv.next = add i64 %indvars.iv, 1
|
|
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
|
|
%exitcond = icmp eq i32 %lftr.wideiv, 256
|
|
br i1 %exitcond, label %for.end, label %for.body
|
|
|
|
for.end:
|
|
%result.0.lcssa = phi float [ %fmul, %for.body ]
|
|
ret float %result.0.lcssa
|
|
}
|
|
|
|
; We can create an in-loop reduction for sub-reductions with a sub and add-reduction
|
|
define i32 @reduction_sub_lhs(ptr noalias nocapture %A) {
|
|
; CHECK-LABEL: define i32 @reduction_sub_lhs(
|
|
; CHECK-SAME: ptr noalias captures(none) [[A:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 3, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD]])
|
|
; CHECK-NEXT: [[TMP5]] = sub i32 [[VEC_PHI]], [[TMP1]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: br label %[[FOR_END:.*]]
|
|
; CHECK: [[FOR_END]]:
|
|
; CHECK-NEXT: ret i32 [[TMP5]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define i32 @reduction_sub_lhs(
|
|
; CHECK-INTERLEAVED-SAME: ptr noalias captures(none) [[A:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 3, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI2:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7]] = sub i32 [[VEC_PHI1]], [[TMP4]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD2]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5]] = sub i32 [[VEC_PHI2]], [[TMP6]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add i32 [[TMP5]], [[TMP7]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_END:.*]]
|
|
; CHECK-INTERLEAVED: [[FOR_END]]:
|
|
; CHECK-INTERLEAVED-NEXT: ret i32 [[BIN_RDX]]
|
|
;
|
|
entry:
|
|
br label %for.body
|
|
|
|
for.body:
|
|
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
|
|
%x.05 = phi i32 [ %sub, %for.body ], [ 3, %entry ]
|
|
%arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
|
|
%l0 = load i32, ptr %arrayidx, align 4
|
|
%sub = sub nsw i32 %x.05, %l0
|
|
%indvars.iv.next = add i64 %indvars.iv, 1
|
|
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
|
|
%exitcond = icmp eq i32 %lftr.wideiv, 256
|
|
br i1 %exitcond, label %for.end, label %for.body
|
|
|
|
for.end:
|
|
%x.0.lcssa = phi i32 [ %sub, %for.body ]
|
|
ret i32 %x.0.lcssa
|
|
}
|
|
|
|
; Conditional reductions with multi-input phis.
|
|
define float @reduction_conditional(ptr %A, ptr %B, ptr %C, float %S) {
|
|
; CHECK-LABEL: define float @reduction_conditional(
|
|
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], float [[S:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x float> zeroinitializer, float [[S]], i32 0
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[PREDPHI3:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
|
|
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
|
|
; CHECK-NEXT: [[TMP3:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
|
|
; CHECK-NEXT: [[TMP4:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD1]], splat (float 1.000000e+00)
|
|
; CHECK-NEXT: [[TMP8:%.*]] = xor <4 x i1> [[TMP4]], splat (i1 true)
|
|
; CHECK-NEXT: [[TMP10:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[TMP8]], <4 x i1> zeroinitializer
|
|
; CHECK-NEXT: [[TMP6:%.*]] = fcmp ule <4 x float> [[WIDE_LOAD]], splat (float 2.000000e+00)
|
|
; CHECK-NEXT: [[TMP7:%.*]] = fadd fast <4 x float> [[VEC_PHI]], [[WIDE_LOAD1]]
|
|
; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP3]], <4 x i1> [[TMP4]], <4 x i1> zeroinitializer
|
|
; CHECK-NEXT: [[TMP9:%.*]] = fadd fast <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]
|
|
; CHECK-NEXT: [[TMP11:%.*]] = select <4 x i1> [[TMP10]], <4 x i1> [[TMP6]], <4 x i1> zeroinitializer
|
|
; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP11]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP7]]
|
|
; CHECK-NEXT: [[PREDPHI2:%.*]] = select <4 x i1> [[TMP5]], <4 x float> [[TMP9]], <4 x float> [[PREDPHI]]
|
|
; CHECK-NEXT: [[PREDPHI3]] = select <4 x i1> [[TMP3]], <4 x float> [[PREDPHI2]], <4 x float> [[VEC_PHI]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
|
|
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: [[TMP13:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[PREDPHI3]])
|
|
; CHECK-NEXT: br label %[[FOR_END:.*]]
|
|
; CHECK: [[FOR_END]]:
|
|
; CHECK-NEXT: ret float [[TMP13]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define float @reduction_conditional(
|
|
; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], float [[S:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = insertelement <4 x float> zeroinitializer, float [[S]], i32 0
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[PREDPHI6:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <4 x float> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI9:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[TMP3]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x float>, ptr [[TMP4]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD3]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD4]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD3]], splat (float 1.000000e+00)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD4]], splat (float 1.000000e+00)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = xor <4 x i1> [[TMP7]], splat (i1 true)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = xor <4 x i1> [[TMP8]], splat (i1 true)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> [[TMP9]], <4 x i1> zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP21:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> [[TMP10]], <4 x i1> zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = fcmp ule <4 x float> [[WIDE_LOAD]], splat (float 2.000000e+00)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = fcmp ule <4 x float> [[WIDE_LOAD2]], splat (float 2.000000e+00)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = fadd fast <4 x float> [[VEC_PHI]], [[WIDE_LOAD3]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = fadd fast <4 x float> [[VEC_PHI1]], [[WIDE_LOAD4]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> [[TMP7]], <4 x i1> zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> [[TMP8]], <4 x i1> zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = fadd fast <4 x float> [[VEC_PHI]], [[WIDE_LOAD]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = fadd fast <4 x float> [[VEC_PHI1]], [[WIDE_LOAD2]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP20:%.*]] = select <4 x i1> [[TMP19]], <4 x i1> [[TMP11]], <4 x i1> zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP22:%.*]] = select <4 x i1> [[TMP21]], <4 x i1> [[TMP12]], <4 x i1> zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP20]], <4 x float> [[VEC_PHI]], <4 x float> [[TMP13]]
|
|
; CHECK-INTERLEAVED-NEXT: [[PREDPHI5:%.*]] = select <4 x i1> [[TMP15]], <4 x float> [[TMP17]], <4 x float> [[PREDPHI]]
|
|
; CHECK-INTERLEAVED-NEXT: [[PREDPHI6]] = select <4 x i1> [[TMP5]], <4 x float> [[PREDPHI5]], <4 x float> [[VEC_PHI]]
|
|
; CHECK-INTERLEAVED-NEXT: [[PREDPHI7:%.*]] = select <4 x i1> [[TMP22]], <4 x float> [[VEC_PHI1]], <4 x float> [[TMP14]]
|
|
; CHECK-INTERLEAVED-NEXT: [[PREDPHI8:%.*]] = select <4 x i1> [[TMP16]], <4 x float> [[TMP18]], <4 x float> [[PREDPHI7]]
|
|
; CHECK-INTERLEAVED-NEXT: [[PREDPHI9]] = select <4 x i1> [[TMP6]], <4 x float> [[PREDPHI8]], <4 x float> [[VEC_PHI1]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = fadd fast <4 x float> [[PREDPHI9]], [[PREDPHI6]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP24:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[BIN_RDX]])
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_END:.*]]
|
|
; CHECK-INTERLEAVED: [[FOR_END]]:
|
|
; CHECK-INTERLEAVED-NEXT: ret float [[TMP24]]
|
|
;
|
|
entry:
|
|
br label %for.body
|
|
|
|
for.body:
|
|
%indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.inc ]
|
|
%sum.033 = phi float [ %S, %entry ], [ %sum.1, %for.inc ]
|
|
%arrayidx = getelementptr inbounds float, ptr %A, i64 %indvars.iv
|
|
%l0 = load float, ptr %arrayidx, align 4
|
|
%arrayidx2 = getelementptr inbounds float, ptr %B, i64 %indvars.iv
|
|
%l1 = load float, ptr %arrayidx2, align 4
|
|
%cmp3 = fcmp ogt float %l0, %l1
|
|
br i1 %cmp3, label %if.then, label %for.inc
|
|
|
|
if.then:
|
|
%cmp6 = fcmp ogt float %l1, 1.000000e+00
|
|
br i1 %cmp6, label %if.then8, label %if.else
|
|
|
|
if.then8:
|
|
%add = fadd fast float %sum.033, %l0
|
|
br label %for.inc
|
|
|
|
if.else:
|
|
%cmp14 = fcmp ogt float %l0, 2.000000e+00
|
|
br i1 %cmp14, label %if.then16, label %for.inc
|
|
|
|
if.then16:
|
|
%add19 = fadd fast float %sum.033, %l1
|
|
br label %for.inc
|
|
|
|
for.inc:
|
|
%sum.1 = phi float [ %add, %if.then8 ], [ %add19, %if.then16 ], [ %sum.033, %if.else ], [ %sum.033, %for.body ]
|
|
%indvars.iv.next = add i64 %indvars.iv, 1
|
|
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
|
|
%exitcond = icmp ne i32 %lftr.wideiv, 128
|
|
br i1 %exitcond, label %for.body, label %for.end
|
|
|
|
for.end:
|
|
%sum.1.lcssa = phi float [ %sum.1, %for.inc ]
|
|
ret float %sum.1.lcssa
|
|
}
|
|
|
|
define i32 @reduction_sum_multiuse(ptr noalias nocapture %A, ptr noalias nocapture %B) {
|
|
; CHECK-LABEL: define i32 @reduction_sum_multiuse(
|
|
; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]]) {
|
|
; CHECK-NEXT: [[_LR_PH1:.*]]:
|
|
; CHECK-NEXT: br label %[[DOTLR_PH:.*]]
|
|
; CHECK: [[_LR_PH:.*:]]
|
|
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[DOTLR_PH]] ], [ 0, %[[_LR_PH1]] ]
|
|
; CHECK-NEXT: [[SUM_02:%.*]] = phi i32 [ [[L10:%.*]], %[[DOTLR_PH]] ], [ 0, %[[_LR_PH1]] ]
|
|
; CHECK-NEXT: [[L2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
|
|
; CHECK-NEXT: [[L3:%.*]] = load i32, ptr [[L2]], align 4
|
|
; CHECK-NEXT: [[L4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]
|
|
; CHECK-NEXT: [[L5:%.*]] = load i32, ptr [[L4]], align 4
|
|
; CHECK-NEXT: [[L6:%.*]] = trunc i64 [[INDVARS_IV]] to i32
|
|
; CHECK-NEXT: [[L7:%.*]] = add i32 [[SUM_02]], [[L6]]
|
|
; CHECK-NEXT: [[L8:%.*]] = add i32 [[L7]], [[L3]]
|
|
; CHECK-NEXT: [[L9:%.*]] = add i32 [[L8]], [[L5]]
|
|
; CHECK-NEXT: [[L10]] = add i32 [[L8]], [[SUM_02]]
|
|
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
|
|
; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
|
|
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 256
|
|
; CHECK-NEXT: br i1 [[EXITCOND]], label %[[END:.*]], label %[[DOTLR_PH]]
|
|
; CHECK: [[END]]:
|
|
; CHECK-NEXT: [[F1:%.*]] = phi i32 [ [[L10]], %[[DOTLR_PH]] ]
|
|
; CHECK-NEXT: ret i32 [[F1]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define i32 @reduction_sum_multiuse(
|
|
; CHECK-INTERLEAVED-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[_LR_PH1:.*]]:
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[DOTLR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[_LR_PH:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[DOTLR_PH]] ], [ 0, %[[_LR_PH1]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[SUM_02:%.*]] = phi i32 [ [[L10:%.*]], %[[DOTLR_PH]] ], [ 0, %[[_LR_PH1]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[L2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV]]
|
|
; CHECK-INTERLEAVED-NEXT: [[L3:%.*]] = load i32, ptr [[L2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[L4:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]]
|
|
; CHECK-INTERLEAVED-NEXT: [[L5:%.*]] = load i32, ptr [[L4]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[L6:%.*]] = trunc i64 [[INDVARS_IV]] to i32
|
|
; CHECK-INTERLEAVED-NEXT: [[L7:%.*]] = add i32 [[SUM_02]], [[L6]]
|
|
; CHECK-INTERLEAVED-NEXT: [[L8:%.*]] = add i32 [[L7]], [[L3]]
|
|
; CHECK-INTERLEAVED-NEXT: [[L9:%.*]] = add i32 [[L8]], [[L5]]
|
|
; CHECK-INTERLEAVED-NEXT: [[L10]] = add i32 [[L8]], [[SUM_02]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
|
|
; CHECK-INTERLEAVED-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
|
|
; CHECK-INTERLEAVED-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], 256
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND]], label %[[END:.*]], label %[[DOTLR_PH]]
|
|
; CHECK-INTERLEAVED: [[END]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[F1:%.*]] = phi i32 [ [[L10]], %[[DOTLR_PH]] ]
|
|
; CHECK-INTERLEAVED-NEXT: ret i32 [[F1]]
|
|
;
|
|
entry:
|
|
br label %.lr.ph
|
|
|
|
.lr.ph:
|
|
%indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %entry ]
|
|
%sum.02 = phi i32 [ %l10, %.lr.ph ], [ 0, %entry ]
|
|
%l2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
|
|
%l3 = load i32, ptr %l2, align 4
|
|
%l4 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv
|
|
%l5 = load i32, ptr %l4, align 4
|
|
%l6 = trunc i64 %indvars.iv to i32
|
|
%l7 = add i32 %sum.02, %l6
|
|
%l8 = add i32 %l7, %l3
|
|
%l9 = add i32 %l8, %l5
|
|
%l10 = add i32 %l8, %sum.02
|
|
%indvars.iv.next = add i64 %indvars.iv, 1
|
|
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
|
|
%exitcond = icmp eq i32 %lftr.wideiv, 256
|
|
br i1 %exitcond, label %end, label %.lr.ph
|
|
|
|
end:
|
|
%f1 = phi i32 [ %l10, %.lr.ph ]
|
|
ret i32 %f1
|
|
}
|
|
|
|
; Predicated loop, cannot (yet) use in-loop reductions.
|
|
define i32 @reduction_predicated(ptr noalias nocapture %A, ptr noalias nocapture %B) {
|
|
; CHECK-LABEL: define i32 @reduction_predicated(
|
|
; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[VEC_IND]])
|
|
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[VEC_PHI]], [[TMP2]]
|
|
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD]])
|
|
; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[TMP3]], [[TMP4]]
|
|
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD1]])
|
|
; CHECK-NEXT: [[TMP7]] = add i32 [[TMP5]], [[TMP6]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
|
|
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: br [[DOT_CRIT_EDGE:label %.*]]
|
|
; CHECK: [[__CRIT_EDGE:.*:]]
|
|
; CHECK-NEXT: ret i32 [[TMP7]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define i32 @reduction_predicated(
|
|
; CHECK-INTERLEAVED-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[VEC_IND]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = add i32 [[VEC_PHI]], [[TMP4]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[STEP_ADD]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = add i32 [[VEC_PHI1]], [[TMP6]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = add i32 [[TMP5]], [[TMP8]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD2]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = add i32 [[TMP7]], [[TMP10]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD3]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP13]] = add i32 [[TMP9]], [[TMP12]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD4]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP15]] = add i32 [[TMP11]], [[TMP14]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[STEP_ADD]], splat (i32 4)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[SUM_0_LCSSA:%.*]] = add i32 [[TMP15]], [[TMP13]]
|
|
; CHECK-INTERLEAVED-NEXT: br [[DOT_CRIT_EDGE:label %.*]]
|
|
; CHECK-INTERLEAVED: [[__CRIT_EDGE:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_0_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %.lr.ph
|
|
|
|
.lr.ph:
|
|
%indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %entry ]
|
|
%sum.02 = phi i32 [ %l9, %.lr.ph ], [ 0, %entry ]
|
|
%l2 = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
|
|
%l3 = load i32, ptr %l2, align 4
|
|
%l4 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv
|
|
%l5 = load i32, ptr %l4, align 4
|
|
%l6 = trunc i64 %indvars.iv to i32
|
|
%l7 = add i32 %sum.02, %l6
|
|
%l8 = add i32 %l7, %l3
|
|
%l9 = add i32 %l8, %l5
|
|
%indvars.iv.next = add i64 %indvars.iv, 1
|
|
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
|
|
%exitcond = icmp eq i32 %lftr.wideiv, 256
|
|
br i1 %exitcond, label %._crit_edge, label %.lr.ph, !llvm.loop !6
|
|
|
|
._crit_edge:
|
|
%sum.0.lcssa = phi i32 [ %l9, %.lr.ph ]
|
|
ret i32 %sum.0.lcssa
|
|
}
|
|
|
|
define i8 @reduction_add_trunc(ptr noalias nocapture %A) {
|
|
; CHECK-LABEL: define i8 @reduction_add_trunc(
|
|
; CHECK-SAME: ptr noalias captures(none) [[A:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ <i32 255, i32 0, i32 0, i32 0>, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = and <4 x i32> [[VEC_PHI]], splat (i32 255)
|
|
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 4
|
|
; CHECK-NEXT: [[TMP2:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i32>
|
|
; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[TMP0]], [[TMP2]]
|
|
; CHECK-NEXT: [[TMP4:%.*]] = trunc <4 x i32> [[TMP6]] to <4 x i8>
|
|
; CHECK-NEXT: [[TMP5]] = zext <4 x i8> [[TMP4]] to <4 x i32>
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
|
|
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
|
|
; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: [[TMP7:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[TMP4]])
|
|
; CHECK-NEXT: [[TMP8:%.*]] = zext i8 [[TMP7]] to i32
|
|
; CHECK-NEXT: br [[DOT_CRIT_EDGE:label %.*]]
|
|
; CHECK: [[__CRIT_EDGE:.*:]]
|
|
; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = trunc i32 [[TMP8]] to i8
|
|
; CHECK-NEXT: ret i8 [[SUM_0_LCSSA]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define i8 @reduction_add_trunc(
|
|
; CHECK-INTERLEAVED-SAME: ptr noalias captures(none) [[A:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ <i32 255, i32 0, i32 0, i32 0>, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = and <4 x i32> [[VEC_PHI]], splat (i32 255)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = and <4 x i32> [[VEC_PHI1]], splat (i32 255)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i32>
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = zext <4 x i8> [[WIDE_LOAD2]] to <4 x i32>
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = add <4 x i32> [[TMP0]], [[TMP4]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = add <4 x i32> [[TMP3]], [[TMP12]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = trunc <4 x i32> [[TMP6]] to <4 x i8>
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = trunc <4 x i32> [[TMP7]] to <4 x i8>
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10]] = zext <4 x i8> [[TMP8]] to <4 x i32>
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP11]] = zext <4 x i8> [[TMP9]] to <4 x i32>
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <4 x i8> [[TMP9]], [[TMP8]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> [[BIN_RDX]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = zext i8 [[TMP13]] to i32
|
|
; CHECK-INTERLEAVED-NEXT: br [[DOT_CRIT_EDGE:label %.*]]
|
|
; CHECK-INTERLEAVED: [[__CRIT_EDGE:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: [[SUM_0_LCSSA:%.*]] = trunc i32 [[TMP14]] to i8
|
|
; CHECK-INTERLEAVED-NEXT: ret i8 [[SUM_0_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %.lr.ph
|
|
|
|
.lr.ph:
|
|
%indvars.iv = phi i32 [ %indvars.iv.next, %.lr.ph ], [ 0, %entry ]
|
|
%sum.02p = phi i32 [ %l9, %.lr.ph ], [ 255, %entry ]
|
|
%sum.02 = and i32 %sum.02p, 255
|
|
%l2 = getelementptr inbounds i8, ptr %A, i32 %indvars.iv
|
|
%l3 = load i8, ptr %l2, align 4
|
|
%l3e = zext i8 %l3 to i32
|
|
%l9 = add i32 %sum.02, %l3e
|
|
%indvars.iv.next = add i32 %indvars.iv, 1
|
|
%exitcond = icmp eq i32 %indvars.iv.next, 256
|
|
br i1 %exitcond, label %._crit_edge, label %.lr.ph
|
|
|
|
._crit_edge:
|
|
%sum.0.lcssa = phi i32 [ %l9, %.lr.ph ]
|
|
%ret = trunc i32 %sum.0.lcssa to i8
|
|
ret i8 %ret
|
|
}
|
|
|
|
|
|
define i8 @reduction_and_trunc(ptr noalias nocapture %A) {
|
|
; CHECK-LABEL: define i8 @reduction_and_trunc(
|
|
; CHECK-SAME: ptr noalias captures(none) [[A:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ <i32 255, i32 -1, i32 -1, i32 -1>, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = and <4 x i32> [[VEC_PHI]], splat (i32 255)
|
|
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 4
|
|
; CHECK-NEXT: [[TMP2:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i32>
|
|
; CHECK-NEXT: [[TMP6:%.*]] = and <4 x i32> [[TMP0]], [[TMP2]]
|
|
; CHECK-NEXT: [[TMP4:%.*]] = trunc <4 x i32> [[TMP6]] to <4 x i8>
|
|
; CHECK-NEXT: [[TMP5]] = zext <4 x i8> [[TMP4]] to <4 x i32>
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
|
|
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
|
|
; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: [[TMP7:%.*]] = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> [[TMP4]])
|
|
; CHECK-NEXT: [[TMP8:%.*]] = zext i8 [[TMP7]] to i32
|
|
; CHECK-NEXT: br [[DOT_CRIT_EDGE:label %.*]]
|
|
; CHECK: [[__CRIT_EDGE:.*:]]
|
|
; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = trunc i32 [[TMP8]] to i8
|
|
; CHECK-NEXT: ret i8 [[SUM_0_LCSSA]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define i8 @reduction_and_trunc(
|
|
; CHECK-INTERLEAVED-SAME: ptr noalias captures(none) [[A:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ <i32 255, i32 -1, i32 -1, i32 -1>, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = and <4 x i32> [[VEC_PHI]], splat (i32 255)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = and <4 x i32> [[VEC_PHI1]], splat (i32 255)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i32>
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = zext <4 x i8> [[WIDE_LOAD2]] to <4 x i32>
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = and <4 x i32> [[TMP0]], [[TMP4]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = and <4 x i32> [[TMP3]], [[TMP12]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = trunc <4 x i32> [[TMP6]] to <4 x i8>
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = trunc <4 x i32> [[TMP7]] to <4 x i8>
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10]] = zext <4 x i8> [[TMP8]] to <4 x i32>
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP11]] = zext <4 x i8> [[TMP9]] to <4 x i32>
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = icmp eq i32 [[INDEX_NEXT]], 256
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = and <4 x i8> [[TMP9]], [[TMP8]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = call i8 @llvm.vector.reduce.and.v4i8(<4 x i8> [[BIN_RDX]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = zext i8 [[TMP13]] to i32
|
|
; CHECK-INTERLEAVED-NEXT: br [[DOT_CRIT_EDGE:label %.*]]
|
|
; CHECK-INTERLEAVED: [[__CRIT_EDGE:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: [[SUM_0_LCSSA:%.*]] = trunc i32 [[TMP14]] to i8
|
|
; CHECK-INTERLEAVED-NEXT: ret i8 [[SUM_0_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %.lr.ph
|
|
|
|
.lr.ph:
|
|
%indvars.iv = phi i32 [ %indvars.iv.next, %.lr.ph ], [ 0, %entry ]
|
|
%sum.02p = phi i32 [ %l9, %.lr.ph ], [ 255, %entry ]
|
|
%sum.02 = and i32 %sum.02p, 255
|
|
%l2 = getelementptr inbounds i8, ptr %A, i32 %indvars.iv
|
|
%l3 = load i8, ptr %l2, align 4
|
|
%l3e = zext i8 %l3 to i32
|
|
%l9 = and i32 %sum.02, %l3e
|
|
%indvars.iv.next = add i32 %indvars.iv, 1
|
|
%exitcond = icmp eq i32 %indvars.iv.next, 256
|
|
br i1 %exitcond, label %._crit_edge, label %.lr.ph
|
|
|
|
._crit_edge:
|
|
%sum.0.lcssa = phi i32 [ %l9, %.lr.ph ]
|
|
%ret = trunc i32 %sum.0.lcssa to i8
|
|
ret i8 %ret
|
|
}
|
|
|
|
; Test case when loop has a call to the llvm.fmuladd intrinsic.
|
|
define float @reduction_fmuladd(ptr %a, ptr %b, i64 %n) {
|
|
; CHECK-LABEL: define float @reduction_fmuladd(
|
|
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*]]:
|
|
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
|
|
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
|
|
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
|
|
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
|
|
; CHECK-NEXT: [[TMP2:%.*]] = fmul <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
|
|
; CHECK-NEXT: [[TMP3:%.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP2]])
|
|
; CHECK-NEXT: [[TMP4]] = fadd float [[VEC_PHI]], [[TMP3]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
|
|
; CHECK-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
|
|
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
|
|
; CHECK: [[SCALAR_PH]]:
|
|
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
|
|
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
|
|
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
|
|
; CHECK: [[FOR_BODY]]:
|
|
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
|
|
; CHECK-NEXT: [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MULADD:%.*]], %[[FOR_BODY]] ]
|
|
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
|
|
; CHECK-NEXT: [[TMP6:%.*]] = load float, ptr [[ARRAYIDX]], align 4
|
|
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
|
|
; CHECK-NEXT: [[TMP7:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
|
|
; CHECK-NEXT: [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[TMP6]], float [[TMP7]], float [[SUM_07]])
|
|
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
|
|
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
|
|
; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
|
|
; CHECK: [[FOR_END]]:
|
|
; CHECK-NEXT: [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], %[[FOR_BODY]] ], [ [[TMP4]], %[[MIDDLE_BLOCK]] ]
|
|
; CHECK-NEXT: ret float [[MULADD_LCSSA]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define float @reduction_fmuladd(
|
|
; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi float [ -0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP9:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[TMP0]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP1]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[TMP2]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = fmul <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD3]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = fmul <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD4]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP4]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7]] = fadd float [[VEC_PHI]], [[TMP6]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP5]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9]] = fadd float [[VEC_PHI1]], [[TMP8]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = fadd float [[TMP9]], [[TMP7]]
|
|
; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[FOR_END:.*]], label %[[SCALAR_PH]]
|
|
; CHECK-INTERLEAVED: [[SCALAR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[FOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[SUM_07:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[MULADD:%.*]], %[[FOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = load float, ptr [[ARRAYIDX]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[TMP11]], float [[TMP12]], float [[SUM_07]])
|
|
; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
|
|
; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END]], label %[[FOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[FOR_END]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], %[[FOR_BODY]] ], [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ]
|
|
; CHECK-INTERLEAVED-NEXT: ret float [[MULADD_LCSSA]]
|
|
;
|
|
|
|
entry:
|
|
br label %for.body
|
|
|
|
for.body:
|
|
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
|
|
%sum.07 = phi float [ 0.000000e+00, %entry ], [ %muladd, %for.body ]
|
|
%arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
|
|
%0 = load float, ptr %arrayidx, align 4
|
|
%arrayidx2 = getelementptr inbounds float, ptr %b, i64 %iv
|
|
%1 = load float, ptr %arrayidx2, align 4
|
|
%muladd = tail call float @llvm.fmuladd.f32(float %0, float %1, float %sum.07)
|
|
%iv.next = add nuw nsw i64 %iv, 1
|
|
%exitcond.not = icmp eq i64 %iv.next, %n
|
|
br i1 %exitcond.not, label %for.end, label %for.body
|
|
|
|
for.end:
|
|
ret float %muladd
|
|
}
|
|
|
|
define float @reduction_fmuladd_recurrence_first_arg(ptr %a, ptr %b, i64 %n) {
|
|
; CHECK-LABEL: define float @reduction_fmuladd_recurrence_first_arg(
|
|
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*]]:
|
|
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
|
|
; CHECK: [[FOR_BODY]]:
|
|
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
|
|
; CHECK-NEXT: [[SUM_07:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[MULADD:%.*]], %[[FOR_BODY]] ]
|
|
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
|
|
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
|
|
; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
|
|
; CHECK-NEXT: [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[SUM_07]], float [[TMP0]], float [[TMP1]])
|
|
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
|
|
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
|
|
; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END:.*]], label %[[FOR_BODY]]
|
|
; CHECK: [[FOR_END]]:
|
|
; CHECK-NEXT: [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], %[[FOR_BODY]] ]
|
|
; CHECK-NEXT: ret float [[MULADD_LCSSA]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define float @reduction_fmuladd_recurrence_first_arg(
|
|
; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[FOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[SUM_07:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[MULADD:%.*]], %[[FOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[SUM_07]], float [[TMP0]], float [[TMP1]])
|
|
; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
|
|
; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END:.*]], label %[[FOR_BODY]]
|
|
; CHECK-INTERLEAVED: [[FOR_END]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], %[[FOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: ret float [[MULADD_LCSSA]]
|
|
;
|
|
|
|
entry:
|
|
br label %for.body
|
|
|
|
for.body:
|
|
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
|
|
%sum.07 = phi float [ 0.000000e+00, %entry ], [ %muladd, %for.body ]
|
|
%arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
|
|
%0 = load float, ptr %arrayidx, align 4
|
|
%arrayidx2 = getelementptr inbounds float, ptr %b, i64 %iv
|
|
%1 = load float, ptr %arrayidx2, align 4
|
|
%muladd = tail call float @llvm.fmuladd.f32(float %sum.07, float %0, float %1)
|
|
%iv.next = add nuw nsw i64 %iv, 1
|
|
%exitcond.not = icmp eq i64 %iv.next, %n
|
|
br i1 %exitcond.not, label %for.end, label %for.body
|
|
|
|
for.end:
|
|
ret float %muladd
|
|
}
|
|
|
|
define float @reduction_fmuladd_recurrence_second_arg(ptr %a, ptr %b, i64 %n) {
|
|
; CHECK-LABEL: define float @reduction_fmuladd_recurrence_second_arg(
|
|
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*]]:
|
|
; CHECK-NEXT: br label %[[FOR_BODY:.*]]
|
|
; CHECK: [[FOR_BODY]]:
|
|
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
|
|
; CHECK-NEXT: [[SUM_07:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[MULADD:%.*]], %[[FOR_BODY]] ]
|
|
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
|
|
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
|
|
; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
|
|
; CHECK-NEXT: [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[TMP0]], float [[SUM_07]], float [[TMP1]])
|
|
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
|
|
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
|
|
; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END:.*]], label %[[FOR_BODY]]
|
|
; CHECK: [[FOR_END]]:
|
|
; CHECK-NEXT: [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], %[[FOR_BODY]] ]
|
|
; CHECK-NEXT: ret float [[MULADD_LCSSA]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define float @reduction_fmuladd_recurrence_second_arg(
|
|
; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[FOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[FOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[SUM_07:%.*]] = phi float [ 0.000000e+00, %[[ENTRY]] ], [ [[MULADD:%.*]], %[[FOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = load float, ptr [[ARRAYIDX]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[MULADD]] = tail call float @llvm.fmuladd.f32(float [[TMP0]], float [[SUM_07]], float [[TMP1]])
|
|
; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
|
|
; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END:.*]], label %[[FOR_BODY]]
|
|
; CHECK-INTERLEAVED: [[FOR_END]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[MULADD_LCSSA:%.*]] = phi float [ [[MULADD]], %[[FOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: ret float [[MULADD_LCSSA]]
|
|
;
|
|
|
|
entry:
|
|
br label %for.body
|
|
|
|
for.body:
|
|
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
|
|
%sum.07 = phi float [ 0.000000e+00, %entry ], [ %muladd, %for.body ]
|
|
%arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
|
|
%0 = load float, ptr %arrayidx, align 4
|
|
%arrayidx2 = getelementptr inbounds float, ptr %b, i64 %iv
|
|
%1 = load float, ptr %arrayidx2, align 4
|
|
%muladd = tail call float @llvm.fmuladd.f32(float %0, float %sum.07, float %1)
|
|
%iv.next = add nuw nsw i64 %iv, 1
|
|
%exitcond.not = icmp eq i64 %iv.next, %n
|
|
br i1 %exitcond.not, label %for.end, label %for.body
|
|
|
|
for.end:
|
|
ret float %muladd
|
|
}
|
|
|
|
define float @reduction_fmuladd_blend(ptr %a, ptr %b, i64 %n, i1 %c) {
|
|
; CHECK-LABEL: define float @reduction_fmuladd_blend(
|
|
; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]], i1 [[C:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*]]:
|
|
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
|
|
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
|
|
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
|
|
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 0
|
|
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[TMP1:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
|
|
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
|
|
; CHECK-NEXT: [[TMP4:%.*]] = fmul <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD1]]
|
|
; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[TMP4]], <4 x float> splat (float -0.000000e+00)
|
|
; CHECK-NEXT: [[TMP6:%.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP5]])
|
|
; CHECK-NEXT: [[TMP7]] = fadd float [[VEC_PHI]], [[TMP6]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
|
|
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
|
|
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
|
|
; CHECK: [[SCALAR_PH]]:
|
|
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
|
|
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
|
|
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
|
|
; CHECK: [[LOOP_HEADER]]:
|
|
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
|
|
; CHECK-NEXT: [[SUM:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], %[[LATCH]] ]
|
|
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
|
|
; CHECK-NEXT: [[TMP9:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
|
|
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
|
|
; CHECK-NEXT: [[TMP10:%.*]] = load float, ptr [[ARRAYIDX]], align 4
|
|
; CHECK-NEXT: br i1 [[C]], label %[[IF:.*]], label %[[ELSE:.*]]
|
|
; CHECK: [[IF]]:
|
|
; CHECK-NEXT: br label %[[LATCH]]
|
|
; CHECK: [[ELSE]]:
|
|
; CHECK-NEXT: [[MULADD:%.*]] = tail call float @llvm.fmuladd.f32(float [[TMP9]], float [[TMP10]], float [[SUM]])
|
|
; CHECK-NEXT: br label %[[LATCH]]
|
|
; CHECK: [[LATCH]]:
|
|
; CHECK-NEXT: [[SUM_NEXT]] = phi float [ [[SUM]], %[[IF]] ], [ [[MULADD]], %[[ELSE]] ]
|
|
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
|
|
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
|
|
; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP22:![0-9]+]]
|
|
; CHECK: [[EXIT]]:
|
|
; CHECK-NEXT: [[SUM_NEXT_LCSSA:%.*]] = phi float [ [[SUM_NEXT]], %[[LATCH]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
|
|
; CHECK-NEXT: ret float [[SUM_NEXT_LCSSA]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define float @reduction_fmuladd_blend(
|
|
; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]], i1 [[C:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
|
|
; CHECK-INTERLEAVED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 0
|
|
; CHECK-INTERLEAVED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true)
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi float [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi float [ -0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[TMP2]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x float>, ptr [[TMP3]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x float>, ptr [[TMP4]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x float>, ptr [[TMP5]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = fmul <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD3]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = fmul <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD4]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[TMP6]], <4 x float> splat (float -0.000000e+00)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP8]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10]] = fadd float [[VEC_PHI]], [[TMP9]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[TMP7]], <4 x float> splat (float -0.000000e+00)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = call float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[TMP11]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP13]] = fadd float [[VEC_PHI1]], [[TMP12]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = fadd float [[TMP13]], [[TMP10]]
|
|
; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
|
|
; CHECK-INTERLEAVED: [[SCALAR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ], [ 0.000000e+00, %[[ENTRY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[LOOP_HEADER:.*]]
|
|
; CHECK-INTERLEAVED: [[LOOP_HEADER]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[SUM:%.*]] = phi float [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_NEXT:%.*]], %[[LATCH]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[IV]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = load float, ptr [[ARRAYIDX]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[C]], label %[[IF:.*]], label %[[ELSE:.*]]
|
|
; CHECK-INTERLEAVED: [[IF]]:
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[LATCH]]
|
|
; CHECK-INTERLEAVED: [[ELSE]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[MULADD:%.*]] = tail call float @llvm.fmuladd.f32(float [[TMP15]], float [[TMP16]], float [[SUM]])
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[LATCH]]
|
|
; CHECK-INTERLEAVED: [[LATCH]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[SUM_NEXT]] = phi float [ [[SUM]], %[[IF]] ], [ [[MULADD]], %[[ELSE]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
|
|
; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP22:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[EXIT]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[SUM_NEXT_LCSSA:%.*]] = phi float [ [[SUM_NEXT]], %[[LATCH]] ], [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ]
|
|
; CHECK-INTERLEAVED-NEXT: ret float [[SUM_NEXT_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %loop.header
|
|
|
|
loop.header:
|
|
%iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
|
|
%sum = phi float [ 0.000000e+00, %entry ], [ %sum.next, %latch ]
|
|
%arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
|
|
%0 = load float, ptr %arrayidx, align 4
|
|
%arrayidx2 = getelementptr inbounds float, ptr %b, i64 %iv
|
|
%1 = load float, ptr %arrayidx2, align 4
|
|
br i1 %c, label %if, label %else
|
|
|
|
if:
|
|
br label %latch
|
|
|
|
else:
|
|
%muladd = tail call float @llvm.fmuladd.f32(float %0, float %1, float %sum)
|
|
br label %latch
|
|
|
|
latch:
|
|
%sum.next = phi float [ %sum, %if ], [ %muladd, %else ]
|
|
%iv.next = add nuw nsw i64 %iv, 1
|
|
%exitcond.not = icmp eq i64 %iv.next, %n
|
|
br i1 %exitcond.not, label %exit, label %loop.header
|
|
|
|
exit:
|
|
ret float %sum.next
|
|
}
|
|
|
|
; This case was previously failing verification due to the mask for the
|
|
; reduction being created after the reduction.
|
|
define i32 @predicated_not_dominates_reduction(ptr nocapture noundef readonly %h, i32 noundef %i) {
|
|
; CHECK-LABEL: define i32 @predicated_not_dominates_reduction(
|
|
; CHECK-SAME: ptr noundef readonly captures(none) [[H:%.*]], i32 noundef [[I:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*]]:
|
|
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[I]], 4
|
|
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[I]], 4
|
|
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[I]], [[N_MOD_VF]]
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[H]], i32 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
|
|
; CHECK-NEXT: [[TMP9:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP2:%.*]] = udiv <4 x i8> [[WIDE_LOAD]], splat (i8 31)
|
|
; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw <4 x i8> [[TMP2]], splat (i8 3)
|
|
; CHECK-NEXT: [[TMP4:%.*]] = udiv <4 x i8> [[TMP3]], splat (i8 31)
|
|
; CHECK-NEXT: [[TMP14:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32>
|
|
; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP9]], <4 x i32> [[TMP14]], <4 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP5]])
|
|
; CHECK-NEXT: [[TMP7]] = add i32 [[VEC_PHI]], [[TMP6]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
|
|
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
|
|
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[I]], [[N_VEC]]
|
|
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END7:.*]], label %[[SCALAR_PH]]
|
|
; CHECK: [[SCALAR_PH]]:
|
|
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
|
|
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP7]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
|
|
; CHECK-NEXT: br label %[[FOR_BODY2:.*]]
|
|
; CHECK: [[FOR_BODY2]]:
|
|
; CHECK-NEXT: [[A_117:%.*]] = phi i32 [ [[INC6:%.*]], %[[FOR_INC5:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
|
|
; CHECK-NEXT: [[G_016:%.*]] = phi i32 [ [[G_1:%.*]], %[[FOR_INC5]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
|
|
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[H]], i32 [[A_117]]
|
|
; CHECK-NEXT: [[TMP10:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
|
|
; CHECK-NEXT: [[TOBOOL3_NOT:%.*]] = icmp eq i8 [[TMP10]], 0
|
|
; CHECK-NEXT: br i1 [[TOBOOL3_NOT]], label %[[FOR_INC5]], label %[[IF_THEN:.*]]
|
|
; CHECK: [[IF_THEN]]:
|
|
; CHECK-NEXT: [[TMP11:%.*]] = udiv i8 [[TMP10]], 31
|
|
; CHECK-NEXT: [[TMP12:%.*]] = shl nuw nsw i8 [[TMP11]], 3
|
|
; CHECK-NEXT: [[TMP13:%.*]] = udiv i8 [[TMP12]], 31
|
|
; CHECK-NEXT: [[DIV4:%.*]] = zext i8 [[TMP13]] to i32
|
|
; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[G_016]], [[DIV4]]
|
|
; CHECK-NEXT: br label %[[FOR_INC5]]
|
|
; CHECK: [[FOR_INC5]]:
|
|
; CHECK-NEXT: [[G_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[G_016]], %[[FOR_BODY2]] ]
|
|
; CHECK-NEXT: [[INC6]] = add nuw nsw i32 [[A_117]], 1
|
|
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC6]], [[I]]
|
|
; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END7]], label %[[FOR_BODY2]], !llvm.loop [[LOOP24:![0-9]+]]
|
|
; CHECK: [[FOR_END7]]:
|
|
; CHECK-NEXT: [[G_1_LCSSA:%.*]] = phi i32 [ [[G_1]], %[[FOR_INC5]] ], [ [[TMP7]], %[[MIDDLE_BLOCK]] ]
|
|
; CHECK-NEXT: ret i32 [[G_1_LCSSA]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define i32 @predicated_not_dominates_reduction(
|
|
; CHECK-INTERLEAVED-SAME: ptr noundef readonly captures(none) [[H:%.*]], i32 noundef [[I:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[I]], 8
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[I]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i32 [[I]], [[N_MOD_VF]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[H]], i32 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP21:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD2]], zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = udiv <4 x i8> [[WIDE_LOAD]], splat (i8 31)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = udiv <4 x i8> [[WIDE_LOAD2]], splat (i8 31)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = shl nuw nsw <4 x i8> [[TMP3]], splat (i8 3)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = shl nuw nsw <4 x i8> [[TMP4]], splat (i8 3)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = udiv <4 x i8> [[TMP5]], splat (i8 31)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = udiv <4 x i8> [[TMP6]], splat (i8 31)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP22:%.*]] = zext <4 x i8> [[TMP7]] to <4 x i32>
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP16]], <4 x i32> [[TMP22]], <4 x i32> zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP9]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP11]] = add i32 [[VEC_PHI]], [[TMP10]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP23:%.*]] = zext <4 x i8> [[TMP8]] to <4 x i32>
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP21]], <4 x i32> [[TMP23]], <4 x i32> zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP12]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP14]] = add i32 [[VEC_PHI1]], [[TMP13]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add i32 [[TMP14]], [[TMP11]]
|
|
; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[I]], [[N_VEC]]
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[FOR_END7:.*]], label %[[SCALAR_PH]]
|
|
; CHECK-INTERLEAVED: [[SCALAR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY2:.*]]
|
|
; CHECK-INTERLEAVED: [[FOR_BODY2]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[A_117:%.*]] = phi i32 [ [[INC6:%.*]], %[[FOR_INC5:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[G_016:%.*]] = phi i32 [ [[G_1:%.*]], %[[FOR_INC5]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[H]], i32 [[A_117]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
|
|
; CHECK-INTERLEAVED-NEXT: [[TOBOOL3_NOT:%.*]] = icmp eq i8 [[TMP17]], 0
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TOBOOL3_NOT]], label %[[FOR_INC5]], label %[[IF_THEN:.*]]
|
|
; CHECK-INTERLEAVED: [[IF_THEN]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = udiv i8 [[TMP17]], 31
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = shl nuw nsw i8 [[TMP18]], 3
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP20:%.*]] = udiv i8 [[TMP19]], 31
|
|
; CHECK-INTERLEAVED-NEXT: [[DIV4:%.*]] = zext i8 [[TMP20]] to i32
|
|
; CHECK-INTERLEAVED-NEXT: [[ADD:%.*]] = add nsw i32 [[G_016]], [[DIV4]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_INC5]]
|
|
; CHECK-INTERLEAVED: [[FOR_INC5]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[G_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[G_016]], %[[FOR_BODY2]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[INC6]] = add nuw nsw i32 [[A_117]], 1
|
|
; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC6]], [[I]]
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END7]], label %[[FOR_BODY2]], !llvm.loop [[LOOP24:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[FOR_END7]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[G_1_LCSSA:%.*]] = phi i32 [ [[G_1]], %[[FOR_INC5]] ], [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ]
|
|
; CHECK-INTERLEAVED-NEXT: ret i32 [[G_1_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %for.body2
|
|
|
|
for.body2:
|
|
%a.117 = phi i32 [ %inc6, %for.inc5 ], [ 0, %entry ]
|
|
%g.016 = phi i32 [ %g.1, %for.inc5 ], [ 0, %entry ]
|
|
%arrayidx = getelementptr inbounds i8, ptr %h, i32 %a.117
|
|
%0 = load i8, ptr %arrayidx, align 1
|
|
%tobool3.not = icmp eq i8 %0, 0
|
|
br i1 %tobool3.not, label %for.inc5, label %if.then
|
|
|
|
if.then:
|
|
%1 = udiv i8 %0, 31
|
|
%2 = shl nuw nsw i8 %1, 3
|
|
%3 = udiv i8 %2, 31
|
|
%div4 = zext i8 %3 to i32
|
|
%add = add nsw i32 %g.016, %div4
|
|
br label %for.inc5
|
|
|
|
for.inc5:
|
|
%g.1 = phi i32 [ %add, %if.then ], [ %g.016, %for.body2 ]
|
|
%inc6 = add nuw nsw i32 %a.117, 1
|
|
%exitcond.not = icmp eq i32 %inc6, %i
|
|
br i1 %exitcond.not, label %for.end7, label %for.body2
|
|
|
|
for.end7:
|
|
%g.1.lcssa = phi i32 [ %g.1, %for.inc5 ]
|
|
ret i32 %g.1.lcssa
|
|
}
|
|
|
|
define i32 @predicated_not_dominates_reduction_twoadd(ptr nocapture noundef readonly %h, i32 noundef %i) {
|
|
; CHECK-LABEL: define i32 @predicated_not_dominates_reduction_twoadd(
|
|
; CHECK-SAME: ptr noundef readonly captures(none) [[H:%.*]], i32 noundef [[I:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*]]:
|
|
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[I]], 4
|
|
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[I]], 4
|
|
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[I]], [[N_MOD_VF]]
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[H]], i32 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
|
|
; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP2:%.*]] = udiv <4 x i8> [[WIDE_LOAD]], splat (i8 31)
|
|
; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw <4 x i8> [[TMP2]], splat (i8 3)
|
|
; CHECK-NEXT: [[TMP4:%.*]] = udiv <4 x i8> [[TMP3]], splat (i8 31)
|
|
; CHECK-NEXT: [[TMP13:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32>
|
|
; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP6]], <4 x i32> [[TMP13]], <4 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP5]])
|
|
; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[VEC_PHI]], [[TMP7]]
|
|
; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP6]], <4 x i32> [[TMP13]], <4 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP9]])
|
|
; CHECK-NEXT: [[TMP11]] = add i32 [[TMP8]], [[TMP10]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
|
|
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
|
|
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[I]], [[N_VEC]]
|
|
; CHECK-NEXT: br i1 [[CMP_N]], label %[[FOR_END7:.*]], label %[[SCALAR_PH]]
|
|
; CHECK: [[SCALAR_PH]]:
|
|
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
|
|
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
|
|
; CHECK-NEXT: br label %[[FOR_BODY2:.*]]
|
|
; CHECK: [[FOR_BODY2]]:
|
|
; CHECK-NEXT: [[A_117:%.*]] = phi i32 [ [[INC6:%.*]], %[[FOR_INC5:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
|
|
; CHECK-NEXT: [[G_016:%.*]] = phi i32 [ [[G_1:%.*]], %[[FOR_INC5]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
|
|
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[H]], i32 [[A_117]]
|
|
; CHECK-NEXT: [[TMP14:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
|
|
; CHECK-NEXT: [[TOBOOL3_NOT:%.*]] = icmp eq i8 [[TMP14]], 0
|
|
; CHECK-NEXT: br i1 [[TOBOOL3_NOT]], label %[[FOR_INC5]], label %[[IF_THEN:.*]]
|
|
; CHECK: [[IF_THEN]]:
|
|
; CHECK-NEXT: [[TMP15:%.*]] = udiv i8 [[TMP14]], 31
|
|
; CHECK-NEXT: [[TMP16:%.*]] = shl nuw nsw i8 [[TMP15]], 3
|
|
; CHECK-NEXT: [[TMP17:%.*]] = udiv i8 [[TMP16]], 31
|
|
; CHECK-NEXT: [[DIV4:%.*]] = zext i8 [[TMP17]] to i32
|
|
; CHECK-NEXT: [[ADD1:%.*]] = add nsw i32 [[G_016]], [[DIV4]]
|
|
; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[ADD1]], [[DIV4]]
|
|
; CHECK-NEXT: br label %[[FOR_INC5]]
|
|
; CHECK: [[FOR_INC5]]:
|
|
; CHECK-NEXT: [[G_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[G_016]], %[[FOR_BODY2]] ]
|
|
; CHECK-NEXT: [[INC6]] = add nuw nsw i32 [[A_117]], 1
|
|
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC6]], [[I]]
|
|
; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END7]], label %[[FOR_BODY2]], !llvm.loop [[LOOP26:![0-9]+]]
|
|
; CHECK: [[FOR_END7]]:
|
|
; CHECK-NEXT: [[G_1_LCSSA:%.*]] = phi i32 [ [[G_1]], %[[FOR_INC5]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
|
|
; CHECK-NEXT: ret i32 [[G_1_LCSSA]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define i32 @predicated_not_dominates_reduction_twoadd(
|
|
; CHECK-INTERLEAVED-SAME: ptr noundef readonly captures(none) [[H:%.*]], i32 noundef [[I:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[I]], 8
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[I]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i32 [[I]], [[N_MOD_VF]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP17:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP20:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[H]], i32 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP27:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP28:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD2]], zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = udiv <4 x i8> [[WIDE_LOAD]], splat (i8 31)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = udiv <4 x i8> [[WIDE_LOAD2]], splat (i8 31)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = shl nuw nsw <4 x i8> [[TMP3]], splat (i8 3)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = shl nuw nsw <4 x i8> [[TMP4]], splat (i8 3)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = udiv <4 x i8> [[TMP5]], splat (i8 31)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = udiv <4 x i8> [[TMP6]], splat (i8 31)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP29:%.*]] = zext <4 x i8> [[TMP7]] to <4 x i32>
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP27]], <4 x i32> [[TMP29]], <4 x i32> zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP9]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = add i32 [[VEC_PHI]], [[TMP10]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP30:%.*]] = zext <4 x i8> [[TMP8]] to <4 x i32>
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = select <4 x i1> [[TMP28]], <4 x i32> [[TMP30]], <4 x i32> zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP12]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = add i32 [[VEC_PHI1]], [[TMP13]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = select <4 x i1> [[TMP27]], <4 x i32> [[TMP29]], <4 x i32> zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP15]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP17]] = add i32 [[TMP11]], [[TMP16]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = select <4 x i1> [[TMP28]], <4 x i32> [[TMP30]], <4 x i32> zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP18]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP20]] = add i32 [[TMP14]], [[TMP19]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add i32 [[TMP20]], [[TMP17]]
|
|
; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[I]], [[N_VEC]]
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[FOR_END7:.*]], label %[[SCALAR_PH]]
|
|
; CHECK-INTERLEAVED: [[SCALAR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY2:.*]]
|
|
; CHECK-INTERLEAVED: [[FOR_BODY2]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[A_117:%.*]] = phi i32 [ [[INC6:%.*]], %[[FOR_INC5:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[G_016:%.*]] = phi i32 [ [[G_1:%.*]], %[[FOR_INC5]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[H]], i32 [[A_117]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP23:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
|
|
; CHECK-INTERLEAVED-NEXT: [[TOBOOL3_NOT:%.*]] = icmp eq i8 [[TMP23]], 0
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TOBOOL3_NOT]], label %[[FOR_INC5]], label %[[IF_THEN:.*]]
|
|
; CHECK-INTERLEAVED: [[IF_THEN]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP24:%.*]] = udiv i8 [[TMP23]], 31
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP25:%.*]] = shl nuw nsw i8 [[TMP24]], 3
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP26:%.*]] = udiv i8 [[TMP25]], 31
|
|
; CHECK-INTERLEAVED-NEXT: [[DIV4:%.*]] = zext i8 [[TMP26]] to i32
|
|
; CHECK-INTERLEAVED-NEXT: [[ADD1:%.*]] = add nsw i32 [[G_016]], [[DIV4]]
|
|
; CHECK-INTERLEAVED-NEXT: [[ADD:%.*]] = add nsw i32 [[ADD1]], [[DIV4]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_INC5]]
|
|
; CHECK-INTERLEAVED: [[FOR_INC5]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[G_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[G_016]], %[[FOR_BODY2]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[INC6]] = add nuw nsw i32 [[A_117]], 1
|
|
; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC6]], [[I]]
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_END7]], label %[[FOR_BODY2]], !llvm.loop [[LOOP26:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[FOR_END7]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[G_1_LCSSA:%.*]] = phi i32 [ [[G_1]], %[[FOR_INC5]] ], [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ]
|
|
; CHECK-INTERLEAVED-NEXT: ret i32 [[G_1_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %for.body2
|
|
|
|
for.body2:
|
|
%a.117 = phi i32 [ %inc6, %for.inc5 ], [ 0, %entry ]
|
|
%g.016 = phi i32 [ %g.1, %for.inc5 ], [ 0, %entry ]
|
|
%arrayidx = getelementptr inbounds i8, ptr %h, i32 %a.117
|
|
%0 = load i8, ptr %arrayidx, align 1
|
|
%tobool3.not = icmp eq i8 %0, 0
|
|
br i1 %tobool3.not, label %for.inc5, label %if.then
|
|
|
|
if.then:
|
|
%1 = udiv i8 %0, 31
|
|
%2 = shl nuw nsw i8 %1, 3
|
|
%3 = udiv i8 %2, 31
|
|
%div4 = zext i8 %3 to i32
|
|
%add1 = add nsw i32 %g.016, %div4
|
|
%add = add nsw i32 %add1, %div4
|
|
br label %for.inc5
|
|
|
|
for.inc5:
|
|
%g.1 = phi i32 [ %add, %if.then ], [ %g.016, %for.body2 ]
|
|
%inc6 = add nuw nsw i32 %a.117, 1
|
|
%exitcond.not = icmp eq i32 %inc6, %i
|
|
br i1 %exitcond.not, label %for.end7, label %for.body2
|
|
|
|
for.end7:
|
|
%g.1.lcssa = phi i32 [ %g.1, %for.inc5 ]
|
|
ret i32 %g.1.lcssa
|
|
}
|
|
|
|
%struct.e = type { i32, i32 }
|
|
define i32 @predicated_or_dominates_reduction(ptr %b) {
|
|
; CHECK-LABEL: define i32 @predicated_or_dominates_reduction(
|
|
; CHECK-SAME: ptr [[B:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP48:%.*]], %[[PRED_LOAD_CONTINUE6]] ]
|
|
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1
|
|
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 2
|
|
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 3
|
|
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[INDEX]], i32 0
|
|
; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP1]], i32 1
|
|
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[TMP9]], i32 [[TMP2]], i32 2
|
|
; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[TMP3]], i32 3
|
|
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [0 x [[STRUCT_E:%.*]]], ptr [[B]], i32 0, i32 [[INDEX]], i32 1
|
|
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds [0 x [[STRUCT_E]]], ptr [[B]], i32 0, i32 [[TMP1]], i32 1
|
|
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [0 x [[STRUCT_E]]], ptr [[B]], i32 0, i32 [[TMP2]], i32 1
|
|
; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [0 x [[STRUCT_E]]], ptr [[B]], i32 0, i32 [[TMP3]], i32 1
|
|
; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP4]], align 4
|
|
; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP6]], align 4
|
|
; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP8]], align 4
|
|
; CHECK-NEXT: [[TMP14:%.*]] = load i32, ptr [[TMP10]], align 4
|
|
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> poison, i32 [[TMP11]], i32 0
|
|
; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP12]], i32 1
|
|
; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP16]], i32 [[TMP13]], i32 2
|
|
; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP14]], i32 3
|
|
; CHECK-NEXT: [[TMP19:%.*]] = icmp eq <4 x i32> [[TMP18]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP19]], i64 0
|
|
; CHECK-NEXT: br i1 [[TMP20]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
|
|
; CHECK: [[PRED_LOAD_IF]]:
|
|
; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds [0 x [[STRUCT_E]]], ptr [[B]], i32 0, i32 [[INDEX]]
|
|
; CHECK-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4
|
|
; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> poison, i32 [[TMP23]], i64 0
|
|
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]]
|
|
; CHECK: [[PRED_LOAD_CONTINUE]]:
|
|
; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP25]], %[[PRED_LOAD_IF]] ]
|
|
; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x i1> [[TMP19]], i64 1
|
|
; CHECK-NEXT: br i1 [[TMP26]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
|
|
; CHECK: [[PRED_LOAD_IF1]]:
|
|
; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds [0 x [[STRUCT_E]]], ptr [[B]], i32 0, i32 [[TMP1]]
|
|
; CHECK-NEXT: [[TMP29:%.*]] = load i32, ptr [[TMP28]], align 4
|
|
; CHECK-NEXT: [[TMP30:%.*]] = insertelement <4 x i32> [[TMP24]], i32 [[TMP29]], i64 1
|
|
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]]
|
|
; CHECK: [[PRED_LOAD_CONTINUE2]]:
|
|
; CHECK-NEXT: [[TMP31:%.*]] = phi <4 x i32> [ [[TMP24]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP30]], %[[PRED_LOAD_IF1]] ]
|
|
; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[TMP19]], i64 2
|
|
; CHECK-NEXT: br i1 [[TMP32]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
|
|
; CHECK: [[PRED_LOAD_IF3]]:
|
|
; CHECK-NEXT: [[TMP34:%.*]] = getelementptr inbounds [0 x [[STRUCT_E]]], ptr [[B]], i32 0, i32 [[TMP2]]
|
|
; CHECK-NEXT: [[TMP35:%.*]] = load i32, ptr [[TMP34]], align 4
|
|
; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x i32> [[TMP31]], i32 [[TMP35]], i64 2
|
|
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE4]]
|
|
; CHECK: [[PRED_LOAD_CONTINUE4]]:
|
|
; CHECK-NEXT: [[TMP36:%.*]] = phi <4 x i32> [ [[TMP31]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP33]], %[[PRED_LOAD_IF3]] ]
|
|
; CHECK-NEXT: [[TMP38:%.*]] = extractelement <4 x i1> [[TMP19]], i64 3
|
|
; CHECK-NEXT: br i1 [[TMP38]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
|
|
; CHECK: [[PRED_LOAD_IF5]]:
|
|
; CHECK-NEXT: [[TMP40:%.*]] = getelementptr inbounds [0 x [[STRUCT_E]]], ptr [[B]], i32 0, i32 [[TMP3]]
|
|
; CHECK-NEXT: [[TMP41:%.*]] = load i32, ptr [[TMP40]], align 4
|
|
; CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> [[TMP36]], i32 [[TMP41]], i64 3
|
|
; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE6]]
|
|
; CHECK: [[PRED_LOAD_CONTINUE6]]:
|
|
; CHECK-NEXT: [[TMP43:%.*]] = phi <4 x i32> [ [[TMP36]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP42]], %[[PRED_LOAD_IF5]] ]
|
|
; CHECK-NEXT: [[TMP44:%.*]] = icmp ne <4 x i32> [[TMP43]], zeroinitializer
|
|
; CHECK-NEXT: [[TMP39:%.*]] = select <4 x i1> [[TMP19]], <4 x i1> [[TMP44]], <4 x i1> zeroinitializer
|
|
; CHECK-NEXT: [[NOT_:%.*]] = xor <4 x i1> [[TMP19]], splat (i1 true)
|
|
; CHECK-NEXT: [[TMP50:%.*]] = or <4 x i1> [[TMP39]], [[NOT_]]
|
|
; CHECK-NEXT: [[TMP51:%.*]] = select <4 x i1> [[TMP50]], <4 x i32> splat (i32 1), <4 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[TMP47:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP51]])
|
|
; CHECK-NEXT: [[TMP48]] = add i32 [[VEC_PHI]], [[TMP47]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
|
|
; CHECK-NEXT: [[TMP49:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000
|
|
; CHECK-NEXT: br i1 [[TMP49]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
|
|
; CHECK: [[FOR_COND_CLEANUP]]:
|
|
; CHECK-NEXT: ret i32 [[TMP48]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define i32 @predicated_or_dominates_reduction(
|
|
; CHECK-INTERLEAVED-SAME: ptr [[B:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE15:.*]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP94:%.*]], %[[PRED_LOAD_CONTINUE15]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP98:%.*]], %[[PRED_LOAD_CONTINUE15]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 2
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 3
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> poison, i32 [[INDEX]], i32 0
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> [[TMP9]], i32 [[TMP1]], i32 1
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP15]], i32 [[TMP2]], i32 2
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> [[TMP17]], i32 [[TMP3]], i32 3
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = add i32 [[INDEX]], 5
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = add i32 [[INDEX]], 6
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = add i32 [[INDEX]], 7
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP37:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP5]], i32 1
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP37]], i32 [[TMP6]], i32 2
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP42:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP7]], i32 3
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = getelementptr inbounds [0 x [[STRUCT_E:%.*]]], ptr [[B]], i32 0, i32 [[INDEX]], i32 1
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = getelementptr inbounds [0 x [[STRUCT_E]]], ptr [[B]], i32 0, i32 [[TMP1]], i32 1
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = getelementptr inbounds [0 x [[STRUCT_E]]], ptr [[B]], i32 0, i32 [[TMP2]], i32 1
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = getelementptr inbounds [0 x [[STRUCT_E]]], ptr [[B]], i32 0, i32 [[TMP3]], i32 1
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = getelementptr inbounds [0 x [[STRUCT_E]]], ptr [[B]], i32 0, i32 [[TMP4]], i32 1
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = getelementptr inbounds [0 x [[STRUCT_E]]], ptr [[B]], i32 0, i32 [[TMP5]], i32 1
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP20:%.*]] = getelementptr inbounds [0 x [[STRUCT_E]]], ptr [[B]], i32 0, i32 [[TMP6]], i32 1
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP22:%.*]] = getelementptr inbounds [0 x [[STRUCT_E]]], ptr [[B]], i32 0, i32 [[TMP7]], i32 1
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP8]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP10]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP25:%.*]] = load i32, ptr [[TMP12]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP26:%.*]] = load i32, ptr [[TMP14]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP27:%.*]] = insertelement <4 x i32> poison, i32 [[TMP23]], i32 0
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> [[TMP27]], i32 [[TMP24]], i32 1
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP35:%.*]] = insertelement <4 x i32> [[TMP21]], i32 [[TMP25]], i32 2
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP30:%.*]] = insertelement <4 x i32> [[TMP35]], i32 [[TMP26]], i32 3
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP31:%.*]] = load i32, ptr [[TMP16]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP32:%.*]] = load i32, ptr [[TMP18]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP33:%.*]] = load i32, ptr [[TMP20]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP34:%.*]] = load i32, ptr [[TMP22]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP28:%.*]] = insertelement <4 x i32> poison, i32 [[TMP31]], i32 0
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP29:%.*]] = insertelement <4 x i32> [[TMP28]], i32 [[TMP32]], i32 1
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP36:%.*]] = insertelement <4 x i32> [[TMP29]], i32 [[TMP33]], i32 2
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP38:%.*]] = insertelement <4 x i32> [[TMP36]], i32 [[TMP34]], i32 3
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP39:%.*]] = icmp eq <4 x i32> [[TMP30]], zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP40:%.*]] = icmp eq <4 x i32> [[TMP38]], zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP41:%.*]] = extractelement <4 x i1> [[TMP39]], i64 0
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP41]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
|
|
; CHECK-INTERLEAVED: [[PRED_LOAD_IF]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP43:%.*]] = getelementptr inbounds [0 x [[STRUCT_E]]], ptr [[B]], i32 0, i32 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP44:%.*]] = load i32, ptr [[TMP43]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP46:%.*]] = insertelement <4 x i32> poison, i32 [[TMP44]], i64 0
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[PRED_LOAD_CONTINUE]]
|
|
; CHECK-INTERLEAVED: [[PRED_LOAD_CONTINUE]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP45:%.*]] = phi <4 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP46]], %[[PRED_LOAD_IF]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP47:%.*]] = extractelement <4 x i1> [[TMP39]], i64 1
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP47]], label %[[PRED_LOAD_IF2:.*]], label %[[PRED_LOAD_CONTINUE3:.*]]
|
|
; CHECK-INTERLEAVED: [[PRED_LOAD_IF2]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP49:%.*]] = getelementptr inbounds [0 x [[STRUCT_E]]], ptr [[B]], i32 0, i32 [[TMP1]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP50:%.*]] = load i32, ptr [[TMP49]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP51:%.*]] = insertelement <4 x i32> [[TMP45]], i32 [[TMP50]], i64 1
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[PRED_LOAD_CONTINUE3]]
|
|
; CHECK-INTERLEAVED: [[PRED_LOAD_CONTINUE3]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP52:%.*]] = phi <4 x i32> [ [[TMP45]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP51]], %[[PRED_LOAD_IF2]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP53:%.*]] = extractelement <4 x i1> [[TMP39]], i64 2
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP53]], label %[[PRED_LOAD_IF4:.*]], label %[[PRED_LOAD_CONTINUE5:.*]]
|
|
; CHECK-INTERLEAVED: [[PRED_LOAD_IF4]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP55:%.*]] = getelementptr inbounds [0 x [[STRUCT_E]]], ptr [[B]], i32 0, i32 [[TMP2]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP56:%.*]] = load i32, ptr [[TMP55]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP54:%.*]] = insertelement <4 x i32> [[TMP52]], i32 [[TMP56]], i64 2
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[PRED_LOAD_CONTINUE5]]
|
|
; CHECK-INTERLEAVED: [[PRED_LOAD_CONTINUE5]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP57:%.*]] = phi <4 x i32> [ [[TMP52]], %[[PRED_LOAD_CONTINUE3]] ], [ [[TMP54]], %[[PRED_LOAD_IF4]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP59:%.*]] = extractelement <4 x i1> [[TMP39]], i64 3
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP59]], label %[[PRED_LOAD_IF6:.*]], label %[[PRED_LOAD_CONTINUE7:.*]]
|
|
; CHECK-INTERLEAVED: [[PRED_LOAD_IF6]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP61:%.*]] = getelementptr inbounds [0 x [[STRUCT_E]]], ptr [[B]], i32 0, i32 [[TMP3]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP62:%.*]] = load i32, ptr [[TMP61]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP60:%.*]] = insertelement <4 x i32> [[TMP57]], i32 [[TMP62]], i64 3
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[PRED_LOAD_CONTINUE7]]
|
|
; CHECK-INTERLEAVED: [[PRED_LOAD_CONTINUE7]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP64:%.*]] = phi <4 x i32> [ [[TMP57]], %[[PRED_LOAD_CONTINUE5]] ], [ [[TMP60]], %[[PRED_LOAD_IF6]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP65:%.*]] = extractelement <4 x i1> [[TMP40]], i64 0
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP65]], label %[[PRED_LOAD_IF8:.*]], label %[[PRED_LOAD_CONTINUE9:.*]]
|
|
; CHECK-INTERLEAVED: [[PRED_LOAD_IF8]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP67:%.*]] = getelementptr inbounds [0 x [[STRUCT_E]]], ptr [[B]], i32 0, i32 [[TMP4]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP68:%.*]] = load i32, ptr [[TMP67]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP66:%.*]] = insertelement <4 x i32> poison, i32 [[TMP68]], i64 0
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[PRED_LOAD_CONTINUE9]]
|
|
; CHECK-INTERLEAVED: [[PRED_LOAD_CONTINUE9]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP72:%.*]] = phi <4 x i32> [ poison, %[[PRED_LOAD_CONTINUE7]] ], [ [[TMP66]], %[[PRED_LOAD_IF8]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP71:%.*]] = extractelement <4 x i1> [[TMP40]], i64 1
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP71]], label %[[PRED_LOAD_IF10:.*]], label %[[PRED_LOAD_CONTINUE11:.*]]
|
|
; CHECK-INTERLEAVED: [[PRED_LOAD_IF10]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP73:%.*]] = getelementptr inbounds [0 x [[STRUCT_E]]], ptr [[B]], i32 0, i32 [[TMP5]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP74:%.*]] = load i32, ptr [[TMP73]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP69:%.*]] = insertelement <4 x i32> [[TMP72]], i32 [[TMP74]], i64 1
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[PRED_LOAD_CONTINUE11]]
|
|
; CHECK-INTERLEAVED: [[PRED_LOAD_CONTINUE11]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP70:%.*]] = phi <4 x i32> [ [[TMP72]], %[[PRED_LOAD_CONTINUE9]] ], [ [[TMP69]], %[[PRED_LOAD_IF10]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP77:%.*]] = extractelement <4 x i1> [[TMP40]], i64 2
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP77]], label %[[PRED_LOAD_IF12:.*]], label %[[PRED_LOAD_CONTINUE13:.*]]
|
|
; CHECK-INTERLEAVED: [[PRED_LOAD_IF12]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP79:%.*]] = getelementptr inbounds [0 x [[STRUCT_E]]], ptr [[B]], i32 0, i32 [[TMP6]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP80:%.*]] = load i32, ptr [[TMP79]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP76:%.*]] = insertelement <4 x i32> [[TMP70]], i32 [[TMP80]], i64 2
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[PRED_LOAD_CONTINUE13]]
|
|
; CHECK-INTERLEAVED: [[PRED_LOAD_CONTINUE13]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP75:%.*]] = phi <4 x i32> [ [[TMP70]], %[[PRED_LOAD_CONTINUE11]] ], [ [[TMP76]], %[[PRED_LOAD_IF12]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP83:%.*]] = extractelement <4 x i1> [[TMP40]], i64 3
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP83]], label %[[PRED_LOAD_IF14:.*]], label %[[PRED_LOAD_CONTINUE15]]
|
|
; CHECK-INTERLEAVED: [[PRED_LOAD_IF14]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP85:%.*]] = getelementptr inbounds [0 x [[STRUCT_E]]], ptr [[B]], i32 0, i32 [[TMP7]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP86:%.*]] = load i32, ptr [[TMP85]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP81:%.*]] = insertelement <4 x i32> [[TMP75]], i32 [[TMP86]], i64 3
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[PRED_LOAD_CONTINUE15]]
|
|
; CHECK-INTERLEAVED: [[PRED_LOAD_CONTINUE15]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP88:%.*]] = phi <4 x i32> [ [[TMP75]], %[[PRED_LOAD_CONTINUE13]] ], [ [[TMP81]], %[[PRED_LOAD_IF14]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP89:%.*]] = icmp ne <4 x i32> [[TMP64]], zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP90:%.*]] = icmp ne <4 x i32> [[TMP88]], zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP78:%.*]] = select <4 x i1> [[TMP39]], <4 x i1> [[TMP89]], <4 x i1> zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP84:%.*]] = select <4 x i1> [[TMP40]], <4 x i1> [[TMP90]], <4 x i1> zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[NOT_:%.*]] = xor <4 x i1> [[TMP39]], splat (i1 true)
|
|
; CHECK-INTERLEAVED-NEXT: [[NOT_18:%.*]] = xor <4 x i1> [[TMP40]], splat (i1 true)
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP91:%.*]] = or <4 x i1> [[TMP78]], [[NOT_]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP92:%.*]] = or <4 x i1> [[TMP84]], [[NOT_18]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP95:%.*]] = select <4 x i1> [[TMP91]], <4 x i32> splat (i32 1), <4 x i32> zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP93:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP95]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP94]] = add i32 [[VEC_PHI]], [[TMP93]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP96:%.*]] = select <4 x i1> [[TMP92]], <4 x i32> splat (i32 1), <4 x i32> zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP97:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP96]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP98]] = add i32 [[VEC_PHI1]], [[TMP97]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP99:%.*]] = icmp eq i32 [[INDEX_NEXT]], 1000
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP99]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add i32 [[TMP98]], [[TMP94]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
|
|
; CHECK-INTERLEAVED: [[FOR_COND_CLEANUP]]:
|
|
; CHECK-INTERLEAVED-NEXT: ret i32 [[BIN_RDX]]
|
|
;
|
|
entry:
|
|
br label %for.body
|
|
|
|
for.cond.cleanup:
|
|
%a.1.lcssa = phi i32 [ %a.1, %for.inc ]
|
|
ret i32 %a.1.lcssa
|
|
|
|
for.body:
|
|
%g.09 = phi i32 [ 0, %entry ], [ %inc3, %for.inc ]
|
|
%a.08 = phi i32 [ 0, %entry ], [ %a.1, %for.inc ]
|
|
%d = getelementptr inbounds [0 x %struct.e], ptr %b, i32 0, i32 %g.09, i32 1
|
|
%0 = load i32, ptr %d, align 4
|
|
%tobool.not = icmp eq i32 %0, 0
|
|
br i1 %tobool.not, label %lor.lhs.false, label %if.then
|
|
|
|
lor.lhs.false:
|
|
%arrayidx = getelementptr inbounds [0 x %struct.e], ptr %b, i32 0, i32 %g.09
|
|
%1 = load i32, ptr %arrayidx, align 4
|
|
%tobool2.not = icmp eq i32 %1, 0
|
|
br i1 %tobool2.not, label %for.inc, label %if.then
|
|
|
|
if.then:
|
|
%inc = add nsw i32 %a.08, 1
|
|
br label %for.inc
|
|
|
|
for.inc:
|
|
%a.1 = phi i32 [ %inc, %if.then ], [ %a.08, %lor.lhs.false ]
|
|
%inc3 = add nuw nsw i32 %g.09, 1
|
|
%exitcond.not = icmp eq i32 %inc3, 1000
|
|
br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
|
|
}
|
|
|
|
; An in-loop add reduction followed by an in-loop sub reduction becomes an add reduction followed by an add reduction with its input negated
|
|
define i32 @reduction_add_sub(ptr noalias nocapture %A, ptr noalias nocapture %B) {
|
|
; CHECK-LABEL: define i32 @reduction_add_sub(
|
|
; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-NEXT: [[TMP2:%.*]] = sub <4 x i32> zeroinitializer, [[WIDE_LOAD1]]
|
|
; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD]])
|
|
; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[VEC_PHI]], [[TMP3]]
|
|
; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]])
|
|
; CHECK-NEXT: [[TMP6]] = add i32 [[TMP4]], [[TMP5]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: br label %[[FOR_END:.*]]
|
|
; CHECK: [[FOR_END]]:
|
|
; CHECK-NEXT: ret i32 [[TMP6]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define i32 @reduction_add_sub(
|
|
; CHECK-INTERLEAVED-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = sub <4 x i32> zeroinitializer, [[WIDE_LOAD3]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = sub <4 x i32> zeroinitializer, [[WIDE_LOAD4]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = add i32 [[VEC_PHI]], [[TMP6]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD2]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = add i32 [[VEC_PHI1]], [[TMP8]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP11]] = add i32 [[TMP7]], [[TMP10]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP5]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP13]] = add i32 [[TMP9]], [[TMP12]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add i32 [[TMP13]], [[TMP11]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_END:.*]]
|
|
; CHECK-INTERLEAVED: [[FOR_END]]:
|
|
; CHECK-INTERLEAVED-NEXT: ret i32 [[BIN_RDX]]
|
|
;
|
|
entry:
|
|
br label %for.body
|
|
|
|
for.body:
|
|
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
|
|
%x.05 = phi i32 [ %sub, %for.body ], [ 0, %entry ]
|
|
%arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
|
|
%arrayidx.b = getelementptr inbounds i32, ptr %B, i64 %indvars.iv
|
|
%l0 = load i32, ptr %arrayidx, align 4
|
|
%l0.b = load i32, ptr %arrayidx.b, align 4
|
|
%add = add nsw i32 %x.05, %l0
|
|
%sub = sub nsw i32 %add, %l0.b
|
|
%indvars.iv.next = add i64 %indvars.iv, 1
|
|
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
|
|
%exitcond = icmp eq i32 %lftr.wideiv, 256
|
|
br i1 %exitcond, label %for.end, label %for.body
|
|
|
|
for.end:
|
|
%x.0.lcssa = phi i32 [ %sub, %for.body ]
|
|
ret i32 %x.0.lcssa
|
|
}
|
|
|
|
; An in-loop sub reduction followed by an in-loop add reduction becomes an add reduction with its input negated
|
|
define i32 @reduction_sub_add(ptr noalias nocapture %A, ptr noalias nocapture %B) {
|
|
; CHECK-LABEL: define i32 @reduction_sub_add(
|
|
; CHECK-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-NEXT: [[TMP2:%.*]] = sub <4 x i32> zeroinitializer, [[WIDE_LOAD]]
|
|
; CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP2]])
|
|
; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[VEC_PHI]], [[TMP3]]
|
|
; CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD1]])
|
|
; CHECK-NEXT: [[TMP6]] = add i32 [[TMP4]], [[TMP5]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: br label %[[FOR_END:.*]]
|
|
; CHECK: [[FOR_END]]:
|
|
; CHECK-NEXT: ret i32 [[TMP6]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define i32 @reduction_sub_add(
|
|
; CHECK-INTERLEAVED-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = sub <4 x i32> zeroinitializer, [[WIDE_LOAD]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = sub <4 x i32> zeroinitializer, [[WIDE_LOAD2]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = add i32 [[VEC_PHI]], [[TMP6]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP5]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = add i32 [[VEC_PHI1]], [[TMP8]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD3]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP11]] = add i32 [[TMP7]], [[TMP10]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[WIDE_LOAD4]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP13]] = add i32 [[TMP9]], [[TMP12]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add i32 [[TMP13]], [[TMP11]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_END:.*]]
|
|
; CHECK-INTERLEAVED: [[FOR_END]]:
|
|
; CHECK-INTERLEAVED-NEXT: ret i32 [[BIN_RDX]]
|
|
;
|
|
entry:
|
|
br label %for.body
|
|
|
|
for.body:
|
|
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
|
|
%x.05 = phi i32 [ %add, %for.body ], [ 0, %entry ]
|
|
%arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
|
|
%arrayidx.b = getelementptr inbounds i32, ptr %B, i64 %indvars.iv
|
|
%l0 = load i32, ptr %arrayidx, align 4
|
|
%l0.b = load i32, ptr %arrayidx.b, align 4
|
|
%sub = sub nsw i32 %x.05, %l0
|
|
%add = add nsw i32 %sub, %l0.b
|
|
%indvars.iv.next = add i64 %indvars.iv, 1
|
|
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
|
|
%exitcond = icmp eq i32 %lftr.wideiv, 256
|
|
br i1 %exitcond, label %for.end, label %for.body
|
|
|
|
for.end:
|
|
%x.0.lcssa = phi i32 [ %add, %for.body ]
|
|
ret i32 %x.0.lcssa
|
|
}
|
|
|
|
; Test that bundling recipes that share an operand into an expression works.
|
|
; In this case the two extends are the recipes that share an operand.
|
|
define i64 @reduction_expression_same_operands(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) {
|
|
; CHECK-LABEL: define i64 @reduction_expression_same_operands(
|
|
; CHECK-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]], i32 [[N:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*]]:
|
|
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 4
|
|
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 4
|
|
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[X]], i32 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 4
|
|
; CHECK-NEXT: [[TMP3:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i64>
|
|
; CHECK-NEXT: [[TMP4:%.*]] = mul nsw <4 x i64> [[TMP3]], [[TMP3]]
|
|
; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP4]])
|
|
; CHECK-NEXT: [[TMP6]] = add i64 [[VEC_PHI]], [[TMP5]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
|
|
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
|
|
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
|
|
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
|
|
; CHECK: [[SCALAR_PH]]:
|
|
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
|
|
; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
|
|
; CHECK-NEXT: br label %[[LOOP:.*]]
|
|
; CHECK: [[LOOP]]:
|
|
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
|
|
; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ [[RDX_NEXT:%.*]], %[[LOOP]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
|
|
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[X]], i32 [[IV]]
|
|
; CHECK-NEXT: [[LOAD0:%.*]] = load i16, ptr [[ARRAYIDX]], align 4
|
|
; CHECK-NEXT: [[CONV0:%.*]] = sext i16 [[LOAD0]] to i32
|
|
; CHECK-NEXT: [[CONV1:%.*]] = sext i16 [[LOAD0]] to i32
|
|
; CHECK-NEXT: [[MUL1:%.*]] = mul nsw i32 [[CONV0]], [[CONV1]]
|
|
; CHECK-NEXT: [[MUL:%.*]] = sext i32 [[MUL1]] to i64
|
|
; CHECK-NEXT: [[RDX_NEXT]] = add nsw i64 [[RDX]], [[MUL]]
|
|
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
|
|
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
|
|
; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP31:![0-9]+]]
|
|
; CHECK: [[EXIT]]:
|
|
; CHECK-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ [[RDX_NEXT]], %[[LOOP]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ]
|
|
; CHECK-NEXT: ret i64 [[R_0_LCSSA]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define i64 @reduction_expression_same_operands(
|
|
; CHECK-INTERLEAVED-SAME: ptr readonly captures(none) [[X:%.*]], ptr readonly captures(none) [[Y:%.*]], i32 [[N:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[X]], i32 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = getelementptr inbounds i16, ptr [[TMP1]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP1]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i16>, ptr [[TMP2]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = sext <4 x i16> [[WIDE_LOAD]] to <4 x i64>
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = mul nsw <4 x i64> [[TMP4]], [[TMP4]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP5]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7]] = add i64 [[VEC_PHI]], [[TMP6]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = sext <4 x i16> [[WIDE_LOAD2]] to <4 x i64>
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = mul nsw <4 x i64> [[TMP9]], [[TMP9]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP10]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP12]] = add i64 [[VEC_PHI1]], [[TMP11]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add i64 [[TMP12]], [[TMP7]]
|
|
; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
|
|
; CHECK-INTERLEAVED: [[SCALAR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[LOOP:.*]]
|
|
; CHECK-INTERLEAVED: [[LOOP]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[RDX:%.*]] = phi i64 [ [[RDX_NEXT:%.*]], %[[LOOP]] ], [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[X]], i32 [[IV]]
|
|
; CHECK-INTERLEAVED-NEXT: [[LOAD0:%.*]] = load i16, ptr [[ARRAYIDX]], align 4
|
|
; CHECK-INTERLEAVED-NEXT: [[CONV0:%.*]] = sext i16 [[LOAD0]] to i32
|
|
; CHECK-INTERLEAVED-NEXT: [[CONV1:%.*]] = sext i16 [[LOAD0]] to i32
|
|
; CHECK-INTERLEAVED-NEXT: [[MUL1:%.*]] = mul nsw i32 [[CONV0]], [[CONV1]]
|
|
; CHECK-INTERLEAVED-NEXT: [[MUL:%.*]] = sext i32 [[MUL1]] to i64
|
|
; CHECK-INTERLEAVED-NEXT: [[RDX_NEXT]] = add nsw i64 [[RDX]], [[MUL]]
|
|
; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
|
|
; CHECK-INTERLEAVED-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP31:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[EXIT]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[R_0_LCSSA:%.*]] = phi i64 [ [[RDX_NEXT]], %[[LOOP]] ], [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ]
|
|
; CHECK-INTERLEAVED-NEXT: ret i64 [[R_0_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %loop
|
|
|
|
loop:
|
|
%iv = phi i32 [ %iv.next, %loop ], [ 0, %entry ]
|
|
%rdx = phi i64 [ %rdx.next, %loop ], [ 0, %entry ]
|
|
%arrayidx = getelementptr inbounds i16, ptr %x, i32 %iv
|
|
%load0 = load i16, ptr %arrayidx, align 4
|
|
%conv0 = sext i16 %load0 to i32
|
|
%conv1 = sext i16 %load0 to i32
|
|
%mul = mul nsw i32 %conv0, %conv1
|
|
%conv = sext i32 %mul to i64
|
|
%rdx.next = add nsw i64 %rdx, %conv
|
|
%iv.next = add nuw nsw i32 %iv, 1
|
|
%exitcond = icmp eq i32 %iv.next, %n
|
|
br i1 %exitcond, label %exit, label %loop
|
|
|
|
exit:
|
|
%r.0.lcssa = phi i64 [ %rdx.next, %loop ]
|
|
ret i64 %r.0.lcssa
|
|
}
|
|
|
|
define i32 @reduction_expression_ext_mulacc_livein(ptr %a, i16 %c) {
|
|
; CHECK-LABEL: define i32 @reduction_expression_ext_mulacc_livein(
|
|
; CHECK-SAME: ptr [[A:%.*]], i16 [[C:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[C]], i64 0
|
|
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
|
|
; CHECK-NEXT: [[TMP1:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i16>
|
|
; CHECK-NEXT: [[TMP2:%.*]] = mul <4 x i16> [[BROADCAST_SPLAT]], [[TMP1]]
|
|
; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i16> [[TMP2]] to <4 x i32>
|
|
; CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]])
|
|
; CHECK-NEXT: [[TMP5]] = add i32 [[VEC_PHI]], [[TMP4]]
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
|
|
; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: br label %[[FOR_EXIT:.*]]
|
|
; CHECK: [[FOR_EXIT]]:
|
|
; CHECK-NEXT: ret i32 [[TMP5]]
|
|
;
|
|
; CHECK-INTERLEAVED-LABEL: define i32 @reduction_expression_ext_mulacc_livein(
|
|
; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], i16 [[C:%.*]]) {
|
|
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_PH:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i16> poison, i16 [[C]], i64 0
|
|
; CHECK-INTERLEAVED-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i16> [[BROADCAST_SPLATINSERT]], <4 x i16> poison, <4 x i32> zeroinitializer
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[TMP0]], i64 4
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
|
|
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = zext <4 x i8> [[WIDE_LOAD]] to <4 x i16>
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[WIDE_LOAD2]] to <4 x i16>
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = mul <4 x i16> [[BROADCAST_SPLAT]], [[TMP2]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = mul <4 x i16> [[BROADCAST_SPLAT]], [[TMP3]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = zext <4 x i16> [[TMP4]] to <4 x i32>
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP6]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP8]] = add i32 [[VEC_PHI]], [[TMP7]]
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = zext <4 x i16> [[TMP5]] to <4 x i32>
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP9]])
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP11]] = add i32 [[VEC_PHI1]], [[TMP10]]
|
|
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
|
|
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
|
|
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
|
|
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
|
|
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add i32 [[TMP11]], [[TMP8]]
|
|
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_EXIT:.*]]
|
|
; CHECK-INTERLEAVED: [[FOR_EXIT]]:
|
|
; CHECK-INTERLEAVED-NEXT: ret i32 [[BIN_RDX]]
|
|
;
|
|
entry:
|
|
br label %for.body
|
|
|
|
for.body:
|
|
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
|
|
%accum = phi i32 [ 0, %entry ], [ %add, %for.body ]
|
|
%gep.a = getelementptr i8, ptr %a, i64 %iv
|
|
%load.a = load i8, ptr %gep.a, align 1
|
|
%ext.a = zext i8 %load.a to i16
|
|
%mul = mul i16 %c, %ext.a
|
|
%mul.ext = zext i16 %mul to i32
|
|
%add = add i32 %mul.ext, %accum
|
|
%iv.next = add i64 %iv, 1
|
|
%exitcond.not = icmp eq i64 %iv.next, 1024
|
|
br i1 %exitcond.not, label %for.exit, label %for.body
|
|
|
|
for.exit:
|
|
ret i32 %add
|
|
}
|
|
|
|
|
|
!6 = distinct !{!6, !7, !8}
|
|
!7 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
|
|
!8 = !{!"llvm.loop.vectorize.enable", i1 true}
|