Doomhowl-Interactive/llvm-project
0
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity
Files
main
llvm-project/llvm/test/Transforms/LoopVectorize/multiple-early-exits.ll
Florian Hahn ede1a9626b [LV] Vectorize early exit loops with multiple exits. (#174864) 
Building on top of the recent changes to introduce BranchOnTwoConds,
this patch adds support for vectorizing loops with multiple early exits,
all dominating a countable latch. The early exits must form a
dominance chain, so we can simply check which early exit has been taken
in dominance order.

Currently LoopVectorizationLegality ensures that all exits other than
the latch must be uncountable. handleUncountableEarlyExits now collects
those uncountable exits and processes each exit.

In the vector region, we compute if any exit has been taken, by taking
the OR of all early exit conditions (EarlyExitConds) and checking if
there's
any active lane.

If the early exit is taken, we exit the loop and compute which early
exit
has been taken. The first taken early exit is the one where its exit
condition is true in the first active lane of EarlyExitConds.

We create a chain of dispatch blocks outside the loop to check this for
the early exit blocks ordered by dominance.

Depends on https://github.com/llvm/llvm-project/pull/174016.

PR: https://github.com/llvm/llvm-project/pull/174864
2026-02-13 16:44:23 +00:00

1536 lines
73 KiB
LLVM
Raw Permalink Blame History

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -S < %s -p loop-vectorize -force-vector-width=4 -force-vector-interleave=1 | FileCheck %s
declare void @init_mem(ptr, i64)
@GlobA = external constant [1024 x i8]
define i64 @two_early_exits_same_exit() {
; CHECK-LABEL: define i64 @two_early_exits_same_exit() {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT1:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[GEP_A]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], splat (i8 42)
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[INDEX_NEXT1]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT1]], 64
; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 false)
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP1]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_1]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_0]]:
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 64, %[[SCALAR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT: [[GEP_A1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV1]]
; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[GEP_A1]], align 1
; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LD1]], 42
; CHECK-NEXT: br i1 [[CMP1]], label %[[EXIT]], label %[[EARLY_EXIT_0:.*]]
; CHECK: [[EARLY_EXIT_0]]:
; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV1]]
; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
; CHECK-NEXT: [[CMP2:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
; CHECK-NEXT: br i1 [[CMP2]], label %[[EXIT]], label %[[LOOP_LATCH]]
; CHECK: [[LOOP_LATCH]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP_HEADER]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[IV1]], %[[LOOP_HEADER]] ], [ 100, %[[EARLY_EXIT_0]] ], [ 43, %[[LOOP_LATCH]] ], [ [[TMP9]], %[[VECTOR_EARLY_EXIT_0]] ], [ 100, %[[VECTOR_EARLY_EXIT_1]] ]
; CHECK-NEXT: ret i64 [[RETVAL]]
;
entry:
%A = alloca [1024 x i8]
%B = alloca [1024 x i8]
call void @init_mem(ptr %A, i64 1024)
call void @init_mem(ptr %B, i64 1024)
br label %loop.header
loop.header:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
%gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
%ld.A = load i8, ptr %gep.A, align 1
%cmp1 = icmp eq i8 %ld.A, 42
br i1 %cmp1, label %exit, label %early.exit.0
early.exit.0:
%gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
%ld.B = load i8, ptr %gep.B, align 1
%cmp2 = icmp eq i8 %ld.A, %ld.B
br i1 %cmp2, label %exit, label %loop.latch
loop.latch:
%iv.next = add i64 %iv, 1
%ec = icmp ne i64 %iv.next, 67
br i1 %ec, label %loop.header, label %exit
exit:
%retval = phi i64 [ %iv, %loop.header ], [ 100, %early.exit.0 ], [ 43, %loop.latch ]
ret i64 %retval
}
define i64 @two_early_exits_different_exits() {
; CHECK-LABEL: define i64 @two_early_exits_different_exits() {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT1:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[GEP_A]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult <4 x i8> [[WIDE_LOAD]], splat (i8 34)
; CHECK-NEXT: [[INDEX_NEXT1]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT1]], 64
; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 false)
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP2]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_1]]:
; CHECK-NEXT: br label %[[EXIT2:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_0]]:
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[EXIT1:.*]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ 64, %[[SCALAR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT: [[GEP_A1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV1]]
; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[GEP_A1]], align 1
; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV1]]
; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
; CHECK-NEXT: br i1 [[CMP1]], label %[[EXIT1]], label %[[EARLY_EXIT_0:.*]]
; CHECK: [[EARLY_EXIT_0]]:
; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i8 [[LD1]], 34
; CHECK-NEXT: br i1 [[CMP2]], label %[[EXIT2]], label %[[LOOP_LATCH]]
; CHECK: [[LOOP_LATCH]]:
; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[IV1]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 67
; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP_HEADER]], label %[[EXIT:.*]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: [[EXIT1]]:
; CHECK-NEXT: [[RET2:%.*]] = phi i64 [ [[IV1]], %[[LOOP_HEADER]] ], [ [[TMP9]], %[[VECTOR_EARLY_EXIT_0]] ]
; CHECK-NEXT: ret i64 [[RET2]]
; CHECK: [[EXIT2]]:
; CHECK-NEXT: [[RET3:%.*]] = phi i64 [ 100, %[[EARLY_EXIT_0]] ], [ 100, %[[VECTOR_EARLY_EXIT_1]] ]
; CHECK-NEXT: ret i64 [[RET3]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i64 43
;
entry:
%A = alloca [1024 x i8]
%B = alloca [1024 x i8]
call void @init_mem(ptr %A, i64 1024)
call void @init_mem(ptr %B, i64 1024)
br label %loop.header
loop.header:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
%gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
%ld.A = load i8, ptr %gep.A, align 1
%gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
%ld.B = load i8, ptr %gep.B, align 1
%cmp1 = icmp eq i8 %ld.A, %ld.B
br i1 %cmp1, label %exit1, label %early.exit.0
early.exit.0:
%cmp2 = icmp ult i8 %ld.A, 34
br i1 %cmp2, label %exit2, label %loop.latch
loop.latch:
%iv.next = add i64 %iv, 1
%ec = icmp ne i64 %iv.next, 67
br i1 %ec, label %loop.header, label %exit
exit1:
%ret1 = phi i64 [ %iv, %loop.header ]
ret i64 %ret1
exit2:
%ret2 = phi i64 [ 100, %early.exit.0 ]
ret i64 %ret2
exit:
ret i64 43
}
define i64 @two_early_exits_different_exits_second_load_not_deref(ptr %B) {
; CHECK-LABEL: define i64 @two_early_exits_different_exits_second_load_not_deref(
; CHECK-SAME: ptr [[B:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[A:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[A]], i64 1024)
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[GEP_A]], align 1
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[LD_B:%.*]] = load i8, ptr [[GEP_B]], align 1
; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LD_A]], [[LD_B]]
; CHECK-NEXT: br i1 [[CMP1]], label %[[EXIT1:.*]], label %[[EARLY_EXIT_0:.*]]
; CHECK: [[EARLY_EXIT_0]]:
; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i8 [[LD_A]], 34
; CHECK-NEXT: br i1 [[CMP2]], label %[[EXIT2:.*]], label %[[LOOP_LATCH]]
; CHECK: [[LOOP_LATCH]]:
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], 67
; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT:.*]]
; CHECK: [[EXIT1]]:
; CHECK-NEXT: [[RET1:%.*]] = phi i64 [ [[IV]], %[[LOOP_HEADER]] ]
; CHECK-NEXT: ret i64 [[RET1]]
; CHECK: [[EXIT2]]:
; CHECK-NEXT: [[RET2:%.*]] = phi i64 [ 100, %[[EARLY_EXIT_0]] ]
; CHECK-NEXT: ret i64 [[RET2]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i64 43
;
entry:
%A = alloca [1024 x i8]
call void @init_mem(ptr %A, i64 1024)
br label %loop.header
loop.header:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
%gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
%ld.A = load i8, ptr %gep.A, align 1
%gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
%ld.B = load i8, ptr %gep.B, align 1
%cmp1 = icmp eq i8 %ld.A, %ld.B
br i1 %cmp1, label %exit1, label %early.exit.0
early.exit.0:
%cmp2 = icmp ult i8 %ld.A, 34
br i1 %cmp2, label %exit2, label %loop.latch
loop.latch:
%iv.next = add i64 %iv, 1
%ec = icmp ne i64 %iv.next, 67
br i1 %ec, label %loop.header, label %exit
exit1:
%ret1 = phi i64 [ %iv, %loop.header ]
ret i64 %ret1
exit2:
%ret2 = phi i64 [ 100, %early.exit.0 ]
ret i64 %ret2
exit:
ret i64 43
}
define i64 @three_early_exits_same_exit() {
; CHECK-LABEL: define i64 @three_early_exits_same_exit() {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX]]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], splat (i8 42)
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[GEP_B]], align 1
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt <4 x i8> [[WIDE_LOAD1]], splat (i8 100)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> splat (i1 true), <4 x i1> [[TMP4]]
; CHECK-NEXT: [[TMP7:%.*]] = freeze <4 x i1> [[TMP6]]
; CHECK-NEXT: [[CMP2:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP7]])
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
; CHECK-NEXT: br i1 [[CMP2]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP1]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_CHECK_0:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK_0]]:
; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP3]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP11]], label %[[VECTOR_EARLY_EXIT_1:.*]], label %[[VECTOR_EARLY_EXIT_2:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_2]]:
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[VECTOR_EARLY_EXIT_1]]:
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[VECTOR_EARLY_EXIT_0]]:
; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: [[TMP13:%.*]] = add i64 3, [[TMP12]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ 200, %[[VECTOR_EARLY_EXIT_2]] ], [ 100, %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP13]], %[[VECTOR_EARLY_EXIT_0]] ], [ 43, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RETVAL]]
;
entry:
%A = alloca [1024 x i8]
%B = alloca [1024 x i8]
call void @init_mem(ptr %A, i64 1024)
call void @init_mem(ptr %B, i64 1024)
br label %loop.header
loop.header:
%iv = phi i64 [ %iv.next, %loop.latch ], [ 3, %entry ]
%gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
%ld.A = load i8, ptr %gep.A, align 1
%cmp1 = icmp eq i8 %ld.A, 42
br i1 %cmp1, label %exit, label %early.exit.0
early.exit.0:
%gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
%ld.B = load i8, ptr %gep.B, align 1
%cmp2 = icmp eq i8 %ld.A, %ld.B
br i1 %cmp2, label %exit, label %early.exit.1
early.exit.1:
%cmp3 = icmp ugt i8 %ld.B, 100
br i1 %cmp3, label %exit, label %loop.latch
loop.latch:
%iv.next = add i64 %iv, 1
%ec = icmp ne i64 %iv.next, 67
br i1 %ec, label %loop.header, label %exit
exit:
%retval = phi i64 [ %iv, %loop.header ], [ 100, %early.exit.0 ], [ 200, %early.exit.1 ], [ 43, %loop.latch ]
ret i64 %retval
}
define i64 @four_early_exits_same_exit() {
; CHECK-LABEL: define i64 @four_early_exits_same_exit() {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P3:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P3]], i64 1024)
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX]]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], splat (i8 42)
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[GEP_B]], align 1
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[P3]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD2]]
; CHECK-NEXT: [[TMP6:%.*]] = icmp ugt <4 x i8> [[WIDE_LOAD2]], splat (i8 100)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP7]], <4 x i1> splat (i1 true), <4 x i1> [[TMP5]]
; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP8]], <4 x i1> splat (i1 true), <4 x i1> [[TMP6]]
; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP9]]
; CHECK-NEXT: [[CMP2:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
; CHECK-NEXT: br i1 [[CMP2]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
; CHECK-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP9]], i1 false)
; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP1]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP13]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_CHECK_0:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK_0]]:
; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP3]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP14]], label %[[VECTOR_EARLY_EXIT_1:.*]], label %[[VECTOR_EARLY_EXIT_CHECK_1:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK_1]]:
; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP5]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP15]], label %[[VECTOR_EARLY_EXIT_2:.*]], label %[[VECTOR_EARLY_EXIT_3:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_3]]:
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[VECTOR_EARLY_EXIT_2]]:
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[VECTOR_EARLY_EXIT_1]]:
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[VECTOR_EARLY_EXIT_0]]:
; CHECK-NEXT: [[TMP16:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: [[TMP17:%.*]] = add i64 3, [[TMP16]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ 300, %[[VECTOR_EARLY_EXIT_3]] ], [ 200, %[[VECTOR_EARLY_EXIT_2]] ], [ 100, %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP17]], %[[VECTOR_EARLY_EXIT_0]] ], [ 43, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RETVAL]]
;
entry:
%A = alloca [1024 x i8]
%B = alloca [1024 x i8]
%C = alloca [1024 x i8]
call void @init_mem(ptr %A, i64 1024)
call void @init_mem(ptr %B, i64 1024)
call void @init_mem(ptr %C, i64 1024)
br label %loop.header
loop.header:
%iv = phi i64 [ %iv.next, %loop.latch ], [ 3, %entry ]
%gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
%ld.A = load i8, ptr %gep.A, align 1
%cmp1 = icmp eq i8 %ld.A, 42
br i1 %cmp1, label %exit, label %early.exit.0
early.exit.0:
%gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
%ld.B = load i8, ptr %gep.B, align 1
%cmp2 = icmp eq i8 %ld.A, %ld.B
br i1 %cmp2, label %exit, label %early.exit.1
early.exit.1:
%gep.C = getelementptr inbounds i8, ptr %C, i64 %iv
%ld.C = load i8, ptr %gep.C, align 1
%cmp3 = icmp eq i8 %ld.B, %ld.C
br i1 %cmp3, label %exit, label %early.exit.2
early.exit.2:
%cmp4 = icmp ugt i8 %ld.C, 100
br i1 %cmp4, label %exit, label %loop.latch
loop.latch:
%iv.next = add i64 %iv, 1
%ec = icmp ne i64 %iv.next, 67
br i1 %ec, label %loop.header, label %exit
exit:
%retval = phi i64 [ %iv, %loop.header ], [ 100, %early.exit.0 ], [ 200, %early.exit.1 ], [ 300, %early.exit.2 ], [ 43, %loop.latch ]
ret i64 %retval
}
define i64 @two_early_exits_with_live_out_values() {
; CHECK-LABEL: define i64 @two_early_exits_with_live_out_values() {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX]]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult <4 x i8> [[TMP3]], splat (i8 34)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> splat (i1 true), <4 x i1> [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64
; CHECK-NEXT: br i1 [[TMP7]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 false)
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP2]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP9]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_1]]:
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i8> [[TMP3]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: [[TMP12:%.*]] = add i64 3, [[TMP11]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[VECTOR_EARLY_EXIT_0]]:
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: [[TMP14:%.*]] = add i64 3, [[TMP13]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[TMP12]], %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP14]], %[[VECTOR_EARLY_EXIT_0]] ], [ 99, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[RETVAL2:%.*]] = phi i8 [ [[TMP10]], %[[VECTOR_EARLY_EXIT_1]] ], [ 0, %[[VECTOR_EARLY_EXIT_0]] ], [ 0, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[EXT:%.*]] = zext i8 [[RETVAL2]] to i64
; CHECK-NEXT: [[RET:%.*]] = add i64 [[RETVAL]], [[EXT]]
; CHECK-NEXT: ret i64 [[RET]]
;
entry:
%A = alloca [1024 x i8]
%B = alloca [1024 x i8]
call void @init_mem(ptr %A, i64 1024)
call void @init_mem(ptr %B, i64 1024)
br label %loop.header
loop.header:
%iv = phi i64 [ %iv.next, %loop.latch ], [ 3, %entry ]
%gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
%ld.A = load i8, ptr %gep.A, align 1
%gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
%ld.B = load i8, ptr %gep.B, align 1
%cmp1 = icmp eq i8 %ld.A, %ld.B
br i1 %cmp1, label %exit, label %early.exit.0
early.exit.0:
%sum = add i8 %ld.A, %ld.B
%cmp2 = icmp ult i8 %sum, 34
br i1 %cmp2, label %exit, label %loop.latch
loop.latch:
%iv.next = add i64 %iv, 1
%ec = icmp ne i64 %iv.next, 67
br i1 %ec, label %loop.header, label %exit
exit:
%retval = phi i64 [ %iv, %loop.header ], [ %iv, %early.exit.0 ], [ 99, %loop.latch ]
%retval2 = phi i8 [ 0, %loop.header ], [ %sum, %early.exit.0 ], [ 0, %loop.latch ]
%ext = zext i8 %retval2 to i64
%ret = add i64 %retval, %ext
ret i64 %ret
}
define i64 @two_early_exits_negated_condition() {
; CHECK-LABEL: define i64 @two_early_exits_negated_condition() {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[IV:%.*]] = add i64 3, [[INDEX]]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[GEP_A]], align 1
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP11]], align 1
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult <4 x i8> [[WIDE_LOAD]], splat (i8 34)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 124
; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 false)
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP2]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_1]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_0]]:
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: [[TMP10:%.*]] = add i64 3, [[TMP9]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ 127, %[[SCALAR_PH]] ]
; CHECK-NEXT: [[GEP_A1:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV1]]
; CHECK-NEXT: [[LD_A:%.*]] = load i8, ptr [[GEP_A1]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV1]]
; CHECK-NEXT: [[LD_B:%.*]] = load i8, ptr [[TMP1]], align 1
; CHECK-NEXT: [[CMP1:%.*]] = icmp ne i8 [[LD_A]], [[LD_B]]
; CHECK-NEXT: br i1 [[CMP1]], label %[[EARLY_EXIT_0:.*]], label %[[EXIT]]
; CHECK: [[EARLY_EXIT_0]]:
; CHECK-NEXT: [[CMP2:%.*]] = icmp uge i8 [[LD_A]], 34
; CHECK-NEXT: br i1 [[CMP2]], label %[[LOOP_LATCH]], label %[[EXIT]]
; CHECK: [[LOOP_LATCH]]:
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV1]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV_NEXT]], 128
; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_HEADER]], label %[[EXIT]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[IV1]], %[[LOOP_HEADER]] ], [ 100, %[[EARLY_EXIT_0]] ], [ 43, %[[LOOP_LATCH]] ], [ [[TMP10]], %[[VECTOR_EARLY_EXIT_0]] ], [ 100, %[[VECTOR_EARLY_EXIT_1]] ]
; CHECK-NEXT: ret i64 [[RETVAL]]
;
entry:
%A = alloca [1024 x i8]
%B = alloca [1024 x i8]
call void @init_mem(ptr %A, i64 1024)
call void @init_mem(ptr %B, i64 1024)
br label %loop.header
loop.header:
%iv = phi i64 [ %iv.next, %loop.latch ], [ 3, %entry ]
%gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
%ld.A = load i8, ptr %gep.A, align 1
%gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
%ld.B = load i8, ptr %gep.B, align 1
%cmp1 = icmp ne i8 %ld.A, %ld.B
br i1 %cmp1, label %early.exit.0, label %exit
early.exit.0:
%cmp2 = icmp uge i8 %ld.A, 34
br i1 %cmp2, label %loop.latch, label %exit
loop.latch:
%iv.next = add i64 %iv, 1
%ec = icmp ne i64 %iv.next, 128
br i1 %ec, label %loop.header, label %exit
exit:
%retval = phi i64 [ %iv, %loop.header ], [ 100, %early.exit.0 ], [ 43, %loop.latch ]
ret i64 %retval
}
; Three early exits to three different exit blocks.
define i64 @three_early_exits_three_exit_blocks() {
; CHECK-LABEL: define i64 @three_early_exits_three_exit_blocks() {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX]]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult <4 x i8> [[WIDE_LOAD]], splat (i8 34)
; CHECK-NEXT: [[TMP4:%.*]] = icmp ugt <4 x i8> [[WIDE_LOAD1]], splat (i8 100)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP5]], <4 x i1> splat (i1 true), <4 x i1> [[TMP4]]
; CHECK-NEXT: [[TMP7:%.*]] = freeze <4 x i1> [[TMP6]]
; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP7]])
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP2]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP11]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_CHECK_0:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK_0]]:
; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP3]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP13]], label %[[VECTOR_EARLY_EXIT_1:.*]], label %[[VECTOR_EARLY_EXIT_2:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_2]]:
; CHECK-NEXT: br label %[[EXIT3:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_1]]:
; CHECK-NEXT: br label %[[EXIT2:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_0]]:
; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: [[RET3:%.*]] = add i64 3, [[TMP12]]
; CHECK-NEXT: br label %[[EXIT1:.*]]
; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret i64 [[RET3]]
; CHECK: [[EXIT2]]:
; CHECK-NEXT: ret i64 100
; CHECK: [[EXIT3]]:
; CHECK-NEXT: ret i64 200
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i64 43
;
entry:
%A = alloca [1024 x i8]
%B = alloca [1024 x i8]
call void @init_mem(ptr %A, i64 1024)
call void @init_mem(ptr %B, i64 1024)
br label %loop.header
loop.header:
%iv = phi i64 [ %iv.next, %loop.latch ], [ 3, %entry ]
%gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
%ld.A = load i8, ptr %gep.A, align 1
%gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
%ld.B = load i8, ptr %gep.B, align 1
%cmp1 = icmp eq i8 %ld.A, %ld.B
br i1 %cmp1, label %exit1, label %early.exit.0
early.exit.0:
%cmp2 = icmp ult i8 %ld.A, 34
br i1 %cmp2, label %exit2, label %early.exit.1
early.exit.1:
%cmp3 = icmp ugt i8 %ld.B, 100
br i1 %cmp3, label %exit3, label %loop.latch
loop.latch:
%iv.next = add i64 %iv, 1
%ec = icmp ne i64 %iv.next, 99
br i1 %ec, label %loop.header, label %exit
exit1:
%ret1 = phi i64 [ %iv, %loop.header ]
ret i64 %ret1
exit2:
%ret2 = phi i64 [ 100, %early.exit.0 ]
ret i64 %ret2
exit3:
%ret3 = phi i64 [ 200, %early.exit.1 ]
ret i64 %ret3
exit:
ret i64 43
}
; Two early exits with IV used as live-out from both exits.
define i64 @two_early_exits_iv_live_out() {
; CHECK-LABEL: define i64 @two_early_exits_iv_live_out() {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult <4 x i8> [[WIDE_LOAD]], splat (i8 34)
; CHECK-NEXT: [[TMP4:%.*]] = or <4 x i1> [[TMP2]], [[TMP3]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[VECTOR_EARLY_EXIT]]:
; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 false)
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP8]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[TMP9]], %[[VECTOR_EARLY_EXIT]] ], [ 128, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RETVAL]]
;
entry:
%A = alloca [1024 x i8]
%B = alloca [1024 x i8]
call void @init_mem(ptr %A, i64 1024)
call void @init_mem(ptr %B, i64 1024)
br label %loop.header
loop.header:
%iv = phi i64 [ %iv.next, %loop.latch ], [ 0, %entry ]
%gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
%ld.A = load i8, ptr %gep.A, align 1
%gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
%ld.B = load i8, ptr %gep.B, align 1
%cmp1 = icmp eq i8 %ld.A, %ld.B
br i1 %cmp1, label %exit, label %early.exit.0
early.exit.0:
%cmp2 = icmp ult i8 %ld.A, 34
br i1 %cmp2, label %exit, label %loop.latch
loop.latch:
%iv.next = add nuw i64 %iv, 1
%ec = icmp eq i64 %iv.next, 128
br i1 %ec, label %exit, label %loop.header
exit:
%retval = phi i64 [ %iv, %loop.header ], [ %iv, %early.exit.0 ], [ 128, %loop.latch ]
ret i64 %retval
}
define i64 @three_early_exits_iv_and_load_live_out() {
; CHECK-LABEL: define i64 @three_early_exits_iv_and_load_live_out() {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult <4 x i8> [[WIDE_LOAD]], splat (i8 34)
; CHECK-NEXT: [[TMP4:%.*]] = or <4 x i1> [[TMP2]], [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = icmp ugt <4 x i8> [[WIDE_LOAD1]], splat (i8 100)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> splat (i1 true), <4 x i1> [[TMP5]]
; CHECK-NEXT: [[TMP7:%.*]] = freeze <4 x i1> [[TMP6]]
; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP7]])
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 false)
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP4]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP10]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_1]]:
; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i8> [[WIDE_LOAD1]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[VECTOR_EARLY_EXIT_0]]:
; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i8> [[WIDE_LOAD]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RETVAL_IV:%.*]] = phi i64 [ [[TMP12]], %[[VECTOR_EARLY_EXIT_1]] ], [ 128, %[[MIDDLE_BLOCK]] ], [ [[TMP14]], %[[VECTOR_EARLY_EXIT_0]] ]
; CHECK-NEXT: [[RETVAL_LD:%.*]] = phi i8 [ [[TMP11]], %[[VECTOR_EARLY_EXIT_1]] ], [ 0, %[[MIDDLE_BLOCK]] ], [ [[TMP13]], %[[VECTOR_EARLY_EXIT_0]] ]
; CHECK-NEXT: [[EXT:%.*]] = zext i8 [[RETVAL_LD]] to i64
; CHECK-NEXT: [[RET:%.*]] = add i64 [[RETVAL_IV]], [[EXT]]
; CHECK-NEXT: ret i64 [[RET]]
;
entry:
%A = alloca [1024 x i8]
%B = alloca [1024 x i8]
call void @init_mem(ptr %A, i64 1024)
call void @init_mem(ptr %B, i64 1024)
br label %loop.header
loop.header:
%iv = phi i64 [ %iv.next, %loop.latch ], [ 0, %entry ]
%gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
%ld.A = load i8, ptr %gep.A, align 1
%gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
%ld.B = load i8, ptr %gep.B, align 1
%cmp1 = icmp eq i8 %ld.A, %ld.B
br i1 %cmp1, label %exit, label %early.exit.0
early.exit.0:
%cmp2 = icmp ult i8 %ld.A, 34
br i1 %cmp2, label %exit, label %early.exit.1
early.exit.1:
%cmp3 = icmp ugt i8 %ld.B, 100
br i1 %cmp3, label %exit, label %loop.latch
loop.latch:
%iv.next = add nuw i64 %iv, 1
%ec = icmp eq i64 %iv.next, 128
br i1 %ec, label %exit, label %loop.header
exit:
%retval.iv = phi i64 [ %iv, %loop.header ], [ %iv, %early.exit.0 ], [ %iv, %early.exit.1 ], [ 128, %loop.latch ]
%retval.ld = phi i8 [ %ld.A, %loop.header ], [ %ld.A, %early.exit.0 ], [ %ld.B, %early.exit.1 ], [ 0, %loop.latch ]
%ext = zext i8 %retval.ld to i64
%ret = add i64 %retval.iv, %ext
ret i64 %ret
}
; Two early exits with different live-out values from each exit.
define i64 @two_early_exits_different_live_outs() {
; CHECK-LABEL: define i64 @two_early_exits_different_live_outs() {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[SUM:%.*]] = phi i64 [ [[SUM_NEXT:%.*]], %[[LOOP_LATCH]] ], [ 0, %[[ENTRY]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1
; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
; CHECK-NEXT: br i1 [[CMP1]], label %[[EXIT1:.*]], label %[[EARLY_EXIT_0:.*]]
; CHECK: [[EARLY_EXIT_0]]:
; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i8 [[LD1]], 34
; CHECK-NEXT: br i1 [[CMP2]], label %[[EXIT2:.*]], label %[[LOOP_LATCH]]
; CHECK: [[LOOP_LATCH]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1
; CHECK-NEXT: [[LD1_EXT:%.*]] = zext i8 [[LD1]] to i64
; CHECK-NEXT: [[SUM_NEXT]] = add i64 [[SUM]], [[LD1_EXT]]
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV_NEXT]], 128
; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT_LATCH:.*]], label %[[LOOP_HEADER]]
; CHECK: [[EXIT1]]:
; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP_HEADER]] ]
; CHECK-NEXT: ret i64 [[IV_LCSSA]]
; CHECK: [[EXIT2]]:
; CHECK-NEXT: [[LD1_LCSSA1:%.*]] = phi i8 [ [[LD1]], %[[EARLY_EXIT_0]] ]
; CHECK-NEXT: [[LD1_EXT2:%.*]] = zext i8 [[LD1_LCSSA1]] to i64
; CHECK-NEXT: ret i64 [[LD1_EXT2]]
; CHECK: [[EXIT_LATCH]]:
; CHECK-NEXT: [[SUM_NEXT_LCSSA:%.*]] = phi i64 [ [[SUM_NEXT]], %[[LOOP_LATCH]] ]
; CHECK-NEXT: ret i64 [[SUM_NEXT_LCSSA]]
;
entry:
%A = alloca [1024 x i8]
%B = alloca [1024 x i8]
call void @init_mem(ptr %A, i64 1024)
call void @init_mem(ptr %B, i64 1024)
br label %loop.header
loop.header:
%iv = phi i64 [ %iv.next, %loop.latch ], [ 0, %entry ]
%sum = phi i64 [ %sum.next, %loop.latch ], [ 0, %entry ]
%gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
%ld.A = load i8, ptr %gep.A, align 1
%gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
%ld.B = load i8, ptr %gep.B, align 1
%cmp1 = icmp eq i8 %ld.A, %ld.B
br i1 %cmp1, label %exit1, label %early.exit.0
early.exit.0:
%cmp2 = icmp ult i8 %ld.A, 34
br i1 %cmp2, label %exit2, label %loop.latch
loop.latch:
%iv.next = add nuw i64 %iv, 1
%ld.A.ext = zext i8 %ld.A to i64
%sum.next = add i64 %sum, %ld.A.ext
%ec = icmp eq i64 %iv.next, 128
br i1 %ec, label %exit.latch, label %loop.header
exit1:
ret i64 %iv
exit2:
%ld.A.ext2 = zext i8 %ld.A to i64
ret i64 %ld.A.ext2
exit.latch:
ret i64 %sum.next
}
; Two early exits to same block, IV live-out with different incoming values.
define i64 @two_early_exits_iv_diff_incoming() {
; CHECK-LABEL: define i64 @two_early_exits_iv_diff_incoming() {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult <4 x i8> [[WIDE_LOAD]], splat (i8 34)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
; CHECK-NEXT: [[TMP14:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; CHECK-NEXT: br i1 [[TMP14]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 false)
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP2]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_1]]:
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[TMP9]], 1
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[VECTOR_EARLY_EXIT_0]]:
; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[TMP10]], %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP11]], %[[VECTOR_EARLY_EXIT_0]] ], [ 200, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RETVAL]]
;
entry:
%A = alloca [1024 x i8]
%B = alloca [1024 x i8]
call void @init_mem(ptr %A, i64 1024)
call void @init_mem(ptr %B, i64 1024)
br label %loop.header
loop.header:
%iv = phi i64 [ %iv.next, %loop.latch ], [ 0, %entry ]
%gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
%ld.A = load i8, ptr %gep.A, align 1
%gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
%ld.B = load i8, ptr %gep.B, align 1
%cmp1 = icmp eq i8 %ld.A, %ld.B
br i1 %cmp1, label %exit, label %early.exit.0
early.exit.0:
%iv.plus1 = add i64 %iv, 1
%cmp2 = icmp ult i8 %ld.A, 34
br i1 %cmp2, label %exit, label %loop.latch
loop.latch:
%iv.next = add nuw i64 %iv, 1
%ec = icmp eq i64 %iv.next, 128
br i1 %ec, label %exit, label %loop.header
exit:
%retval = phi i64 [ %iv, %loop.header ], [ %iv.plus1, %early.exit.0 ], [ 200, %loop.latch ]
ret i64 %retval
}
define { i64, i64 } @three_early_exits_multiple_live_outs() {
; CHECK-LABEL: define { i64, i64 } @three_early_exits_multiple_live_outs() {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult <4 x i8> [[TMP3]], splat (i8 34)
; CHECK-NEXT: [[TMP5:%.*]] = sub <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP6:%.*]] = icmp ugt <4 x i8> [[TMP5]], splat (i8 100)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP2]], <4 x i1> splat (i1 true), <4 x i1> [[TMP4]]
; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP7]], <4 x i1> splat (i1 true), <4 x i1> [[TMP6]]
; CHECK-NEXT: [[TMP9:%.*]] = freeze <4 x i1> [[TMP8]]
; CHECK-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP9]])
; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; CHECK-NEXT: br i1 [[TMP10]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP8]], i1 false)
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP2]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_CHECK_0:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK_0]]:
; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP4]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP13]], label %[[VECTOR_EARLY_EXIT_1:.*]], label %[[VECTOR_EARLY_EXIT_2:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_2]]:
; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i8> [[TMP5]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[VECTOR_EARLY_EXIT_1]]:
; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i8> [[TMP3]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: [[TMP17:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[VECTOR_EARLY_EXIT_0]]:
; CHECK-NEXT: [[TMP18:%.*]] = extractelement <4 x i8> [[WIDE_LOAD]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[INDEX]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RETVAL_IV:%.*]] = phi i64 [ [[TMP15]], %[[VECTOR_EARLY_EXIT_2]] ], [ [[TMP17]], %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP19]], %[[VECTOR_EARLY_EXIT_0]] ], [ 128, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[RETVAL_VAL:%.*]] = phi i8 [ [[TMP14]], %[[VECTOR_EARLY_EXIT_2]] ], [ [[TMP16]], %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP18]], %[[VECTOR_EARLY_EXIT_0]] ], [ 0, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[EXT:%.*]] = zext i8 [[RETVAL_VAL]] to i64
; CHECK-NEXT: [[R1:%.*]] = insertvalue { i64, i64 } undef, i64 [[RETVAL_IV]], 0
; CHECK-NEXT: [[R2:%.*]] = insertvalue { i64, i64 } [[R1]], i64 [[EXT]], 1
; CHECK-NEXT: ret { i64, i64 } [[R2]]
;
entry:
%A = alloca [1024 x i8]
%B = alloca [1024 x i8]
call void @init_mem(ptr %A, i64 1024)
call void @init_mem(ptr %B, i64 1024)
br label %loop.header
loop.header:
%iv = phi i64 [ %iv.next, %loop.latch ], [ 0, %entry ]
%gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
%ld.A = load i8, ptr %gep.A, align 1
%gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
%ld.B = load i8, ptr %gep.B, align 1
%cmp1 = icmp eq i8 %ld.A, %ld.B
br i1 %cmp1, label %exit, label %early.exit.0
early.exit.0:
%sum = add i8 %ld.A, %ld.B
%cmp2 = icmp ult i8 %sum, 34
br i1 %cmp2, label %exit, label %early.exit.1
early.exit.1:
%diff = sub i8 %ld.A, %ld.B
%cmp3 = icmp ugt i8 %diff, 100
br i1 %cmp3, label %exit, label %loop.latch
loop.latch:
%iv.next = add nuw i64 %iv, 1
%ec = icmp eq i64 %iv.next, 128
br i1 %ec, label %exit, label %loop.header
exit:
%retval.iv = phi i64 [ %iv, %loop.header ], [ %iv, %early.exit.0 ], [ %iv, %early.exit.1 ], [ 128, %loop.latch ]
%retval.val = phi i8 [ %ld.A, %loop.header ], [ %sum, %early.exit.0 ], [ %diff, %early.exit.1 ], [ 0, %loop.latch ]
%ext = zext i8 %retval.val to i64
%r1 = insertvalue { i64, i64 } undef, i64 %retval.iv, 0
%r2 = insertvalue { i64, i64 } %r1, i64 %ext, 1
ret { i64, i64 } %r2
}
; Two early exits with second load only executed conditionally (after first exit check).
define i64 @two_early_exits_load_in_early_exit_block() {
; CHECK-LABEL: define i64 @two_early_exits_load_in_early_exit_block() {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], splat (i8 42)
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[GEP_B]], align 1
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]]
; CHECK-NEXT: [[CMP2:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; CHECK-NEXT: br i1 [[CMP2]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
; CHECK-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 false)
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP1]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP8]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_1:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_1]]:
; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[VECTOR_EARLY_EXIT_0]]:
; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[TMP9]], %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP10]], %[[VECTOR_EARLY_EXIT_0]] ], [ 128, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RETVAL]]
;
entry:
%A = alloca [1024 x i8]
%B = alloca [1024 x i8]
call void @init_mem(ptr %A, i64 1024)
call void @init_mem(ptr %B, i64 1024)
br label %loop.header
loop.header:
%iv = phi i64 [ %iv.next, %loop.latch ], [ 0, %entry ]
%gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
%ld.A = load i8, ptr %gep.A, align 1
%cmp1 = icmp eq i8 %ld.A, 42
br i1 %cmp1, label %exit, label %early.exit.0
early.exit.0:
%gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
%ld.B = load i8, ptr %gep.B, align 1
%cmp2 = icmp eq i8 %ld.A, %ld.B
br i1 %cmp2, label %exit, label %loop.latch
loop.latch:
%iv.next = add nuw i64 %iv, 1
%ec = icmp eq i64 %iv.next, 128
br i1 %ec, label %exit, label %loop.header
exit:
%retval = phi i64 [ %iv, %loop.header ], [ %iv, %early.exit.0 ], [ 128, %loop.latch ]
ret i64 %retval
}
define i64 @three_early_exits_loads_in_different_blocks() {
; CHECK-LABEL: define i64 @three_early_exits_loads_in_different_blocks() {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P3:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P3]], i64 1024)
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], splat (i8 42)
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[GEP_B]], align 1
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD1]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[P3]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD1]], [[WIDE_LOAD2]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
; CHECK-NEXT: [[TMP7:%.*]] = select <4 x i1> [[TMP6]], <4 x i1> splat (i1 true), <4 x i1> [[TMP5]]
; CHECK-NEXT: [[TMP8:%.*]] = freeze <4 x i1> [[TMP7]]
; CHECK-NEXT: [[CMP2:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; CHECK-NEXT: br i1 [[CMP2]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP7]], i1 false)
; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x i1> [[TMP1]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP11]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_CHECK_0:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK_0]]:
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP3]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP12]], label %[[VECTOR_EARLY_EXIT_1:.*]], label %[[VECTOR_EARLY_EXIT_2:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_2]]:
; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[VECTOR_EARLY_EXIT_1]]:
; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[VECTOR_EARLY_EXIT_0]]:
; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[TMP13]], %[[VECTOR_EARLY_EXIT_2]] ], [ [[TMP14]], %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP15]], %[[VECTOR_EARLY_EXIT_0]] ], [ 128, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[RETVAL]]
;
entry:
%A = alloca [1024 x i8]
%B = alloca [1024 x i8]
%C = alloca [1024 x i8]
call void @init_mem(ptr %A, i64 1024)
call void @init_mem(ptr %B, i64 1024)
call void @init_mem(ptr %C, i64 1024)
br label %loop.header
loop.header:
%iv = phi i64 [ %iv.next, %loop.latch ], [ 0, %entry ]
%gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
%ld.A = load i8, ptr %gep.A, align 1
%cmp1 = icmp eq i8 %ld.A, 42
br i1 %cmp1, label %exit, label %early.exit.0
early.exit.0:
%gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
%ld.B = load i8, ptr %gep.B, align 1
%cmp2 = icmp eq i8 %ld.A, %ld.B
br i1 %cmp2, label %exit, label %early.exit.1
early.exit.1:
%gep.C = getelementptr inbounds i8, ptr %C, i64 %iv
%ld.C = load i8, ptr %gep.C, align 1
%cmp3 = icmp eq i8 %ld.B, %ld.C
br i1 %cmp3, label %exit, label %loop.latch
loop.latch:
%iv.next = add nuw i64 %iv, 1
%ec = icmp eq i64 %iv.next, 128
br i1 %ec, label %exit, label %loop.header
exit:
%retval = phi i64 [ %iv, %loop.header ], [ %iv, %early.exit.0 ], [ %iv, %early.exit.1 ], [ 128, %loop.latch ]
ret i64 %retval
}
define { i64, i8 } @four_early_exits_with_conditional_loads() {
; CHECK-LABEL: define { i64, i8 } @four_early_exits_with_conditional_loads() {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P1:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P2:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P3:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[P4:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[P1]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P2]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P3]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[P4]], i64 1024)
; CHECK-NEXT: br label %[[VECTOR_PH:.*]]
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY_INTERIM:.*]] ]
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], splat (i8 10)
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i8>, ptr [[GEP_B]], align 1
; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD1]], splat (i8 20)
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[P3]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD2]], splat (i8 30)
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[P4]], i64 [[IV]]
; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP6]], align 1
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD3]], splat (i8 40)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
; CHECK-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP1]], <4 x i1> splat (i1 true), <4 x i1> [[TMP3]]
; CHECK-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP8]], <4 x i1> splat (i1 true), <4 x i1> [[TMP5]]
; CHECK-NEXT: [[TMP10:%.*]] = select <4 x i1> [[TMP9]], <4 x i1> splat (i1 true), <4 x i1> [[TMP7]]
; CHECK-NEXT: [[TMP11:%.*]] = freeze <4 x i1> [[TMP10]]
; CHECK-NEXT: [[CMP2:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP11]])
; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128
; CHECK-NEXT: br i1 [[CMP2]], label %[[VECTOR_EARLY_EXIT_CHECK:.*]], label %[[VECTOR_BODY_INTERIM]]
; CHECK: [[VECTOR_BODY_INTERIM]]:
; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK]]:
; CHECK-NEXT: [[FIRST_ACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP10]], i1 false)
; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP1]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP14]], label %[[VECTOR_EARLY_EXIT_0:.*]], label %[[VECTOR_EARLY_EXIT_CHECK_0:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK_0]]:
; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP3]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP15]], label %[[VECTOR_EARLY_EXIT_1:.*]], label %[[VECTOR_EARLY_EXIT_CHECK_1:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_CHECK_1]]:
; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x i1> [[TMP5]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br i1 [[TMP16]], label %[[VECTOR_EARLY_EXIT_2:.*]], label %[[VECTOR_EARLY_EXIT_3:.*]]
; CHECK: [[VECTOR_EARLY_EXIT_3]]:
; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i8> [[WIDE_LOAD3]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[VECTOR_EARLY_EXIT_2]]:
; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i8> [[WIDE_LOAD2]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: [[TMP20:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[VECTOR_EARLY_EXIT_1]]:
; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i8> [[WIDE_LOAD1]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: [[TMP22:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[VECTOR_EARLY_EXIT_0]]:
; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i8> [[WIDE_LOAD]], i64 [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[IV]], [[FIRST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RETVAL_IV:%.*]] = phi i64 [ [[TMP18]], %[[VECTOR_EARLY_EXIT_3]] ], [ [[TMP20]], %[[VECTOR_EARLY_EXIT_2]] ], [ [[TMP22]], %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP24]], %[[VECTOR_EARLY_EXIT_0]] ], [ 128, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[RETVAL_VAL:%.*]] = phi i8 [ [[TMP17]], %[[VECTOR_EARLY_EXIT_3]] ], [ [[TMP19]], %[[VECTOR_EARLY_EXIT_2]] ], [ [[TMP21]], %[[VECTOR_EARLY_EXIT_1]] ], [ [[TMP23]], %[[VECTOR_EARLY_EXIT_0]] ], [ 0, %[[MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[R1:%.*]] = insertvalue { i64, i8 } undef, i64 [[RETVAL_IV]], 0
; CHECK-NEXT: [[R2:%.*]] = insertvalue { i64, i8 } [[R1]], i8 [[RETVAL_VAL]], 1
; CHECK-NEXT: ret { i64, i8 } [[R2]]
;
entry:
%A = alloca [1024 x i8]
%B = alloca [1024 x i8]
%C = alloca [1024 x i8]
%D = alloca [1024 x i8]
call void @init_mem(ptr %A, i64 1024)
call void @init_mem(ptr %B, i64 1024)
call void @init_mem(ptr %C, i64 1024)
call void @init_mem(ptr %D, i64 1024)
br label %loop.header
loop.header:
%iv = phi i64 [ %iv.next, %loop.latch ], [ 0, %entry ]
%gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
%ld.A = load i8, ptr %gep.A, align 1
%cmp1 = icmp eq i8 %ld.A, 10
br i1 %cmp1, label %exit, label %early.exit.0
early.exit.0:
%gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
%ld.B = load i8, ptr %gep.B, align 1
%cmp2 = icmp eq i8 %ld.B, 20
br i1 %cmp2, label %exit, label %early.exit.1
early.exit.1:
%gep.C = getelementptr inbounds i8, ptr %C, i64 %iv
%ld.C = load i8, ptr %gep.C, align 1
%cmp3 = icmp eq i8 %ld.C, 30
br i1 %cmp3, label %exit, label %early.exit.2
early.exit.2:
%gep.D = getelementptr inbounds i8, ptr %D, i64 %iv
%ld.D = load i8, ptr %gep.D, align 1
%cmp4 = icmp eq i8 %ld.D, 40
br i1 %cmp4, label %exit, label %loop.latch
loop.latch:
%iv.next = add nuw i64 %iv, 1
%ec = icmp eq i64 %iv.next, 128
br i1 %ec, label %exit, label %loop.header
exit:
%retval.iv = phi i64 [ %iv, %loop.header ], [ %iv, %early.exit.0 ], [ %iv, %early.exit.1 ], [ %iv, %early.exit.2 ], [ 128, %loop.latch ]
%retval.val = phi i8 [ %ld.A, %loop.header ], [ %ld.B, %early.exit.0 ], [ %ld.C, %early.exit.1 ], [ %ld.D, %early.exit.2 ], [ 0, %loop.latch ]
%r1 = insertvalue { i64, i8 } undef, i64 %retval.iv, 0
%r2 = insertvalue { i64, i8 } %r1, i8 %retval.val, 1
ret { i64, i8 } %r2
}
define i64 @two_early_exits_variable_trip_count(i64 %n) {
; CHECK-LABEL: define i64 @two_early_exits_variable_trip_count(
; CHECK-SAME: i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[N]], 1024
; CHECK-NEXT: call void @llvm.assume(i1 [[CMP]])
; CHECK-NEXT: [[A:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: [[B:%.*]] = alloca [1024 x i8], align 1
; CHECK-NEXT: call void @init_mem(ptr [[A]], i64 1024)
; CHECK-NEXT: call void @init_mem(ptr [[B]], i64 1024)
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]]
; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[GEP_A]], align 1
; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LD1]], 42
; CHECK-NEXT: br i1 [[CMP1]], label %[[EXIT:.*]], label %[[EARLY_EXIT_0:.*]]
; CHECK: [[EARLY_EXIT_0]]:
; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[IV]]
; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[GEP_B]], align 1
; CHECK-NEXT: [[CMP2:%.*]] = icmp eq i8 [[LD1]], [[LD2]]
; CHECK-NEXT: br i1 [[CMP2]], label %[[EXIT]], label %[[LOOP_LATCH]]
; CHECK: [[LOOP_LATCH]]:
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[IV_NEXT]], [[N]]
; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP_HEADER]], label %[[EXIT]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[IV]], %[[LOOP_HEADER]] ], [ 100, %[[EARLY_EXIT_0]] ], [ [[N]], %[[LOOP_LATCH]] ]
; CHECK-NEXT: ret i64 [[RETVAL]]
;
entry:
%cmp = icmp ult i64 %n, 1024
call void @llvm.assume(i1 %cmp)
%A = alloca [1024 x i8]
%B = alloca [1024 x i8]
call void @init_mem(ptr %A, i64 1024)
call void @init_mem(ptr %B, i64 1024)
br label %loop.header
loop.header:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
%gep.A = getelementptr inbounds i8, ptr %A, i64 %iv
%ld.A = load i8, ptr %gep.A, align 1
%cmp1 = icmp eq i8 %ld.A, 42
br i1 %cmp1, label %exit, label %early.exit.0
early.exit.0:
%gep.B = getelementptr inbounds i8, ptr %B, i64 %iv
%ld.B = load i8, ptr %gep.B, align 1
%cmp2 = icmp eq i8 %ld.A, %ld.B
br i1 %cmp2, label %exit, label %loop.latch
loop.latch:
%iv.next = add i64 %iv, 1
%ec = icmp ne i64 %iv.next, %n
br i1 %ec, label %loop.header, label %exit
exit:
%retval = phi i64 [ %iv, %loop.header ], [ 100, %early.exit.0 ], [ %n, %loop.latch ]
ret i64 %retval
}
define i64 @three_early_exits_same_block_no_latch_exit(i1 %cmp0) {
; CHECK-LABEL: define i64 @three_early_exits_same_block_no_latch_exit(
; CHECK-SAME: i1 [[CMP0:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT: br i1 [[CMP0]], label %[[EARLY_EXIT:.*]], label %[[LOOP_BODY_0:.*]]
; CHECK: [[LOOP_BODY_0]]:
; CHECK-NEXT: [[GEP:%.*]] = getelementptr [24 x i8], ptr @GlobA, i64 [[IV]]
; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr i8, ptr [[GEP]], i64 4
; CHECK-NEXT: [[L_0:%.*]] = load i8, ptr [[GEP_1]], align 1
; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[L_0]], 0
; CHECK-NEXT: br i1 [[CMP1]], label %[[EARLY_EXIT]], label %[[LOOP_BODY_1:.*]]
; CHECK: [[LOOP_BODY_1]]:
; CHECK-NEXT: [[L_1:%.*]] = load i8, ptr [[GEP]], align 1
; CHECK-NEXT: [[CMP2:%.*]] = icmp eq i8 [[L_1]], 0
; CHECK-NEXT: br i1 [[CMP2]], label %[[EARLY_EXIT]], label %[[LOOP_LATCH]]
; CHECK: [[LOOP_LATCH]]:
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 0
; CHECK-NEXT: br i1 [[EC]], label %[[LATCH_EXIT:.*]], label %[[LOOP_HEADER]]
; CHECK: [[LATCH_EXIT]]:
; CHECK-NEXT: ret i64 0
; CHECK: [[EARLY_EXIT]]:
; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP_BODY_1]] ], [ [[IV]], %[[LOOP_BODY_0]] ], [ [[IV]], %[[LOOP_HEADER]] ]
; CHECK-NEXT: ret i64 [[IV_LCSSA]]
;
entry:
br label %loop.header
loop.header:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]
br i1 %cmp0, label %early.exit, label %loop.body.0
loop.body.0:
%gep = getelementptr [24 x i8], ptr @GlobA, i64 %iv
%gep.1 = getelementptr i8, ptr %gep, i64 4
%l.0 = load i8, ptr %gep.1, align 1
%cmp1 = icmp eq i8 %l.0, 0
br i1 %cmp1, label %early.exit, label %loop.body.1
loop.body.1:
%l.1 = load i8, ptr %gep, align 1
%cmp2 = icmp eq i8 %l.1, 0
br i1 %cmp2, label %early.exit, label %loop.latch
loop.latch:
%iv.next = add nuw nsw i64 %iv, 1
%ec = icmp eq i64 %iv, 0
br i1 %ec, label %latch.exit, label %loop.header
latch.exit:
ret i64 0
early.exit:
ret i64 %iv
}
Reference in New Issue View Git Blame Copy Permalink