bab0dc4d48
Reapply 8a115b6934 with an update to tests handling remarks.
The patch now directly emits a clear remark when we bail out
due to the memory check threshold.
Original message:
When GeneratedRTChecks::create bails out due to exceeding the cost
threshold, no runtime checks are generated and we must not proceed
assuming checks have been generated.
Mark the checks as never succeeding, to make sure we don't try to
vectorize assuming the runtime checks hold. This fixes a case where we
previously incorrectly vectorized assuming runtime checks had been
generated when forcing vectorization via metadate.
Fixes the mis-compile mentioned in
https://github.com/llvm/llvm-project/pull/166247#issuecomment-3631471588
99 lines
5.0 KiB
LLVM
99 lines
5.0 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
|
|
; RUN: opt -p loop-vectorize -vectorize-memory-check-threshold=0 -S %s | FileCheck --check-prefix=LIMIT0 %s
|
|
; RUN: opt -p loop-vectorize -vectorize-memory-check-threshold=1 -S %s | FileCheck --check-prefix=LIMIT1 %s
|
|
|
|
; Make sure we do not incorrectly vectorize with -vectorize-memory-check-threshold=0;
|
|
; no runtime check is generated and the loop should not be vectorized.
|
|
define i16 @runtime_checks_needed(ptr %src, ptr %dst) {
|
|
; LIMIT0-LABEL: define i16 @runtime_checks_needed(
|
|
; LIMIT0-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]]) {
|
|
; LIMIT0-NEXT: [[ENTRY:.*]]:
|
|
; LIMIT0-NEXT: br label %[[LOOP:.*]]
|
|
; LIMIT0: [[LOOP]]:
|
|
; LIMIT0-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
|
|
; LIMIT0-NEXT: [[L:%.*]] = load i16, ptr [[SRC]], align 1
|
|
; LIMIT0-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[INDEX]]
|
|
; LIMIT0-NEXT: store i16 [[L]], ptr [[TMP1]], align 1
|
|
; LIMIT0-NEXT: [[INDEX_NEXT]] = add nuw nsw i64 [[INDEX]], 1
|
|
; LIMIT0-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
|
|
; LIMIT0-NEXT: br i1 [[TMP2]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP0:![0-9]+]]
|
|
; LIMIT0: [[EXIT]]:
|
|
; LIMIT0-NEXT: [[TMP0:%.*]] = phi i16 [ [[L]], %[[LOOP]] ]
|
|
; LIMIT0-NEXT: ret i16 [[TMP0]]
|
|
;
|
|
; LIMIT1-LABEL: define i16 @runtime_checks_needed(
|
|
; LIMIT1-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]]) {
|
|
; LIMIT1-NEXT: [[ENTRY:.*:]]
|
|
; LIMIT1-NEXT: br label %[[VECTOR_MEMCHECK:.*]]
|
|
; LIMIT1: [[VECTOR_MEMCHECK]]:
|
|
; LIMIT1-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 2000
|
|
; LIMIT1-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[SRC]], i64 2
|
|
; LIMIT1-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP1]]
|
|
; LIMIT1-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[SRC]], [[SCEVGEP]]
|
|
; LIMIT1-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
|
|
; LIMIT1-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
|
|
; LIMIT1: [[VECTOR_PH]]:
|
|
; LIMIT1-NEXT: [[TMP0:%.*]] = load i16, ptr [[SRC]], align 1, !alias.scope [[META0:![0-9]+]]
|
|
; LIMIT1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[TMP0]], i64 0
|
|
; LIMIT1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT]], <2 x i16> poison, <2 x i32> zeroinitializer
|
|
; LIMIT1-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; LIMIT1: [[VECTOR_BODY]]:
|
|
; LIMIT1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
|
|
; LIMIT1-NEXT: [[TMP1:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[INDEX]]
|
|
; LIMIT1-NEXT: store <2 x i16> [[BROADCAST_SPLAT]], ptr [[TMP1]], align 1, !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
|
|
; LIMIT1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
|
|
; LIMIT1-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
|
|
; LIMIT1-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
|
|
; LIMIT1: [[MIDDLE_BLOCK]]:
|
|
; LIMIT1-NEXT: br label %[[EXIT:.*]]
|
|
; LIMIT1: [[SCALAR_PH]]:
|
|
; LIMIT1-NEXT: br label %[[LOOP:.*]]
|
|
; LIMIT1: [[LOOP]]:
|
|
; LIMIT1-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
|
|
; LIMIT1-NEXT: [[L:%.*]] = load i16, ptr [[SRC]], align 1
|
|
; LIMIT1-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i16, ptr [[DST]], i64 [[IV]]
|
|
; LIMIT1-NEXT: store i16 [[L]], ptr [[GEP_DST]], align 1
|
|
; LIMIT1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
|
|
; LIMIT1-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
|
|
; LIMIT1-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
|
|
; LIMIT1: [[EXIT]]:
|
|
; LIMIT1-NEXT: [[L_LCSSA:%.*]] = phi i16 [ [[L]], %[[LOOP]] ], [ [[TMP0]], %[[MIDDLE_BLOCK]] ]
|
|
; LIMIT1-NEXT: ret i16 [[L_LCSSA]]
|
|
;
|
|
entry:
|
|
br label %loop
|
|
|
|
loop:
|
|
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
|
|
%l = load i16, ptr %src, align 1
|
|
%gep.dst = getelementptr inbounds i16, ptr %dst, i64 %iv
|
|
store i16 %l, ptr %gep.dst, align 1
|
|
%iv.next = add nsw nuw i64 %iv, 1
|
|
%ec = icmp eq i64 %iv.next, 1000
|
|
br i1 %ec, label %exit, label %loop, !llvm.loop !0
|
|
|
|
exit:
|
|
ret i16 %l
|
|
}
|
|
|
|
!0 = distinct !{!0, !2, !3}
|
|
!1 = !{!"llvm.loop.mustprogress"}
|
|
!2 = !{!"llvm.loop.vectorize.width", i32 2}
|
|
!3 = !{!"llvm.loop.vectorize.enable", i1 true}
|
|
|
|
;.
|
|
; LIMIT0: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
|
|
; LIMIT0: [[META1]] = !{!"llvm.loop.vectorize.width", i32 2}
|
|
; LIMIT0: [[META2]] = !{!"llvm.loop.vectorize.enable", i1 true}
|
|
;.
|
|
; LIMIT1: [[META0]] = !{[[META1:![0-9]+]]}
|
|
; LIMIT1: [[META1]] = distinct !{[[META1]], [[META2:![0-9]+]]}
|
|
; LIMIT1: [[META2]] = distinct !{[[META2]], !"LVerDomain"}
|
|
; LIMIT1: [[META3]] = !{[[META4:![0-9]+]]}
|
|
; LIMIT1: [[META4]] = distinct !{[[META4]], [[META2]]}
|
|
; LIMIT1: [[LOOP5]] = distinct !{[[LOOP5]], [[META6:![0-9]+]], [[META7:![0-9]+]]}
|
|
; LIMIT1: [[META6]] = !{!"llvm.loop.isvectorized", i32 1}
|
|
; LIMIT1: [[META7]] = !{!"llvm.loop.unroll.runtime.disable"}
|
|
; LIMIT1: [[LOOP8]] = distinct !{[[LOOP8]], [[META6]]}
|
|
;.
|