184f236a18
The fix does the following in expandPartwordCmpXchg and insertRMWCmpXchgLoop. - Issues volatile operations in the emulation loops if the original operation is volatile. - A preheader load is used for initializing the "cmp" and "new" values of the cmpxchg in the loop. Makes this load atomic. This is done under a target hook (`issueAtomicInitLoadForAtomicEmulation()`) , to allow backends to migrate independently. - `processAtomicInstr` is called on this load, to massage it into something that can be lowered in SelectionDAG / GISel. - This caused 3 kinds of failures. 1. Caused by change to codegen: updated these either using the scripts, or mechanically (using claude) to match the new codegen. 2. Crashes caused by newly created atomic loads not being processed by AtomicExpandPass. (The atomic load if tested in an independent test does not cause a crash). To fix these, added recursive calls to processAtomicInstr on the newly created atomic loads. These calls convert the loads to libcalls, or cast them to integer types. 3. Crashes in X86, AMDGPU, and AArch64 caused by unhandled vector types. These loads crash even with upstream LLVM, due to the lack of support in these targets for vector atomic loads (the corresponding vector atomicrmw instructions are supported). Disabled issuing atomic loads for these backends. Will follow up with individual PRs to revert to default behavior.
179 lines
9.6 KiB
LLVM
179 lines
9.6 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
|
|
; RUN: opt -S --mtriple=loongarch64 --passes='require<libcall-lowering-info>,atomic-expand' --mattr=+d %s | FileCheck %s
|
|
|
|
define float @atomicrmw_fadd_float(ptr %ptr, float %value) {
|
|
; CHECK-LABEL: @atomicrmw_fadd_float(
|
|
; CHECK-NEXT: [[TMP6:%.*]] = load atomic i32, ptr [[PTR:%.*]] monotonic, align 4
|
|
; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[TMP6]] to float
|
|
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
|
|
; CHECK: atomicrmw.start:
|
|
; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
|
|
; CHECK-NEXT: [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
|
|
; CHECK-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32
|
|
; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32
|
|
; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4
|
|
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
|
|
; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
|
|
; CHECK-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float
|
|
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
|
|
; CHECK: atomicrmw.end:
|
|
; CHECK-NEXT: ret float [[TMP5]]
|
|
;
|
|
%res = atomicrmw fadd ptr %ptr, float %value seq_cst
|
|
ret float %res
|
|
}
|
|
|
|
define float @atomicrmw_fsub_float(ptr %ptr, float %value) {
|
|
; CHECK-LABEL: @atomicrmw_fsub_float(
|
|
; CHECK-NEXT: [[TMP6:%.*]] = load atomic i32, ptr [[PTR:%.*]] monotonic, align 4
|
|
; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[TMP6]] to float
|
|
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
|
|
; CHECK: atomicrmw.start:
|
|
; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
|
|
; CHECK-NEXT: [[NEW:%.*]] = fsub float [[LOADED]], [[VALUE:%.*]]
|
|
; CHECK-NEXT: [[TMP2:%.*]] = bitcast float [[NEW]] to i32
|
|
; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[LOADED]] to i32
|
|
; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP3]], i32 [[TMP2]] seq_cst seq_cst, align 4
|
|
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP4]], 1
|
|
; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP4]], 0
|
|
; CHECK-NEXT: [[TMP5]] = bitcast i32 [[NEWLOADED]] to float
|
|
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
|
|
; CHECK: atomicrmw.end:
|
|
; CHECK-NEXT: ret float [[TMP5]]
|
|
;
|
|
%res = atomicrmw fsub ptr %ptr, float %value seq_cst
|
|
ret float %res
|
|
}
|
|
|
|
define float @atomicrmw_fmin_float(ptr %ptr, float %value) {
|
|
; CHECK-LABEL: @atomicrmw_fmin_float(
|
|
; CHECK-NEXT: [[TMP7:%.*]] = load atomic i32, ptr [[PTR:%.*]] monotonic, align 4
|
|
; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[TMP7]] to float
|
|
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
|
|
; CHECK: atomicrmw.start:
|
|
; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
|
|
; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.minnum.f32(float [[LOADED]], float [[VALUE:%.*]])
|
|
; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
|
|
; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
|
|
; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
|
|
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
|
|
; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
|
|
; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
|
|
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
|
|
; CHECK: atomicrmw.end:
|
|
; CHECK-NEXT: ret float [[TMP6]]
|
|
;
|
|
%res = atomicrmw fmin ptr %ptr, float %value seq_cst
|
|
ret float %res
|
|
}
|
|
|
|
define float @atomicrmw_fmax_float(ptr %ptr, float %value) {
|
|
; CHECK-LABEL: @atomicrmw_fmax_float(
|
|
; CHECK-NEXT: [[TMP7:%.*]] = load atomic i32, ptr [[PTR:%.*]] monotonic, align 4
|
|
; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[TMP7]] to float
|
|
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
|
|
; CHECK: atomicrmw.start:
|
|
; CHECK-NEXT: [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
|
|
; CHECK-NEXT: [[TMP2:%.*]] = call float @llvm.maxnum.f32(float [[LOADED]], float [[VALUE:%.*]])
|
|
; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[TMP2]] to i32
|
|
; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
|
|
; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst, align 4
|
|
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
|
|
; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
|
|
; CHECK-NEXT: [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
|
|
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
|
|
; CHECK: atomicrmw.end:
|
|
; CHECK-NEXT: ret float [[TMP6]]
|
|
;
|
|
%res = atomicrmw fmax ptr %ptr, float %value seq_cst
|
|
ret float %res
|
|
}
|
|
|
|
define double @atomicrmw_fadd_double(ptr %ptr, double %value) {
|
|
; CHECK-LABEL: @atomicrmw_fadd_double(
|
|
; CHECK-NEXT: [[TMP6:%.*]] = load atomic i64, ptr [[PTR:%.*]] monotonic, align 8
|
|
; CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP6]] to double
|
|
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
|
|
; CHECK: atomicrmw.start:
|
|
; CHECK-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
|
|
; CHECK-NEXT: [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
|
|
; CHECK-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; CHECK-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8
|
|
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; CHECK-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
|
|
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
|
|
; CHECK: atomicrmw.end:
|
|
; CHECK-NEXT: ret double [[TMP5]]
|
|
;
|
|
%res = atomicrmw fadd ptr %ptr, double %value seq_cst
|
|
ret double %res
|
|
}
|
|
|
|
define double @atomicrmw_fsub_double(ptr %ptr, double %value) {
|
|
; CHECK-LABEL: @atomicrmw_fsub_double(
|
|
; CHECK-NEXT: [[TMP6:%.*]] = load atomic i64, ptr [[PTR:%.*]] monotonic, align 8
|
|
; CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP6]] to double
|
|
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
|
|
; CHECK: atomicrmw.start:
|
|
; CHECK-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
|
|
; CHECK-NEXT: [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE:%.*]]
|
|
; CHECK-NEXT: [[TMP2:%.*]] = bitcast double [[NEW]] to i64
|
|
; CHECK-NEXT: [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
|
|
; CHECK-NEXT: [[TMP4:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP3]], i64 [[TMP2]] seq_cst seq_cst, align 8
|
|
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
|
|
; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
|
|
; CHECK-NEXT: [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
|
|
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
|
|
; CHECK: atomicrmw.end:
|
|
; CHECK-NEXT: ret double [[TMP5]]
|
|
;
|
|
%res = atomicrmw fsub ptr %ptr, double %value seq_cst
|
|
ret double %res
|
|
}
|
|
|
|
define double @atomicrmw_fmin_double(ptr %ptr, double %value) {
|
|
; CHECK-LABEL: @atomicrmw_fmin_double(
|
|
; CHECK-NEXT: [[TMP7:%.*]] = load atomic i64, ptr [[PTR:%.*]] monotonic, align 8
|
|
; CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP7]] to double
|
|
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
|
|
; CHECK: atomicrmw.start:
|
|
; CHECK-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
|
|
; CHECK-NEXT: [[TMP2:%.*]] = call double @llvm.minnum.f64(double [[LOADED]], double [[VALUE:%.*]])
|
|
; CHECK-NEXT: [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
|
|
; CHECK-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
|
|
; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
|
|
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
|
|
; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
|
|
; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
|
|
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
|
|
; CHECK: atomicrmw.end:
|
|
; CHECK-NEXT: ret double [[TMP6]]
|
|
;
|
|
%res = atomicrmw fmin ptr %ptr, double %value seq_cst
|
|
ret double %res
|
|
}
|
|
|
|
define double @atomicrmw_fmax_double(ptr %ptr, double %value) {
|
|
; CHECK-LABEL: @atomicrmw_fmax_double(
|
|
; CHECK-NEXT: [[TMP7:%.*]] = load atomic i64, ptr [[PTR:%.*]] monotonic, align 8
|
|
; CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[TMP7]] to double
|
|
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
|
|
; CHECK: atomicrmw.start:
|
|
; CHECK-NEXT: [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
|
|
; CHECK-NEXT: [[TMP2:%.*]] = call double @llvm.maxnum.f64(double [[LOADED]], double [[VALUE:%.*]])
|
|
; CHECK-NEXT: [[TMP3:%.*]] = bitcast double [[TMP2]] to i64
|
|
; CHECK-NEXT: [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
|
|
; CHECK-NEXT: [[TMP5:%.*]] = cmpxchg ptr [[PTR]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst, align 8
|
|
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
|
|
; CHECK-NEXT: [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
|
|
; CHECK-NEXT: [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
|
|
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
|
|
; CHECK: atomicrmw.end:
|
|
; CHECK-NEXT: ret double [[TMP6]]
|
|
;
|
|
%res = atomicrmw fmax ptr %ptr, double %value seq_cst
|
|
ret double %res
|
|
}
|