Florian Hahn
fd6524963b
Teach VPInstruction::getNumOperandsForOpcode about AtomicRMW, AtomicCmpXchg and Fence, so VPlan construction over an outer loop that still contains these operations does not crash before later legality rejects the loop. Fixes crashes in the added tests.
169 lines
6.7 KiB
LLVM
169 lines
6.7 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
|
|
; RUN: opt -passes=loop-vectorize -enable-vplan-native-path -S %s | FileCheck %s
|
|
|
|
define void @unsafe_atomicrmw(ptr noalias %A, ptr %ctr, i64 %N, i64 %M) {
|
|
; CHECK-LABEL: define void @unsafe_atomicrmw(
|
|
; CHECK-SAME: ptr noalias [[A:%.*]], ptr [[CTR:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*]]:
|
|
; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
|
|
; CHECK: [[OUTER_HEADER]]:
|
|
; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
|
|
; CHECK-NEXT: [[OLD:%.*]] = atomicrmw add ptr [[CTR]], i32 1 seq_cst, align 4
|
|
; CHECK-NEXT: br label %[[INNER_BODY:.*]]
|
|
; CHECK: [[INNER_BODY]]:
|
|
; CHECK-NEXT: [[J:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[J_NEXT:%.*]], %[[INNER_BODY]] ]
|
|
; CHECK-NEXT: [[I_MUL_M:%.*]] = mul nsw i64 [[I]], [[M]]
|
|
; CHECK-NEXT: [[IDX:%.*]] = add nsw i64 [[I_MUL_M]], [[J]]
|
|
; CHECK-NEXT: [[A_PTR:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IDX]]
|
|
; CHECK-NEXT: [[V:%.*]] = trunc i64 [[I]] to i32
|
|
; CHECK-NEXT: store i32 [[V]], ptr [[A_PTR]], align 4
|
|
; CHECK-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J]], 1
|
|
; CHECK-NEXT: [[J_CMP:%.*]] = icmp eq i64 [[J_NEXT]], [[M]]
|
|
; CHECK-NEXT: br i1 [[J_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
|
|
; CHECK: [[OUTER_LATCH]]:
|
|
; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
|
|
; CHECK-NEXT: [[I_CMP:%.*]] = icmp eq i64 [[I_NEXT]], [[N]]
|
|
; CHECK-NEXT: br i1 [[I_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP0:![0-9]+]]
|
|
; CHECK: [[EXIT]]:
|
|
; CHECK-NEXT: ret void
|
|
;
|
|
entry:
|
|
br label %outer.header
|
|
|
|
outer.header:
|
|
%i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
|
|
%old = atomicrmw add ptr %ctr, i32 1 seq_cst, align 4
|
|
br label %inner.body
|
|
|
|
inner.body:
|
|
%j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ]
|
|
%i.mul.M = mul nsw i64 %i, %M
|
|
%idx = add nsw i64 %i.mul.M, %j
|
|
%A.ptr = getelementptr inbounds i32, ptr %A, i64 %idx
|
|
%v = trunc i64 %i to i32
|
|
store i32 %v, ptr %A.ptr, align 4
|
|
%j.next = add nuw nsw i64 %j, 1
|
|
%j.cmp = icmp eq i64 %j.next, %M
|
|
br i1 %j.cmp, label %outer.latch, label %inner.body
|
|
|
|
outer.latch:
|
|
%i.next = add nuw nsw i64 %i, 1
|
|
%i.cmp = icmp eq i64 %i.next, %N
|
|
br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0
|
|
|
|
exit:
|
|
ret void
|
|
}
|
|
|
|
define void @unsafe_cmpxchg(ptr noalias %A, ptr %ctr, i64 %N, i64 %M) {
|
|
; CHECK-LABEL: define void @unsafe_cmpxchg(
|
|
; CHECK-SAME: ptr noalias [[A:%.*]], ptr [[CTR:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*]]:
|
|
; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
|
|
; CHECK: [[OUTER_HEADER]]:
|
|
; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
|
|
; CHECK-NEXT: [[PAIR:%.*]] = cmpxchg ptr [[CTR]], i32 0, i32 1 seq_cst seq_cst, align 4
|
|
; CHECK-NEXT: br label %[[INNER_BODY:.*]]
|
|
; CHECK: [[INNER_BODY]]:
|
|
; CHECK-NEXT: [[J:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[J_NEXT:%.*]], %[[INNER_BODY]] ]
|
|
; CHECK-NEXT: [[I_MUL_M:%.*]] = mul nsw i64 [[I]], [[M]]
|
|
; CHECK-NEXT: [[IDX:%.*]] = add nsw i64 [[I_MUL_M]], [[J]]
|
|
; CHECK-NEXT: [[A_PTR:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IDX]]
|
|
; CHECK-NEXT: [[V:%.*]] = trunc i64 [[I]] to i32
|
|
; CHECK-NEXT: store i32 [[V]], ptr [[A_PTR]], align 4
|
|
; CHECK-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J]], 1
|
|
; CHECK-NEXT: [[J_CMP:%.*]] = icmp eq i64 [[J_NEXT]], [[M]]
|
|
; CHECK-NEXT: br i1 [[J_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
|
|
; CHECK: [[OUTER_LATCH]]:
|
|
; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
|
|
; CHECK-NEXT: [[I_CMP:%.*]] = icmp eq i64 [[I_NEXT]], [[N]]
|
|
; CHECK-NEXT: br i1 [[I_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP0]]
|
|
; CHECK: [[EXIT]]:
|
|
; CHECK-NEXT: ret void
|
|
;
|
|
entry:
|
|
br label %outer.header
|
|
|
|
outer.header:
|
|
%i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
|
|
%pair = cmpxchg ptr %ctr, i32 0, i32 1 seq_cst seq_cst, align 4
|
|
br label %inner.body
|
|
|
|
inner.body:
|
|
%j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ]
|
|
%i.mul.M = mul nsw i64 %i, %M
|
|
%idx = add nsw i64 %i.mul.M, %j
|
|
%A.ptr = getelementptr inbounds i32, ptr %A, i64 %idx
|
|
%v = trunc i64 %i to i32
|
|
store i32 %v, ptr %A.ptr, align 4
|
|
%j.next = add nuw nsw i64 %j, 1
|
|
%j.cmp = icmp eq i64 %j.next, %M
|
|
br i1 %j.cmp, label %outer.latch, label %inner.body
|
|
|
|
outer.latch:
|
|
%i.next = add nuw nsw i64 %i, 1
|
|
%i.cmp = icmp eq i64 %i.next, %N
|
|
br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0
|
|
|
|
exit:
|
|
ret void
|
|
}
|
|
|
|
define void @unsafe_fence(ptr noalias %A, i64 %N, i64 %M) {
|
|
; CHECK-LABEL: define void @unsafe_fence(
|
|
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*]]:
|
|
; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
|
|
; CHECK: [[OUTER_HEADER]]:
|
|
; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
|
|
; CHECK-NEXT: fence seq_cst
|
|
; CHECK-NEXT: br label %[[INNER_BODY:.*]]
|
|
; CHECK: [[INNER_BODY]]:
|
|
; CHECK-NEXT: [[J:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[J_NEXT:%.*]], %[[INNER_BODY]] ]
|
|
; CHECK-NEXT: [[I_MUL_M:%.*]] = mul nsw i64 [[I]], [[M]]
|
|
; CHECK-NEXT: [[IDX:%.*]] = add nsw i64 [[I_MUL_M]], [[J]]
|
|
; CHECK-NEXT: [[A_PTR:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IDX]]
|
|
; CHECK-NEXT: [[V:%.*]] = trunc i64 [[I]] to i32
|
|
; CHECK-NEXT: store i32 [[V]], ptr [[A_PTR]], align 4
|
|
; CHECK-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J]], 1
|
|
; CHECK-NEXT: [[J_CMP:%.*]] = icmp eq i64 [[J_NEXT]], [[M]]
|
|
; CHECK-NEXT: br i1 [[J_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
|
|
; CHECK: [[OUTER_LATCH]]:
|
|
; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
|
|
; CHECK-NEXT: [[I_CMP:%.*]] = icmp eq i64 [[I_NEXT]], [[N]]
|
|
; CHECK-NEXT: br i1 [[I_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP0]]
|
|
; CHECK: [[EXIT]]:
|
|
; CHECK-NEXT: ret void
|
|
;
|
|
entry:
|
|
br label %outer.header
|
|
|
|
outer.header:
|
|
%i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
|
|
fence seq_cst
|
|
br label %inner.body
|
|
|
|
inner.body:
|
|
%j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ]
|
|
%i.mul.M = mul nsw i64 %i, %M
|
|
%idx = add nsw i64 %i.mul.M, %j
|
|
%A.ptr = getelementptr inbounds i32, ptr %A, i64 %idx
|
|
%v = trunc i64 %i to i32
|
|
store i32 %v, ptr %A.ptr, align 4
|
|
%j.next = add nuw nsw i64 %j, 1
|
|
%j.cmp = icmp eq i64 %j.next, %M
|
|
br i1 %j.cmp, label %outer.latch, label %inner.body
|
|
|
|
outer.latch:
|
|
%i.next = add nuw nsw i64 %i, 1
|
|
%i.cmp = icmp eq i64 %i.next, %N
|
|
br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0
|
|
|
|
exit:
|
|
ret void
|
|
}
|
|
|
|
!0 = distinct !{!0, !1, !2}
|
|
!1 = !{!"llvm.loop.vectorize.width", i32 4}
|
|
!2 = !{!"llvm.loop.vectorize.enable", i1 true}
|