; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -passes=loop-vectorize -enable-vplan-native-path -S %s | FileCheck %s

define void @unsafe_atomicrmw(ptr noalias %A, ptr %ctr, i64 %N, i64 %M) {
; CHECK-LABEL: define void @unsafe_atomicrmw(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr [[CTR:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    [[OLD:%.*]] = atomicrmw add ptr [[CTR]], i32 1 seq_cst, align 4
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[J:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[J_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[I_MUL_M:%.*]] = mul nsw i64 [[I]], [[M]]
; CHECK-NEXT:    [[IDX:%.*]] = add nsw i64 [[I_MUL_M]], [[J]]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IDX]]
; CHECK-NEXT:    [[V:%.*]] = trunc i64 [[I]] to i32
; CHECK-NEXT:    store i32 [[V]], ptr [[A_PTR]], align 4
; CHECK-NEXT:    [[J_NEXT]] = add nuw nsw i64 [[J]], 1
; CHECK-NEXT:    [[J_CMP:%.*]] = icmp eq i64 [[J_NEXT]], [[M]]
; CHECK-NEXT:    br i1 [[J_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; CHECK-NEXT:    [[I_CMP:%.*]] = icmp eq i64 [[I_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[I_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
  %old = atomicrmw add ptr %ctr, i32 1 seq_cst, align 4
  br label %inner.body

inner.body:
  %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ]
  %i.mul.M = mul nsw i64 %i, %M
  %idx = add nsw i64 %i.mul.M, %j
  %A.ptr = getelementptr inbounds i32, ptr %A, i64 %idx
  %v = trunc i64 %i to i32
  store i32 %v, ptr %A.ptr, align 4
  %j.next = add nuw nsw i64 %j, 1
  %j.cmp = icmp eq i64 %j.next, %M
  br i1 %j.cmp, label %outer.latch, label %inner.body

outer.latch:
  %i.next = add nuw nsw i64 %i, 1
  %i.cmp = icmp eq i64 %i.next, %N
  br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

define void @unsafe_cmpxchg(ptr noalias %A, ptr %ctr, i64 %N, i64 %M) {
; CHECK-LABEL: define void @unsafe_cmpxchg(
; CHECK-SAME: ptr noalias [[A:%.*]], ptr [[CTR:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    [[PAIR:%.*]] = cmpxchg ptr [[CTR]], i32 0, i32 1 seq_cst seq_cst, align 4
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[J:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[J_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[I_MUL_M:%.*]] = mul nsw i64 [[I]], [[M]]
; CHECK-NEXT:    [[IDX:%.*]] = add nsw i64 [[I_MUL_M]], [[J]]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IDX]]
; CHECK-NEXT:    [[V:%.*]] = trunc i64 [[I]] to i32
; CHECK-NEXT:    store i32 [[V]], ptr [[A_PTR]], align 4
; CHECK-NEXT:    [[J_NEXT]] = add nuw nsw i64 [[J]], 1
; CHECK-NEXT:    [[J_CMP:%.*]] = icmp eq i64 [[J_NEXT]], [[M]]
; CHECK-NEXT:    br i1 [[J_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; CHECK-NEXT:    [[I_CMP:%.*]] = icmp eq i64 [[I_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[I_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP0]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
  %pair = cmpxchg ptr %ctr, i32 0, i32 1 seq_cst seq_cst, align 4
  br label %inner.body

inner.body:
  %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ]
  %i.mul.M = mul nsw i64 %i, %M
  %idx = add nsw i64 %i.mul.M, %j
  %A.ptr = getelementptr inbounds i32, ptr %A, i64 %idx
  %v = trunc i64 %i to i32
  store i32 %v, ptr %A.ptr, align 4
  %j.next = add nuw nsw i64 %j, 1
  %j.cmp = icmp eq i64 %j.next, %M
  br i1 %j.cmp, label %outer.latch, label %inner.body

outer.latch:
  %i.next = add nuw nsw i64 %i, 1
  %i.cmp = icmp eq i64 %i.next, %N
  br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

define void @unsafe_fence(ptr noalias %A, i64 %N, i64 %M) {
; CHECK-LABEL: define void @unsafe_fence(
; CHECK-SAME: ptr noalias [[A:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
; CHECK-NEXT:  [[ENTRY:.*]]:
; CHECK-NEXT:    br label %[[OUTER_HEADER:.*]]
; CHECK:       [[OUTER_HEADER]]:
; CHECK-NEXT:    [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
; CHECK-NEXT:    fence seq_cst
; CHECK-NEXT:    br label %[[INNER_BODY:.*]]
; CHECK:       [[INNER_BODY]]:
; CHECK-NEXT:    [[J:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[J_NEXT:%.*]], %[[INNER_BODY]] ]
; CHECK-NEXT:    [[I_MUL_M:%.*]] = mul nsw i64 [[I]], [[M]]
; CHECK-NEXT:    [[IDX:%.*]] = add nsw i64 [[I_MUL_M]], [[J]]
; CHECK-NEXT:    [[A_PTR:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IDX]]
; CHECK-NEXT:    [[V:%.*]] = trunc i64 [[I]] to i32
; CHECK-NEXT:    store i32 [[V]], ptr [[A_PTR]], align 4
; CHECK-NEXT:    [[J_NEXT]] = add nuw nsw i64 [[J]], 1
; CHECK-NEXT:    [[J_CMP:%.*]] = icmp eq i64 [[J_NEXT]], [[M]]
; CHECK-NEXT:    br i1 [[J_CMP]], label %[[OUTER_LATCH]], label %[[INNER_BODY]]
; CHECK:       [[OUTER_LATCH]]:
; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
; CHECK-NEXT:    [[I_CMP:%.*]] = icmp eq i64 [[I_NEXT]], [[N]]
; CHECK-NEXT:    br i1 [[I_CMP]], label %[[EXIT:.*]], label %[[OUTER_HEADER]], !llvm.loop [[LOOP0]]
; CHECK:       [[EXIT]]:
; CHECK-NEXT:    ret void
;
entry:
  br label %outer.header

outer.header:
  %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ]
  fence seq_cst
  br label %inner.body

inner.body:
  %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.body ]
  %i.mul.M = mul nsw i64 %i, %M
  %idx = add nsw i64 %i.mul.M, %j
  %A.ptr = getelementptr inbounds i32, ptr %A, i64 %idx
  %v = trunc i64 %i to i32
  store i32 %v, ptr %A.ptr, align 4
  %j.next = add nuw nsw i64 %j, 1
  %j.cmp = icmp eq i64 %j.next, %M
  br i1 %j.cmp, label %outer.latch, label %inner.body

outer.latch:
  %i.next = add nuw nsw i64 %i, 1
  %i.cmp = icmp eq i64 %i.next, %N
  br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0

exit:
  ret void
}

!0 = distinct !{!0, !1, !2}
!1 = !{!"llvm.loop.vectorize.width", i32 4}
!2 = !{!"llvm.loop.vectorize.enable", i1 true}