289 lines
15 KiB
LLVM
289 lines
15 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6
|
|
; RUN: opt -S -passes=loop-vectorize -enable-vplan-native-path < %s | FileCheck %s
|
|
|
|
; Test coverage for contiguous access detection in outer loop vectorization.
|
|
; Tests various stride and type combinations.
|
|
|
|
; --- stride-1 float load ---
|
|
; for (i = 0; i < N; i++)
|
|
; for (j = 0; j < M; j++)
|
|
; A[i * M + j] *= scale[i];
|
|
;
|
|
; scale[i] is a stride-1 access w.r.t. the outer loop IV.
|
|
define void @scale_rows(ptr noalias %A, ptr noalias %scale, i64 %N, i64 %M) {
|
|
; CHECK-LABEL: define void @scale_rows(
|
|
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[SCALE:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: [[CMP_OUTER:%.*]] = icmp sgt i64 [[N]], 0
|
|
; CHECK-NEXT: br i1 [[CMP_OUTER]], label %[[OUTER_PH:.*]], [[EXIT:label %.*]]
|
|
; CHECK: [[OUTER_PH]]:
|
|
; CHECK-NEXT: [[CMP_INNER:%.*]] = icmp sgt i64 [[M]], 0
|
|
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
|
|
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
|
|
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
|
|
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
|
|
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
|
|
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[SCALE]], <4 x i64> [[VEC_IND]]
|
|
; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 4 [[TMP0]], <4 x i1> splat (i1 true), <4 x float> poison)
|
|
; CHECK-NEXT: br i1 [[CMP_INNER]], label %[[INNER_PH1:.*]], label %[[VECTOR_LATCH]]
|
|
; CHECK: [[INNER_PH1]]:
|
|
; CHECK-NEXT: [[TMP1:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
|
|
; CHECK-NEXT: br label %[[INNER_BODY2:.*]]
|
|
; CHECK: [[INNER_BODY2]]:
|
|
; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[INNER_PH1]] ], [ [[TMP6:%.*]], %[[INNER_BODY2]] ]
|
|
; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i64> [[TMP1]], [[TMP2]]
|
|
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A]], <4 x i64> [[TMP3]]
|
|
; CHECK-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 4 [[TMP4]], <4 x i1> splat (i1 true), <4 x float> poison)
|
|
; CHECK-NEXT: [[TMP5:%.*]] = fmul <4 x float> [[WIDE_MASKED_GATHER3]], [[WIDE_MASKED_GATHER]]
|
|
; CHECK-NEXT: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> [[TMP5]], <4 x ptr> align 4 [[TMP4]], <4 x i1> splat (i1 true))
|
|
; CHECK-NEXT: [[TMP6]] = add nuw nsw <4 x i64> [[TMP2]], splat (i64 1)
|
|
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <4 x i64> [[TMP6]], [[BROADCAST_SPLAT]]
|
|
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP7]], i64 0
|
|
; CHECK-NEXT: br i1 [[TMP8]], label %[[OUTER_LATCH_LOOPEXIT4:.*]], label %[[INNER_BODY2]]
|
|
; CHECK: [[OUTER_LATCH_LOOPEXIT4]]:
|
|
; CHECK-NEXT: br label %[[VECTOR_LATCH]]
|
|
; CHECK: [[VECTOR_LATCH]]:
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
|
|
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
|
|
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
|
|
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT_LOOPEXIT:label %.*]], label %[[SCALAR_PH]]
|
|
; CHECK: [[SCALAR_PH]]:
|
|
;
|
|
entry:
|
|
%cmp.outer = icmp sgt i64 %N, 0
|
|
br i1 %cmp.outer, label %outer.ph, label %exit
|
|
|
|
outer.ph:
|
|
%cmp.inner = icmp sgt i64 %M, 0
|
|
br label %outer.header
|
|
|
|
outer.header:
|
|
%i = phi i64 [ 0, %outer.ph ], [ %i.next, %outer.latch ]
|
|
%scale.ptr = getelementptr inbounds float, ptr %scale, i64 %i
|
|
%scale.val = load float, ptr %scale.ptr, align 4
|
|
br i1 %cmp.inner, label %inner.ph, label %outer.latch
|
|
|
|
inner.ph:
|
|
%i.mul.M = mul nsw i64 %i, %M
|
|
br label %inner.body
|
|
|
|
inner.body:
|
|
%j = phi i64 [ 0, %inner.ph ], [ %j.next, %inner.body ]
|
|
%idx = add nsw i64 %i.mul.M, %j
|
|
%A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
|
|
%A.val = load float, ptr %A.ptr, align 4
|
|
%mul = fmul float %A.val, %scale.val
|
|
store float %mul, ptr %A.ptr, align 4
|
|
%j.next = add nuw nsw i64 %j, 1
|
|
%j.cmp = icmp eq i64 %j.next, %M
|
|
br i1 %j.cmp, label %outer.latch, label %inner.body
|
|
|
|
outer.latch:
|
|
%i.next = add nuw nsw i64 %i, 1
|
|
%i.cmp = icmp eq i64 %i.next, %N
|
|
br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0
|
|
|
|
exit:
|
|
ret void
|
|
}
|
|
|
|
; --- stride-2 float load (NOT contiguous) ---
|
|
; for (i = 0; i < N; i++)
|
|
; for (j = 0; j < M; j++)
|
|
; A[i*M+j] *= scale[2*i];
|
|
;
|
|
; scale[2*i] has stride 2*sizeof(float) = 8 bytes, not matching
|
|
; sizeof(float) = 4. Should remain a gather.
|
|
define void @stride2_float_load(ptr noalias %A, ptr noalias %scale, i64 %N, i64 %M) {
|
|
; CHECK-LABEL: define void @stride2_float_load(
|
|
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[SCALE:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: [[CMP_OUTER:%.*]] = icmp sgt i64 [[N]], 0
|
|
; CHECK-NEXT: br i1 [[CMP_OUTER]], label %[[OUTER_PH:.*]], [[EXIT:label %.*]]
|
|
; CHECK: [[OUTER_PH]]:
|
|
; CHECK-NEXT: [[CMP_INNER:%.*]] = icmp sgt i64 [[M]], 0
|
|
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
|
|
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
|
|
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
|
|
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
|
|
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
|
|
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = shl nuw nsw <4 x i64> [[VEC_IND]], splat (i64 1)
|
|
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[SCALE]], <4 x i64> [[TMP0]]
|
|
; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 4 [[TMP1]], <4 x i1> splat (i1 true), <4 x float> poison)
|
|
; CHECK-NEXT: br i1 [[CMP_INNER]], label %[[INNER_PH1:.*]], label %[[VECTOR_LATCH]]
|
|
; CHECK: [[INNER_PH1]]:
|
|
; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
|
|
; CHECK-NEXT: br label %[[INNER_BODY2:.*]]
|
|
; CHECK: [[INNER_BODY2]]:
|
|
; CHECK-NEXT: [[TMP3:%.*]] = phi <4 x i64> [ zeroinitializer, %[[INNER_PH1]] ], [ [[TMP7:%.*]], %[[INNER_BODY2]] ]
|
|
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <4 x i64> [[TMP2]], [[TMP3]]
|
|
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[A]], <4 x i64> [[TMP4]]
|
|
; CHECK-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 4 [[TMP5]], <4 x i1> splat (i1 true), <4 x float> poison)
|
|
; CHECK-NEXT: [[TMP6:%.*]] = fmul <4 x float> [[WIDE_MASKED_GATHER3]], [[WIDE_MASKED_GATHER]]
|
|
; CHECK-NEXT: call void @llvm.masked.scatter.v4f32.v4p0(<4 x float> [[TMP6]], <4 x ptr> align 4 [[TMP5]], <4 x i1> splat (i1 true))
|
|
; CHECK-NEXT: [[TMP7]] = add nuw nsw <4 x i64> [[TMP3]], splat (i64 1)
|
|
; CHECK-NEXT: [[TMP8:%.*]] = icmp eq <4 x i64> [[TMP7]], [[BROADCAST_SPLAT]]
|
|
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP8]], i64 0
|
|
; CHECK-NEXT: br i1 [[TMP9]], label %[[OUTER_LATCH_LOOPEXIT4:.*]], label %[[INNER_BODY2]]
|
|
; CHECK: [[OUTER_LATCH_LOOPEXIT4]]:
|
|
; CHECK-NEXT: br label %[[VECTOR_LATCH]]
|
|
; CHECK: [[VECTOR_LATCH]]:
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
|
|
; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
|
|
; CHECK-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
|
|
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT_LOOPEXIT:label %.*]], label %[[SCALAR_PH]]
|
|
; CHECK: [[SCALAR_PH]]:
|
|
;
|
|
entry:
|
|
%cmp.outer = icmp sgt i64 %N, 0
|
|
br i1 %cmp.outer, label %outer.ph, label %exit
|
|
|
|
outer.ph:
|
|
%cmp.inner = icmp sgt i64 %M, 0
|
|
br label %outer.header
|
|
|
|
outer.header:
|
|
%i = phi i64 [ 0, %outer.ph ], [ %i.next, %outer.latch ]
|
|
%i2 = shl nuw nsw i64 %i, 1
|
|
%scale.ptr = getelementptr inbounds float, ptr %scale, i64 %i2
|
|
%scale.val = load float, ptr %scale.ptr, align 4
|
|
br i1 %cmp.inner, label %inner.ph, label %outer.latch
|
|
|
|
inner.ph:
|
|
%i.mul.M = mul nsw i64 %i, %M
|
|
br label %inner.body
|
|
|
|
inner.body:
|
|
%j = phi i64 [ 0, %inner.ph ], [ %j.next, %inner.body ]
|
|
%idx = add nsw i64 %i.mul.M, %j
|
|
%A.ptr = getelementptr inbounds float, ptr %A, i64 %idx
|
|
%A.val = load float, ptr %A.ptr, align 4
|
|
%mul = fmul float %A.val, %scale.val
|
|
store float %mul, ptr %A.ptr, align 4
|
|
%j.next = add nuw nsw i64 %j, 1
|
|
%j.cmp = icmp eq i64 %j.next, %M
|
|
br i1 %j.cmp, label %outer.latch, label %inner.body
|
|
|
|
outer.latch:
|
|
%i.next = add nuw nsw i64 %i, 1
|
|
%i.cmp = icmp eq i64 %i.next, %N
|
|
br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0
|
|
|
|
exit:
|
|
ret void
|
|
}
|
|
|
|
; --- stride-1 double load ---
|
|
; for (i = 0; i < N; i++)
|
|
; for (j = 0; j < M; j++)
|
|
; A[i*M+j] *= scale[i]; // double type
|
|
;
|
|
; scale[i] has stride sizeof(double) = 8, matching the element type.
|
|
define void @stride1_double_load(ptr noalias %A, ptr noalias %scale, i64 %N, i64 %M) {
|
|
; CHECK-LABEL: define void @stride1_double_load(
|
|
; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[SCALE:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: [[CMP_OUTER:%.*]] = icmp sgt i64 [[N]], 0
|
|
; CHECK-NEXT: br i1 [[CMP_OUTER]], label %[[OUTER_PH:.*]], [[EXIT:label %.*]]
|
|
; CHECK: [[OUTER_PH]]:
|
|
; CHECK-NEXT: [[CMP_INNER:%.*]] = icmp sgt i64 [[M]], 0
|
|
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
|
|
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
|
|
; CHECK: [[VECTOR_PH]]:
|
|
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
|
|
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
|
|
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[M]], i64 0
|
|
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
|
|
; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
|
|
; CHECK: [[VECTOR_BODY]]:
|
|
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_LATCH:.*]] ]
|
|
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_LATCH]] ]
|
|
; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds double, ptr [[SCALE]], <4 x i64> [[VEC_IND]]
|
|
; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> align 8 [[TMP0]], <4 x i1> splat (i1 true), <4 x double> poison)
|
|
; CHECK-NEXT: br i1 [[CMP_INNER]], label %[[INNER_PH1:.*]], label %[[VECTOR_LATCH]]
|
|
; CHECK: [[INNER_PH1]]:
|
|
; CHECK-NEXT: [[TMP1:%.*]] = mul nsw <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
|
|
; CHECK-NEXT: br label %[[INNER_BODY2:.*]]
|
|
; CHECK: [[INNER_BODY2]]:
|
|
; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i64> [ zeroinitializer, %[[INNER_PH1]] ], [ [[TMP6:%.*]], %[[INNER_BODY2]] ]
|
|
; CHECK-NEXT: [[TMP3:%.*]] = add nsw <4 x i64> [[TMP1]], [[TMP2]]
|
|
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds double, ptr [[A]], <4 x i64> [[TMP3]]
|
|
; CHECK-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> align 8 [[TMP4]], <4 x i1> splat (i1 true), <4 x double> poison)
|
|
; CHECK-NEXT: [[TMP5:%.*]] = fmul <4 x double> [[WIDE_MASKED_GATHER3]], [[WIDE_MASKED_GATHER]]
|
|
; CHECK-NEXT: call void @llvm.masked.scatter.v4f64.v4p0(<4 x double> [[TMP5]], <4 x ptr> align 8 [[TMP4]], <4 x i1> splat (i1 true))
|
|
; CHECK-NEXT: [[TMP6]] = add nuw nsw <4 x i64> [[TMP2]], splat (i64 1)
|
|
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq <4 x i64> [[TMP6]], [[BROADCAST_SPLAT]]
|
|
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP7]], i64 0
|
|
; CHECK-NEXT: br i1 [[TMP8]], label %[[OUTER_LATCH_LOOPEXIT4:.*]], label %[[INNER_BODY2]]
|
|
; CHECK: [[OUTER_LATCH_LOOPEXIT4]]:
|
|
; CHECK-NEXT: br label %[[VECTOR_LATCH]]
|
|
; CHECK: [[VECTOR_LATCH]]:
|
|
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
|
|
; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4)
|
|
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
|
|
; CHECK-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
|
|
; CHECK: [[MIDDLE_BLOCK]]:
|
|
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
|
|
; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT_LOOPEXIT:label %.*]], label %[[SCALAR_PH]]
|
|
; CHECK: [[SCALAR_PH]]:
|
|
;
|
|
entry:
|
|
%cmp.outer = icmp sgt i64 %N, 0
|
|
br i1 %cmp.outer, label %outer.ph, label %exit
|
|
|
|
outer.ph:
|
|
%cmp.inner = icmp sgt i64 %M, 0
|
|
br label %outer.header
|
|
|
|
outer.header:
|
|
%i = phi i64 [ 0, %outer.ph ], [ %i.next, %outer.latch ]
|
|
%scale.ptr = getelementptr inbounds double, ptr %scale, i64 %i
|
|
%scale.val = load double, ptr %scale.ptr, align 8
|
|
br i1 %cmp.inner, label %inner.ph, label %outer.latch
|
|
|
|
inner.ph:
|
|
%i.mul.M = mul nsw i64 %i, %M
|
|
br label %inner.body
|
|
|
|
inner.body:
|
|
%j = phi i64 [ 0, %inner.ph ], [ %j.next, %inner.body ]
|
|
%idx = add nsw i64 %i.mul.M, %j
|
|
%A.ptr = getelementptr inbounds double, ptr %A, i64 %idx
|
|
%A.val = load double, ptr %A.ptr, align 8
|
|
%mul = fmul double %A.val, %scale.val
|
|
store double %mul, ptr %A.ptr, align 8
|
|
%j.next = add nuw nsw i64 %j, 1
|
|
%j.cmp = icmp eq i64 %j.next, %M
|
|
br i1 %j.cmp, label %outer.latch, label %inner.body
|
|
|
|
outer.latch:
|
|
%i.next = add nuw nsw i64 %i, 1
|
|
%i.cmp = icmp eq i64 %i.next, %N
|
|
br i1 %i.cmp, label %exit, label %outer.header, !llvm.loop !0
|
|
|
|
exit:
|
|
ret void
|
|
}
|
|
|
|
!0 = distinct !{!0, !1, !2}
|
|
!1 = !{!"llvm.loop.vectorize.width", i32 4}
|
|
!2 = !{!"llvm.loop.vectorize.enable", i1 true}
|