[MachineLICM][AArch64] Hoist COPY instructions with other uses in the loop (#71403)

When there is a COPY instruction in the loop with other uses, we want to
hoist the COPY, which in turn leads to the users being hoisted as well.

Co-authored-by David Green : David.Green@arm.com
This commit is contained in:
Rin authored and GitHub committed 2023-11-20 10:01:04 +00:00
1 parent f7b5c25507
commit befa925aca
27 files changed
+3270 -3571

No files matched your search

+12
View File
@@ -1262,6 +1262,18 @@ bool MachineLICMBase::IsProfitableToHoist(MachineInstr &MI,
return false;
}
// If we have a COPY with other uses in the loop, hoist to allow the users to
// also be hoisted.
if (MI.isCopy() && MI.getOperand(0).isReg() &&
MI.getOperand(0).getReg().isVirtual() && MI.getOperand(1).isReg() &&
MI.getOperand(1).getReg().isVirtual() &&
IsLoopInvariantInst(MI, CurLoop) &&
any_of(MRI->use_nodbg_instructions(MI.getOperand(0).getReg()),
[&CurLoop](MachineInstr &UseMI) {
return CurLoop->contains(&UseMI);
}))
return true;
// High register pressure situation, only hoist if the instruction is going
// to be remat'ed.
if (!isTriviallyReMaterializable(MI) &&
+63 -63
View File
@@ -52,19 +52,19 @@ define void @loop1(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n
; CHECK-NEXT: b.eq .LBB0_8
; CHECK-NEXT: .LBB0_6: // %for.body.preheader1
; CHECK-NEXT: movi d0, #0000000000000000
; CHECK-NEXT: sub w10, w2, w10
; CHECK-NEXT: mov w11, #1132396544 // =0x437f0000
; CHECK-NEXT: sub w10, w2, w10
; CHECK-NEXT: fmov s1, w11
; CHECK-NEXT: .LBB0_7: // %for.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: fmov s2, w11
; CHECK-NEXT: ldr s1, [x8], #4
; CHECK-NEXT: fcmp s1, s2
; CHECK-NEXT: fcsel s2, s2, s1, gt
; CHECK-NEXT: fcmp s1, #0.0
; CHECK-NEXT: fcsel s1, s0, s2, mi
; CHECK-NEXT: ldr s2, [x8], #4
; CHECK-NEXT: fcmp s2, s1
; CHECK-NEXT: fcsel s3, s1, s2, gt
; CHECK-NEXT: fcmp s2, #0.0
; CHECK-NEXT: fcsel s2, s0, s3, mi
; CHECK-NEXT: subs w10, w10, #1
; CHECK-NEXT: fcvtzs w12, s1
; CHECK-NEXT: strb w12, [x9], #1
; CHECK-NEXT: fcvtzs w11, s2
; CHECK-NEXT: strb w11, [x9], #1
; CHECK-NEXT: b.ne .LBB0_7
; CHECK-NEXT: .LBB0_8: // %for.cond.cleanup
; CHECK-NEXT: ret
@@ -165,25 +165,25 @@ define void @loop2(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n
; CHECK-NEXT: mov x9, x0
; CHECK-NEXT: .LBB1_5: // %for.body.preheader1
; CHECK-NEXT: movi d0, #0000000000000000
; CHECK-NEXT: sub w10, w2, w10
; CHECK-NEXT: mov w11, #1132396544 // =0x437f0000
; CHECK-NEXT: sub w10, w2, w10
; CHECK-NEXT: fmov s1, w11
; CHECK-NEXT: .LBB1_6: // %for.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ldp s1, s3, [x8], #8
; CHECK-NEXT: fmov s2, w11
; CHECK-NEXT: fcmp s1, s2
; CHECK-NEXT: fcsel s4, s2, s1, gt
; CHECK-NEXT: fcmp s1, #0.0
; CHECK-NEXT: fcsel s1, s0, s4, mi
; CHECK-NEXT: fcmp s3, s2
; CHECK-NEXT: fcsel s2, s2, s3, gt
; CHECK-NEXT: ldp s2, s3, [x8], #8
; CHECK-NEXT: fcmp s2, s1
; CHECK-NEXT: fcsel s4, s1, s2, gt
; CHECK-NEXT: fcmp s2, #0.0
; CHECK-NEXT: fcsel s2, s0, s4, mi
; CHECK-NEXT: fcmp s3, s1
; CHECK-NEXT: fcsel s4, s1, s3, gt
; CHECK-NEXT: fcmp s3, #0.0
; CHECK-NEXT: fcvtzs w12, s1
; CHECK-NEXT: fcsel s2, s0, s2, mi
; CHECK-NEXT: fcvtzs w11, s2
; CHECK-NEXT: fcsel s3, s0, s4, mi
; CHECK-NEXT: subs w10, w10, #1
; CHECK-NEXT: strb w12, [x9]
; CHECK-NEXT: fcvtzs w13, s2
; CHECK-NEXT: strb w13, [x9, #1]
; CHECK-NEXT: strb w11, [x9]
; CHECK-NEXT: fcvtzs w12, s3
; CHECK-NEXT: strb w12, [x9, #1]
; CHECK-NEXT: add x9, x9, #2
; CHECK-NEXT: b.ne .LBB1_6
; CHECK-NEXT: .LBB1_7: // %for.cond.cleanup
@@ -380,33 +380,33 @@ define void @loop3(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n
; CHECK-NEXT: mov x9, x0
; CHECK-NEXT: .LBB2_7: // %for.body.preheader1
; CHECK-NEXT: movi d0, #0000000000000000
; CHECK-NEXT: sub w10, w2, w10
; CHECK-NEXT: mov w11, #1132396544 // =0x437f0000
; CHECK-NEXT: sub w10, w2, w10
; CHECK-NEXT: fmov s1, w11
; CHECK-NEXT: .LBB2_8: // %for.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ldp s1, s3, [x8]
; CHECK-NEXT: fmov s2, w11
; CHECK-NEXT: fcmp s1, s2
; CHECK-NEXT: fcsel s4, s2, s1, gt
; CHECK-NEXT: fcmp s1, #0.0
; CHECK-NEXT: fcsel s1, s0, s4, mi
; CHECK-NEXT: fcmp s3, s2
; CHECK-NEXT: fcsel s4, s2, s3, gt
; CHECK-NEXT: ldp s2, s3, [x8]
; CHECK-NEXT: fcmp s2, s1
; CHECK-NEXT: fcsel s4, s1, s2, gt
; CHECK-NEXT: fcmp s2, #0.0
; CHECK-NEXT: fcsel s2, s0, s4, mi
; CHECK-NEXT: fcmp s3, s1
; CHECK-NEXT: fcsel s4, s1, s3, gt
; CHECK-NEXT: fcmp s3, #0.0
; CHECK-NEXT: ldr s3, [x8, #8]
; CHECK-NEXT: fcvtzs w12, s1
; CHECK-NEXT: fcvtzs w11, s2
; CHECK-NEXT: add x8, x8, #12
; CHECK-NEXT: fcsel s4, s0, s4, mi
; CHECK-NEXT: fcmp s3, s2
; CHECK-NEXT: strb w12, [x9]
; CHECK-NEXT: fcsel s2, s2, s3, gt
; CHECK-NEXT: fcmp s3, s1
; CHECK-NEXT: strb w11, [x9]
; CHECK-NEXT: fcsel s5, s1, s3, gt
; CHECK-NEXT: fcmp s3, #0.0
; CHECK-NEXT: fcvtzs w13, s4
; CHECK-NEXT: fcsel s2, s0, s2, mi
; CHECK-NEXT: fcvtzs w12, s4
; CHECK-NEXT: fcsel s3, s0, s5, mi
; CHECK-NEXT: subs w10, w10, #1
; CHECK-NEXT: strb w13, [x9, #1]
; CHECK-NEXT: fcvtzs w14, s2
; CHECK-NEXT: strb w14, [x9, #2]
; CHECK-NEXT: strb w12, [x9, #1]
; CHECK-NEXT: fcvtzs w13, s3
; CHECK-NEXT: strb w13, [x9, #2]
; CHECK-NEXT: add x9, x9, #3
; CHECK-NEXT: b.ne .LBB2_8
; CHECK-NEXT: .LBB2_9: // %for.cond.cleanup
@@ -549,39 +549,39 @@ define void @loop4(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n
; CHECK-NEXT: mov x9, x0
; CHECK-NEXT: .LBB3_5: // %for.body.preheader1
; CHECK-NEXT: movi d0, #0000000000000000
; CHECK-NEXT: sub w10, w2, w10
; CHECK-NEXT: mov w11, #1132396544 // =0x437f0000
; CHECK-NEXT: sub w10, w2, w10
; CHECK-NEXT: fmov s1, w11
; CHECK-NEXT: .LBB3_6: // %for.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-NEXT: ldp s1, s3, [x8]
; CHECK-NEXT: fmov s2, w11
; CHECK-NEXT: fcmp s1, s2
; CHECK-NEXT: fcsel s4, s2, s1, gt
; CHECK-NEXT: fcmp s1, #0.0
; CHECK-NEXT: fcsel s1, s0, s4, mi
; CHECK-NEXT: fcmp s3, s2
; CHECK-NEXT: fcsel s4, s2, s3, gt
; CHECK-NEXT: ldp s2, s3, [x8]
; CHECK-NEXT: fcmp s2, s1
; CHECK-NEXT: fcsel s4, s1, s2, gt
; CHECK-NEXT: fcmp s2, #0.0
; CHECK-NEXT: fcsel s2, s0, s4, mi
; CHECK-NEXT: fcmp s3, s1
; CHECK-NEXT: fcsel s4, s1, s3, gt
; CHECK-NEXT: fcmp s3, #0.0
; CHECK-NEXT: ldp s3, s5, [x8, #8]
; CHECK-NEXT: fcvtzs w12, s1
; CHECK-NEXT: fcvtzs w11, s2
; CHECK-NEXT: add x8, x8, #16
; CHECK-NEXT: fcsel s4, s0, s4, mi
; CHECK-NEXT: fcmp s3, s2
; CHECK-NEXT: strb w12, [x9]
; CHECK-NEXT: fcsel s6, s2, s3, gt
; CHECK-NEXT: fcmp s3, s1
; CHECK-NEXT: strb w11, [x9]
; CHECK-NEXT: fcsel s6, s1, s3, gt
; CHECK-NEXT: fcmp s3, #0.0
; CHECK-NEXT: fcvtzs w13, s4
; CHECK-NEXT: fcvtzs w12, s4
; CHECK-NEXT: fcsel s3, s0, s6, mi
; CHECK-NEXT: fcmp s5, s2
; CHECK-NEXT: strb w13, [x9, #1]
; CHECK-NEXT: fcsel s2, s2, s5, gt
; CHECK-NEXT: fcmp s5, s1
; CHECK-NEXT: strb w12, [x9, #1]
; CHECK-NEXT: fcsel s6, s1, s5, gt
; CHECK-NEXT: fcmp s5, #0.0
; CHECK-NEXT: fcvtzs w14, s3
; CHECK-NEXT: fcsel s2, s0, s2, mi
; CHECK-NEXT: fcvtzs w13, s3
; CHECK-NEXT: fcsel s5, s0, s6, mi
; CHECK-NEXT: subs w10, w10, #1
; CHECK-NEXT: strb w14, [x9, #2]
; CHECK-NEXT: fcvtzs w15, s2
; CHECK-NEXT: strb w15, [x9, #3]
; CHECK-NEXT: strb w13, [x9, #2]
; CHECK-NEXT: fcvtzs w14, s5
; CHECK-NEXT: strb w14, [x9, #3]
; CHECK-NEXT: add x9, x9, #4
; CHECK-NEXT: b.ne .LBB3_6
; CHECK-NEXT: .LBB3_7: // %for.cond.cleanup
+18 -16
View File
@@ -2756,17 +2756,18 @@ exit:
define i32 @test_pr62620_widening_instr(ptr %p1, ptr %p2, i64 %lx, i32 %h) {
; CHECK-LABEL: test_pr62620_widening_instr:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: lsl x8, x2, #4
; CHECK-NEXT: ldr q0, [x0, x8]
; CHECK-NEXT: ldr q1, [x1, x8]
; CHECK-NEXT: lsl x9, x2, #4
; CHECK-NEXT: mov x8, x0
; CHECK-NEXT: mov w0, wzr
; CHECK-NEXT: ldr q0, [x8, x9]
; CHECK-NEXT: ldr q1, [x1, x9]
; CHECK-NEXT: uabdl.8h v2, v0, v1
; CHECK-NEXT: uabal2.8h v2, v0, v1
; CHECK-NEXT: uaddlv.8h s0, v2
; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: LBB23_1: ; %loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: uabdl.8h v2, v0, v1
; CHECK-NEXT: subs w3, w3, #1
; CHECK-NEXT: uabal2.8h v2, v0, v1
; CHECK-NEXT: uaddlv.8h s2, v2
; CHECK-NEXT: fmov w8, s2
; CHECK-NEXT: add w0, w8, w0
; CHECK-NEXT: b.ne LBB23_1
; CHECK-NEXT: ; %bb.2: ; %exit
@@ -2774,19 +2775,20 @@ define i32 @test_pr62620_widening_instr(ptr %p1, ptr %p2, i64 %lx, i32 %h) {
;
; CHECK-BE-LABEL: test_pr62620_widening_instr:
; CHECK-BE: // %bb.0: // %entry
; CHECK-BE-NEXT: lsl x8, x2, #4
; CHECK-BE-NEXT: add x9, x0, x8
; CHECK-BE-NEXT: add x8, x1, x8
; CHECK-BE-NEXT: lsl x9, x2, #4
; CHECK-BE-NEXT: mov x8, x0
; CHECK-BE-NEXT: mov w0, wzr
; CHECK-BE-NEXT: ld1 { v0.16b }, [x9]
; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]
; CHECK-BE-NEXT: add x8, x8, x9
; CHECK-BE-NEXT: add x9, x1, x9
; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]
; CHECK-BE-NEXT: ld1 { v1.16b }, [x9]
; CHECK-BE-NEXT: uabdl v2.8h, v0.8b, v1.8b
; CHECK-BE-NEXT: uabal2 v2.8h, v0.16b, v1.16b
; CHECK-BE-NEXT: uaddlv s0, v2.8h
; CHECK-BE-NEXT: fmov w8, s0
; CHECK-BE-NEXT: .LBB23_1: // %loop
; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1
; CHECK-BE-NEXT: uabdl v2.8h, v0.8b, v1.8b
; CHECK-BE-NEXT: subs w3, w3, #1
; CHECK-BE-NEXT: uabal2 v2.8h, v0.16b, v1.16b
; CHECK-BE-NEXT: uaddlv s2, v2.8h
; CHECK-BE-NEXT: fmov w8, s2
; CHECK-BE-NEXT: add w0, w8, w0
; CHECK-BE-NEXT: b.ne .LBB23_1
; CHECK-BE-NEXT: // %bb.2: // %exit
@@ -1444,15 +1444,14 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat(ptr %ptr) #1 {
; GFX90A-LABEL: flat_atomic_fadd_f64_noret_pat:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT: s_mov_b64 s[2:3], 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GFX90A-NEXT: s_mov_b64 s[0:1], 0
; GFX90A-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -1460,9 +1459,9 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat(ptr %ptr) #1 {
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX90A-NEXT: s_cbranch_execnz .LBB50_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_endpgm
@@ -1519,15 +1518,14 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_system(ptr %ptr) #1 {
; GFX90A-LABEL: flat_atomic_fadd_f64_noret_pat_system:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT: s_mov_b64 s[2:3], 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GFX90A-NEXT: s_mov_b64 s[0:1], 0
; GFX90A-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -1536,9 +1534,9 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_system(ptr %ptr) #1 {
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX90A-NEXT: s_cbranch_execnz .LBB52_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_endpgm
@@ -1719,23 +1717,22 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_agent_safe(ptr %ptr) {
; GFX90A-LABEL: flat_atomic_fadd_f64_noret_pat_agent_safe:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT: s_mov_b64 s[2:3], 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GFX90A-NEXT: s_mov_b64 s[0:1], 0
; GFX90A-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX90A-NEXT: s_cbranch_execnz .LBB58_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_endpgm
@@ -1948,22 +1945,21 @@ main_body:
define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrspace(3) %ptr) #4 {
; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_load_dword s2, s[0:1], 0x24
; GFX90A-NEXT: s_mov_b64 s[0:1], 0
; GFX90A-NEXT: s_load_dword s0, s[0:1], 0x24
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v0, s2
; GFX90A-NEXT: ds_read_b64 v[0:1], v0
; GFX90A-NEXT: v_mov_b32_e32 v2, s0
; GFX90A-NEXT: ds_read_b64 v[0:1], v2
; GFX90A-NEXT: s_mov_b64 s[0:1], 0
; GFX90A-NEXT: .LBB67_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_add_f64 v[2:3], v[0:1], 4.0
; GFX90A-NEXT: v_mov_b32_e32 v4, s2
; GFX90A-NEXT: v_add_f64 v[4:5], v[0:1], 4.0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: ds_cmpst_rtn_b64 v[2:3], v4, v[0:1], v[2:3]
; GFX90A-NEXT: ds_cmpst_rtn_b64 v[4:5], v2, v[0:1], v[4:5]
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[0:1]
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[0:1]
; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX90A-NEXT: s_cbranch_execnz .LBB67_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1971,22 +1967,21 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
;
; GFX940-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe:
; GFX940: ; %bb.0: ; %main_body
; GFX940-NEXT: s_load_dword s2, s[0:1], 0x24
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: s_load_dword s0, s[0:1], 0x24
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v0, s2
; GFX940-NEXT: ds_read_b64 v[0:1], v0
; GFX940-NEXT: v_mov_b32_e32 v2, s0
; GFX940-NEXT: ds_read_b64 v[0:1], v2
; GFX940-NEXT: s_mov_b64 s[0:1], 0
; GFX940-NEXT: .LBB67_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
; GFX940-NEXT: v_add_f64 v[2:3], v[0:1], 4.0
; GFX940-NEXT: v_mov_b32_e32 v4, s2
; GFX940-NEXT: v_add_f64 v[4:5], v[0:1], 4.0
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
; GFX940-NEXT: ds_cmpst_rtn_b64 v[2:3], v4, v[0:1], v[2:3]
; GFX940-NEXT: ds_cmpst_rtn_b64 v[4:5], v2, v[0:1], v[4:5]
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[0:1]
; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[0:1]
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[2:3]
; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_cbranch_execnz .LBB67_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+28 -28
View File
@@ -1534,16 +1534,16 @@ main_body:
define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat(ptr %ptr) #1 {
; GFX90A-LABEL: flat_atomic_fadd_f64_noret_pat:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT: s_mov_b64 s[2:3], 0
; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
; GFX90A-NEXT: s_mov_b64 s[0:1], 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -1551,9 +1551,9 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat(ptr %ptr) #1 {
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX90A-NEXT: s_cbranch_execnz .LBB53_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_endpgm
@@ -1609,16 +1609,16 @@ main_body:
define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_system(ptr %ptr) #1 {
; GFX90A-LABEL: flat_atomic_fadd_f64_noret_pat_system:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT: s_mov_b64 s[2:3], 0
; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
; GFX90A-NEXT: s_mov_b64 s[0:1], 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -1627,9 +1627,9 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_system(ptr %ptr) #1 {
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX90A-NEXT: s_cbranch_execnz .LBB55_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_endpgm
@@ -1813,24 +1813,24 @@ main_body:
define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_agent_safe(ptr %ptr) {
; GFX90A-LABEL: flat_atomic_fadd_f64_noret_pat_agent_safe:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GFX90A-NEXT: s_mov_b64 s[2:3], 0
; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
; GFX90A-NEXT: s_mov_b64 s[0:1], 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX90A-NEXT: s_cbranch_execnz .LBB61_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_endpgm
@@ -2104,17 +2104,17 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v0, s2
; GFX90A-NEXT: ds_read_b64 v[0:1], v0
; GFX90A-NEXT: v_mov_b32_e32 v2, s2
; GFX90A-NEXT: .LBB72_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_add_f64 v[2:3], v[0:1], 4.0
; GFX90A-NEXT: v_mov_b32_e32 v4, s2
; GFX90A-NEXT: v_add_f64 v[4:5], v[0:1], 4.0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: ds_cmpst_rtn_b64 v[2:3], v4, v[0:1], v[2:3]
; GFX90A-NEXT: ds_cmpst_rtn_b64 v[4:5], v2, v[0:1], v[4:5]
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[0:1]
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[0:1]
; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX90A-NEXT: s_cbranch_execnz .LBB72_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2127,17 +2127,17 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v0, s2
; GFX940-NEXT: ds_read_b64 v[0:1], v0
; GFX940-NEXT: v_mov_b32_e32 v2, s2
; GFX940-NEXT: .LBB72_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
; GFX940-NEXT: v_add_f64 v[2:3], v[0:1], 4.0
; GFX940-NEXT: v_mov_b32_e32 v4, s2
; GFX940-NEXT: v_add_f64 v[4:5], v[0:1], 4.0
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
; GFX940-NEXT: ds_cmpst_rtn_b64 v[2:3], v4, v[0:1], v[2:3]
; GFX940-NEXT: ds_cmpst_rtn_b64 v[4:5], v2, v[0:1], v[4:5]
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[0:1]
; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[0:1]
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[2:3]
; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_cbranch_execnz .LBB72_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -56,20 +56,19 @@ define amdgpu_kernel void @test_move_load_address_to_vgpr_d16_hi(ptr addrspace(1
; GCN: ; %bb.0: ; %bb
; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_movk_i32 s2, 0x100
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: global_load_ushort v0, v1, s[0:1] glc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v2, s1
; GCN-NEXT: s_movk_i32 s1, 0x100
; GCN-NEXT: .LBB1_1: ; %bb3
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN-NEXT: v_lshlrev_b64 v[3:4], 1, v[0:1]
; GCN-NEXT: v_add_co_u32_e32 v3, vcc, s0, v3
; GCN-NEXT: v_addc_co_u32_e32 v4, vcc, v2, v4, vcc
; GCN-NEXT: global_load_short_d16_hi v0, v[3:4], off glc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_lshlrev_b64 v[2:3], 1, v[0:1]
; GCN-NEXT: v_mov_b32_e32 v0, s1
; GCN-NEXT: v_add_co_u32_e32 v2, vcc, s0, v2
; GCN-NEXT: v_addc_co_u32_e32 v3, vcc, v0, v3, vcc
; GCN-NEXT: global_load_short_d16_hi v0, v[2:3], off glc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, s2, v0
; GCN-NEXT: v_cmp_eq_u32_e32 vcc, s1, v0
; GCN-NEXT: s_cbranch_vccz .LBB1_1
; GCN-NEXT: ; %bb.2: ; %bb2
; GCN-NEXT: s_endpgm
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+104 -116
View File
@@ -331,22 +331,21 @@ define amdgpu_kernel void @lds_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3) %
; GFX7-NEXT: s_cbranch_execz .LBB2_4
; GFX7-NEXT: ; %bb.1:
; GFX7-NEXT: s_lshl_b32 s8, s3, 3
; GFX7-NEXT: v_mov_b32_e32 v1, s8
; GFX7-NEXT: ds_read_b32 v1, v1
; GFX7-NEXT: v_mov_b32_e32 v2, s8
; GFX7-NEXT: ds_read_b32 v1, v2
; GFX7-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s6
; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v3, s6
; GFX7-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v3, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s8
; GFX7-NEXT: v_add_f32_e32 v4, v3, v2
; GFX7-NEXT: v_mov_b32_e32 v4, v1
; GFX7-NEXT: v_add_f32_e32 v1, v4, v3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v1, v3, v4
; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB2_2
@@ -364,47 +363,45 @@ define amdgpu_kernel void @lds_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3) %
; GFX7-NEXT: ; %bb.5:
; GFX7-NEXT: s_lshl_b32 s3, s3, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s3
; GFX7-NEXT: ds_read_b32 v2, v1
; GFX7-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v1, s6
; GFX7-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX7-NEXT: ds_read_b32 v3, v1
; GFX7-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: .LBB2_6: ; %atomicrmw.start2
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_add_f32_e32 v3, v2, v1
; GFX7-NEXT: v_mov_b32_e32 v4, s3
; GFX7-NEXT: v_add_f32_e32 v4, v3, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v4, v2, v3
; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: v_mov_b32_e32 v2, v3
; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB2_6
; GFX7-NEXT: .LBB2_7: ; %Flow14
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v1, s2
; GFX7-NEXT: ds_read_b32 v1, v1
; GFX7-NEXT: v_mov_b32_e32 v2, s2
; GFX7-NEXT: ds_read_b32 v1, v2
; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX7-NEXT: v_add_f32_e32 v0, s8, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_mov_b64 s[2:3], 0
; GFX7-NEXT: .LBB2_8: ; %atomicrmw.start8
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v2, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s2
; GFX7-NEXT: v_add_f32_e32 v3, v2, v0
; GFX7-NEXT: v_mov_b32_e32 v3, v1
; GFX7-NEXT: v_add_f32_e32 v1, v3, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v1, v2, v3
; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX7-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX7-NEXT: s_cbranch_execnz .LBB2_8
; GFX7-NEXT: ; %bb.9: ; %atomicrmw.end7
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
@@ -427,22 +424,21 @@ define amdgpu_kernel void @lds_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3) %
; GFX8-NEXT: s_cbranch_execz .LBB2_4
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_lshl_b32 s8, s3, 3
; GFX8-NEXT: v_mov_b32_e32 v1, s8
; GFX8-NEXT: ds_read_b32 v1, v1
; GFX8-NEXT: v_mov_b32_e32 v2, s8
; GFX8-NEXT: ds_read_b32 v1, v2
; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v2, s6
; GFX8-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v3, s6
; GFX8-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s8
; GFX8-NEXT: v_add_f32_e32 v4, v3, v2
; GFX8-NEXT: v_mov_b32_e32 v4, v1
; GFX8-NEXT: v_add_f32_e32 v1, v4, v3
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v1, v3, v4
; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB2_2
@@ -460,47 +456,45 @@ define amdgpu_kernel void @lds_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3) %
; GFX8-NEXT: ; %bb.5:
; GFX8-NEXT: s_lshl_b32 s3, s3, 4
; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: ds_read_b32 v2, v1
; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s6
; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX8-NEXT: ds_read_b32 v3, v1
; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
; GFX8-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB2_6: ; %atomicrmw.start2
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_add_f32_e32 v3, v2, v1
; GFX8-NEXT: v_mov_b32_e32 v4, s3
; GFX8-NEXT: v_add_f32_e32 v4, v3, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v4, v2, v3
; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v2, v3
; GFX8-NEXT: v_mov_b32_e32 v3, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB2_6
; GFX8-NEXT: .LBB2_7: ; %Flow16
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v1, s2
; GFX8-NEXT: ds_read_b32 v1, v1
; GFX8-NEXT: v_mov_b32_e32 v2, s2
; GFX8-NEXT: ds_read_b32 v1, v2
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX8-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX8-NEXT: v_add_f32_e32 v0, s8, v0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: s_mov_b64 s[2:3], 0
; GFX8-NEXT: .LBB2_8: ; %atomicrmw.start8
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v2, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s2
; GFX8-NEXT: v_add_f32_e32 v3, v2, v0
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: v_add_f32_e32 v1, v3, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v1, v2, v3
; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX8-NEXT: s_cbranch_execnz .LBB2_8
; GFX8-NEXT: ; %bb.9: ; %atomicrmw.end7
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
@@ -698,21 +692,20 @@ define amdgpu_kernel void @lds_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrspa
; GFX7-NEXT: s_cbranch_execz .LBB3_4
; GFX7-NEXT: ; %bb.1:
; GFX7-NEXT: s_lshl_b32 s8, s3, 3
; GFX7-NEXT: v_mov_b32_e32 v1, s8
; GFX7-NEXT: ds_read_b32 v1, v1
; GFX7-NEXT: v_mov_b32_e32 v2, s8
; GFX7-NEXT: ds_read_b32 v1, v2
; GFX7-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s6
; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v3, s6
; GFX7-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v3, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s8
; GFX7-NEXT: v_add_f32_e32 v4, v3, v2
; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v1, v3, v4
; GFX7-NEXT: v_mov_b32_e32 v4, v1
; GFX7-NEXT: v_add_f32_e32 v1, v4, v3
; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB3_2
@@ -730,45 +723,43 @@ define amdgpu_kernel void @lds_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrspa
; GFX7-NEXT: ; %bb.5:
; GFX7-NEXT: s_lshl_b32 s3, s3, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s3
; GFX7-NEXT: ds_read_b32 v2, v1
; GFX7-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v1, s6
; GFX7-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX7-NEXT: ds_read_b32 v3, v1
; GFX7-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: .LBB3_6: ; %atomicrmw.start2
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_add_f32_e32 v3, v2, v1
; GFX7-NEXT: v_mov_b32_e32 v4, s3
; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v4, v2, v3
; GFX7-NEXT: v_add_f32_e32 v4, v3, v2
; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: v_mov_b32_e32 v2, v3
; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB3_6
; GFX7-NEXT: .LBB3_7: ; %Flow14
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v1, s2
; GFX7-NEXT: ds_read_b32 v1, v1
; GFX7-NEXT: v_mov_b32_e32 v2, s2
; GFX7-NEXT: ds_read_b32 v1, v2
; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX7-NEXT: v_add_f32_e32 v0, s8, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_mov_b64 s[2:3], 0
; GFX7-NEXT: .LBB3_8: ; %atomicrmw.start8
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v2, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s2
; GFX7-NEXT: v_add_f32_e32 v3, v2, v0
; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v1, v2, v3
; GFX7-NEXT: v_mov_b32_e32 v3, v1
; GFX7-NEXT: v_add_f32_e32 v1, v3, v0
; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX7-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX7-NEXT: s_cbranch_execnz .LBB3_8
; GFX7-NEXT: ; %bb.9: ; %atomicrmw.end7
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
@@ -791,21 +782,20 @@ define amdgpu_kernel void @lds_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrspa
; GFX8-NEXT: s_cbranch_execz .LBB3_4
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_lshl_b32 s8, s3, 3
; GFX8-NEXT: v_mov_b32_e32 v1, s8
; GFX8-NEXT: ds_read_b32 v1, v1
; GFX8-NEXT: v_mov_b32_e32 v2, s8
; GFX8-NEXT: ds_read_b32 v1, v2
; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v2, s6
; GFX8-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v3, s6
; GFX8-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s8
; GFX8-NEXT: v_add_f32_e32 v4, v3, v2
; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v1, v3, v4
; GFX8-NEXT: v_mov_b32_e32 v4, v1
; GFX8-NEXT: v_add_f32_e32 v1, v4, v3
; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB3_2
@@ -823,45 +813,43 @@ define amdgpu_kernel void @lds_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrspa
; GFX8-NEXT: ; %bb.5:
; GFX8-NEXT: s_lshl_b32 s3, s3, 4
; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: ds_read_b32 v2, v1
; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s6
; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX8-NEXT: ds_read_b32 v3, v1
; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
; GFX8-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB3_6: ; %atomicrmw.start2
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_add_f32_e32 v3, v2, v1
; GFX8-NEXT: v_mov_b32_e32 v4, s3
; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v4, v2, v3
; GFX8-NEXT: v_add_f32_e32 v4, v3, v2
; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v2, v3
; GFX8-NEXT: v_mov_b32_e32 v3, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB3_6
; GFX8-NEXT: .LBB3_7: ; %Flow16
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v1, s2
; GFX8-NEXT: ds_read_b32 v1, v1
; GFX8-NEXT: v_mov_b32_e32 v2, s2
; GFX8-NEXT: ds_read_b32 v1, v2
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX8-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX8-NEXT: v_add_f32_e32 v0, s8, v0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: s_mov_b64 s[2:3], 0
; GFX8-NEXT: .LBB3_8: ; %atomicrmw.start8
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v2, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s2
; GFX8-NEXT: v_add_f32_e32 v3, v2, v0
; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v1, v2, v3
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: v_add_f32_e32 v1, v3, v0
; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX8-NEXT: s_cbranch_execnz .LBB3_8
; GFX8-NEXT: ; %bb.9: ; %atomicrmw.end7
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
@@ -69,21 +69,21 @@ define void @issue63986(i64 %0, i64 %idxprom) {
; CHECK-NEXT: v_lshlrev_b64 v[2:3], 6, v[2:3]
; CHECK-NEXT: s_cbranch_execnz .LBB0_9
; CHECK-NEXT: .LBB0_6: ; %loop-memcpy-residual.preheader
; CHECK-NEXT: v_mov_b32_e32 v7, s5
; CHECK-NEXT: v_or_b32_e32 v2, 32, v4
; CHECK-NEXT: v_mov_b32_e32 v3, v5
; CHECK-NEXT: s_mov_b64 s[6:7], 0
; CHECK-NEXT: v_mov_b32_e32 v6, s4
; CHECK-NEXT: .LBB0_7: ; %loop-memcpy-residual
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_add_u32 s8, 32, s6
; CHECK-NEXT: s_addc_u32 s9, 0, s7
; CHECK-NEXT: v_mov_b32_e32 v6, s8
; CHECK-NEXT: v_mov_b32_e32 v7, s9
; CHECK-NEXT: flat_load_ubyte v10, v[6:7]
; CHECK-NEXT: s_add_u32 s4, 32, s6
; CHECK-NEXT: s_addc_u32 s5, 0, s7
; CHECK-NEXT: v_mov_b32_e32 v9, s5
; CHECK-NEXT: v_mov_b32_e32 v8, s4
; CHECK-NEXT: flat_load_ubyte v10, v[8:9]
; CHECK-NEXT: v_mov_b32_e32 v9, s7
; CHECK-NEXT: v_mov_b32_e32 v7, s5
; CHECK-NEXT: v_add_co_u32_e32 v8, vcc, s6, v2
; CHECK-NEXT: s_add_u32 s6, s6, 1
; CHECK-NEXT: v_mov_b32_e32 v6, s4
; CHECK-NEXT: v_addc_co_u32_e32 v9, vcc, v3, v9, vcc
; CHECK-NEXT: s_addc_u32 s7, s7, 0
; CHECK-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[6:7]
@@ -236,13 +236,13 @@ define void @issue63986_reduced_expanded(i64 %idxprom) {
; CHECK-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
; CHECK-NEXT: s_cbranch_execnz .LBB1_8
; CHECK-NEXT: .LBB1_5: ; %loop-memcpy-residual.preheader
; CHECK-NEXT: s_mov_b64 s[6:7], 0
; CHECK-NEXT: .LBB1_6: ; %loop-memcpy-residual
; CHECK-NEXT: s_add_u32 s6, s6, 1
; CHECK-NEXT: v_mov_b32_e32 v0, s4
; CHECK-NEXT: s_mov_b64 s[6:7], 0
; CHECK-NEXT: v_mov_b32_e32 v1, s5
; CHECK-NEXT: s_addc_u32 s7, s7, 0
; CHECK-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
; CHECK-NEXT: .LBB1_6: ; %loop-memcpy-residual
; CHECK-NEXT: s_add_u32 s4, s6, 1
; CHECK-NEXT: s_addc_u32 s5, s7, 0
; CHECK-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]
; CHECK-NEXT: s_mov_b64 s[6:7], 1
; CHECK-NEXT: s_cbranch_vccnz .LBB1_6
; CHECK-NEXT: ; %bb.7: ; %Flow
@@ -9,7 +9,8 @@ define amdgpu_kernel void @negated_cond(ptr addrspace(1) %arg1) {
; GCN-NEXT: s_mov_b32 s10, -1
; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_mov_b64 s[8:9], s[4:5]
; GCN-NEXT: s_mov_b32 s9, s5
; GCN-NEXT: s_mov_b32 s8, s4
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_branch .LBB0_2
; GCN-NEXT: .LBB0_1: ; %loop.exit.guard
@@ -496,7 +496,8 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1) %buffer) {
; GFX900-NEXT: v_mov_b32_e32 v2, s35
; GFX900-NEXT: v_add_co_u32_e32 v1, vcc, s34, v1
; GFX900-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
; GFX900-NEXT: v_add_co_u32_e32 v1, vcc, 0x5000, v1
; GFX900-NEXT: s_movk_i32 s0, 0x5000
; GFX900-NEXT: v_add_co_u32_e32 v1, vcc, s0, v1
; GFX900-NEXT: v_mov_b32_e32 v3, 0
; GFX900-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
; GFX900-NEXT: s_movk_i32 s2, 0x7f
@@ -723,7 +724,8 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1) %buffer) {
; GFX90A-NEXT: v_mov_b32_e32 v2, s35
; GFX90A-NEXT: v_add_co_u32_e32 v1, vcc, s34, v1
; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x5000, v1
; GFX90A-NEXT: s_movk_i32 s0, 0x5000
; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, s0, v1
; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX90A-NEXT: s_movk_i32 s2, 0x7f
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], 0, 0
@@ -3,7 +3,7 @@
; GCN-LABEL: {{^}}test_remat_sgpr:
; GCN-NOT: v_writelane_b32
; GCN-COUNT-6: s_mov_b32 s{{[0-9]+}}, 0x
; GCN-COUNT-4: s_mov_b32 s{{[0-9]+}}, 0x
; GCN: {{^}}[[LOOP:.LBB[0-9_]+]]:
; GCN-NOT: v_writelane_b32
; GCN: s_cbranch_{{[^ ]+}} [[LOOP]]
+1 -1
View File
@@ -1720,8 +1720,8 @@ define i64 @v_test_sdiv_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_addc_u32_e64 v8, s[4:5], 0, -1, vcc
; GCN-IR-NEXT: s_mov_b64 s[10:11], 0
; GCN-IR-NEXT: v_mov_b32_e32 v12, 0
; GCN-IR-NEXT: v_mov_b32_e32 v6, 0
; GCN-IR-NEXT: s_movk_i32 s12, 0x7fff
; GCN-IR-NEXT: v_mov_b32_e32 v6, 0
; GCN-IR-NEXT: .LBB13_3: ; %udiv-do-while
; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN-IR-NEXT: v_lshl_b64 v[9:10], v[9:10], 1
+13 -13
View File
@@ -2103,19 +2103,19 @@ define void @crash_lshlrevb16_not_reg_op() {
; NOSDWA: ; %bb.0: ; %bb0
; NOSDWA-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; NOSDWA-NEXT: s_mov_b64 s[4:5], 0
; NOSDWA-NEXT: v_mov_b32_e32 v0, 0xff
; NOSDWA-NEXT: v_and_b32_e32 v0, s4, v0
; NOSDWA-NEXT: v_lshlrev_b16_e64 v1, 8, 1
; NOSDWA-NEXT: v_or_b32_e32 v0, v0, v1
; NOSDWA-NEXT: s_and_b64 vcc, exec, -1
; NOSDWA-NEXT: v_lshlrev_b16_e64 v3, 8, 1
; NOSDWA-NEXT: .LBB22_1: ; %bb1
; NOSDWA-NEXT: ; =>This Inner Loop Header: Depth=1
; NOSDWA-NEXT: v_mov_b32_e32 v0, s4
; NOSDWA-NEXT: v_mov_b32_e32 v2, 0xff
; NOSDWA-NEXT: s_lshl_b32 s6, s4, 3
; NOSDWA-NEXT: v_mov_b32_e32 v1, s5
; NOSDWA-NEXT: v_mov_b32_e32 v1, s4
; NOSDWA-NEXT: v_mov_b32_e32 v2, s5
; NOSDWA-NEXT: s_mov_b64 s[4:5], 1
; NOSDWA-NEXT: v_and_b32_e32 v2, s4, v2
; NOSDWA-NEXT: v_or_b32_e32 v2, v2, v3
; NOSDWA-NEXT: v_lshrrev_b16_e32 v2, s6, v2
; NOSDWA-NEXT: flat_store_byte v[0:1], v2
; NOSDWA-NEXT: v_lshrrev_b16_e32 v3, s6, v0
; NOSDWA-NEXT: flat_store_byte v[1:2], v3
; NOSDWA-NEXT: s_mov_b64 vcc, vcc
; NOSDWA-NEXT: s_cbranch_vccnz .LBB22_1
; NOSDWA-NEXT: ; %bb.2: ; %DummyReturnBlock
@@ -2126,17 +2126,17 @@ define void @crash_lshlrevb16_not_reg_op() {
; GFX89: ; %bb.0: ; %bb0
; GFX89-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX89-NEXT: s_mov_b64 s[4:5], 0
; GFX89-NEXT: s_and_b64 vcc, exec, -1
; GFX89-NEXT: v_lshlrev_b16_e64 v0, 8, 1
; GFX89-NEXT: v_mov_b32_e32 v1, s4
; GFX89-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX89-NEXT: s_and_b64 vcc, exec, -1
; GFX89-NEXT: .LBB22_1: ; %bb1
; GFX89-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX89-NEXT: v_mov_b32_e32 v3, s4
; GFX89-NEXT: s_lshl_b32 s6, s4, 3
; GFX89-NEXT: v_mov_b32_e32 v1, s4
; GFX89-NEXT: v_or_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX89-NEXT: v_mov_b32_e32 v2, s5
; GFX89-NEXT: s_mov_b64 s[4:5], 1
; GFX89-NEXT: v_lshrrev_b16_e32 v3, s6, v3
; GFX89-NEXT: v_lshrrev_b16_e32 v3, s6, v0
; GFX89-NEXT: flat_store_byte v[1:2], v3
; GFX89-NEXT: s_mov_b64 vcc, vcc
; GFX89-NEXT: s_cbranch_vccnz .LBB22_1
@@ -2149,8 +2149,8 @@ define void @crash_lshlrevb16_not_reg_op() {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: v_lshlrev_b16_e64 v0, 8, 1
; GFX9-NEXT: s_and_b64 vcc, exec, -1
; GFX9-NEXT: v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-NEXT: s_and_b64 vcc, exec, -1
; GFX9-NEXT: .LBB22_1: ; %bb1
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_lshl_b32 s6, s4, 3
+1 -1
View File
@@ -1912,8 +1912,8 @@ define i64 @v_test_srem_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_addc_u32_e64 v9, s[4:5], 0, -1, vcc
; GCN-IR-NEXT: s_mov_b64 s[10:11], 0
; GCN-IR-NEXT: v_mov_b32_e32 v13, 0
; GCN-IR-NEXT: v_mov_b32_e32 v7, 0
; GCN-IR-NEXT: s_movk_i32 s12, 0x7fff
; GCN-IR-NEXT: v_mov_b32_e32 v7, 0
; GCN-IR-NEXT: .LBB13_3: ; %udiv-do-while
; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN-IR-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
+1 -1
View File
@@ -1281,8 +1281,8 @@ define i64 @v_test_udiv_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_addc_u32_e64 v1, s[4:5], 0, -1, vcc
; GCN-IR-NEXT: s_mov_b64 s[10:11], 0
; GCN-IR-NEXT: v_mov_b32_e32 v10, 0
; GCN-IR-NEXT: v_mov_b32_e32 v5, 0
; GCN-IR-NEXT: s_movk_i32 s12, 0x7fff
; GCN-IR-NEXT: v_mov_b32_e32 v5, 0
; GCN-IR-NEXT: .LBB10_3: ; %udiv-do-while
; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN-IR-NEXT: v_lshl_b64 v[7:8], v[7:8], 1
+1 -1
View File
@@ -1280,8 +1280,8 @@ define i64 @v_test_urem_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_addc_u32_e64 v7, s[4:5], 0, -1, vcc
; GCN-IR-NEXT: s_mov_b64 s[10:11], 0
; GCN-IR-NEXT: v_mov_b32_e32 v11, 0
; GCN-IR-NEXT: v_mov_b32_e32 v5, 0
; GCN-IR-NEXT: s_movk_i32 s12, 0x7fff
; GCN-IR-NEXT: v_mov_b32_e32 v5, 0
; GCN-IR-NEXT: .LBB9_3: ; %udiv-do-while
; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN-IR-NEXT: v_lshl_b64 v[8:9], v[8:9], 1
@@ -48,14 +48,18 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
; CHECK-NEXT: js .LBB0_10
; CHECK-NEXT: # %bb.2: # %b158
; CHECK-NEXT: movslq (%r9), %rsi
; CHECK-NEXT: xorl %edi, %edi
; CHECK-NEXT: movl %esi, %edi
; CHECK-NEXT: orl %eax, %edi
; CHECK-NEXT: movl %edi, %r8d
; CHECK-NEXT: shrl $31, %r8d
; CHECK-NEXT: xorl %r9d, %r9d
; CHECK-NEXT: xorps %xmm0, %xmm0
; CHECK-NEXT: movb $1, %r8b
; CHECK-NEXT: movb $1, %r10b
; CHECK-NEXT: jmp .LBB0_3
; CHECK-NEXT: .p2align 4, 0x90
; CHECK-NEXT: .LBB0_9: # %b1606
; CHECK-NEXT: # in Loop: Header=BB0_3 Depth=1
; CHECK-NEXT: testb %dil, %dil
; CHECK-NEXT: testb %r9b, %r9b
; CHECK-NEXT: je .LBB0_10
; CHECK-NEXT: .LBB0_3: # %a29b173
; CHECK-NEXT: # =>This Loop Header: Depth=1
@@ -83,47 +87,42 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
; CHECK-NEXT: .LBB0_36: # %a30b
; CHECK-NEXT: # Parent Loop BB0_3 Depth=1
; CHECK-NEXT: # => This Inner Loop Header: Depth=2
; CHECK-NEXT: testb %dil, %dil
; CHECK-NEXT: testb %r9b, %r9b
; CHECK-NEXT: je .LBB0_36
; CHECK-NEXT: .LBB0_18: # %b188
; CHECK-NEXT: # in Loop: Header=BB0_3 Depth=1
; CHECK-NEXT: testb %r8b, %r8b
; CHECK-NEXT: testb %r10b, %r10b
; CHECK-NEXT: jne .LBB0_4
; CHECK-NEXT: .p2align 4, 0x90
; CHECK-NEXT: .LBB0_19: # %a30b294
; CHECK-NEXT: # Parent Loop BB0_3 Depth=1
; CHECK-NEXT: # => This Inner Loop Header: Depth=2
; CHECK-NEXT: testb %dil, %dil
; CHECK-NEXT: testb %r9b, %r9b
; CHECK-NEXT: je .LBB0_19
; CHECK-NEXT: .LBB0_4: # %a33b
; CHECK-NEXT: # in Loop: Header=BB0_3 Depth=1
; CHECK-NEXT: movl %esi, %r9d
; CHECK-NEXT: orl %eax, %r9d
; CHECK-NEXT: testl %edi, %edi
; CHECK-NEXT: jns .LBB0_20
; CHECK-NEXT: .LBB0_5: # %a50b
; CHECK-NEXT: # in Loop: Header=BB0_3 Depth=1
; CHECK-NEXT: movl %eax, %r10d
; CHECK-NEXT: orl %esi, %r10d
; CHECK-NEXT: testl %edi, %edi
; CHECK-NEXT: jns .LBB0_25
; CHECK-NEXT: .LBB0_6: # %a57b
; CHECK-NEXT: # in Loop: Header=BB0_3 Depth=1
; CHECK-NEXT: shrl $31, %r9d
; CHECK-NEXT: testb %r9b, %r9b
; CHECK-NEXT: testb %r8b, %r8b
; CHECK-NEXT: je .LBB0_29
; CHECK-NEXT: .LBB0_7: # %a66b
; CHECK-NEXT: # in Loop: Header=BB0_3 Depth=1
; CHECK-NEXT: shrl $31, %r10d
; CHECK-NEXT: testb %r10b, %r10b
; CHECK-NEXT: testb %r8b, %r8b
; CHECK-NEXT: jne .LBB0_8
; CHECK-NEXT: .p2align 4, 0x90
; CHECK-NEXT: .LBB0_33: # %a74b
; CHECK-NEXT: # Parent Loop BB0_3 Depth=1
; CHECK-NEXT: # => This Inner Loop Header: Depth=2
; CHECK-NEXT: testb %dil, %dil
; CHECK-NEXT: testb %r9b, %r9b
; CHECK-NEXT: jne .LBB0_33
; CHECK-NEXT: # %bb.34: # %b1582
; CHECK-NEXT: # in Loop: Header=BB0_33 Depth=2
; CHECK-NEXT: testb %dil, %dil
; CHECK-NEXT: jne .LBB0_33
; CHECK-NEXT: .LBB0_8: # %a93b
; CHECK-NEXT: # in Loop: Header=BB0_3 Depth=1
@@ -140,13 +139,13 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
; CHECK-NEXT: addss %xmm0, %xmm2
; CHECK-NEXT: addss %xmm1, %xmm2
; CHECK-NEXT: movss %xmm2, i6000(%rip)
; CHECK-NEXT: testb %dil, %dil
; CHECK-NEXT: testb %r9b, %r9b
; CHECK-NEXT: jne .LBB0_35
; CHECK-NEXT: jmp .LBB0_9
; CHECK-NEXT: .p2align 4, 0x90
; CHECK-NEXT: .LBB0_21: # %b377
; CHECK-NEXT: # in Loop: Header=BB0_20 Depth=2
; CHECK-NEXT: testb %dil, %dil
; CHECK-NEXT: testb %r9b, %r9b
; CHECK-NEXT: je .LBB0_22
; CHECK-NEXT: .LBB0_20: # %b341
; CHECK-NEXT: # Parent Loop BB0_3 Depth=1
@@ -159,7 +158,7 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
; CHECK-NEXT: # Parent Loop BB0_3 Depth=1
; CHECK-NEXT: # Parent Loop BB0_20 Depth=2
; CHECK-NEXT: # => This Inner Loop Header: Depth=3
; CHECK-NEXT: testb %dil, %dil
; CHECK-NEXT: testb %r9b, %r9b
; CHECK-NEXT: je .LBB0_37
; CHECK-NEXT: jmp .LBB0_21
; CHECK-NEXT: .p2align 4, 0x90
@@ -179,11 +178,11 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
; CHECK-NEXT: # Parent Loop BB0_3 Depth=1
; CHECK-NEXT: # Parent Loop BB0_25 Depth=2
; CHECK-NEXT: # => This Inner Loop Header: Depth=3
; CHECK-NEXT: testb %dil, %dil
; CHECK-NEXT: testb %r9b, %r9b
; CHECK-NEXT: je .LBB0_38
; CHECK-NEXT: .LBB0_26: # %b879
; CHECK-NEXT: # in Loop: Header=BB0_25 Depth=2
; CHECK-NEXT: testb %r8b, %r8b
; CHECK-NEXT: testb %r10b, %r10b
; CHECK-NEXT: jne .LBB0_27
; CHECK-NEXT: .p2align 4, 0x90
; CHECK-NEXT: .LBB0_28: # %a53b1019
@@ -227,7 +226,7 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
; CHECK-NEXT: .p2align 4, 0x90
; CHECK-NEXT: .LBB0_24: # %b712
; CHECK-NEXT: # in Loop: Header=BB0_22 Depth=2
; CHECK-NEXT: testb %dil, %dil
; CHECK-NEXT: testb %r9b, %r9b
; CHECK-NEXT: je .LBB0_5
; CHECK-NEXT: .LBB0_22: # %b535
; CHECK-NEXT: # Parent Loop BB0_3 Depth=1
@@ -240,7 +239,7 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
; CHECK-NEXT: # Parent Loop BB0_3 Depth=1
; CHECK-NEXT: # Parent Loop BB0_22 Depth=2
; CHECK-NEXT: # => This Inner Loop Header: Depth=3
; CHECK-NEXT: testb %dil, %dil
; CHECK-NEXT: testb %r9b, %r9b
; CHECK-NEXT: je .LBB0_23
; CHECK-NEXT: jmp .LBB0_24
; CHECK-NEXT: .LBB0_10: # %a109b
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; REQUIRES: asserts
; RUN: llc < %s -mattr=+sse3,+sse4.1 -mcpu=penryn -stats 2>&1 | grep "5 machinelicm"
; RUN: llc < %s -mattr=+sse3,+sse4.1 -mcpu=penryn -stats 2>&1 | grep "6 machinelicm"
; RUN: llc < %s -mattr=+sse3,+sse4.1 -mcpu=penryn | FileCheck %s
; rdar://6627786
; rdar://7792037
+1 -5
View File
@@ -11,7 +11,7 @@ define void @func() {
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: testb %al, %al
; CHECK-NEXT: je .LBB0_1
; CHECK-NEXT: # %bb.4: # %L_30
; CHECK-NEXT: # %bb.3: # %L_30
; CHECK-NEXT: retq
; CHECK-NEXT: .LBB0_1: # %bb56
; CHECK-NEXT: xorl %eax, %eax
@@ -19,10 +19,6 @@ define void @func() {
; CHECK-NEXT: .LBB0_2: # %bb33
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-NEXT: testb %al, %al
; CHECK-NEXT: jne .LBB0_2
; CHECK-NEXT: # %bb.3: # %bb35
; CHECK-NEXT: # in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: testb %al, %al
; CHECK-NEXT: jmp .LBB0_2
bb1:
br i1 undef, label %L_10, label %L_10
-1
View File
@@ -118,7 +118,6 @@ define dso_local void @fn() {
; CHECK-NEXT: jne .LBB0_22
; CHECK-NEXT: # %bb.23: # %for.cond47
; CHECK-NEXT: # in Loop: Header=BB0_22 Depth=2
; CHECK-NEXT: testb %bl, %bl
; CHECK-NEXT: jne .LBB0_22
; CHECK-NEXT: .LBB0_9: # %ae
; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1
+6 -6
View File
@@ -9,15 +9,15 @@ define void @PR53842() {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0
; CHECK-NEXT: vpmovzxbq {{.*#+}} zmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero,mem[4],zero,zero,zero,zero,zero,zero,zero,mem[5],zero,zero,zero,zero,zero,zero,zero,mem[6],zero,zero,zero,zero,zero,zero,zero,mem[7],zero,zero,zero,zero,zero,zero,zero
; CHECK-NEXT: vextracti64x4 $1, %zmm1, %ymm3
; CHECK-NEXT: vpxor %xmm2, %xmm2, %xmm2
; CHECK-NEXT: vpcmpeqq %ymm2, %ymm3, %ymm3
; CHECK-NEXT: vextracti64x4 $1, %zmm1, %ymm2
; CHECK-NEXT: vpxor %xmm3, %xmm3, %xmm3
; CHECK-NEXT: vpcmpeqq %ymm3, %ymm2, %ymm2
; CHECK-NEXT: vpcmpeqq %ymm3, %ymm1, %ymm1
; CHECK-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1
; CHECK-NEXT: .p2align 4, 0x90
; CHECK-NEXT: .LBB0_1: # %vector.body
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-NEXT: vpcmpeqq %ymm2, %ymm1, %ymm4
; CHECK-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm4
; CHECK-NEXT: vpsubq %zmm4, %zmm0, %zmm0
; CHECK-NEXT: vpsubq %zmm1, %zmm0, %zmm0
; CHECK-NEXT: jmp .LBB0_1
entry:
br label %vector.body
+4 -4
View File
@@ -15,17 +15,17 @@ define i32 @PR63108() {
; SSE-NEXT: jmp .LBB0_5
; SSE-NEXT: .LBB0_2: # %vector.body.preheader
; SSE-NEXT: pxor %xmm0, %xmm0
; SSE-NEXT: movdqa {{.*#+}} xmm2 = [57339,0,0,0]
; SSE-NEXT: movdqa {{.*#+}} xmm1 = [57339,0,0,0]
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: .p2align 4, 0x90
; SSE-NEXT: .LBB0_3: # %vector.body
; SSE-NEXT: # =>This Inner Loop Header: Depth=1
; SSE-NEXT: movdqa %xmm2, %xmm1
; SSE-NEXT: movdqa %xmm1, %xmm2
; SSE-NEXT: testb %al, %al
; SSE-NEXT: pxor %xmm2, %xmm2
; SSE-NEXT: pxor %xmm1, %xmm1
; SSE-NEXT: jne .LBB0_3
; SSE-NEXT: # %bb.4: # %middle.block
; SSE-NEXT: pxor %xmm1, %xmm0
; SSE-NEXT: pxor %xmm2, %xmm0
; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSE-NEXT: pxor %xmm0, %xmm1
; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
@@ -152,87 +152,89 @@ define void @vector_variable_shift_left_loop(ptr nocapture %arr, ptr nocapture r
; AVX1-NEXT: .LBB0_3: # %vector.ph
; AVX1-NEXT: movl %r9d, %edx
; AVX1-NEXT: andl $-32, %edx
; AVX1-NEXT: vmovd %eax, %xmm0
; AVX1-NEXT: vmovd %r8d, %xmm1
; AVX1-NEXT: vmovd %eax, %xmm7
; AVX1-NEXT: vmovd %r8d, %xmm8
; AVX1-NEXT: xorl %ecx, %ecx
; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero
; AVX1-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero
; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero
; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm6 = xmm1[0],zero,xmm1[1],zero
; AVX1-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm7 = xmm0[0],zero,xmm0[1],zero
; AVX1-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm8 = xmm1[0],zero,xmm1[1],zero
; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm7[0],zero,xmm7[1],zero
; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm8[0],zero,xmm8[1],zero
; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm7[0],zero,xmm7[1],zero
; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm8[0],zero,xmm8[1],zero
; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm7[0],zero,xmm7[1],zero
; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm6 = xmm8[0],zero,xmm8[1],zero
; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm7 = xmm7[0],zero,xmm7[1],zero
; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm8 = xmm8[0],zero,xmm8[1],zero
; AVX1-NEXT: .p2align 4, 0x90
; AVX1-NEXT: .LBB0_4: # %vector.body
; AVX1-NEXT: # =>This Inner Loop Header: Depth=1
; AVX1-NEXT: vmovq {{.*#+}} xmm9 = mem[0],zero
; AVX1-NEXT: vmovq {{.*#+}} xmm10 = mem[0],zero
; AVX1-NEXT: vmovq {{.*#+}} xmm13 = mem[0],zero
; AVX1-NEXT: vmovq {{.*#+}} xmm14 = mem[0],zero
; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX1-NEXT: vpcmpeqb %xmm3, %xmm9, %xmm9
; AVX1-NEXT: vmovq {{.*#+}} xmm11 = mem[0],zero
; AVX1-NEXT: vmovq {{.*#+}} xmm12 = mem[0],zero
; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX1-NEXT: vpcmpeqb %xmm2, %xmm9, %xmm9
; AVX1-NEXT: vpmovsxbd %xmm9, %xmm14
; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[1,1,1,1]
; AVX1-NEXT: vpmovsxbd %xmm9, %xmm15
; AVX1-NEXT: vpcmpeqb %xmm2, %xmm10, %xmm9
; AVX1-NEXT: vpmovsxbd %xmm9, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[1,1,1,1]
; AVX1-NEXT: vpmovsxbd %xmm9, %xmm1
; AVX1-NEXT: vpcmpeqb %xmm2, %xmm11, %xmm9
; AVX1-NEXT: vpmovsxbd %xmm9, %xmm13
; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[1,1,1,1]
; AVX1-NEXT: vpmovsxbd %xmm9, %xmm12
; AVX1-NEXT: vpcmpeqb %xmm3, %xmm10, %xmm9
; AVX1-NEXT: vpmovsxbd %xmm9, %xmm11
; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[1,1,1,1]
; AVX1-NEXT: vpcmpeqb %xmm2, %xmm12, %xmm9
; AVX1-NEXT: vpmovsxbd %xmm9, %xmm10
; AVX1-NEXT: vpcmpeqb %xmm3, %xmm13, %xmm13
; AVX1-NEXT: vpmovzxdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
; AVX1-NEXT: # xmm0 = mem[0],zero,mem[1],zero
; AVX1-NEXT: vmovdqu (%rdi,%rcx,4), %xmm9
; AVX1-NEXT: vpslld %xmm0, %xmm9, %xmm1
; AVX1-NEXT: vpmovzxdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
; AVX1-NEXT: # xmm2 = mem[0],zero,mem[1],zero
; AVX1-NEXT: vpslld %xmm2, %xmm9, %xmm9
; AVX1-NEXT: vblendvps %xmm15, %xmm1, %xmm9, %xmm9
; AVX1-NEXT: vpmovsxbd %xmm13, %xmm1
; AVX1-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[1,1,1,1]
; AVX1-NEXT: vpmovsxbd %xmm13, %xmm13
; AVX1-NEXT: vpcmpeqb %xmm3, %xmm14, %xmm14
; AVX1-NEXT: vmovdqu 16(%rdi,%rcx,4), %xmm15
; AVX1-NEXT: vpslld %xmm0, %xmm15, %xmm0
; AVX1-NEXT: vpslld %xmm2, %xmm15, %xmm2
; AVX1-NEXT: vpmovsxbd %xmm14, %xmm15
; AVX1-NEXT: vpshufd {{.*#+}} xmm14 = xmm14[1,1,1,1]
; AVX1-NEXT: vpmovsxbd %xmm14, %xmm14
; AVX1-NEXT: vblendvps %xmm12, %xmm0, %xmm2, %xmm0
; AVX1-NEXT: vmovdqu 32(%rdi,%rcx,4), %xmm2
; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[1,1,1,1]
; AVX1-NEXT: vpmovsxbd %xmm9, %xmm9
; AVX1-NEXT: vmovdqu (%rdi,%rcx,4), %xmm12
; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
; AVX1-NEXT: vpslld %xmm3, %xmm2, %xmm12
; AVX1-NEXT: vpslld %xmm3, %xmm12, %xmm2
; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
; AVX1-NEXT: vpslld %xmm4, %xmm12, %xmm12
; AVX1-NEXT: vblendvps %xmm14, %xmm2, %xmm12, %xmm12
; AVX1-NEXT: vmovdqu 16(%rdi,%rcx,4), %xmm2
; AVX1-NEXT: vpslld %xmm3, %xmm2, %xmm14
; AVX1-NEXT: vpslld %xmm4, %xmm2, %xmm2
; AVX1-NEXT: vblendvps %xmm11, %xmm12, %xmm2, %xmm2
; AVX1-NEXT: vmovdqu 48(%rdi,%rcx,4), %xmm11
; AVX1-NEXT: vpslld %xmm3, %xmm11, %xmm12
; AVX1-NEXT: vpslld %xmm4, %xmm11, %xmm11
; AVX1-NEXT: vblendvps %xmm10, %xmm12, %xmm11, %xmm10
; AVX1-NEXT: vmovdqu 64(%rdi,%rcx,4), %xmm11
; AVX1-NEXT: vpslld %xmm5, %xmm11, %xmm12
; AVX1-NEXT: vpslld %xmm6, %xmm11, %xmm11
; AVX1-NEXT: vblendvps %xmm1, %xmm12, %xmm11, %xmm1
; AVX1-NEXT: vmovdqu 80(%rdi,%rcx,4), %xmm11
; AVX1-NEXT: vpslld %xmm5, %xmm11, %xmm12
; AVX1-NEXT: vpslld %xmm6, %xmm11, %xmm11
; AVX1-NEXT: vblendvps %xmm13, %xmm12, %xmm11, %xmm11
; AVX1-NEXT: vmovdqu 96(%rdi,%rcx,4), %xmm12
; AVX1-NEXT: vpslld %xmm7, %xmm12, %xmm13
; AVX1-NEXT: vpslld %xmm8, %xmm12, %xmm12
; AVX1-NEXT: vblendvps %xmm15, %xmm13, %xmm12, %xmm12
; AVX1-NEXT: vmovdqu 112(%rdi,%rcx,4), %xmm13
; AVX1-NEXT: vpslld %xmm7, %xmm13, %xmm15
; AVX1-NEXT: vpslld %xmm8, %xmm13, %xmm13
; AVX1-NEXT: vblendvps %xmm14, %xmm15, %xmm13, %xmm13
; AVX1-NEXT: vmovups %xmm9, (%rdi,%rcx,4)
; AVX1-NEXT: vmovups %xmm0, 16(%rdi,%rcx,4)
; AVX1-NEXT: vmovups %xmm2, 32(%rdi,%rcx,4)
; AVX1-NEXT: vmovups %xmm10, 48(%rdi,%rcx,4)
; AVX1-NEXT: vmovups %xmm1, 64(%rdi,%rcx,4)
; AVX1-NEXT: vblendvps %xmm15, %xmm14, %xmm2, %xmm2
; AVX1-NEXT: vmovdqu 32(%rdi,%rcx,4), %xmm14
; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
; AVX1-NEXT: vpslld %xmm3, %xmm14, %xmm15
; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
; AVX1-NEXT: vpslld %xmm4, %xmm14, %xmm14
; AVX1-NEXT: vblendvps %xmm0, %xmm15, %xmm14, %xmm0
; AVX1-NEXT: vmovdqu 48(%rdi,%rcx,4), %xmm14
; AVX1-NEXT: vpslld %xmm3, %xmm14, %xmm15
; AVX1-NEXT: vpslld %xmm4, %xmm14, %xmm14
; AVX1-NEXT: vblendvps %xmm1, %xmm15, %xmm14, %xmm1
; AVX1-NEXT: vmovdqu 64(%rdi,%rcx,4), %xmm14
; AVX1-NEXT: vpslld %xmm5, %xmm14, %xmm15
; AVX1-NEXT: vpslld %xmm6, %xmm14, %xmm14
; AVX1-NEXT: vblendvps %xmm13, %xmm15, %xmm14, %xmm13
; AVX1-NEXT: vmovdqu 80(%rdi,%rcx,4), %xmm14
; AVX1-NEXT: vpslld %xmm5, %xmm14, %xmm15
; AVX1-NEXT: vpslld %xmm6, %xmm14, %xmm14
; AVX1-NEXT: vblendvps %xmm11, %xmm15, %xmm14, %xmm11
; AVX1-NEXT: vmovdqu 96(%rdi,%rcx,4), %xmm14
; AVX1-NEXT: vpslld %xmm7, %xmm14, %xmm15
; AVX1-NEXT: vpslld %xmm8, %xmm14, %xmm14
; AVX1-NEXT: vblendvps %xmm10, %xmm15, %xmm14, %xmm10
; AVX1-NEXT: vmovdqu 112(%rdi,%rcx,4), %xmm14
; AVX1-NEXT: vpslld %xmm7, %xmm14, %xmm15
; AVX1-NEXT: vpslld %xmm8, %xmm14, %xmm14
; AVX1-NEXT: vblendvps %xmm9, %xmm15, %xmm14, %xmm9
; AVX1-NEXT: vmovups %xmm12, (%rdi,%rcx,4)
; AVX1-NEXT: vmovups %xmm2, 16(%rdi,%rcx,4)
; AVX1-NEXT: vmovups %xmm0, 32(%rdi,%rcx,4)
; AVX1-NEXT: vmovups %xmm1, 48(%rdi,%rcx,4)
; AVX1-NEXT: vmovups %xmm13, 64(%rdi,%rcx,4)
; AVX1-NEXT: vmovups %xmm11, 80(%rdi,%rcx,4)
; AVX1-NEXT: vmovups %xmm12, 96(%rdi,%rcx,4)
; AVX1-NEXT: vmovups %xmm13, 112(%rdi,%rcx,4)
; AVX1-NEXT: vmovups %xmm10, 96(%rdi,%rcx,4)
; AVX1-NEXT: vmovups %xmm9, 112(%rdi,%rcx,4)
; AVX1-NEXT: addq $32, %rcx
; AVX1-NEXT: cmpq %rcx, %rdx
; AVX1-NEXT: jne .LBB0_4