aco/gfx12: insert wait between VMEM WaW

llvm/llvm-project#105549

fossil-db (gfx1200):
Totals from 1783 (2.25% of 79395) affected shaders:
Instrs: 7398391 -> 7404566 (+0.08%); split: -0.00%, +0.08%
CodeSize: 38862456 -> 38886364 (+0.06%); split: -0.00%, +0.06%
Latency: 83191513 -> 84211504 (+1.23%); split: -0.00%, +1.23%
InvThroughput: 15185936 -> 15345744 (+1.05%); split: -0.01%, +1.06%

Signed-off-by: Rhys Perry <[email protected]>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/32373>

Author: pendingchaos

src/amd/compiler/aco_insert_waitcnt.cpp

@@ -278,9 +278,13 @@ check_instr(wait_ctx& ctx, wait_imm& wait, Instruction* instr)
 
          wait_imm reg_imm = it->second.imm;
 
-         /* Vector Memory reads and writes return in the order they were issued */
+         /* Vector Memory reads and writes decrease the counter in the order they were issued.
+          * Before GFX12, they also write VGPRs in order if they're of the same type.
+          * TODO: We can do this for GFX12 and different types for GFX11 if we know that the two
+          * VMEM loads do not write the same lanes. Since GFX11, we track VMEM operations on the
+          * linear CFG, so this is difficult */
          uint8_t vmem_type = get_vmem_type(ctx.gfx_level, instr);
-         if (vmem_type) {
+         if (vmem_type && ctx.gfx_level < GFX12) {
             wait_event event = get_vmem_event(ctx, instr, vmem_type);
             wait_type type = (wait_type)(ffs(ctx.info->get_counters_for_event(event)) - 1);
             if ((it->second.events & ctx.info->events[type]) == event &&

src/amd/compiler/tests/test_insert_waitcnt.cpp

@@ -192,6 +192,7 @@ BEGIN_TEST(insert_waitcnt.waw.vmem_types)
 
       //>> p_unit_test 0
       //! v1: %0:v[4] = buffer_load_dword %0:s[0-3], %0:v[0], 0
+      //~gfx12! s_wait_loadcnt imm:0
       //! v1: %0:v[4] = buffer_load_dword %0:s[0-3], %0:v[0], 0
       bld.pseudo(aco_opcode::p_unit_test, Operand::c32(0));
       bld.mubuf(aco_opcode::buffer_load_dword, def_v4, desc_s4, op_v0, Operand::zero(), 0, false);
@@ -220,6 +221,7 @@ BEGIN_TEST(insert_waitcnt.waw.vmem_types)
 
       //>> p_unit_test 3
       //! v1: %0:v[4] = image_sample %0:s[8-15], %0:s[0-3],  v1: undef, %0:v[0] 1d
+      //~gfx12! s_wait_samplecnt imm:0
       //! v1: %0:v[4] = image_sample %0:s[8-15], %0:s[0-3],  v1: undef, %0:v[0] 1d
       bld.reset(program->create_and_insert_block());
       bld.pseudo(aco_opcode::p_unit_test, Operand::c32(3));
@@ -249,6 +251,7 @@ BEGIN_TEST(insert_waitcnt.waw.vmem_types)
 
       //>> p_unit_test 6
       //! v1: %0:v[4] = image_bvh64_intersect_ray %0:s[0-3],  s4: undef,  v1: undef, %0:v[16-26] 1d
+      //~gfx12! s_wait_bvhcnt imm:0
       //! v1: %0:v[4] = image_bvh64_intersect_ray %0:s[0-3],  s4: undef,  v1: undef, %0:v[16-26] 1d
       bld.reset(program->create_and_insert_block());
       bld.pseudo(aco_opcode::p_unit_test, Operand::c32(6));
@@ -300,6 +303,7 @@ BEGIN_TEST(insert_waitcnt.waw.vmem_types)
       //>> BB11
       //! /* logical preds: BB9, BB10, / linear preds: BB9, BB10, / kind: uniform, */
       //! p_unit_test 9
+      //~gfx12! s_wait_loadcnt imm:0
       //! v1: %0:v[4] = buffer_load_dword %0:s[0-3], %0:v[0], 0
       bld.pseudo(aco_opcode::p_unit_test, Operand::c32(9));
       bld.mubuf(aco_opcode::buffer_load_dword, def_v4, desc_s4, op_v0, Operand::zero(), 0, false);
@@ -326,6 +330,7 @@ BEGIN_TEST(insert_waitcnt.waw.vmem_types)
       //! /* logical preds: BB12, BB13, / linear preds: BB12, BB13, / kind: uniform, */
       //! p_unit_test 10
       //~gfx11! s_waitcnt vmcnt(0)
+      //~gfx12! s_wait_loadcnt imm:0
       //~gfx12! s_wait_samplecnt imm:0
       //! v1: %0:v[4] = buffer_load_dword %0:s[0-3], %0:v[0], 0
       bld.pseudo(aco_opcode::p_unit_test, Operand::c32(10));