From 2f099abd0dfc5a6839b85204f4e683ca562ecdd3 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Fri, 7 Jun 2024 23:17:13 +0200 Subject: [PATCH 1/3] DAG: Replace bitwidth with type in suffix in atomic tablegen ops For FP atomics involving bfloat vs. half, we need to distinguish the type and not rely on the bitwidth alone. For my purposes, an alternative would be to relax the atomic predicate MemoryVT pattern check with a memory size only check. Since there are no extending operations involved, the pattern value check should be unambiguous. For some reason when using the _32 variants for atomicrmw fadd, I was able to select v2f16 but v2bf16 would fail. Changes mostly done with sed, e.g. sed -E -i -r 's/atomic_load_(add|swap|sub|and|clr|or|xor|nand|min|max|umin|umax|swap)_([0-9]+)/atomic_load_\1_i\2/' llvm/lib/Target/*/*.td --- .../include/llvm/Target/TargetSelectionDAG.td | 74 ++- .../lib/Target/AArch64/AArch64InstrFormats.td | 60 +-- llvm/lib/Target/AArch64/AArch64InstrGISel.td | 8 +- llvm/lib/Target/AMDGPU/AMDGPUInstructions.td | 31 +- llvm/lib/Target/AMDGPU/BUFInstructions.td | 4 +- llvm/lib/Target/AMDGPU/DSInstructions.td | 46 +- .../Target/AMDGPU/EvergreenInstructions.td | 42 +- llvm/lib/Target/AMDGPU/FLATInstructions.td | 12 +- llvm/lib/Target/AMDGPU/SIInstrInfo.td | 25 +- llvm/lib/Target/AVR/AVRInstrInfo.td | 20 +- llvm/lib/Target/BPF/BPFInstrInfo.td | 30 +- .../Target/LoongArch/LoongArchInstrInfo.td | 64 +-- llvm/lib/Target/Mips/Mips64InstrInfo.td | 24 +- llvm/lib/Target/Mips/MipsInstrInfo.td | 78 ++-- llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 442 +++++++++--------- llvm/lib/Target/PowerPC/PPCInstr64Bit.td | 34 +- llvm/lib/Target/PowerPC/PPCInstrInfo.td | 74 +-- llvm/lib/Target/RISCV/RISCVInstrInfoA.td | 82 ++-- llvm/lib/Target/RISCV/RISCVInstrInfoZa.td | 46 +- llvm/lib/Target/Sparc/SparcInstr64Bit.td | 2 +- llvm/lib/Target/Sparc/SparcInstrInfo.td | 8 +- llvm/lib/Target/SystemZ/SystemZInstrInfo.td | 16 +- llvm/lib/Target/VE/VEInstrInfo.td | 8 +- .../WebAssembly/WebAssemblyInstrAtomics.td | 46 +- llvm/lib/Target/X86/X86InstrCompiler.td | 8 +- llvm/lib/Target/X86/X86InstrMisc.td | 8 +- llvm/test/TableGen/HasNoUse.td | 6 +- 27 files changed, 650 insertions(+), 648 deletions(-) diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td index 15e02eb49271d..f15737e95f97c 100644 --- a/llvm/include/llvm/Target/TargetSelectionDAG.td +++ b/llvm/include/llvm/Target/TargetSelectionDAG.td @@ -1680,60 +1680,38 @@ multiclass ternary_atomic_op_ord { } } -multiclass binary_atomic_op { - def _8 : PatFrag<(ops node:$ptr, node:$val), - (atomic_op node:$ptr, node:$val)> { - let IsAtomic = true; - let MemoryVT = !if(IsInt, i8, ?); - } - def _16 : PatFrag<(ops node:$ptr, node:$val), - (atomic_op node:$ptr, node:$val)> { - let IsAtomic = true; - let MemoryVT = !if(IsInt, i16, f16); - } - def _32 : PatFrag<(ops node:$ptr, node:$val), - (atomic_op node:$ptr, node:$val)> { - let IsAtomic = true; - let MemoryVT = !if(IsInt, i32, f32); - } - def _64 : PatFrag<(ops node:$ptr, node:$val), - (atomic_op node:$ptr, node:$val)> { - let IsAtomic = true; - let MemoryVT = !if(IsInt, i64, f64); +multiclass binary_atomic_op { + foreach vt = [ i8, i16, i32, i64 ] in { + def _#vt : PatFrag<(ops node:$ptr, node:$val), + (atomic_op node:$ptr, node:$val)> { + let IsAtomic = true; + let MemoryVT = vt; + } + + defm NAME#_#vt : binary_atomic_op_ord; } +} - defm NAME#_8 : binary_atomic_op_ord; - defm NAME#_16 : binary_atomic_op_ord; - defm NAME#_32 : binary_atomic_op_ord; - defm NAME#_64 : binary_atomic_op_ord; +multiclass binary_atomic_op_fp { + foreach vt = [ f16, bf16, v2f16, v2bf16, f32, f64 ] in { + def _#vt : PatFrag<(ops node:$ptr, node:$val), + (atomic_op node:$ptr, node:$val)> { + let IsAtomic = true; + let MemoryVT = vt; + } + } } multiclass ternary_atomic_op { - def _8 : PatFrag<(ops node:$ptr, node:$cmp, node:$val), - (atomic_op node:$ptr, node:$cmp, node:$val)> { - let IsAtomic = true; - let MemoryVT = i8; - } - def _16 : PatFrag<(ops node:$ptr, node:$cmp, node:$val), - (atomic_op node:$ptr, node:$cmp, node:$val)> { - let IsAtomic = true; - let MemoryVT = i16; - } - def _32 : PatFrag<(ops node:$ptr, node:$cmp, node:$val), - (atomic_op node:$ptr, node:$cmp, node:$val)> { - let IsAtomic = true; - let MemoryVT = i32; - } - def _64 : PatFrag<(ops node:$ptr, node:$cmp, node:$val), - (atomic_op node:$ptr, node:$cmp, node:$val)> { - let IsAtomic = true; - let MemoryVT = i64; + foreach vt = [ i8, i16, i32, i64 ] in { + def _#vt : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (atomic_op node:$ptr, node:$cmp, node:$val)> { + let IsAtomic = true; + let MemoryVT = vt; + } + + defm NAME#_#vt : ternary_atomic_op_ord; } - - defm NAME#_8 : ternary_atomic_op_ord; - defm NAME#_16 : ternary_atomic_op_ord; - defm NAME#_32 : ternary_atomic_op_ord; - defm NAME#_64 : ternary_atomic_op_ord; } defm atomic_load_add : binary_atomic_op; diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td index 1f437d0ed6f8d..17d011086634c 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td +++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td @@ -11887,79 +11887,79 @@ multiclass LDOPregister opc, string op, bits<1> Acq, bits<1> Rel, // complex DAG for DstRHS. let Predicates = [HasLSE] in multiclass LDOPregister_patterns_ord_dag { - def : Pat<(!cast(op#"_"#size#"_monotonic") GPR64sp:$Rn, SrcRHS), + ValueType vt, dag SrcRHS, dag DstRHS> { + def : Pat<(!cast(op#"_"#vt#"_monotonic") GPR64sp:$Rn, SrcRHS), (!cast(inst # suffix) DstRHS, GPR64sp:$Rn)>; - def : Pat<(!cast(op#"_"#size#"_acquire") GPR64sp:$Rn, SrcRHS), + def : Pat<(!cast(op#"_"#vt#"_acquire") GPR64sp:$Rn, SrcRHS), (!cast(inst # "A" # suffix) DstRHS, GPR64sp:$Rn)>; - def : Pat<(!cast(op#"_"#size#"_release") GPR64sp:$Rn, SrcRHS), + def : Pat<(!cast(op#"_"#vt#"_release") GPR64sp:$Rn, SrcRHS), (!cast(inst # "L" # suffix) DstRHS, GPR64sp:$Rn)>; - def : Pat<(!cast(op#"_"#size#"_acq_rel") GPR64sp:$Rn, SrcRHS), + def : Pat<(!cast(op#"_"#vt#"_acq_rel") GPR64sp:$Rn, SrcRHS), (!cast(inst # "AL" # suffix) DstRHS, GPR64sp:$Rn)>; - def : Pat<(!cast(op#"_"#size#"_seq_cst") GPR64sp:$Rn, SrcRHS), + def : Pat<(!cast(op#"_"#vt#"_seq_cst") GPR64sp:$Rn, SrcRHS), (!cast(inst # "AL" # suffix) DstRHS, GPR64sp:$Rn)>; } multiclass LDOPregister_patterns_ord { - defm : LDOPregister_patterns_ord_dag; + ValueType vt, dag RHS> { + defm : LDOPregister_patterns_ord_dag; } multiclass LDOPregister_patterns_ord_mod { - defm : LDOPregister_patterns_ord_dag; + ValueType vt, dag LHS, dag RHS> { + defm : LDOPregister_patterns_ord_dag; } multiclass LDOPregister_patterns { - defm : LDOPregister_patterns_ord; - defm : LDOPregister_patterns_ord; - defm : LDOPregister_patterns_ord; - defm : LDOPregister_patterns_ord; + defm : LDOPregister_patterns_ord; + defm : LDOPregister_patterns_ord; + defm : LDOPregister_patterns_ord; + defm : LDOPregister_patterns_ord; } multiclass LDOPregister_patterns_mod { - defm : LDOPregister_patterns_ord_mod(mod#Xrr) XZR, GPR64:$Rm))>; - defm : LDOPregister_patterns_ord_mod(mod#Wrr) WZR, GPR32:$Rm))>; - defm : LDOPregister_patterns_ord_mod(mod#Wrr) WZR, GPR32:$Rm))>; - defm : LDOPregister_patterns_ord_mod(mod#Wrr) WZR, GPR32:$Rm))>; } let Predicates = [HasLSE] in multiclass CASregister_patterns_ord_dag { - def : Pat<(!cast(op#"_"#size#"_monotonic") GPR64sp:$Rn, OLD, NEW), + ValueType vt, dag OLD, dag NEW> { + def : Pat<(!cast(op#"_"#vt#"_monotonic") GPR64sp:$Rn, OLD, NEW), (!cast(inst # suffix) OLD, NEW, GPR64sp:$Rn)>; - def : Pat<(!cast(op#"_"#size#"_acquire") GPR64sp:$Rn, OLD, NEW), + def : Pat<(!cast(op#"_"#vt#"_acquire") GPR64sp:$Rn, OLD, NEW), (!cast(inst # "A" # suffix) OLD, NEW, GPR64sp:$Rn)>; - def : Pat<(!cast(op#"_"#size#"_release") GPR64sp:$Rn, OLD, NEW), + def : Pat<(!cast(op#"_"#vt#"_release") GPR64sp:$Rn, OLD, NEW), (!cast(inst # "L" # suffix) OLD, NEW, GPR64sp:$Rn)>; - def : Pat<(!cast(op#"_"#size#"_acq_rel") GPR64sp:$Rn, OLD, NEW), + def : Pat<(!cast(op#"_"#vt#"_acq_rel") GPR64sp:$Rn, OLD, NEW), (!cast(inst # "AL" # suffix) OLD, NEW, GPR64sp:$Rn)>; - def : Pat<(!cast(op#"_"#size#"_seq_cst") GPR64sp:$Rn, OLD, NEW), + def : Pat<(!cast(op#"_"#vt#"_seq_cst") GPR64sp:$Rn, OLD, NEW), (!cast(inst # "AL" # suffix) OLD, NEW, GPR64sp:$Rn)>; } multiclass CASregister_patterns_ord { - defm : CASregister_patterns_ord_dag; + ValueType vt, dag OLD, dag NEW> { + defm : CASregister_patterns_ord_dag; } multiclass CASregister_patterns { - defm : CASregister_patterns_ord; - defm : CASregister_patterns_ord; - defm : CASregister_patterns_ord; - defm : CASregister_patterns_ord; } diff --git a/llvm/lib/Target/AArch64/AArch64InstrGISel.td b/llvm/lib/Target/AArch64/AArch64InstrGISel.td index 58ca52f37b63b..2d2b2bee99ec4 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrGISel.td +++ b/llvm/lib/Target/AArch64/AArch64InstrGISel.td @@ -346,16 +346,16 @@ let Predicates = [HasNEON] in { } let Predicates = [HasNoLSE] in { -def : Pat<(atomic_cmp_swap_8 GPR64:$addr, GPR32:$desired, GPR32:$new), +def : Pat<(atomic_cmp_swap_i8 GPR64:$addr, GPR32:$desired, GPR32:$new), (CMP_SWAP_8 GPR64:$addr, GPR32:$desired, GPR32:$new)>; -def : Pat<(atomic_cmp_swap_16 GPR64:$addr, GPR32:$desired, GPR32:$new), +def : Pat<(atomic_cmp_swap_i16 GPR64:$addr, GPR32:$desired, GPR32:$new), (CMP_SWAP_16 GPR64:$addr, GPR32:$desired, GPR32:$new)>; -def : Pat<(atomic_cmp_swap_32 GPR64:$addr, GPR32:$desired, GPR32:$new), +def : Pat<(atomic_cmp_swap_i32 GPR64:$addr, GPR32:$desired, GPR32:$new), (CMP_SWAP_32 GPR64:$addr, GPR32:$desired, GPR32:$new)>; -def : Pat<(atomic_cmp_swap_64 GPR64:$addr, GPR64:$desired, GPR64:$new), +def : Pat<(atomic_cmp_swap_i64 GPR64:$addr, GPR64:$desired, GPR64:$new), (CMP_SWAP_64 GPR64:$addr, GPR64:$desired, GPR64:$new)>; } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td index fa7492ac6cbe1..bd348f11007a0 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td @@ -637,9 +637,14 @@ defm int_amdgcn_atomic_cond_sub_u32 : local_addr_space_atomic_op; defm int_amdgcn_atomic_cond_sub_u32 : flat_addr_space_atomic_op; defm int_amdgcn_atomic_cond_sub_u32 : global_addr_space_atomic_op; -multiclass noret_binary_atomic_op { +multiclass noret_binary_atomic_op { let HasNoUse = true in - defm "_noret" : binary_atomic_op; + defm "_noret" : binary_atomic_op; +} + +multiclass noret_binary_atomic_op_fp { + let HasNoUse = true in + defm "_noret" : binary_atomic_op_fp; } multiclass noret_ternary_atomic_op { @@ -647,11 +652,21 @@ multiclass noret_ternary_atomic_op { defm "_noret" : ternary_atomic_op; } -multiclass binary_atomic_op_all_as { - foreach as = [ "global", "flat", "constant", "local", "private", "region" ] in { +defvar atomic_addrspace_names = [ "global", "flat", "constant", "local", "private", "region" ]; + +multiclass binary_atomic_op_all_as { + foreach as = atomic_addrspace_names in { + let AddressSpaces = !cast("LoadAddress_"#as).AddrSpaces in { + defm "_"#as : binary_atomic_op; + defm "_"#as : noret_binary_atomic_op; + } + } +} +multiclass binary_atomic_op_fp_all_as { + foreach as = atomic_addrspace_names in { let AddressSpaces = !cast("LoadAddress_"#as).AddrSpaces in { - defm "_"#as : binary_atomic_op; - defm "_"#as : noret_binary_atomic_op; + defm "_"#as : binary_atomic_op_fp; + defm "_"#as : noret_binary_atomic_op_fp; } } } @@ -666,11 +681,11 @@ defm atomic_load_sub : binary_atomic_op_all_as; defm atomic_load_umax : binary_atomic_op_all_as; defm atomic_load_umin : binary_atomic_op_all_as; defm atomic_load_xor : binary_atomic_op_all_as; -defm atomic_load_fadd : binary_atomic_op_all_as; +defm atomic_load_fadd : binary_atomic_op_fp_all_as; defm atomic_load_uinc_wrap : binary_atomic_op_all_as; defm atomic_load_udec_wrap : binary_atomic_op_all_as; let MemoryVT = v2f16 in -defm atomic_load_fadd_v2f16 : binary_atomic_op_all_as; +defm atomic_load_fadd_v2f16 : binary_atomic_op_fp_all_as; defm AMDGPUatomic_cmp_swap : binary_atomic_op_all_as; def load_align8_local : PatFrag<(ops node:$ptr), (load_local node:$ptr)>, diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index b05834e5803a2..01d649c784d70 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -1545,7 +1545,7 @@ multiclass BufferAtomicPat_Common(OpPrefix # !if(!eq(RtnMode, "ret"), "", "_noret") - # !if(isIntr, "", "_" # vt.Size)); + # !if(isIntr, "", "_" # vt)); defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in { @@ -1582,7 +1582,7 @@ multiclass BufferAtomicCmpSwapPat_Common("AMDGPUatomic_cmp_swap_global" # !if(!eq(RtnMode, "ret"), "", "_noret") - # "_" # vt.Size); + # "_" # vt); defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", ""); defvar data_vt_RC = getVregSrcForVT.ret.RegClass; diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td index 19bb4300531cf..b61f3d7427279 100644 --- a/llvm/lib/Target/AMDGPU/DSInstructions.td +++ b/llvm/lib/Target/AMDGPU/DSInstructions.td @@ -965,16 +965,16 @@ defm : DSWritePat_mc ; multiclass DSAtomicRetPat_mc { let OtherPredicates = [LDSRequiresM0Init] in { - def : DSAtomicRetPat(frag#"_local_m0_"#vt.Size)>; + def : DSAtomicRetPat(frag#"_local_m0_"#vt)>; } let OtherPredicates = [NotLDSRequiresM0Init] in { def : DSAtomicRetPat(!cast(inst)#"_gfx9"), vt, - !cast(frag#"_local_"#vt.Size)>; + !cast(frag#"_local_"#vt)>; } let OtherPredicates = [HasGDS] in { - def : DSAtomicRetPat(frag#"_region_m0_"#vt.Size), + def : DSAtomicRetPat(frag#"_region_m0_"#vt), /* complexity */ 0, /* gds */ 1>; } } @@ -983,24 +983,24 @@ multiclass DSAtomicRetNoRetPat_mc { let OtherPredicates = [LDSRequiresM0Init] in { def : DSAtomicRetPat(frag#"_local_m0_"#vt.Size)>; + !cast(frag#"_local_m0_"#vt)>; def : DSAtomicRetPat(frag#"_local_m0_noret_"#vt.Size), /* complexity */ 1>; + !cast(frag#"_local_m0_noret_"#vt), /* complexity */ 1>; } let OtherPredicates = [NotLDSRequiresM0Init] in { def : DSAtomicRetPat(!cast(inst)#"_gfx9"), vt, - !cast(frag#"_local_"#vt.Size)>; + !cast(frag#"_local_"#vt)>; def : DSAtomicRetPat(!cast(noRetInst)#"_gfx9"), vt, - !cast(frag#"_local_noret_"#vt.Size), /* complexity */ 1>; + !cast(frag#"_local_noret_"#vt), /* complexity */ 1>; } let OtherPredicates = [HasGDS] in { def : DSAtomicRetPat(frag#"_region_m0_"#vt.Size), + !cast(frag#"_region_m0_"#vt), /* complexity */ 0, /* gds */ 1>; def : DSAtomicRetPat(frag#"_region_m0_noret_"#vt.Size), + !cast(frag#"_region_m0_noret_"#vt), /* complexity */ 1, /* gds */ 1>; } } @@ -1019,23 +1019,23 @@ class DSAtomicCmpXChgSwapped { let OtherPredicates = [LDSRequiresM0Init] in { - def : DSAtomicCmpXChgSwapped(frag#"_local_m0_"#vt.Size)>; - def : DSAtomicCmpXChgSwapped(frag#"_local_m0_noret_"#vt.Size), + def : DSAtomicCmpXChgSwapped(frag#"_local_m0_"#vt)>; + def : DSAtomicCmpXChgSwapped(frag#"_local_m0_noret_"#vt), /* complexity */ 1>; } let OtherPredicates = [NotLDSRequiresM0Init] in { def : DSAtomicCmpXChgSwapped(!cast(inst)#"_gfx9"), vt, - !cast(frag#"_local_"#vt.Size)>; + !cast(frag#"_local_"#vt)>; def : DSAtomicCmpXChgSwapped(!cast(noRetInst)#"_gfx9"), vt, - !cast(frag#"_local_noret_"#vt.Size), + !cast(frag#"_local_noret_"#vt), /* complexity */ 1>; } let OtherPredicates = [HasGDS] in { - def : DSAtomicCmpXChgSwapped(frag#"_region_m0_"#vt.Size), + def : DSAtomicCmpXChgSwapped(frag#"_region_m0_"#vt), /* complexity */ 0, /* gds */ 1>; - def : DSAtomicCmpXChgSwapped(frag#"_region_m0_noret_"#vt.Size), + def : DSAtomicCmpXChgSwapped(frag#"_region_m0_noret_"#vt), /* complexity */ 1, /* gds */ 1>; } } @@ -1053,14 +1053,14 @@ class DSAtomicCmpXChg { def : DSAtomicCmpXChg(!cast(inst)#"_gfx9"), vt, - !cast(frag#"_local_"#vt.Size)>; + !cast(frag#"_local_"#vt)>; def : DSAtomicCmpXChg(!cast(noRetInst)#"_gfx9"), vt, - !cast(frag#"_local_noret_"#vt.Size), /* complexity */ 1>; + !cast(frag#"_local_noret_"#vt), /* complexity */ 1>; let OtherPredicates = [HasGDS] in { - def : DSAtomicCmpXChg(frag#"_region_m0_"#vt.Size), + def : DSAtomicCmpXChg(frag#"_region_m0_"#vt), /* complexity */ 0, /* gds */ 1>; - def : DSAtomicCmpXChg(frag#"_region_m0_noret_"#vt.Size), + def : DSAtomicCmpXChg(frag#"_region_m0_noret_"#vt), /* complexity */ 1, /* gds */ 1>; } } @@ -1119,9 +1119,9 @@ defm : DSAtomicCmpXChg_mc; +def : DSAtomicRetPat; let AddedComplexity = 1 in -def : DSAtomicRetPat; +def : DSAtomicRetPat; class DSAtomicRetPatIntrinsic : GCNPat < @@ -1135,9 +1135,9 @@ def : DSAtomicRetPatIntrinsic; +def : DSAtomicRetPat; let AddedComplexity = 1 in -def : DSAtomicRetPat; +def : DSAtomicRetPat; def : GCNPat < (v2i16 (int_amdgcn_ds_fadd_v2bf16 i32:$ptr, v2i16:$src)), (DS_PK_ADD_RTN_BF16 VGPR_32:$ptr, VGPR_32:$src, 0, 0) diff --git a/llvm/lib/Target/AMDGPU/EvergreenInstructions.td b/llvm/lib/Target/AMDGPU/EvergreenInstructions.td index 3767dd0b6d478..280def5440c81 100644 --- a/llvm/lib/Target/AMDGPU/EvergreenInstructions.td +++ b/llvm/lib/Target/AMDGPU/EvergreenInstructions.td @@ -322,25 +322,25 @@ def : EGOrCaymanPat<(i32 (atomic_cmp_swap_global_noret i32:$ptr, i32:$cmp, i32:$ $ptr), sub1)>; defm AtomicSwapPat : AtomicPat ; + atomic_swap_global_noret_i32>; defm AtomicAddPat : AtomicPat ; + atomic_load_add_global_noret_i32>; defm AtomicSubPat : AtomicPat ; + atomic_load_sub_global_noret_i32>; defm AtomicMinPat : AtomicPat ; + atomic_load_min_global_noret_i32>; defm AtomicUMinPat : AtomicPat ; + atomic_load_umin_global_noret_i32>; defm AtomicMaxPat : AtomicPat ; + atomic_load_max_global_noret_i32>; defm AtomicUMaxPat : AtomicPat ; + atomic_load_umax_global_noret_i32>; defm AtomicAndPat : AtomicPat ; + atomic_load_and_global_noret_i32>; defm AtomicOrPat : AtomicPat ; + atomic_load_or_global_noret_i32>; defm AtomicXorPat : AtomicPat ; + atomic_load_xor_global_noret_i32>; // Should be predicated on FeatureFP64 // def FMA_64 : R600_3OP < @@ -712,37 +712,37 @@ def LDS_SHORT_WRITE : R600_LDS_1A1D_NORET<0x13, "LDS_SHORT_WRITE", [(truncstorei16_local i32:$src1, i32:$src0)] >; def LDS_ADD_RET : R600_LDS_1A1D_RET <0x20, "LDS_ADD", - [(set i32:$dst, (atomic_load_add_local_32 i32:$src0, i32:$src1))] + [(set i32:$dst, (atomic_load_add_local_i32 i32:$src0, i32:$src1))] >; def LDS_SUB_RET : R600_LDS_1A1D_RET <0x21, "LDS_SUB", - [(set i32:$dst, (atomic_load_sub_local_32 i32:$src0, i32:$src1))] + [(set i32:$dst, (atomic_load_sub_local_i32 i32:$src0, i32:$src1))] >; def LDS_AND_RET : R600_LDS_1A1D_RET <0x29, "LDS_AND", - [(set i32:$dst, (atomic_load_and_local_32 i32:$src0, i32:$src1))] + [(set i32:$dst, (atomic_load_and_local_i32 i32:$src0, i32:$src1))] >; def LDS_OR_RET : R600_LDS_1A1D_RET <0x2a, "LDS_OR", - [(set i32:$dst, (atomic_load_or_local_32 i32:$src0, i32:$src1))] + [(set i32:$dst, (atomic_load_or_local_i32 i32:$src0, i32:$src1))] >; def LDS_XOR_RET : R600_LDS_1A1D_RET <0x2b, "LDS_XOR", - [(set i32:$dst, (atomic_load_xor_local_32 i32:$src0, i32:$src1))] + [(set i32:$dst, (atomic_load_xor_local_i32 i32:$src0, i32:$src1))] >; def LDS_MIN_INT_RET : R600_LDS_1A1D_RET <0x25, "LDS_MIN_INT", - [(set i32:$dst, (atomic_load_min_local_32 i32:$src0, i32:$src1))] + [(set i32:$dst, (atomic_load_min_local_i32 i32:$src0, i32:$src1))] >; def LDS_MAX_INT_RET : R600_LDS_1A1D_RET <0x26, "LDS_MAX_INT", - [(set i32:$dst, (atomic_load_max_local_32 i32:$src0, i32:$src1))] + [(set i32:$dst, (atomic_load_max_local_i32 i32:$src0, i32:$src1))] >; def LDS_MIN_UINT_RET : R600_LDS_1A1D_RET <0x27, "LDS_MIN_UINT", - [(set i32:$dst, (atomic_load_umin_local_32 i32:$src0, i32:$src1))] + [(set i32:$dst, (atomic_load_umin_local_i32 i32:$src0, i32:$src1))] >; def LDS_MAX_UINT_RET : R600_LDS_1A1D_RET <0x28, "LDS_MAX_UINT", - [(set i32:$dst, (atomic_load_umax_local_32 i32:$src0, i32:$src1))] + [(set i32:$dst, (atomic_load_umax_local_i32 i32:$src0, i32:$src1))] >; def LDS_WRXCHG_RET : R600_LDS_1A1D_RET <0x2d, "LDS_WRXCHG", - [(set i32:$dst, (atomic_swap_local_32 i32:$src0, i32:$src1))] + [(set i32:$dst, (atomic_swap_local_i32 i32:$src0, i32:$src1))] >; def LDS_CMPST_RET : R600_LDS_1A2D_RET <0x30, "LDS_CMPST", - [(set i32:$dst, (atomic_cmp_swap_local_32 i32:$src0, i32:$src1, i32:$src2))] + [(set i32:$dst, (atomic_cmp_swap_local_i32 i32:$src0, i32:$src1, i32:$src2))] >; def LDS_READ_RET : R600_LDS_1A <0x32, "LDS_READ_RET", [(set (i32 R600_Reg32:$dst), (load_local R600_Reg32:$src0))] diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td index aab19b8adc275..818cbde592432 100644 --- a/llvm/lib/Target/AMDGPU/FLATInstructions.td +++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td @@ -1105,7 +1105,7 @@ multiclass FlatAtomicNoRtnPatWithAddrSpace : - FlatAtomicNoRtnPatBase; + FlatAtomicNoRtnPatBase; multiclass FlatAtomicRtnPatBase : - FlatAtomicRtnPatBase; + FlatAtomicRtnPatBase; multiclass FlatAtomicPat { - defvar rtnNode = !cast(node # !if(isIntr, "", "_" # vt.Size)); - defvar noRtnNode = !cast(node # "_noret" # !if(isIntr, "", "_" # vt.Size)); + defvar rtnNode = !cast(node # !if(isIntr, "", "_" # vt)); + defvar noRtnNode = !cast(node # "_noret" # !if(isIntr, "", "_" # vt)); let AddedComplexity = complexity in def : FlatSignedAtomicPatBase(inst#"_RTN"), rtnNode, vt, data_vt>; @@ -1280,11 +1280,11 @@ multiclass GlobalFLATAtomicPatsRtnBase : - GlobalFLATAtomicPatsNoRtnBase; + GlobalFLATAtomicPatsNoRtnBase; multiclass GlobalFLATAtomicPatsRtn : - GlobalFLATAtomicPatsRtnBase; + GlobalFLATAtomicPatsRtnBase; multiclass GlobalFLATAtomicPats : diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td index 40289f2addfdf..d778195faf23b 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td @@ -318,8 +318,8 @@ class isIntType { // PatFrags for global memory operations //===----------------------------------------------------------------------===// -defm atomic_load_fmin : binary_atomic_op_all_as; -defm atomic_load_fmax : binary_atomic_op_all_as; +defm atomic_load_fmin : binary_atomic_op_fp_all_as; +defm atomic_load_fmax : binary_atomic_op_fp_all_as; //===----------------------------------------------------------------------===// // SDNodes PatFrags for loads/stores with a glue input. @@ -709,15 +709,24 @@ multiclass SIAtomicM0Glue2 ; let AddressSpaces = StoreAddress_local.AddrSpaces in { - defm _local_m0 : binary_atomic_op (NAME#"_glue"), IsInt>; - defm _local_m0 : noret_binary_atomic_op (NAME#"_glue"), - IsInt>; + + if IsInt then { + defm _local_m0 : binary_atomic_op (NAME#"_glue")>; + defm _local_m0 : noret_binary_atomic_op (NAME#"_glue")>; + } else { + defm _local_m0 : binary_atomic_op_fp (NAME#"_glue")>; + defm _local_m0 : noret_binary_atomic_op_fp (NAME#"_glue")>; + } } let AddressSpaces = StoreAddress_region.AddrSpaces in { - defm _region_m0 : binary_atomic_op (NAME#"_glue"), IsInt>; - defm _region_m0 : noret_binary_atomic_op (NAME#"_glue"), - IsInt>; + if IsInt then { + defm _region_m0 : binary_atomic_op (NAME#"_glue")>; + defm _region_m0 : noret_binary_atomic_op (NAME#"_glue")>; + } else { + defm _region_m0 : binary_atomic_op_fp (NAME#"_glue")>; + defm _region_m0 : noret_binary_atomic_op_fp (NAME#"_glue")>; + } } } diff --git a/llvm/lib/Target/AVR/AVRInstrInfo.td b/llvm/lib/Target/AVR/AVRInstrInfo.td index 88b1989ef9170..4d0df9f1f683d 100644 --- a/llvm/lib/Target/AVR/AVRInstrInfo.td +++ b/llvm/lib/Target/AVR/AVRInstrInfo.td @@ -1156,16 +1156,16 @@ class AtomicLoadOp8 : AtomicLoadOp; class AtomicLoadOp16 : AtomicLoadOp; let usesCustomInserter=1 in { - def AtomicLoadAdd8 : AtomicLoadOp8; - def AtomicLoadAdd16 : AtomicLoadOp16; - def AtomicLoadSub8 : AtomicLoadOp8; - def AtomicLoadSub16 : AtomicLoadOp16; - def AtomicLoadAnd8 : AtomicLoadOp8; - def AtomicLoadAnd16 : AtomicLoadOp16; - def AtomicLoadOr8 : AtomicLoadOp8; - def AtomicLoadOr16 : AtomicLoadOp16; - def AtomicLoadXor8 : AtomicLoadOp8; - def AtomicLoadXor16 : AtomicLoadOp16; + def AtomicLoadAdd8 : AtomicLoadOp8; + def AtomicLoadAdd16 : AtomicLoadOp16; + def AtomicLoadSub8 : AtomicLoadOp8; + def AtomicLoadSub16 : AtomicLoadOp16; + def AtomicLoadAnd8 : AtomicLoadOp8; + def AtomicLoadAnd16 : AtomicLoadOp16; + def AtomicLoadOr8 : AtomicLoadOp8; + def AtomicLoadOr16 : AtomicLoadOp16; + def AtomicLoadXor8 : AtomicLoadOp8; + def AtomicLoadXor16 : AtomicLoadOp16; } def AtomicFence : Pseudo<(outs), (ins), "atomic_fence", [(atomic_fence timm, timm)]>; diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.td b/llvm/lib/Target/BPF/BPFInstrInfo.td index 66c57952a7f10..55989f5eb6a3c 100644 --- a/llvm/lib/Target/BPF/BPFInstrInfo.td +++ b/llvm/lib/Target/BPF/BPFInstrInfo.td @@ -807,7 +807,7 @@ class XADD let Constraints = "$dst = $val" in { let Predicates = [BPFNoALU32] in { - def XADDW : XADD; + def XADDW : XADD; } } @@ -897,23 +897,23 @@ class XFALU32; - def XFANDW32 : XFALU32; - def XFORW32 : XFALU32; - def XFXORW32 : XFALU32; + def XFADDW32 : XFALU32; + def XFANDW32 : XFALU32; + def XFORW32 : XFALU32; + def XFXORW32 : XFALU32; } - def XFADDD : XFALU64; - def XFANDD : XFALU64; - def XFORD : XFALU64; - def XFXORD : XFALU64; + def XFADDD : XFALU64; + def XFANDD : XFALU64; + def XFORD : XFALU64; + def XFXORD : XFALU64; } // atomic_load_sub can be represented as a neg followed // by an atomic_load_add. -def : Pat<(atomic_load_sub_32 ADDRri:$addr, GPR32:$val), +def : Pat<(atomic_load_sub_i32 ADDRri:$addr, GPR32:$val), (XFADDW32 ADDRri:$addr, (NEG_32 GPR32:$val))>; -def : Pat<(atomic_load_sub_64 ADDRri:$addr, GPR:$val), +def : Pat<(atomic_load_sub_i64 ADDRri:$addr, GPR:$val), (XFADDD ADDRri:$addr, (NEG_64 GPR:$val))>; // Atomic Exchange @@ -953,10 +953,10 @@ class XCHG32 let Constraints = "$dst = $val" in { let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in { - def XCHGW32 : XCHG32; + def XCHGW32 : XCHG32; } - def XCHGD : XCHG; + def XCHGD : XCHG; } // Compare-And-Exchange @@ -996,11 +996,11 @@ class CMPXCHG32 let Predicates = [BPFHasALU32], Defs = [W0], Uses = [W0], DecoderNamespace = "BPFALU32" in { - def CMPXCHGW32 : CMPXCHG32; + def CMPXCHGW32 : CMPXCHG32; } let Defs = [R0], Uses = [R0] in { - def CMPXCHGD : CMPXCHG; + def CMPXCHGD : CMPXCHG; } // bswap16, bswap32, bswap64 diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td index f72f46e39e2a4..6b0b819a417ed 100644 --- a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td +++ b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td @@ -2041,60 +2041,60 @@ multiclass ternary_atomic_op_failure_ord { }]>; } -defm atomic_cmp_swap_32 : ternary_atomic_op_failure_ord; -defm atomic_cmp_swap_64 : ternary_atomic_op_failure_ord; +defm atomic_cmp_swap_i32 : ternary_atomic_op_failure_ord; +defm atomic_cmp_swap_i64 : ternary_atomic_op_failure_ord; let Predicates = [IsLA64] in { def : AtomicPat; -def : Pat<(atomic_swap_32 GPR:$addr, GPR:$incr), +def : Pat<(atomic_swap_i32 GPR:$addr, GPR:$incr), (AMSWAP__DB_W GPR:$incr, GPR:$addr)>; -def : Pat<(atomic_swap_64 GPR:$addr, GPR:$incr), +def : Pat<(atomic_swap_i64 GPR:$addr, GPR:$incr), (AMSWAP__DB_D GPR:$incr, GPR:$addr)>; -def : Pat<(atomic_load_add_64 GPR:$rj, GPR:$rk), +def : Pat<(atomic_load_add_i64 GPR:$rj, GPR:$rk), (AMADD__DB_D GPR:$rk, GPR:$rj)>; def : AtomicPat; -def : Pat<(atomic_load_sub_32 GPR:$rj, GPR:$rk), +def : Pat<(atomic_load_sub_i32 GPR:$rj, GPR:$rk), (AMADD__DB_W (SUB_W R0, GPR:$rk), GPR:$rj)>; -def : Pat<(atomic_load_sub_64 GPR:$rj, GPR:$rk), +def : Pat<(atomic_load_sub_i64 GPR:$rj, GPR:$rk), (AMADD__DB_D (SUB_D R0, GPR:$rk), GPR:$rj)>; def : AtomicPat; -defm : PseudoBinPat<"atomic_load_nand_64", PseudoAtomicLoadNand64>; +defm : PseudoBinPat<"atomic_load_nand_i64", PseudoAtomicLoadNand64>; def : AtomicPat; -def : Pat<(atomic_load_add_32 GPR:$rj, GPR:$rk), +def : Pat<(atomic_load_add_i32 GPR:$rj, GPR:$rk), (AMADD__DB_W GPR:$rk, GPR:$rj)>; -def : Pat<(atomic_load_and_32 GPR:$rj, GPR:$rk), +def : Pat<(atomic_load_and_i32 GPR:$rj, GPR:$rk), (AMAND__DB_W GPR:$rk, GPR:$rj)>; -def : Pat<(atomic_load_and_64 GPR:$rj, GPR:$rk), +def : Pat<(atomic_load_and_i64 GPR:$rj, GPR:$rk), (AMAND__DB_D GPR:$rk, GPR:$rj)>; -def : Pat<(atomic_load_or_32 GPR:$rj, GPR:$rk), +def : Pat<(atomic_load_or_i32 GPR:$rj, GPR:$rk), (AMOR__DB_W GPR:$rk, GPR:$rj)>; -def : Pat<(atomic_load_or_64 GPR:$rj, GPR:$rk), +def : Pat<(atomic_load_or_i64 GPR:$rj, GPR:$rk), (AMOR__DB_D GPR:$rk, GPR:$rj)>; -def : Pat<(atomic_load_xor_32 GPR:$rj, GPR:$rk), +def : Pat<(atomic_load_xor_i32 GPR:$rj, GPR:$rk), (AMXOR__DB_W GPR:$rk, GPR:$rj)>; -def : Pat<(atomic_load_xor_64 GPR:$rj, GPR:$rk), +def : Pat<(atomic_load_xor_i64 GPR:$rj, GPR:$rk), (AMXOR__DB_D GPR:$rk, GPR:$rj)>; -def : Pat<(atomic_load_umin_32 GPR:$rj, GPR:$rk), +def : Pat<(atomic_load_umin_i32 GPR:$rj, GPR:$rk), (AMMIN__DB_WU GPR:$rk, GPR:$rj)>; -def : Pat<(atomic_load_umin_64 GPR:$rj, GPR:$rk), +def : Pat<(atomic_load_umin_i64 GPR:$rj, GPR:$rk), (AMMIN__DB_DU GPR:$rk, GPR:$rj)>; -def : Pat<(atomic_load_umax_32 GPR:$rj, GPR:$rk), +def : Pat<(atomic_load_umax_i32 GPR:$rj, GPR:$rk), (AMMAX__DB_WU GPR:$rk, GPR:$rj)>; -def : Pat<(atomic_load_umax_64 GPR:$rj, GPR:$rk), +def : Pat<(atomic_load_umax_i64 GPR:$rj, GPR:$rk), (AMMAX__DB_DU GPR:$rk, GPR:$rj)>; -def : Pat<(atomic_load_min_32 GPR:$rj, GPR:$rk), +def : Pat<(atomic_load_min_i32 GPR:$rj, GPR:$rk), (AMMIN__DB_W GPR:$rk, GPR:$rj)>; -def : Pat<(atomic_load_min_64 GPR:$rj, GPR:$rk), +def : Pat<(atomic_load_min_i64 GPR:$rj, GPR:$rk), (AMMIN__DB_D GPR:$rk, GPR:$rj)>; -def : Pat<(atomic_load_max_32 GPR:$rj, GPR:$rk), +def : Pat<(atomic_load_max_i32 GPR:$rj, GPR:$rk), (AMMAX__DB_W GPR:$rk, GPR:$rj)>; -def : Pat<(atomic_load_max_64 GPR:$rj, GPR:$rk), +def : Pat<(atomic_load_max_i64 GPR:$rj, GPR:$rk), (AMMAX__DB_D GPR:$rk, GPR:$rj)>; def : AtomicPat; } -defm : PseudoCmpXchgPat<"atomic_cmp_swap_32", PseudoCmpXchg32>; -defm : PseudoCmpXchgPat<"atomic_cmp_swap_64", PseudoCmpXchg64, i64>; +defm : PseudoCmpXchgPat<"atomic_cmp_swap_i32", PseudoCmpXchg32>; +defm : PseudoCmpXchgPat<"atomic_cmp_swap_i64", PseudoCmpXchg64, i64>; def : Pat<(int_loongarch_masked_cmpxchg_i64 GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, timm:$fail_order), (PseudoMaskedCmpXchg32 @@ -2131,23 +2131,23 @@ def : PseudoMaskedAMMinMaxPat; } // Predicates = [IsLA64] -defm : PseudoBinPat<"atomic_load_nand_32", PseudoAtomicLoadNand32>; +defm : PseudoBinPat<"atomic_load_nand_i32", PseudoAtomicLoadNand32>; let Predicates = [IsLA32] in { def : AtomicPat; -defm : PseudoBinPat<"atomic_swap_32", PseudoAtomicSwap32>; +defm : PseudoBinPat<"atomic_swap_i32", PseudoAtomicSwap32>; def : AtomicPat; def : AtomicPat; def : AtomicPat; -defm : PseudoBinPat<"atomic_load_add_32", PseudoAtomicLoadAdd32>; -defm : PseudoBinPat<"atomic_load_sub_32", PseudoAtomicLoadSub32>; -defm : PseudoBinPat<"atomic_load_and_32", PseudoAtomicLoadAnd32>; -defm : PseudoBinPat<"atomic_load_or_32", PseudoAtomicLoadOr32>; -defm : PseudoBinPat<"atomic_load_xor_32", PseudoAtomicLoadXor32>; +defm : PseudoBinPat<"atomic_load_add_i32", PseudoAtomicLoadAdd32>; +defm : PseudoBinPat<"atomic_load_sub_i32", PseudoAtomicLoadSub32>; +defm : PseudoBinPat<"atomic_load_and_i32", PseudoAtomicLoadAnd32>; +defm : PseudoBinPat<"atomic_load_or_i32", PseudoAtomicLoadOr32>; +defm : PseudoBinPat<"atomic_load_xor_i32", PseudoAtomicLoadXor32>; } // Predicates = [IsLA32] /// Intrinsics diff --git a/llvm/lib/Target/Mips/Mips64InstrInfo.td b/llvm/lib/Target/Mips/Mips64InstrInfo.td index c0e7eef8dd9d5..f6ac3091a3ba8 100644 --- a/llvm/lib/Target/Mips/Mips64InstrInfo.td +++ b/llvm/lib/Target/Mips/Mips64InstrInfo.td @@ -75,18 +75,18 @@ def assertzext_lt_i32 : PatFrag<(ops node:$src), (assertzext node:$src), [{ // Instructions specific format //===----------------------------------------------------------------------===// let usesCustomInserter = 1 in { - def ATOMIC_LOAD_ADD_I64 : Atomic2Ops; - def ATOMIC_LOAD_SUB_I64 : Atomic2Ops; - def ATOMIC_LOAD_AND_I64 : Atomic2Ops; - def ATOMIC_LOAD_OR_I64 : Atomic2Ops; - def ATOMIC_LOAD_XOR_I64 : Atomic2Ops; - def ATOMIC_LOAD_NAND_I64 : Atomic2Ops; - def ATOMIC_SWAP_I64 : Atomic2Ops; - def ATOMIC_CMP_SWAP_I64 : AtomicCmpSwap; - def ATOMIC_LOAD_MIN_I64 : Atomic2Ops; - def ATOMIC_LOAD_MAX_I64 : Atomic2Ops; - def ATOMIC_LOAD_UMIN_I64 : Atomic2Ops; - def ATOMIC_LOAD_UMAX_I64 : Atomic2Ops; + def ATOMIC_LOAD_ADD_I64 : Atomic2Ops; + def ATOMIC_LOAD_SUB_I64 : Atomic2Ops; + def ATOMIC_LOAD_AND_I64 : Atomic2Ops; + def ATOMIC_LOAD_OR_I64 : Atomic2Ops; + def ATOMIC_LOAD_XOR_I64 : Atomic2Ops; + def ATOMIC_LOAD_NAND_I64 : Atomic2Ops; + def ATOMIC_SWAP_I64 : Atomic2Ops; + def ATOMIC_CMP_SWAP_I64 : AtomicCmpSwap; + def ATOMIC_LOAD_MIN_I64 : Atomic2Ops; + def ATOMIC_LOAD_MAX_I64 : Atomic2Ops; + def ATOMIC_LOAD_UMIN_I64 : Atomic2Ops; + def ATOMIC_LOAD_UMAX_I64 : Atomic2Ops; } def ATOMIC_LOAD_ADD_I64_POSTRA : Atomic2OpsPostRA; diff --git a/llvm/lib/Target/Mips/MipsInstrInfo.td b/llvm/lib/Target/Mips/MipsInstrInfo.td index 23e04c442bf6f..85e3e78d2a4d8 100644 --- a/llvm/lib/Target/Mips/MipsInstrInfo.td +++ b/llvm/lib/Target/Mips/MipsInstrInfo.td @@ -1904,45 +1904,45 @@ def ADJCALLSTACKUP : MipsPseudo<(outs), (ins i32imm:$amt1, i32imm:$amt2), } let usesCustomInserter = 1 in { - def ATOMIC_LOAD_ADD_I8 : Atomic2Ops; - def ATOMIC_LOAD_ADD_I16 : Atomic2Ops; - def ATOMIC_LOAD_ADD_I32 : Atomic2Ops; - def ATOMIC_LOAD_SUB_I8 : Atomic2Ops; - def ATOMIC_LOAD_SUB_I16 : Atomic2Ops; - def ATOMIC_LOAD_SUB_I32 : Atomic2Ops; - def ATOMIC_LOAD_AND_I8 : Atomic2Ops; - def ATOMIC_LOAD_AND_I16 : Atomic2Ops; - def ATOMIC_LOAD_AND_I32 : Atomic2Ops; - def ATOMIC_LOAD_OR_I8 : Atomic2Ops; - def ATOMIC_LOAD_OR_I16 : Atomic2Ops; - def ATOMIC_LOAD_OR_I32 : Atomic2Ops; - def ATOMIC_LOAD_XOR_I8 : Atomic2Ops; - def ATOMIC_LOAD_XOR_I16 : Atomic2Ops; - def ATOMIC_LOAD_XOR_I32 : Atomic2Ops; - def ATOMIC_LOAD_NAND_I8 : Atomic2Ops; - def ATOMIC_LOAD_NAND_I16 : Atomic2Ops; - def ATOMIC_LOAD_NAND_I32 : Atomic2Ops; - - def ATOMIC_SWAP_I8 : Atomic2Ops; - def ATOMIC_SWAP_I16 : Atomic2Ops; - def ATOMIC_SWAP_I32 : Atomic2Ops; - - def ATOMIC_CMP_SWAP_I8 : AtomicCmpSwap; - def ATOMIC_CMP_SWAP_I16 : AtomicCmpSwap; - def ATOMIC_CMP_SWAP_I32 : AtomicCmpSwap; - - def ATOMIC_LOAD_MIN_I8 : Atomic2Ops; - def ATOMIC_LOAD_MIN_I16 : Atomic2Ops; - def ATOMIC_LOAD_MIN_I32 : Atomic2Ops; - def ATOMIC_LOAD_MAX_I8 : Atomic2Ops; - def ATOMIC_LOAD_MAX_I16 : Atomic2Ops; - def ATOMIC_LOAD_MAX_I32 : Atomic2Ops; - def ATOMIC_LOAD_UMIN_I8 : Atomic2Ops; - def ATOMIC_LOAD_UMIN_I16 : Atomic2Ops; - def ATOMIC_LOAD_UMIN_I32 : Atomic2Ops; - def ATOMIC_LOAD_UMAX_I8 : Atomic2Ops; - def ATOMIC_LOAD_UMAX_I16 : Atomic2Ops; - def ATOMIC_LOAD_UMAX_I32 : Atomic2Ops; + def ATOMIC_LOAD_ADD_I8 : Atomic2Ops; + def ATOMIC_LOAD_ADD_I16 : Atomic2Ops; + def ATOMIC_LOAD_ADD_I32 : Atomic2Ops; + def ATOMIC_LOAD_SUB_I8 : Atomic2Ops; + def ATOMIC_LOAD_SUB_I16 : Atomic2Ops; + def ATOMIC_LOAD_SUB_I32 : Atomic2Ops; + def ATOMIC_LOAD_AND_I8 : Atomic2Ops; + def ATOMIC_LOAD_AND_I16 : Atomic2Ops; + def ATOMIC_LOAD_AND_I32 : Atomic2Ops; + def ATOMIC_LOAD_OR_I8 : Atomic2Ops; + def ATOMIC_LOAD_OR_I16 : Atomic2Ops; + def ATOMIC_LOAD_OR_I32 : Atomic2Ops; + def ATOMIC_LOAD_XOR_I8 : Atomic2Ops; + def ATOMIC_LOAD_XOR_I16 : Atomic2Ops; + def ATOMIC_LOAD_XOR_I32 : Atomic2Ops; + def ATOMIC_LOAD_NAND_I8 : Atomic2Ops; + def ATOMIC_LOAD_NAND_I16 : Atomic2Ops; + def ATOMIC_LOAD_NAND_I32 : Atomic2Ops; + + def ATOMIC_SWAP_I8 : Atomic2Ops; + def ATOMIC_SWAP_I16 : Atomic2Ops; + def ATOMIC_SWAP_I32 : Atomic2Ops; + + def ATOMIC_CMP_SWAP_I8 : AtomicCmpSwap; + def ATOMIC_CMP_SWAP_I16 : AtomicCmpSwap; + def ATOMIC_CMP_SWAP_I32 : AtomicCmpSwap; + + def ATOMIC_LOAD_MIN_I8 : Atomic2Ops; + def ATOMIC_LOAD_MIN_I16 : Atomic2Ops; + def ATOMIC_LOAD_MIN_I32 : Atomic2Ops; + def ATOMIC_LOAD_MAX_I8 : Atomic2Ops; + def ATOMIC_LOAD_MAX_I16 : Atomic2Ops; + def ATOMIC_LOAD_MAX_I32 : Atomic2Ops; + def ATOMIC_LOAD_UMIN_I8 : Atomic2Ops; + def ATOMIC_LOAD_UMIN_I16 : Atomic2Ops; + def ATOMIC_LOAD_UMIN_I32 : Atomic2Ops; + def ATOMIC_LOAD_UMAX_I8 : Atomic2Ops; + def ATOMIC_LOAD_UMAX_I16 : Atomic2Ops; + def ATOMIC_LOAD_UMAX_I32 : Atomic2Ops; } def ATOMIC_LOAD_ADD_I8_POSTRA : Atomic2OpsSubwordPostRA; diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td index 440af085cb8e9..a65170e56aa24 100644 --- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td +++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td @@ -265,7 +265,7 @@ multiclass MATCH_ANY_SYNC, Requires<[hasPTX<62>, hasSM<30>]>; @@ -1618,18 +1618,18 @@ multiclass F_ATOMIC_3; -def atomic_load_add_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), - (atomic_load_add_32 node:$a, node:$b)>; -def atomic_load_add_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), - (atomic_load_add_32 node:$a, node:$b)>; -def atomic_load_add_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), - (atomic_load_add_64 node:$a, node:$b)>; -def atomic_load_add_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), - (atomic_load_add_64 node:$a, node:$b)>; -def atomic_load_add_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), - (atomic_load_add_64 node:$a, node:$b)>; +def atomic_load_add_i32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), + (atomic_load_add_i32 node:$a, node:$b)>; +def atomic_load_add_i32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), + (atomic_load_add_i32 node:$a, node:$b)>; +def atomic_load_add_i32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), + (atomic_load_add_i32 node:$a, node:$b)>; +def atomic_load_add_i64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), + (atomic_load_add_i64 node:$a, node:$b)>; +def atomic_load_add_i64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), + (atomic_load_add_i64 node:$a, node:$b)>; +def atomic_load_add_i64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), + (atomic_load_add_i64 node:$a, node:$b)>; def atomic_load_add_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), (atomic_load_fadd node:$a, node:$b)>; def atomic_load_add_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), @@ -1638,22 +1638,22 @@ def atomic_load_add_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), (atomic_load_fadd node:$a, node:$b)>; defm INT_PTX_ATOM_ADD_G_32 : F_ATOMIC_2; + atomic_load_add_i32_g, i32imm, imm>; defm INT_PTX_ATOM_ADD_S_32 : F_ATOMIC_2; + atomic_load_add_i32_s, i32imm, imm>; defm INT_PTX_ATOM_ADD_GEN_32 : F_ATOMIC_2; + atomic_load_add_i32_gen, i32imm, imm>; defm INT_PTX_ATOM_ADD_GEN_32_USE_G : F_ATOMIC_2; + ".add", atomic_load_add_i32_gen, i32imm, imm>; defm INT_PTX_ATOM_ADD_G_64 : F_ATOMIC_2; + atomic_load_add_i64_g, i64imm, imm>; defm INT_PTX_ATOM_ADD_S_64 : F_ATOMIC_2; + atomic_load_add_i64_s, i64imm, imm>; defm INT_PTX_ATOM_ADD_GEN_64 : F_ATOMIC_2; + atomic_load_add_i64_gen, i64imm, imm>; defm INT_PTX_ATOM_ADD_GEN_64_USE_G : F_ATOMIC_2; + ".add", atomic_load_add_i64_gen, i64imm, imm>; defm INT_PTX_ATOM_ADD_G_F16 : F_ATOMIC_2, hasPTX<63>]>; @@ -1685,187 +1685,187 @@ defm INT_PTX_ATOM_ADD_GEN_F64 : F_ATOMIC_2; -def atomic_load_sub_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), - (atomic_load_sub_32 node:$a, node:$b)>; -def atomic_load_sub_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), - (atomic_load_sub_32 node:$a, node:$b)>; -def atomic_load_sub_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), - (atomic_load_sub_64 node:$a, node:$b)>; -def atomic_load_sub_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), - (atomic_load_sub_64 node:$a, node:$b)>; -def atomic_load_sub_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), - (atomic_load_sub_64 node:$a, node:$b)>; +def atomic_load_sub_i32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), + (atomic_load_sub_i32 node:$a, node:$b)>; +def atomic_load_sub_i32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), + (atomic_load_sub_i32 node:$a, node:$b)>; +def atomic_load_sub_i32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), + (atomic_load_sub_i32 node:$a, node:$b)>; +def atomic_load_sub_i64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), + (atomic_load_sub_i64 node:$a, node:$b)>; +def atomic_load_sub_i64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), + (atomic_load_sub_i64 node:$a, node:$b)>; +def atomic_load_sub_i64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), + (atomic_load_sub_i64 node:$a, node:$b)>; defm INT_PTX_ATOM_SUB_G_32 : F_ATOMIC_2_NEG; + atomic_load_sub_i32_g>; defm INT_PTX_ATOM_SUB_G_64 : F_ATOMIC_2_NEG; + atomic_load_sub_i64_g>; defm INT_PTX_ATOM_SUB_GEN_32 : F_ATOMIC_2_NEG; + atomic_load_sub_i32_gen>; defm INT_PTX_ATOM_SUB_GEN_32_USE_G : F_ATOMIC_2_NEG; + ".add", atomic_load_sub_i32_gen>; defm INT_PTX_ATOM_SUB_S_32 : F_ATOMIC_2_NEG; + atomic_load_sub_i32_s>; defm INT_PTX_ATOM_SUB_S_64 : F_ATOMIC_2_NEG; + atomic_load_sub_i64_s>; defm INT_PTX_ATOM_SUB_GEN_64 : F_ATOMIC_2_NEG; + atomic_load_sub_i64_gen>; defm INT_PTX_ATOM_SUB_GEN_64_USE_G : F_ATOMIC_2_NEG; + ".add", atomic_load_sub_i64_gen>; // atom_swap -def atomic_swap_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), - (atomic_swap_32 node:$a, node:$b)>; -def atomic_swap_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), - (atomic_swap_32 node:$a, node:$b)>; -def atomic_swap_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), - (atomic_swap_32 node:$a, node:$b)>; -def atomic_swap_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), - (atomic_swap_64 node:$a, node:$b)>; -def atomic_swap_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), - (atomic_swap_64 node:$a, node:$b)>; -def atomic_swap_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), - (atomic_swap_64 node:$a, node:$b)>; +def atomic_swap_i32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), + (atomic_swap_i32 node:$a, node:$b)>; +def atomic_swap_i32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), + (atomic_swap_i32 node:$a, node:$b)>; +def atomic_swap_i32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), + (atomic_swap_i32 node:$a, node:$b)>; +def atomic_swap_i64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), + (atomic_swap_i64 node:$a, node:$b)>; +def atomic_swap_i64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), + (atomic_swap_i64 node:$a, node:$b)>; +def atomic_swap_i64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), + (atomic_swap_i64 node:$a, node:$b)>; defm INT_PTX_ATOM_SWAP_G_32 : F_ATOMIC_2; + atomic_swap_i32_g, i32imm, imm>; defm INT_PTX_ATOM_SWAP_S_32 : F_ATOMIC_2; + atomic_swap_i32_s, i32imm, imm>; defm INT_PTX_ATOM_SWAP_GEN_32 : F_ATOMIC_2; + atomic_swap_i32_gen, i32imm, imm>; defm INT_PTX_ATOM_SWAP_GEN_32_USE_G : F_ATOMIC_2; + ".exch", atomic_swap_i32_gen, i32imm, imm>; defm INT_PTX_ATOM_SWAP_G_64 : F_ATOMIC_2; + atomic_swap_i64_g, i64imm, imm>; defm INT_PTX_ATOM_SWAP_S_64 : F_ATOMIC_2; + atomic_swap_i64_s, i64imm, imm>; defm INT_PTX_ATOM_SWAP_GEN_64 : F_ATOMIC_2; + atomic_swap_i64_gen, i64imm, imm>; defm INT_PTX_ATOM_SWAP_GEN_64_USE_G : F_ATOMIC_2; + ".exch", atomic_swap_i64_gen, i64imm, imm>; // atom_max -def atomic_load_max_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b) - , (atomic_load_max_32 node:$a, node:$b)>; -def atomic_load_max_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), - (atomic_load_max_32 node:$a, node:$b)>; -def atomic_load_max_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), - (atomic_load_max_32 node:$a, node:$b)>; -def atomic_load_max_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b) - , (atomic_load_max_64 node:$a, node:$b)>; -def atomic_load_max_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), - (atomic_load_max_64 node:$a, node:$b)>; -def atomic_load_max_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), - (atomic_load_max_64 node:$a, node:$b)>; -def atomic_load_umax_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), - (atomic_load_umax_32 node:$a, node:$b)>; -def atomic_load_umax_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), - (atomic_load_umax_32 node:$a, node:$b)>; -def atomic_load_umax_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), - (atomic_load_umax_32 node:$a, node:$b)>; -def atomic_load_umax_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), - (atomic_load_umax_64 node:$a, node:$b)>; -def atomic_load_umax_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), - (atomic_load_umax_64 node:$a, node:$b)>; -def atomic_load_umax_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), - (atomic_load_umax_64 node:$a, node:$b)>; +def atomic_load_max_i32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b) + , (atomic_load_max_i32 node:$a, node:$b)>; +def atomic_load_max_i32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), + (atomic_load_max_i32 node:$a, node:$b)>; +def atomic_load_max_i32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), + (atomic_load_max_i32 node:$a, node:$b)>; +def atomic_load_max_i64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b) + , (atomic_load_max_i64 node:$a, node:$b)>; +def atomic_load_max_i64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), + (atomic_load_max_i64 node:$a, node:$b)>; +def atomic_load_max_i64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), + (atomic_load_max_i64 node:$a, node:$b)>; +def atomic_load_umax_i32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), + (atomic_load_umax_i32 node:$a, node:$b)>; +def atomic_load_umax_i32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), + (atomic_load_umax_i32 node:$a, node:$b)>; +def atomic_load_umax_i32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), + (atomic_load_umax_i32 node:$a, node:$b)>; +def atomic_load_umax_i64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), + (atomic_load_umax_i64 node:$a, node:$b)>; +def atomic_load_umax_i64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), + (atomic_load_umax_i64 node:$a, node:$b)>; +def atomic_load_umax_i64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), + (atomic_load_umax_i64 node:$a, node:$b)>; defm INT_PTX_ATOM_LOAD_MAX_G_32 : F_ATOMIC_2; + ".max", atomic_load_max_i32_g, i32imm, imm>; defm INT_PTX_ATOM_LOAD_MAX_S_32 : F_ATOMIC_2; + ".max", atomic_load_max_i32_s, i32imm, imm>; defm INT_PTX_ATOM_LOAD_MAX_GEN_32 : F_ATOMIC_2; + atomic_load_max_i32_gen, i32imm, imm>; defm INT_PTX_ATOM_LOAD_MAX_GEN_32_USE_G : F_ATOMIC_2; + ".s32", ".max", atomic_load_max_i32_gen, i32imm, imm>; defm INT_PTX_ATOM_LOAD_MAX_G_64 : F_ATOMIC_2]>; + ".max", atomic_load_max_i64_g, i64imm, imm, [hasSM<32>]>; defm INT_PTX_ATOM_LOAD_MAX_S_64 : F_ATOMIC_2]>; + ".max", atomic_load_max_i64_s, i64imm, imm, [hasSM<32>]>; defm INT_PTX_ATOM_LOAD_MAX_GEN_64 : F_ATOMIC_2]>; + atomic_load_max_i64_gen, i64imm, imm, [hasSM<32>]>; defm INT_PTX_ATOM_LOAD_MAX_GEN_64_USE_G : F_ATOMIC_2]>; + ".s64", ".max", atomic_load_max_i64_gen, i64imm, imm, [hasSM<32>]>; defm INT_PTX_ATOM_LOAD_UMAX_G_32 : F_ATOMIC_2; + ".max", atomic_load_umax_i32_g, i32imm, imm>; defm INT_PTX_ATOM_LOAD_UMAX_S_32 : F_ATOMIC_2; + ".max", atomic_load_umax_i32_s, i32imm, imm>; defm INT_PTX_ATOM_LOAD_UMAX_GEN_32 : F_ATOMIC_2; + atomic_load_umax_i32_gen, i32imm, imm>; defm INT_PTX_ATOM_LOAD_UMAX_GEN_32_USE_G : F_ATOMIC_2; + ".u32", ".max", atomic_load_umax_i32_gen, i32imm, imm>; defm INT_PTX_ATOM_LOAD_UMAX_G_64 : F_ATOMIC_2]>; + ".max", atomic_load_umax_i64_g, i64imm, imm, [hasSM<32>]>; defm INT_PTX_ATOM_LOAD_UMAX_S_64 : F_ATOMIC_2]>; + ".max", atomic_load_umax_i64_s, i64imm, imm, [hasSM<32>]>; defm INT_PTX_ATOM_LOAD_UMAX_GEN_64 : F_ATOMIC_2]>; + atomic_load_umax_i64_gen, i64imm, imm, [hasSM<32>]>; defm INT_PTX_ATOM_LOAD_UMAX_GEN_64_USE_G : F_ATOMIC_2]>; + ".u64", ".max", atomic_load_umax_i64_gen, i64imm, imm, [hasSM<32>]>; // atom_min -def atomic_load_min_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), - (atomic_load_min_32 node:$a, node:$b)>; -def atomic_load_min_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), - (atomic_load_min_32 node:$a, node:$b)>; -def atomic_load_min_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), - (atomic_load_min_32 node:$a, node:$b)>; -def atomic_load_min_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), - (atomic_load_min_64 node:$a, node:$b)>; -def atomic_load_min_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), - (atomic_load_min_64 node:$a, node:$b)>; -def atomic_load_min_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), - (atomic_load_min_64 node:$a, node:$b)>; -def atomic_load_umin_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), - (atomic_load_umin_32 node:$a, node:$b)>; -def atomic_load_umin_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), - (atomic_load_umin_32 node:$a, node:$b)>; -def atomic_load_umin_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), - (atomic_load_umin_32 node:$a, node:$b)>; -def atomic_load_umin_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), - (atomic_load_umin_64 node:$a, node:$b)>; -def atomic_load_umin_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), - (atomic_load_umin_64 node:$a, node:$b)>; -def atomic_load_umin_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), - (atomic_load_umin_64 node:$a, node:$b)>; +def atomic_load_min_i32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), + (atomic_load_min_i32 node:$a, node:$b)>; +def atomic_load_min_i32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), + (atomic_load_min_i32 node:$a, node:$b)>; +def atomic_load_min_i32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), + (atomic_load_min_i32 node:$a, node:$b)>; +def atomic_load_min_i64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), + (atomic_load_min_i64 node:$a, node:$b)>; +def atomic_load_min_i64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), + (atomic_load_min_i64 node:$a, node:$b)>; +def atomic_load_min_i64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), + (atomic_load_min_i64 node:$a, node:$b)>; +def atomic_load_umin_i32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), + (atomic_load_umin_i32 node:$a, node:$b)>; +def atomic_load_umin_i32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), + (atomic_load_umin_i32 node:$a, node:$b)>; +def atomic_load_umin_i32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), + (atomic_load_umin_i32 node:$a, node:$b)>; +def atomic_load_umin_i64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), + (atomic_load_umin_i64 node:$a, node:$b)>; +def atomic_load_umin_i64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), + (atomic_load_umin_i64 node:$a, node:$b)>; +def atomic_load_umin_i64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), + (atomic_load_umin_i64 node:$a, node:$b)>; defm INT_PTX_ATOM_LOAD_MIN_G_32 : F_ATOMIC_2; + ".min", atomic_load_min_i32_g, i32imm, imm>; defm INT_PTX_ATOM_LOAD_MIN_S_32 : F_ATOMIC_2; + ".min", atomic_load_min_i32_s, i32imm, imm>; defm INT_PTX_ATOM_LOAD_MIN_GEN_32 : F_ATOMIC_2; + atomic_load_min_i32_gen, i32imm, imm>; defm INT_PTX_ATOM_LOAD_MIN_GEN_32_USE_G : F_ATOMIC_2; + ".s32", ".min", atomic_load_min_i32_gen, i32imm, imm>; defm INT_PTX_ATOM_LOAD_MIN_G_64 : F_ATOMIC_2]>; + ".min", atomic_load_min_i64_g, i64imm, imm, [hasSM<32>]>; defm INT_PTX_ATOM_LOAD_MIN_S_64 : F_ATOMIC_2]>; + ".min", atomic_load_min_i64_s, i64imm, imm, [hasSM<32>]>; defm INT_PTX_ATOM_LOAD_MIN_GEN_64 : F_ATOMIC_2]>; + atomic_load_min_i64_gen, i64imm, imm, [hasSM<32>]>; defm INT_PTX_ATOM_LOAD_MIN_GEN_64_USE_G : F_ATOMIC_2]>; + ".s64", ".min", atomic_load_min_i64_gen, i64imm, imm, [hasSM<32>]>; defm INT_PTX_ATOM_LOAD_UMIN_G_32 : F_ATOMIC_2; + ".min", atomic_load_umin_i32_g, i32imm, imm>; defm INT_PTX_ATOM_LOAD_UMIN_S_32 : F_ATOMIC_2; + ".min", atomic_load_umin_i32_s, i32imm, imm>; defm INT_PTX_ATOM_LOAD_UMIN_GEN_32 : F_ATOMIC_2; + atomic_load_umin_i32_gen, i32imm, imm>; defm INT_PTX_ATOM_LOAD_UMIN_GEN_32_USE_G : F_ATOMIC_2; + ".u32", ".min", atomic_load_umin_i32_gen, i32imm, imm>; defm INT_PTX_ATOM_LOAD_UMIN_G_64 : F_ATOMIC_2]>; + ".min", atomic_load_umin_i64_g, i64imm, imm, [hasSM<32>]>; defm INT_PTX_ATOM_LOAD_UMIN_S_64 : F_ATOMIC_2]>; + ".min", atomic_load_umin_i64_s, i64imm, imm, [hasSM<32>]>; defm INT_PTX_ATOM_LOAD_UMIN_GEN_64 : F_ATOMIC_2]>; + atomic_load_umin_i64_gen, i64imm, imm, [hasSM<32>]>; defm INT_PTX_ATOM_LOAD_UMIN_GEN_64_USE_G : F_ATOMIC_2]>; + ".u64", ".min", atomic_load_umin_i64_gen, i64imm, imm, [hasSM<32>]>; // atom_inc atom_dec @@ -1901,131 +1901,131 @@ defm INT_PTX_ATOM_DEC_GEN_32_USE_G : F_ATOMIC_2; -def atomic_load_and_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), - (atomic_load_and_32 node:$a, node:$b)>; -def atomic_load_and_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), - (atomic_load_and_32 node:$a, node:$b)>; -def atomic_load_and_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), - (atomic_load_and_64 node:$a, node:$b)>; -def atomic_load_and_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), - (atomic_load_and_64 node:$a, node:$b)>; -def atomic_load_and_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), - (atomic_load_and_64 node:$a, node:$b)>; +def atomic_load_and_i32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), + (atomic_load_and_i32 node:$a, node:$b)>; +def atomic_load_and_i32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), + (atomic_load_and_i32 node:$a, node:$b)>; +def atomic_load_and_i32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), + (atomic_load_and_i32 node:$a, node:$b)>; +def atomic_load_and_i64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), + (atomic_load_and_i64 node:$a, node:$b)>; +def atomic_load_and_i64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), + (atomic_load_and_i64 node:$a, node:$b)>; +def atomic_load_and_i64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), + (atomic_load_and_i64 node:$a, node:$b)>; defm INT_PTX_ATOM_AND_G_32 : F_ATOMIC_2; + atomic_load_and_i32_g, i32imm, imm>; defm INT_PTX_ATOM_AND_S_32 : F_ATOMIC_2; + atomic_load_and_i32_s, i32imm, imm>; defm INT_PTX_ATOM_AND_GEN_32 : F_ATOMIC_2; + atomic_load_and_i32_gen, i32imm, imm>; defm INT_PTX_ATOM_AND_GEN_32_USE_G : F_ATOMIC_2; + ".and", atomic_load_and_i32_gen, i32imm, imm>; defm INT_PTX_ATOM_AND_G_64 : F_ATOMIC_2]>; + atomic_load_and_i64_g, i64imm, imm, [hasSM<32>]>; defm INT_PTX_ATOM_AND_S_64 : F_ATOMIC_2]>; + atomic_load_and_i64_s, i64imm, imm, [hasSM<32>]>; defm INT_PTX_ATOM_AND_GEN_64 : F_ATOMIC_2]>; + atomic_load_and_i64_gen, i64imm, imm, [hasSM<32>]>; defm INT_PTX_ATOM_AND_GEN_64_USE_G : F_ATOMIC_2]>; + ".and", atomic_load_and_i64_gen, i64imm, imm, [hasSM<32>]>; // atom_or -def atomic_load_or_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), - (atomic_load_or_32 node:$a, node:$b)>; -def atomic_load_or_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), - (atomic_load_or_32 node:$a, node:$b)>; -def atomic_load_or_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), - (atomic_load_or_32 node:$a, node:$b)>; -def atomic_load_or_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), - (atomic_load_or_64 node:$a, node:$b)>; -def atomic_load_or_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), - (atomic_load_or_64 node:$a, node:$b)>; -def atomic_load_or_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), - (atomic_load_or_64 node:$a, node:$b)>; +def atomic_load_or_i32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), + (atomic_load_or_i32 node:$a, node:$b)>; +def atomic_load_or_i32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), + (atomic_load_or_i32 node:$a, node:$b)>; +def atomic_load_or_i32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), + (atomic_load_or_i32 node:$a, node:$b)>; +def atomic_load_or_i64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), + (atomic_load_or_i64 node:$a, node:$b)>; +def atomic_load_or_i64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), + (atomic_load_or_i64 node:$a, node:$b)>; +def atomic_load_or_i64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), + (atomic_load_or_i64 node:$a, node:$b)>; defm INT_PTX_ATOM_OR_G_32 : F_ATOMIC_2; + atomic_load_or_i32_g, i32imm, imm>; defm INT_PTX_ATOM_OR_GEN_32 : F_ATOMIC_2; + atomic_load_or_i32_gen, i32imm, imm>; defm INT_PTX_ATOM_OR_GEN_32_USE_G : F_ATOMIC_2; + ".or", atomic_load_or_i32_gen, i32imm, imm>; defm INT_PTX_ATOM_OR_S_32 : F_ATOMIC_2; + atomic_load_or_i32_s, i32imm, imm>; defm INT_PTX_ATOM_OR_G_64 : F_ATOMIC_2]>; + atomic_load_or_i64_g, i64imm, imm, [hasSM<32>]>; defm INT_PTX_ATOM_OR_GEN_64 : F_ATOMIC_2]>; + atomic_load_or_i64_gen, i64imm, imm, [hasSM<32>]>; defm INT_PTX_ATOM_OR_GEN_64_USE_G : F_ATOMIC_2]>; + ".or", atomic_load_or_i64_gen, i64imm, imm, [hasSM<32>]>; defm INT_PTX_ATOM_OR_S_64 : F_ATOMIC_2]>; + atomic_load_or_i64_s, i64imm, imm, [hasSM<32>]>; // atom_xor -def atomic_load_xor_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), - (atomic_load_xor_32 node:$a, node:$b)>; -def atomic_load_xor_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), - (atomic_load_xor_32 node:$a, node:$b)>; -def atomic_load_xor_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), - (atomic_load_xor_32 node:$a, node:$b)>; -def atomic_load_xor_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), - (atomic_load_xor_64 node:$a, node:$b)>; -def atomic_load_xor_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), - (atomic_load_xor_64 node:$a, node:$b)>; -def atomic_load_xor_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), - (atomic_load_xor_64 node:$a, node:$b)>; +def atomic_load_xor_i32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), + (atomic_load_xor_i32 node:$a, node:$b)>; +def atomic_load_xor_i32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), + (atomic_load_xor_i32 node:$a, node:$b)>; +def atomic_load_xor_i32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), + (atomic_load_xor_i32 node:$a, node:$b)>; +def atomic_load_xor_i64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b), + (atomic_load_xor_i64 node:$a, node:$b)>; +def atomic_load_xor_i64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b), + (atomic_load_xor_i64 node:$a, node:$b)>; +def atomic_load_xor_i64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b), + (atomic_load_xor_i64 node:$a, node:$b)>; defm INT_PTX_ATOM_XOR_G_32 : F_ATOMIC_2; + atomic_load_xor_i32_g, i32imm, imm>; defm INT_PTX_ATOM_XOR_S_32 : F_ATOMIC_2; + atomic_load_xor_i32_s, i32imm, imm>; defm INT_PTX_ATOM_XOR_GEN_32 : F_ATOMIC_2; + atomic_load_xor_i32_gen, i32imm, imm>; defm INT_PTX_ATOM_XOR_GEN_32_USE_G : F_ATOMIC_2; + ".xor", atomic_load_xor_i32_gen, i32imm, imm>; defm INT_PTX_ATOM_XOR_G_64 : F_ATOMIC_2]>; + atomic_load_xor_i64_g, i64imm, imm, [hasSM<32>]>; defm INT_PTX_ATOM_XOR_S_64 : F_ATOMIC_2]>; + atomic_load_xor_i64_s, i64imm, imm, [hasSM<32>]>; defm INT_PTX_ATOM_XOR_GEN_64 : F_ATOMIC_2]>; + atomic_load_xor_i64_gen, i64imm, imm, [hasSM<32>]>; defm INT_PTX_ATOM_XOR_GEN_64_USE_G : F_ATOMIC_2]>; + ".xor", atomic_load_xor_i64_gen, i64imm, imm, [hasSM<32>]>; // atom_cas -def atomic_cmp_swap_32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b, node:$c), - (atomic_cmp_swap_32 node:$a, node:$b, node:$c)>; -def atomic_cmp_swap_32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b, node:$c), - (atomic_cmp_swap_32 node:$a, node:$b, node:$c)>; -def atomic_cmp_swap_32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b, node:$c), - (atomic_cmp_swap_32 node:$a, node:$b, node:$c)>; -def atomic_cmp_swap_64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b, node:$c), - (atomic_cmp_swap_64 node:$a, node:$b, node:$c)>; -def atomic_cmp_swap_64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b, node:$c), - (atomic_cmp_swap_64 node:$a, node:$b, node:$c)>; -def atomic_cmp_swap_64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b, node:$c), - (atomic_cmp_swap_64 node:$a, node:$b, node:$c)>; +def atomic_cmp_swap_i32_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b, node:$c), + (atomic_cmp_swap_i32 node:$a, node:$b, node:$c)>; +def atomic_cmp_swap_i32_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b, node:$c), + (atomic_cmp_swap_i32 node:$a, node:$b, node:$c)>; +def atomic_cmp_swap_i32_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b, node:$c), + (atomic_cmp_swap_i32 node:$a, node:$b, node:$c)>; +def atomic_cmp_swap_i64_g: ATOMIC_GLOBAL_CHK<(ops node:$a, node:$b, node:$c), + (atomic_cmp_swap_i64 node:$a, node:$b, node:$c)>; +def atomic_cmp_swap_i64_s: ATOMIC_SHARED_CHK<(ops node:$a, node:$b, node:$c), + (atomic_cmp_swap_i64 node:$a, node:$b, node:$c)>; +def atomic_cmp_swap_i64_gen: ATOMIC_GENERIC_CHK<(ops node:$a, node:$b, node:$c), + (atomic_cmp_swap_i64 node:$a, node:$b, node:$c)>; defm INT_PTX_ATOM_CAS_G_32 : F_ATOMIC_3; + atomic_cmp_swap_i32_g, i32imm>; defm INT_PTX_ATOM_CAS_S_32 : F_ATOMIC_3; + atomic_cmp_swap_i32_s, i32imm>; defm INT_PTX_ATOM_CAS_GEN_32 : F_ATOMIC_3; + atomic_cmp_swap_i32_gen, i32imm>; defm INT_PTX_ATOM_CAS_GEN_32_USE_G : F_ATOMIC_3; + ".cas", atomic_cmp_swap_i32_gen, i32imm>; defm INT_PTX_ATOM_CAS_G_64 : F_ATOMIC_3; + atomic_cmp_swap_i64_g, i64imm>; defm INT_PTX_ATOM_CAS_S_64 : F_ATOMIC_3; + atomic_cmp_swap_i64_s, i64imm>; defm INT_PTX_ATOM_CAS_GEN_64 : F_ATOMIC_3; + atomic_cmp_swap_i64_gen, i64imm>; defm INT_PTX_ATOM_CAS_GEN_64_USE_G : F_ATOMIC_3; + ".cas", atomic_cmp_swap_i64_gen, i64imm>; // Support for scoped atomic operations. Matches // int_nvvm_atomic_{op}_{space}_{type}_{scope} diff --git a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td index eda5eb975e700..8f5afbae01de1 100644 --- a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td +++ b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td @@ -292,42 +292,42 @@ def : Pat<(PPCcall_nop_rm (i64 mcsym:$dst)), let Defs = [CR0] in { def ATOMIC_LOAD_ADD_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$incr), "#ATOMIC_LOAD_ADD_I64", - [(set i64:$dst, (atomic_load_add_64 ForceXForm:$ptr, i64:$incr))]>; + [(set i64:$dst, (atomic_load_add_i64 ForceXForm:$ptr, i64:$incr))]>; def ATOMIC_LOAD_SUB_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$incr), "#ATOMIC_LOAD_SUB_I64", - [(set i64:$dst, (atomic_load_sub_64 ForceXForm:$ptr, i64:$incr))]>; + [(set i64:$dst, (atomic_load_sub_i64 ForceXForm:$ptr, i64:$incr))]>; def ATOMIC_LOAD_OR_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$incr), "#ATOMIC_LOAD_OR_I64", - [(set i64:$dst, (atomic_load_or_64 ForceXForm:$ptr, i64:$incr))]>; + [(set i64:$dst, (atomic_load_or_i64 ForceXForm:$ptr, i64:$incr))]>; def ATOMIC_LOAD_XOR_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$incr), "#ATOMIC_LOAD_XOR_I64", - [(set i64:$dst, (atomic_load_xor_64 ForceXForm:$ptr, i64:$incr))]>; + [(set i64:$dst, (atomic_load_xor_i64 ForceXForm:$ptr, i64:$incr))]>; def ATOMIC_LOAD_AND_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$incr), "#ATOMIC_LOAD_AND_i64", - [(set i64:$dst, (atomic_load_and_64 ForceXForm:$ptr, i64:$incr))]>; + [(set i64:$dst, (atomic_load_and_i64 ForceXForm:$ptr, i64:$incr))]>; def ATOMIC_LOAD_NAND_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$incr), "#ATOMIC_LOAD_NAND_I64", - [(set i64:$dst, (atomic_load_nand_64 ForceXForm:$ptr, i64:$incr))]>; + [(set i64:$dst, (atomic_load_nand_i64 ForceXForm:$ptr, i64:$incr))]>; def ATOMIC_LOAD_MIN_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$incr), "#ATOMIC_LOAD_MIN_I64", - [(set i64:$dst, (atomic_load_min_64 ForceXForm:$ptr, i64:$incr))]>; + [(set i64:$dst, (atomic_load_min_i64 ForceXForm:$ptr, i64:$incr))]>; def ATOMIC_LOAD_MAX_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$incr), "#ATOMIC_LOAD_MAX_I64", - [(set i64:$dst, (atomic_load_max_64 ForceXForm:$ptr, i64:$incr))]>; + [(set i64:$dst, (atomic_load_max_i64 ForceXForm:$ptr, i64:$incr))]>; def ATOMIC_LOAD_UMIN_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$incr), "#ATOMIC_LOAD_UMIN_I64", - [(set i64:$dst, (atomic_load_umin_64 ForceXForm:$ptr, i64:$incr))]>; + [(set i64:$dst, (atomic_load_umin_i64 ForceXForm:$ptr, i64:$incr))]>; def ATOMIC_LOAD_UMAX_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$incr), "#ATOMIC_LOAD_UMAX_I64", - [(set i64:$dst, (atomic_load_umax_64 ForceXForm:$ptr, i64:$incr))]>; + [(set i64:$dst, (atomic_load_umax_i64 ForceXForm:$ptr, i64:$incr))]>; def ATOMIC_CMP_SWAP_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$old, g8rc:$new), "#ATOMIC_CMP_SWAP_I64", - [(set i64:$dst, (atomic_cmp_swap_64 ForceXForm:$ptr, i64:$old, i64:$new))]>; + [(set i64:$dst, (atomic_cmp_swap_i64 ForceXForm:$ptr, i64:$old, i64:$new))]>; def ATOMIC_SWAP_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$new), "#ATOMIC_SWAP_I64", - [(set i64:$dst, (atomic_swap_64 ForceXForm:$ptr, i64:$new))]>; + [(set i64:$dst, (atomic_swap_i64 ForceXForm:$ptr, i64:$new))]>; } // Instructions to support atomic operations @@ -1036,7 +1036,7 @@ defm DIVDE : XOForm_1rcr<31, 425, 0, (outs g8rc:$RT), (ins g8rc:$RA, g8rc:$RB), let Predicates = [IsISA3_0] in { def MADDHD : VAForm_1a<48, (outs g8rc:$RT), (ins g8rc:$RA, g8rc:$RB, g8rc:$RC), "maddhd $RT, $RA, $RB, $RC", IIC_IntMulHD, []>, isPPC64; -def MADDHDU : VAForm_1a<49, +def MADDHDU : VAForm_1a<49, (outs g8rc:$RT), (ins g8rc:$RA, g8rc:$RB, g8rc:$RC), "maddhdu $RT, $RA, $RB, $RC", IIC_IntMulHD, []>, isPPC64; def MADDLD : VAForm_1a<51, (outs gprc:$RT), (ins gprc:$RA, gprc:$RB, gprc:$RC), @@ -1044,7 +1044,7 @@ def MADDLD : VAForm_1a<51, (outs gprc:$RT), (ins gprc:$RA, gprc:$RB, gprc:$RC), [(set i32:$RT, (add_without_simm16 (mul_without_simm16 i32:$RA, i32:$RB), i32:$RC))]>, isPPC64; let Interpretation64Bit = 1, isCodeGenOnly = 1 in { - def MADDLD8 : VAForm_1a<51, + def MADDLD8 : VAForm_1a<51, (outs g8rc:$RT), (ins g8rc:$RA, g8rc:$RB, g8rc:$RC), "maddld $RT, $RA, $RB, $RC", IIC_IntMulHD, [(set i64:$RT, (add_without_simm16 (mul_without_simm16 i64:$RA, i64:$RB), i64:$RC))]>, @@ -1349,8 +1349,8 @@ def LWZX8 : XForm_1_memOp<31, 23, (outs g8rc:$RST), (ins (memrr $RA, $RB):$addr "lwzx $RST, $addr", IIC_LdStLoad, [(set i64:$RST, (zextloadi32 XForm:$addr))]>, ZExt32To64; - - + + // Update forms. let mayLoad = 1, hasSideEffects = 0 in { def LBZU8 : DForm_1<35, (outs g8rc:$RST, ptr_rc_nor0:$ea_result), @@ -1635,7 +1635,7 @@ def PADDIdtprel : PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc_nox0:$reg, s16imm let PPC970_Unit = 2 in { let Interpretation64Bit = 1, isCodeGenOnly = 1 in { -// Truncating stores. +// Truncating stores. def STB8 : DForm_1<38, (outs), (ins g8rc:$RST, (memri $D, $RA):$addr), "stb $RST, $addr", IIC_LdStStore, [(truncstorei8 i64:$RST, DForm:$addr)]>; diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td index 09f829943528c..1686249c0f89d 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td @@ -723,7 +723,7 @@ def PCRelativeMemops : Predicate<"Subtarget->hasPCRelativeMemops()">; def IsNotISA3_1 : Predicate<"!Subtarget->isISA3_1()">; // AIX assembler may not be modern enough to support some extended mne. -def ModernAs: Predicate<"!Subtarget->isAIXABI() || Subtarget->HasModernAIXAs">, +def ModernAs: Predicate<"!Subtarget->isAIXABI() || Subtarget->HasModernAIXAs">, AssemblerPredicate<(any_of (not AIXOS), FeatureModernAIXAs)>; def IsAIX : Predicate<"Subtarget->isAIXABI()">; def NotAIX : Predicate<"!Subtarget->isAIXABI()">; @@ -1747,114 +1747,114 @@ def : Pat<(int_ppc_dcbtst_with_hint xoaddr:$dst, i32:$TH), let Defs = [CR0] in { def ATOMIC_LOAD_ADD_I8 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_ADD_I8", - [(set i32:$dst, (atomic_load_add_8 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_add_i8 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_SUB_I8 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_SUB_I8", - [(set i32:$dst, (atomic_load_sub_8 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_sub_i8 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_AND_I8 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_AND_I8", - [(set i32:$dst, (atomic_load_and_8 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_and_i8 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_OR_I8 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_OR_I8", - [(set i32:$dst, (atomic_load_or_8 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_or_i8 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_XOR_I8 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "ATOMIC_LOAD_XOR_I8", - [(set i32:$dst, (atomic_load_xor_8 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_xor_i8 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_NAND_I8 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_NAND_I8", - [(set i32:$dst, (atomic_load_nand_8 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_nand_i8 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_MIN_I8 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_MIN_I8", - [(set i32:$dst, (atomic_load_min_8 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_min_i8 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_MAX_I8 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_MAX_I8", - [(set i32:$dst, (atomic_load_max_8 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_max_i8 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_UMIN_I8 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_UMIN_I8", - [(set i32:$dst, (atomic_load_umin_8 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_umin_i8 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_UMAX_I8 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_UMAX_I8", - [(set i32:$dst, (atomic_load_umax_8 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_umax_i8 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_ADD_I16 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_ADD_I16", - [(set i32:$dst, (atomic_load_add_16 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_add_i16 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_SUB_I16 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_SUB_I16", - [(set i32:$dst, (atomic_load_sub_16 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_sub_i16 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_AND_I16 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_AND_I16", - [(set i32:$dst, (atomic_load_and_16 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_and_i16 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_OR_I16 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_OR_I16", - [(set i32:$dst, (atomic_load_or_16 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_or_i16 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_XOR_I16 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_XOR_I16", - [(set i32:$dst, (atomic_load_xor_16 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_xor_i16 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_NAND_I16 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_NAND_I16", - [(set i32:$dst, (atomic_load_nand_16 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_nand_i16 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_MIN_I16 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_MIN_I16", - [(set i32:$dst, (atomic_load_min_16 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_min_i16 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_MAX_I16 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_MAX_I16", - [(set i32:$dst, (atomic_load_max_16 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_max_i16 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_UMIN_I16 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_UMIN_I16", - [(set i32:$dst, (atomic_load_umin_16 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_umin_i16 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_UMAX_I16 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_UMAX_I16", - [(set i32:$dst, (atomic_load_umax_16 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_umax_i16 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_ADD_I32 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_ADD_I32", - [(set i32:$dst, (atomic_load_add_32 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_add_i32 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_SUB_I32 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_SUB_I32", - [(set i32:$dst, (atomic_load_sub_32 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_sub_i32 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_AND_I32 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_AND_I32", - [(set i32:$dst, (atomic_load_and_32 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_and_i32 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_OR_I32 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_OR_I32", - [(set i32:$dst, (atomic_load_or_32 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_or_i32 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_XOR_I32 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_XOR_I32", - [(set i32:$dst, (atomic_load_xor_32 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_xor_i32 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_NAND_I32 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_NAND_I32", - [(set i32:$dst, (atomic_load_nand_32 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_nand_i32 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_MIN_I32 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_MIN_I32", - [(set i32:$dst, (atomic_load_min_32 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_min_i32 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_MAX_I32 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_MAX_I32", - [(set i32:$dst, (atomic_load_max_32 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_max_i32 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_UMIN_I32 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_UMIN_I32", - [(set i32:$dst, (atomic_load_umin_32 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_umin_i32 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_LOAD_UMAX_I32 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$incr), "#ATOMIC_LOAD_UMAX_I32", - [(set i32:$dst, (atomic_load_umax_32 ForceXForm:$ptr, i32:$incr))]>; + [(set i32:$dst, (atomic_load_umax_i32 ForceXForm:$ptr, i32:$incr))]>; def ATOMIC_CMP_SWAP_I8 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$old, gprc:$new), "#ATOMIC_CMP_SWAP_I8", - [(set i32:$dst, (atomic_cmp_swap_8 ForceXForm:$ptr, i32:$old, i32:$new))]>; + [(set i32:$dst, (atomic_cmp_swap_i8 ForceXForm:$ptr, i32:$old, i32:$new))]>; def ATOMIC_CMP_SWAP_I16 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$old, gprc:$new), "#ATOMIC_CMP_SWAP_I16 $dst $ptr $old $new", - [(set i32:$dst, (atomic_cmp_swap_16 ForceXForm:$ptr, i32:$old, i32:$new))]>; + [(set i32:$dst, (atomic_cmp_swap_i16 ForceXForm:$ptr, i32:$old, i32:$new))]>; def ATOMIC_CMP_SWAP_I32 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$old, gprc:$new), "#ATOMIC_CMP_SWAP_I32 $dst $ptr $old $new", - [(set i32:$dst, (atomic_cmp_swap_32 ForceXForm:$ptr, i32:$old, i32:$new))]>; + [(set i32:$dst, (atomic_cmp_swap_i32 ForceXForm:$ptr, i32:$old, i32:$new))]>; def ATOMIC_SWAP_I8 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$new), "#ATOMIC_SWAP_i8", - [(set i32:$dst, (atomic_swap_8 ForceXForm:$ptr, i32:$new))]>; + [(set i32:$dst, (atomic_swap_i8 ForceXForm:$ptr, i32:$new))]>; def ATOMIC_SWAP_I16 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$new), "#ATOMIC_SWAP_I16", - [(set i32:$dst, (atomic_swap_16 ForceXForm:$ptr, i32:$new))]>; + [(set i32:$dst, (atomic_swap_i16 ForceXForm:$ptr, i32:$new))]>; def ATOMIC_SWAP_I32 : PPCCustomInserterPseudo< (outs gprc:$dst), (ins memrr:$ptr, gprc:$new), "#ATOMIC_SWAP_I32", - [(set i32:$dst, (atomic_swap_32 ForceXForm:$ptr, i32:$new))]>; + [(set i32:$dst, (atomic_swap_i32 ForceXForm:$ptr, i32:$new))]>; } def : Pat<(PPCatomicCmpSwap_8 ForceXForm:$ptr, i32:$old, i32:$new), diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoA.td b/llvm/lib/Target/RISCV/RISCVInstrInfoA.td index 814e0ddf111e6..493e1a5fdc74a 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoA.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoA.td @@ -166,25 +166,25 @@ let Predicates = !listconcat([HasStdExtA, HasStdExtZtso], ExtraPreds) in { } } -defm : AMOPat<"atomic_swap_32", "AMOSWAP_W">; -defm : AMOPat<"atomic_load_add_32", "AMOADD_W">; -defm : AMOPat<"atomic_load_and_32", "AMOAND_W">; -defm : AMOPat<"atomic_load_or_32", "AMOOR_W">; -defm : AMOPat<"atomic_load_xor_32", "AMOXOR_W">; -defm : AMOPat<"atomic_load_max_32", "AMOMAX_W">; -defm : AMOPat<"atomic_load_min_32", "AMOMIN_W">; -defm : AMOPat<"atomic_load_umax_32", "AMOMAXU_W">; -defm : AMOPat<"atomic_load_umin_32", "AMOMINU_W">; - -defm : AMOPat<"atomic_swap_64", "AMOSWAP_D", i64, [IsRV64]>; -defm : AMOPat<"atomic_load_add_64", "AMOADD_D", i64, [IsRV64]>; -defm : AMOPat<"atomic_load_and_64", "AMOAND_D", i64, [IsRV64]>; -defm : AMOPat<"atomic_load_or_64", "AMOOR_D", i64, [IsRV64]>; -defm : AMOPat<"atomic_load_xor_64", "AMOXOR_D", i64, [IsRV64]>; -defm : AMOPat<"atomic_load_max_64", "AMOMAX_D", i64, [IsRV64]>; -defm : AMOPat<"atomic_load_min_64", "AMOMIN_D", i64, [IsRV64]>; -defm : AMOPat<"atomic_load_umax_64", "AMOMAXU_D", i64, [IsRV64]>; -defm : AMOPat<"atomic_load_umin_64", "AMOMINU_D", i64, [IsRV64]>; +defm : AMOPat<"atomic_swap_i32", "AMOSWAP_W">; +defm : AMOPat<"atomic_load_add_i32", "AMOADD_W">; +defm : AMOPat<"atomic_load_and_i32", "AMOAND_W">; +defm : AMOPat<"atomic_load_or_i32", "AMOOR_W">; +defm : AMOPat<"atomic_load_xor_i32", "AMOXOR_W">; +defm : AMOPat<"atomic_load_max_i32", "AMOMAX_W">; +defm : AMOPat<"atomic_load_min_i32", "AMOMIN_W">; +defm : AMOPat<"atomic_load_umax_i32", "AMOMAXU_W">; +defm : AMOPat<"atomic_load_umin_i32", "AMOMINU_W">; + +defm : AMOPat<"atomic_swap_i64", "AMOSWAP_D", i64, [IsRV64]>; +defm : AMOPat<"atomic_load_add_i64", "AMOADD_D", i64, [IsRV64]>; +defm : AMOPat<"atomic_load_and_i64", "AMOAND_D", i64, [IsRV64]>; +defm : AMOPat<"atomic_load_or_i64", "AMOOR_D", i64, [IsRV64]>; +defm : AMOPat<"atomic_load_xor_i64", "AMOXOR_D", i64, [IsRV64]>; +defm : AMOPat<"atomic_load_max_i64", "AMOMAX_D", i64, [IsRV64]>; +defm : AMOPat<"atomic_load_min_i64", "AMOMIN_D", i64, [IsRV64]>; +defm : AMOPat<"atomic_load_umax_i64", "AMOMAXU_D", i64, [IsRV64]>; +defm : AMOPat<"atomic_load_umin_i64", "AMOMINU_D", i64, [IsRV64]>; /// Pseudo AMOs @@ -243,15 +243,15 @@ let Size = 20 in def PseudoAtomicLoadNand32 : PseudoAMO; // Ordering constants must be kept in sync with the AtomicOrdering enum in // AtomicOrdering.h. -def : Pat<(XLenVT (atomic_load_nand_32_monotonic GPR:$addr, GPR:$incr)), +def : Pat<(XLenVT (atomic_load_nand_i32_monotonic GPR:$addr, GPR:$incr)), (PseudoAtomicLoadNand32 GPR:$addr, GPR:$incr, 2)>; -def : Pat<(XLenVT (atomic_load_nand_32_acquire GPR:$addr, GPR:$incr)), +def : Pat<(XLenVT (atomic_load_nand_i32_acquire GPR:$addr, GPR:$incr)), (PseudoAtomicLoadNand32 GPR:$addr, GPR:$incr, 4)>; -def : Pat<(XLenVT (atomic_load_nand_32_release GPR:$addr, GPR:$incr)), +def : Pat<(XLenVT (atomic_load_nand_i32_release GPR:$addr, GPR:$incr)), (PseudoAtomicLoadNand32 GPR:$addr, GPR:$incr, 5)>; -def : Pat<(XLenVT (atomic_load_nand_32_acq_rel GPR:$addr, GPR:$incr)), +def : Pat<(XLenVT (atomic_load_nand_i32_acq_rel GPR:$addr, GPR:$incr)), (PseudoAtomicLoadNand32 GPR:$addr, GPR:$incr, 6)>; -def : Pat<(XLenVT (atomic_load_nand_32_seq_cst GPR:$addr, GPR:$incr)), +def : Pat<(XLenVT (atomic_load_nand_i32_seq_cst GPR:$addr, GPR:$incr)), (PseudoAtomicLoadNand32 GPR:$addr, GPR:$incr, 7)>; let Size = 28 in @@ -294,15 +294,15 @@ let Size = 20 in def PseudoAtomicLoadNand64 : PseudoAMO; // Ordering constants must be kept in sync with the AtomicOrdering enum in // AtomicOrdering.h. -def : Pat<(i64 (atomic_load_nand_64_monotonic GPR:$addr, GPR:$incr)), +def : Pat<(i64 (atomic_load_nand_i64_monotonic GPR:$addr, GPR:$incr)), (PseudoAtomicLoadNand64 GPR:$addr, GPR:$incr, 2)>; -def : Pat<(i64 (atomic_load_nand_64_acquire GPR:$addr, GPR:$incr)), +def : Pat<(i64 (atomic_load_nand_i64_acquire GPR:$addr, GPR:$incr)), (PseudoAtomicLoadNand64 GPR:$addr, GPR:$incr, 4)>; -def : Pat<(i64 (atomic_load_nand_64_release GPR:$addr, GPR:$incr)), +def : Pat<(i64 (atomic_load_nand_i64_release GPR:$addr, GPR:$incr)), (PseudoAtomicLoadNand64 GPR:$addr, GPR:$incr, 5)>; -def : Pat<(i64 (atomic_load_nand_64_acq_rel GPR:$addr, GPR:$incr)), +def : Pat<(i64 (atomic_load_nand_i64_acq_rel GPR:$addr, GPR:$incr)), (PseudoAtomicLoadNand64 GPR:$addr, GPR:$incr, 6)>; -def : Pat<(i64 (atomic_load_nand_64_seq_cst GPR:$addr, GPR:$incr)), +def : Pat<(i64 (atomic_load_nand_i64_seq_cst GPR:$addr, GPR:$incr)), (PseudoAtomicLoadNand64 GPR:$addr, GPR:$incr, 7)>; def : PseudoMaskedAMOPat; +defm : PseudoCmpXchgPat<"atomic_cmp_swap_i32", PseudoCmpXchg32>; } let Predicates = [HasStdExtA, NoStdExtZacas, IsRV64] in { def PseudoCmpXchg64 : PseudoCmpXchg; -defm : PseudoCmpXchgPat<"atomic_cmp_swap_64", PseudoCmpXchg64, i64>; +defm : PseudoCmpXchgPat<"atomic_cmp_swap_i64", PseudoCmpXchg64, i64>; } let Predicates = [HasStdExtA] in { @@ -422,18 +422,18 @@ let Predicates = !listconcat([HasStdExtA, HasStdExtZtso], ExtraPreds) in { } } -defm : AMOPat2<"atomic_swap_32", "AMOSWAP_W", i32>; -defm : AMOPat2<"atomic_load_add_32", "AMOADD_W", i32>; -defm : AMOPat2<"atomic_load_and_32", "AMOAND_W", i32>; -defm : AMOPat2<"atomic_load_or_32", "AMOOR_W", i32>; -defm : AMOPat2<"atomic_load_xor_32", "AMOXOR_W", i32>; -defm : AMOPat2<"atomic_load_max_32", "AMOMAX_W", i32>; -defm : AMOPat2<"atomic_load_min_32", "AMOMIN_W", i32>; -defm : AMOPat2<"atomic_load_umax_32", "AMOMAXU_W", i32>; -defm : AMOPat2<"atomic_load_umin_32", "AMOMINU_W", i32>; +defm : AMOPat2<"atomic_swap_i32", "AMOSWAP_W", i32>; +defm : AMOPat2<"atomic_load_add_i32", "AMOADD_W", i32>; +defm : AMOPat2<"atomic_load_and_i32", "AMOAND_W", i32>; +defm : AMOPat2<"atomic_load_or_i32", "AMOOR_W", i32>; +defm : AMOPat2<"atomic_load_xor_i32", "AMOXOR_W", i32>; +defm : AMOPat2<"atomic_load_max_i32", "AMOMAX_W", i32>; +defm : AMOPat2<"atomic_load_min_i32", "AMOMIN_W", i32>; +defm : AMOPat2<"atomic_load_umax_i32", "AMOMAXU_W", i32>; +defm : AMOPat2<"atomic_load_umin_i32", "AMOMINU_W", i32>; let Predicates = [HasStdExtA, IsRV64] in -defm : PseudoCmpXchgPat<"atomic_cmp_swap_32", PseudoCmpXchg32, i32>; +defm : PseudoCmpXchgPat<"atomic_cmp_swap_i32", PseudoCmpXchg32, i32>; let Predicates = [HasAtomicLdSt] in { def : LdPat; diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZa.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZa.td index 0cd41cac218f9..1ee78359bc4a5 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoZa.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZa.td @@ -116,8 +116,8 @@ multiclass AMOCASPat; -defm : AMOCASPat<"atomic_cmp_swap_64", "AMOCAS_D_RV64", i64, [IsRV64]>; +defm : AMOCASPat<"atomic_cmp_swap_i32", "AMOCAS_W">; +defm : AMOCASPat<"atomic_cmp_swap_i64", "AMOCAS_D_RV64", i64, [IsRV64]>; //===----------------------------------------------------------------------===// // Zawrs (Wait-on-Reservation-Set) @@ -188,27 +188,27 @@ defm AMOCAS_H : AMO_cas_aq_rl<0b00101, 0b001, "amocas.h", GPR>; /// AMOs -defm : AMOPat<"atomic_swap_8", "AMOSWAP_B", XLenVT, [HasStdExtZabha]>; -defm : AMOPat<"atomic_load_add_8", "AMOADD_B", XLenVT, [HasStdExtZabha]>; -defm : AMOPat<"atomic_load_and_8", "AMOAND_B", XLenVT, [HasStdExtZabha]>; -defm : AMOPat<"atomic_load_or_8", "AMOOR_B", XLenVT, [HasStdExtZabha]>; -defm : AMOPat<"atomic_load_xor_8", "AMOXOR_B", XLenVT, [HasStdExtZabha]>; -defm : AMOPat<"atomic_load_max_8", "AMOMAX_B", XLenVT, [HasStdExtZabha]>; -defm : AMOPat<"atomic_load_min_8", "AMOMIN_B", XLenVT, [HasStdExtZabha]>; -defm : AMOPat<"atomic_load_umax_8", "AMOMAXU_B", XLenVT, [HasStdExtZabha]>; -defm : AMOPat<"atomic_load_umin_8", "AMOMINU_B", XLenVT, [HasStdExtZabha]>; - -defm : AMOPat<"atomic_swap_16", "AMOSWAP_H", XLenVT, [HasStdExtZabha]>; -defm : AMOPat<"atomic_load_add_16", "AMOADD_H", XLenVT, [HasStdExtZabha]>; -defm : AMOPat<"atomic_load_and_16", "AMOAND_H", XLenVT, [HasStdExtZabha]>; -defm : AMOPat<"atomic_load_or_16", "AMOOR_H", XLenVT, [HasStdExtZabha]>; -defm : AMOPat<"atomic_load_xor_16", "AMOXOR_H", XLenVT, [HasStdExtZabha]>; -defm : AMOPat<"atomic_load_max_16", "AMOMAX_H", XLenVT, [HasStdExtZabha]>; -defm : AMOPat<"atomic_load_min_16", "AMOMIN_H", XLenVT, [HasStdExtZabha]>; -defm : AMOPat<"atomic_load_umax_16", "AMOMAXU_H", XLenVT, [HasStdExtZabha]>; -defm : AMOPat<"atomic_load_umin_16", "AMOMINU_H", XLenVT, [HasStdExtZabha]>; +defm : AMOPat<"atomic_swap_i8", "AMOSWAP_B", XLenVT, [HasStdExtZabha]>; +defm : AMOPat<"atomic_load_add_i8", "AMOADD_B", XLenVT, [HasStdExtZabha]>; +defm : AMOPat<"atomic_load_and_i8", "AMOAND_B", XLenVT, [HasStdExtZabha]>; +defm : AMOPat<"atomic_load_or_i8", "AMOOR_B", XLenVT, [HasStdExtZabha]>; +defm : AMOPat<"atomic_load_xor_i8", "AMOXOR_B", XLenVT, [HasStdExtZabha]>; +defm : AMOPat<"atomic_load_max_i8", "AMOMAX_B", XLenVT, [HasStdExtZabha]>; +defm : AMOPat<"atomic_load_min_i8", "AMOMIN_B", XLenVT, [HasStdExtZabha]>; +defm : AMOPat<"atomic_load_umax_i8", "AMOMAXU_B", XLenVT, [HasStdExtZabha]>; +defm : AMOPat<"atomic_load_umin_i8", "AMOMINU_B", XLenVT, [HasStdExtZabha]>; + +defm : AMOPat<"atomic_swap_i16", "AMOSWAP_H", XLenVT, [HasStdExtZabha]>; +defm : AMOPat<"atomic_load_add_i16", "AMOADD_H", XLenVT, [HasStdExtZabha]>; +defm : AMOPat<"atomic_load_and_i16", "AMOAND_H", XLenVT, [HasStdExtZabha]>; +defm : AMOPat<"atomic_load_or_i16", "AMOOR_H", XLenVT, [HasStdExtZabha]>; +defm : AMOPat<"atomic_load_xor_i16", "AMOXOR_H", XLenVT, [HasStdExtZabha]>; +defm : AMOPat<"atomic_load_max_i16", "AMOMAX_H", XLenVT, [HasStdExtZabha]>; +defm : AMOPat<"atomic_load_min_i16", "AMOMIN_H", XLenVT, [HasStdExtZabha]>; +defm : AMOPat<"atomic_load_umax_i16", "AMOMAXU_H", XLenVT, [HasStdExtZabha]>; +defm : AMOPat<"atomic_load_umin_i16", "AMOMINU_H", XLenVT, [HasStdExtZabha]>; /// AMOCAS -defm : AMOCASPat<"atomic_cmp_swap_8", "AMOCAS_B", XLenVT, [HasStdExtZabha]>; -defm : AMOCASPat<"atomic_cmp_swap_16", "AMOCAS_H", XLenVT, [HasStdExtZabha]>; +defm : AMOCASPat<"atomic_cmp_swap_i8", "AMOCAS_B", XLenVT, [HasStdExtZabha]>; +defm : AMOCASPat<"atomic_cmp_swap_i16", "AMOCAS_H", XLenVT, [HasStdExtZabha]>; diff --git a/llvm/lib/Target/Sparc/SparcInstr64Bit.td b/llvm/lib/Target/Sparc/SparcInstr64Bit.td index 93862414fb352..6b78137451650 100644 --- a/llvm/lib/Target/Sparc/SparcInstr64Bit.td +++ b/llvm/lib/Target/Sparc/SparcInstr64Bit.td @@ -478,7 +478,7 @@ def : Pat<(i64 (atomic_load_64 ADDRri:$src)), (LDXri ADDRri:$src)>; def : Pat<(atomic_store_64 i64:$val, ADDRrr:$dst), (STXrr ADDRrr:$dst, $val)>; def : Pat<(atomic_store_64 i64:$val, ADDRri:$dst), (STXri ADDRri:$dst, $val)>; -def : Pat<(atomic_cmp_swap_64 i64:$rs1, i64:$rs2, i64:$swap), +def : Pat<(atomic_cmp_swap_i64 i64:$rs1, i64:$rs2, i64:$swap), (CASXArr $rs1, $rs2, $swap, 0x80)>; } // Predicates = [Is64Bit] diff --git a/llvm/lib/Target/Sparc/SparcInstrInfo.td b/llvm/lib/Target/Sparc/SparcInstrInfo.td index cac96a1398721..7b074231ec62e 100644 --- a/llvm/lib/Target/Sparc/SparcInstrInfo.td +++ b/llvm/lib/Target/Sparc/SparcInstrInfo.td @@ -744,11 +744,11 @@ let Constraints = "$val = $rd" in { def SWAPrr : F3_1<3, 0b001111, (outs IntRegs:$rd), (ins (MEMrr $rs1, $rs2):$addr, IntRegs:$val), "swap [$addr], $rd", - [(set i32:$rd, (atomic_swap_32 ADDRrr:$addr, i32:$val))]>; + [(set i32:$rd, (atomic_swap_i32 ADDRrr:$addr, i32:$val))]>; def SWAPri : F3_2<3, 0b001111, (outs IntRegs:$rd), (ins (MEMri $rs1, $simm13):$addr, IntRegs:$val), "swap [$addr], $rd", - [(set i32:$rd, (atomic_swap_32 ADDRri:$addr, i32:$val))]>; + [(set i32:$rd, (atomic_swap_i32 ADDRri:$addr, i32:$val))]>; def SWAPArr : F3_1_asi<3, 0b011111, (outs IntRegs:$rd), (ins (MEMrr $rs1, $rs2):$addr, ASITag:$asi, IntRegs:$val), "swapa [$addr] $asi, $rd", @@ -1913,12 +1913,12 @@ def : Pat<(atomic_store_32 i32:$val, ADDRrr:$dst), (STrr ADDRrr:$dst, $val)>; def : Pat<(atomic_store_32 i32:$val, ADDRri:$dst), (STri ADDRri:$dst, $val)>; let Predicates = [HasV9] in -def : Pat<(atomic_cmp_swap_32 iPTR:$rs1, i32:$rs2, i32:$swap), +def : Pat<(atomic_cmp_swap_i32 iPTR:$rs1, i32:$rs2, i32:$swap), (CASArr $rs1, $rs2, $swap, 0x80)>; // Same pattern as CASArr above, but with a different ASI. let Predicates = [HasLeonCASA] in -def : Pat<(atomic_cmp_swap_32 iPTR:$rs1, i32:$rs2, i32:$swap), +def : Pat<(atomic_cmp_swap_i32 iPTR:$rs1, i32:$rs2, i32:$swap), (CASArr $rs1, $rs2, $swap, 0x0A)>; // A register pair with zero upper half. diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.td b/llvm/lib/Target/SystemZ/SystemZInstrInfo.td index 7f3a143aad970..7c6ab3f9b1ab5 100644 --- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.td +++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.td @@ -1733,16 +1733,16 @@ let hasSideEffects = 1 in def Serialize : Alias<2, (outs), (ins), []>; let Predicates = [FeatureInterlockedAccess1], Defs = [CC] in { - def LAA : LoadAndOpRSY<"laa", 0xEBF8, atomic_load_add_32, GR32>; - def LAAG : LoadAndOpRSY<"laag", 0xEBE8, atomic_load_add_64, GR64>; + def LAA : LoadAndOpRSY<"laa", 0xEBF8, atomic_load_add_i32, GR32>; + def LAAG : LoadAndOpRSY<"laag", 0xEBE8, atomic_load_add_i64, GR64>; def LAAL : LoadAndOpRSY<"laal", 0xEBFA, null_frag, GR32>; def LAALG : LoadAndOpRSY<"laalg", 0xEBEA, null_frag, GR64>; - def LAN : LoadAndOpRSY<"lan", 0xEBF4, atomic_load_and_32, GR32>; - def LANG : LoadAndOpRSY<"lang", 0xEBE4, atomic_load_and_64, GR64>; - def LAO : LoadAndOpRSY<"lao", 0xEBF6, atomic_load_or_32, GR32>; - def LAOG : LoadAndOpRSY<"laog", 0xEBE6, atomic_load_or_64, GR64>; - def LAX : LoadAndOpRSY<"lax", 0xEBF7, atomic_load_xor_32, GR32>; - def LAXG : LoadAndOpRSY<"laxg", 0xEBE7, atomic_load_xor_64, GR64>; + def LAN : LoadAndOpRSY<"lan", 0xEBF4, atomic_load_and_i32, GR32>; + def LANG : LoadAndOpRSY<"lang", 0xEBE4, atomic_load_and_i64, GR64>; + def LAO : LoadAndOpRSY<"lao", 0xEBF6, atomic_load_or_i32, GR32>; + def LAOG : LoadAndOpRSY<"laog", 0xEBE6, atomic_load_or_i64, GR64>; + def LAX : LoadAndOpRSY<"lax", 0xEBF7, atomic_load_xor_i32, GR32>; + def LAXG : LoadAndOpRSY<"laxg", 0xEBE7, atomic_load_xor_i64, GR64>; } def ATOMIC_SWAPW : AtomicLoadWBinaryReg; diff --git a/llvm/lib/Target/VE/VEInstrInfo.td b/llvm/lib/Target/VE/VEInstrInfo.td index cbad5a0eafb27..75ef3b7336dbd 100644 --- a/llvm/lib/Target/VE/VEInstrInfo.td +++ b/llvm/lib/Target/VE/VEInstrInfo.td @@ -1158,9 +1158,9 @@ defm ATMAM : RRCASm<"atmam", 0x53, I64, i64, uimm0to2>; // Section 8.2.20 - CAS (Compare and Swap) let DecoderMethod = "DecodeCASI64" in -defm CASL : RRCASm<"cas.l", 0x62, I64, i64, simm7, atomic_cmp_swap_64>; +defm CASL : RRCASm<"cas.l", 0x62, I64, i64, simm7, atomic_cmp_swap_i64>; let DecoderMethod = "DecodeCASI32", cx = 1 in -defm CASW : RRCASm<"cas.w", 0x62, I32, i32, simm7, atomic_cmp_swap_32>; +defm CASW : RRCASm<"cas.w", 0x62, I32, i32, simm7, atomic_cmp_swap_i32>; //----------------------------------------------------------------------------- // Section 8.3 - Transfer Control Instructions @@ -1896,9 +1896,9 @@ defm : TRATMSTm; // Atomic swaps def : Pat<(i32 (ts1am i64:$src, i32:$flag, i32:$new)), (TS1AMWrir $src, 0, $flag, $new)>; -def : Pat<(i32 (atomic_swap_32 ADDRri:$src, i32:$new)), +def : Pat<(i32 (atomic_swap_i32 ADDRri:$src, i32:$new)), (TS1AMWrii MEMriRRM:$src, 15, $new)>; -def : Pat<(i64 (atomic_swap_64 ADDRri:$src, i64:$new)), +def : Pat<(i64 (atomic_swap_i64 ADDRri:$src, i64:$new)), (TS1AMLrir MEMriRRM:$src, (LEAzii 0, 0, 255), i64:$new)>; //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td index 4623ce9b5c381..46bd5e42a9d52 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td +++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td @@ -351,17 +351,17 @@ multiclass BinRMWPattern; } -defm : BinRMWPattern; -defm : BinRMWPattern; -defm : BinRMWPattern; -defm : BinRMWPattern; -defm : BinRMWPattern; -defm : BinRMWPattern; // Truncating & zero-extending binary RMW patterns. @@ -408,27 +408,27 @@ multiclass BinRMWTruncExtPattern< } defm : BinRMWTruncExtPattern< - atomic_load_add_8, atomic_load_add_16, atomic_load_add_32, + atomic_load_add_i8, atomic_load_add_i16, atomic_load_add_i32, "ATOMIC_RMW8_U_ADD_I32", "ATOMIC_RMW16_U_ADD_I32", "ATOMIC_RMW8_U_ADD_I64", "ATOMIC_RMW16_U_ADD_I64", "ATOMIC_RMW32_U_ADD_I64">; defm : BinRMWTruncExtPattern< - atomic_load_sub_8, atomic_load_sub_16, atomic_load_sub_32, + atomic_load_sub_i8, atomic_load_sub_i16, atomic_load_sub_i32, "ATOMIC_RMW8_U_SUB_I32", "ATOMIC_RMW16_U_SUB_I32", "ATOMIC_RMW8_U_SUB_I64", "ATOMIC_RMW16_U_SUB_I64", "ATOMIC_RMW32_U_SUB_I64">; defm : BinRMWTruncExtPattern< - atomic_load_and_8, atomic_load_and_16, atomic_load_and_32, + atomic_load_and_i8, atomic_load_and_i16, atomic_load_and_i32, "ATOMIC_RMW8_U_AND_I32", "ATOMIC_RMW16_U_AND_I32", "ATOMIC_RMW8_U_AND_I64", "ATOMIC_RMW16_U_AND_I64", "ATOMIC_RMW32_U_AND_I64">; defm : BinRMWTruncExtPattern< - atomic_load_or_8, atomic_load_or_16, atomic_load_or_32, + atomic_load_or_i8, atomic_load_or_i16, atomic_load_or_i32, "ATOMIC_RMW8_U_OR_I32", "ATOMIC_RMW16_U_OR_I32", "ATOMIC_RMW8_U_OR_I64", "ATOMIC_RMW16_U_OR_I64", "ATOMIC_RMW32_U_OR_I64">; defm : BinRMWTruncExtPattern< - atomic_load_xor_8, atomic_load_xor_16, atomic_load_xor_32, + atomic_load_xor_i8, atomic_load_xor_i16, atomic_load_xor_i32, "ATOMIC_RMW8_U_XOR_I32", "ATOMIC_RMW16_U_XOR_I32", "ATOMIC_RMW8_U_XOR_I64", "ATOMIC_RMW16_U_XOR_I64", "ATOMIC_RMW32_U_XOR_I64">; defm : BinRMWTruncExtPattern< - atomic_swap_8, atomic_swap_16, atomic_swap_32, + atomic_swap_i8, atomic_swap_i16, atomic_swap_i32, "ATOMIC_RMW8_U_XCHG_I32", "ATOMIC_RMW16_U_XCHG_I32", "ATOMIC_RMW8_U_XCHG_I64", "ATOMIC_RMW16_U_XCHG_I64", "ATOMIC_RMW32_U_XCHG_I64">; @@ -485,8 +485,8 @@ multiclass TerRMWPat { Requires<[HasAddr64, HasAtomics]>; } -defm : TerRMWPat; -defm : TerRMWPat; +defm : TerRMWPat; +defm : TerRMWPat; // Truncating & zero-extending ternary RMW patterns. // DAG legalization & optimization before instruction selection may introduce @@ -524,13 +524,13 @@ class sext_ter_rmw_8_64 : class sext_ter_rmw_16_64 : sext_ter_rmw_8_64; // 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s -defm : TerRMWPat, "ATOMIC_RMW8_U_CMPXCHG_I32">; -defm : TerRMWPat, "ATOMIC_RMW16_U_CMPXCHG_I32">; -defm : TerRMWPat, "ATOMIC_RMW8_U_CMPXCHG_I64">; -defm : TerRMWPat, "ATOMIC_RMW16_U_CMPXCHG_I64">; -defm : TerRMWPat, "ATOMIC_RMW32_U_CMPXCHG_I64">; +defm : TerRMWPat, "ATOMIC_RMW8_U_CMPXCHG_I32">; +defm : TerRMWPat, "ATOMIC_RMW16_U_CMPXCHG_I32">; +defm : TerRMWPat, "ATOMIC_RMW8_U_CMPXCHG_I64">; +defm : TerRMWPat, "ATOMIC_RMW16_U_CMPXCHG_I64">; +defm : TerRMWPat, "ATOMIC_RMW32_U_CMPXCHG_I64">; -defm : TerRMWPat, "ATOMIC_RMW8_U_CMPXCHG_I32">; -defm : TerRMWPat, "ATOMIC_RMW16_U_CMPXCHG_I32">; -defm : TerRMWPat, "ATOMIC_RMW8_U_CMPXCHG_I64">; -defm : TerRMWPat, "ATOMIC_RMW16_U_CMPXCHG_I64">; +defm : TerRMWPat, "ATOMIC_RMW8_U_CMPXCHG_I32">; +defm : TerRMWPat, "ATOMIC_RMW16_U_CMPXCHG_I32">; +defm : TerRMWPat, "ATOMIC_RMW8_U_CMPXCHG_I64">; +defm : TerRMWPat, "ATOMIC_RMW16_U_CMPXCHG_I64">; diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td index 6fb6e1633b0c9..5a8177e2b3607 100644 --- a/llvm/lib/Target/X86/X86InstrCompiler.td +++ b/llvm/lib/Target/X86/X86InstrCompiler.td @@ -1035,27 +1035,27 @@ multiclass ATOMIC_RMW_BINOP opc8, bits<8> opc, string mnemonic, (ins GR8:$val, i8mem:$ptr), !strconcat(mnemonic, "{b}\t{$val, $ptr|$ptr, $val}"), [(set GR8:$dst, - (!cast(frag # "_8") addr:$ptr, GR8:$val))]>; + (!cast(frag # "_i8") addr:$ptr, GR8:$val))]>; def NAME#16 : I(frag # "_16") addr:$ptr, GR16:$val))]>, + (!cast(frag # "_i16") addr:$ptr, GR16:$val))]>, OpSize16; def NAME#32 : I(frag # "_32") addr:$ptr, GR32:$val))]>, + (!cast(frag # "_i32") addr:$ptr, GR32:$val))]>, OpSize32; def NAME#64 : RI(frag # "_64") addr:$ptr, GR64:$val))]>; + (!cast(frag # "_i64") addr:$ptr, GR64:$val))]>; } } diff --git a/llvm/lib/Target/X86/X86InstrMisc.td b/llvm/lib/Target/X86/X86InstrMisc.td index c4da0e50a1dd8..c9ff8abb02efd 100644 --- a/llvm/lib/Target/X86/X86InstrMisc.td +++ b/llvm/lib/Target/X86/X86InstrMisc.td @@ -823,27 +823,27 @@ multiclass ATOMIC_SWAP opc8, bits<8> opc, string mnemonic, string frag> !strconcat(mnemonic, "{b}\t{$val, $ptr|$ptr, $val}"), [(set GR8:$dst, - (!cast(frag # "_8") addr:$ptr, GR8:$val))]>; + (!cast(frag # "_i8") addr:$ptr, GR8:$val))]>; def NAME#16rm : I(frag # "_16") addr:$ptr, GR16:$val))]>, + (!cast(frag # "_i16") addr:$ptr, GR16:$val))]>, OpSize16; def NAME#32rm : I(frag # "_32") addr:$ptr, GR32:$val))]>, + (!cast(frag # "_i32") addr:$ptr, GR32:$val))]>, OpSize32; def NAME#64rm : RI(frag # "_64") addr:$ptr, GR64:$val))]>; + (!cast(frag # "_i64") addr:$ptr, GR64:$val))]>; } } diff --git a/llvm/test/TableGen/HasNoUse.td b/llvm/test/TableGen/HasNoUse.td index 030598d1cbeec..6e6bcc2a81df7 100644 --- a/llvm/test/TableGen/HasNoUse.td +++ b/llvm/test/TableGen/HasNoUse.td @@ -9,7 +9,7 @@ include "GlobalISelEmitterCommon.td" def NO_RET_ATOMIC_ADD : I<(outs), (ins GPR32Op:$src0, GPR32Op:$src1), []>; // SDAG: case 0: { -// SDAG-NEXT: // Predicate_atomic_load_add_no_ret_32 +// SDAG-NEXT: // Predicate_atomic_load_add_no_ret_i32 // SDAG-NEXT: SDNode *N = Node; // SDAG-NEXT: (void)N; // SDAG-NEXT: if (cast(N)->getMemoryVT() != MVT::i32) return false; @@ -23,7 +23,7 @@ def NO_RET_ATOMIC_ADD : I<(outs), (ins GPR32Op:$src0, GPR32Op:$src1), []>; // GISEL-NEXT: GIM_CheckHasNoUse, /*MI*/0, // GISEL-NEXT: // MIs[0] src0 // GISEL-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/0, -// GISEL-NEXT: // (atomic_load_add:{ *:[i32] } iPTR:{ *:[iPTR] }:$src0, i32:{ *:[i32] }:$src1)<> => (NO_RET_ATOMIC_ADD GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1) +// GISEL-NEXT: // (atomic_load_add:{ *:[i32] } iPTR:{ *:[iPTR] }:$src0, i32:{ *:[i32] }:$src1)<> => (NO_RET_ATOMIC_ADD GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1) // GISEL-NEXT: GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(MyTarget::NO_RET_ATOMIC_ADD), // GISEL-NEXT: GIR_RootToRootCopy, /*OpIdx*/1, // src0 // GISEL-NEXT: GIR_RootToRootCopy, /*OpIdx*/2, // src1 @@ -35,6 +35,6 @@ let HasNoUse = true in defm atomic_load_add_no_ret : binary_atomic_op; def : Pat < - (atomic_load_add_no_ret_32 iPTR:$src0, i32:$src1), + (atomic_load_add_no_ret_i32 iPTR:$src0, i32:$src1), (NO_RET_ATOMIC_ADD GPR32:$src0, GPR32:$src1) >; From 2f1eda0b9e5ce96704793287dd3d158c5f9630c4 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Mon, 10 Jun 2024 16:34:00 +0200 Subject: [PATCH 2/3] Remove AMDGPU custom v2f16 wrapper --- llvm/lib/Target/AMDGPU/AMDGPUInstructions.td | 2 -- llvm/lib/Target/AMDGPU/DSInstructions.td | 5 ++--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td index bd348f11007a0..783bc9d7ef593 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td @@ -684,8 +684,6 @@ defm atomic_load_xor : binary_atomic_op_all_as; defm atomic_load_fadd : binary_atomic_op_fp_all_as; defm atomic_load_uinc_wrap : binary_atomic_op_all_as; defm atomic_load_udec_wrap : binary_atomic_op_all_as; -let MemoryVT = v2f16 in -defm atomic_load_fadd_v2f16 : binary_atomic_op_fp_all_as; defm AMDGPUatomic_cmp_swap : binary_atomic_op_all_as; def load_align8_local : PatFrag<(ops node:$ptr), (load_local node:$ptr)>, diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td index b61f3d7427279..5150641c50ea2 100644 --- a/llvm/lib/Target/AMDGPU/DSInstructions.td +++ b/llvm/lib/Target/AMDGPU/DSInstructions.td @@ -1135,9 +1135,8 @@ def : DSAtomicRetPatIntrinsic; -let AddedComplexity = 1 in -def : DSAtomicRetPat; +defm : DSAtomicRetNoRetPat_mc; + def : GCNPat < (v2i16 (int_amdgcn_ds_fadd_v2bf16 i32:$ptr, v2i16:$src)), (DS_PK_ADD_RTN_BF16 VGPR_32:$ptr, VGPR_32:$src, 0, 0) From fad8630790db6a4d8294751058fb8c2c8efde135 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Wed, 12 Jun 2024 09:17:39 +0200 Subject: [PATCH 3/3] Indentation --- llvm/include/llvm/Target/TargetSelectionDAG.td | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td index f15737e95f97c..4ea02e6aa7f00 100644 --- a/llvm/include/llvm/Target/TargetSelectionDAG.td +++ b/llvm/include/llvm/Target/TargetSelectionDAG.td @@ -1683,7 +1683,7 @@ multiclass ternary_atomic_op_ord { multiclass binary_atomic_op { foreach vt = [ i8, i16, i32, i64 ] in { def _#vt : PatFrag<(ops node:$ptr, node:$val), - (atomic_op node:$ptr, node:$val)> { + (atomic_op node:$ptr, node:$val)> { let IsAtomic = true; let MemoryVT = vt; } @@ -1695,7 +1695,7 @@ multiclass binary_atomic_op { multiclass binary_atomic_op_fp { foreach vt = [ f16, bf16, v2f16, v2bf16, f32, f64 ] in { def _#vt : PatFrag<(ops node:$ptr, node:$val), - (atomic_op node:$ptr, node:$val)> { + (atomic_op node:$ptr, node:$val)> { let IsAtomic = true; let MemoryVT = vt; } @@ -1705,7 +1705,7 @@ multiclass binary_atomic_op_fp { multiclass ternary_atomic_op { foreach vt = [ i8, i16, i32, i64 ] in { def _#vt : PatFrag<(ops node:$ptr, node:$cmp, node:$val), - (atomic_op node:$ptr, node:$cmp, node:$val)> { + (atomic_op node:$ptr, node:$cmp, node:$val)> { let IsAtomic = true; let MemoryVT = vt; }