From 1355f5e80092ce82695f7ed73aaa690b0c7ca793 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Wed, 16 Oct 2024 08:07:38 -0700 Subject: [PATCH 1/9] [WebAssembly] Protect memory.fill and memory.copy from zero-length ranges. WebAssembly's `memory.fill` and `memory.copy` instructions trap if the pointers are out of bounds, even if the length is zero. This is different from LLVM, which expects that it can call `memcpy` on arbitrary invalid pointers if the length is zero. To avoid spurious traps, branch around `memory.fill` and `memory.copy` when the length is zero. --- .../lib/Target/WebAssembly/WebAssemblyISD.def | 4 + .../WebAssembly/WebAssemblyISelLowering.cpp | 140 +++++++++++++++++ .../WebAssembly/WebAssemblyInstrBulkMemory.td | 99 ++++++++++-- .../WebAssemblySelectionDAGInfo.cpp | 19 ++- llvm/test/CodeGen/WebAssembly/bulk-memory.ll | 102 ++++++++++--- .../test/CodeGen/WebAssembly/bulk-memory64.ll | 141 +++++++++++++----- 6 files changed, 434 insertions(+), 71 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def index f1ef58ef1f349..90c373e59d187 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def +++ b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def @@ -54,3 +54,7 @@ HANDLE_MEM_NODETYPE(GLOBAL_GET) HANDLE_MEM_NODETYPE(GLOBAL_SET) HANDLE_MEM_NODETYPE(TABLE_GET) HANDLE_MEM_NODETYPE(TABLE_SET) + +// Bulk memory instructions that require branching to handle empty ranges. +HANDLE_NODETYPE(MEMCPY) +HANDLE_NODETYPE(MEMSET) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp index 8611c05be89a1..5c9b356b84e01 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp @@ -568,6 +568,138 @@ static MachineBasicBlock *LowerFPToInt(MachineInstr &MI, DebugLoc DL, return DoneMBB; } +// Lower a `MEMCPY` instruction into a CFG triangle around a `MEMORY_COPY` +// instuction to handle the zero-length case. +static MachineBasicBlock *LowerMemcpy(MachineInstr &MI, DebugLoc DL, + MachineBasicBlock *BB, + const TargetInstrInfo &TII, bool Int64) { + MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); + + MachineOperand DstMem = MI.getOperand(0); + MachineOperand SrcMem = MI.getOperand(1); + MachineOperand Dst = MI.getOperand(2); + MachineOperand Src = MI.getOperand(3); + MachineOperand Len = MI.getOperand(4); + + // We're going to add an extra use to `Len` to test if it's zero; that + // use shouldn't be a kill, even if the original use is. + MachineOperand NoKillLen = Len; + NoKillLen.setIsKill(false); + + // Decide on which `MachineInstr` opcode we're going to use. + unsigned Eqz = Int64 ? WebAssembly::EQZ_I64 : WebAssembly::EQZ_I32; + unsigned MemoryCopy = + Int64 ? WebAssembly::MEMORY_COPY_A64 : WebAssembly::MEMORY_COPY_A32; + + // Create two new basic blocks; one for the new `memory.fill` that we can + // branch over, and one for the rest of the instructions after the original + // `memory.fill`. + const BasicBlock *LLVMBB = BB->getBasicBlock(); + MachineFunction *F = BB->getParent(); + MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB); + MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB); + + MachineFunction::iterator It = ++BB->getIterator(); + F->insert(It, TrueMBB); + F->insert(It, DoneMBB); + + // Transfer the remainder of BB and its successor edges to DoneMBB. + DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end()); + DoneMBB->transferSuccessorsAndUpdatePHIs(BB); + + // Connect the CFG edges. + BB->addSuccessor(TrueMBB); + BB->addSuccessor(DoneMBB); + TrueMBB->addSuccessor(DoneMBB); + + // Create a virtual register for the `Eqz` result. + unsigned EqzReg; + EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass); + + // Erase the original `memory.copy`. + MI.eraseFromParent(); + + // Test if `Len` is zero. + BuildMI(BB, DL, TII.get(Eqz), EqzReg).add(NoKillLen); + + // Insert a new `memory.copy`. + BuildMI(TrueMBB, DL, TII.get(MemoryCopy)) + .add(DstMem) + .add(SrcMem) + .add(Dst) + .add(Src) + .add(Len); + + // Create the CFG triangle. + BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(DoneMBB).addReg(EqzReg); + BuildMI(TrueMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB); + + return DoneMBB; +} + +// Lower a `MEMSET` instruction into a CFG triangle around a `MEMORY_FILL` +// instuction to handle the zero-length case. +static MachineBasicBlock *LowerMemset(MachineInstr &MI, DebugLoc DL, + MachineBasicBlock *BB, + const TargetInstrInfo &TII, bool Int64) { + MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); + + MachineOperand Mem = MI.getOperand(0); + MachineOperand Dst = MI.getOperand(1); + MachineOperand Val = MI.getOperand(2); + MachineOperand Len = MI.getOperand(3); + + // We're going to add an extra use to `Len` to test if it's zero; that + // use shouldn't be a kill, even if the original use is. + MachineOperand NoKillLen = Len; + NoKillLen.setIsKill(false); + + // Decide on which `MachineInstr` opcode we're going to use. + unsigned Eqz = Int64 ? WebAssembly::EQZ_I64 : WebAssembly::EQZ_I32; + unsigned MemoryFill = + Int64 ? WebAssembly::MEMORY_FILL_A64 : WebAssembly::MEMORY_FILL_A32; + + // Create two new basic blocks; one for the new `memory.fill` that we can + // branch over, and one for the rest of the instructions after the original + // `memory.fill`. + const BasicBlock *LLVMBB = BB->getBasicBlock(); + MachineFunction *F = BB->getParent(); + MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB); + MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB); + + MachineFunction::iterator It = ++BB->getIterator(); + F->insert(It, TrueMBB); + F->insert(It, DoneMBB); + + // Transfer the remainder of BB and its successor edges to DoneMBB. + DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end()); + DoneMBB->transferSuccessorsAndUpdatePHIs(BB); + + // Connect the CFG edges. + BB->addSuccessor(TrueMBB); + BB->addSuccessor(DoneMBB); + TrueMBB->addSuccessor(DoneMBB); + + // Create a virtual register for the `Eqz` result. + unsigned EqzReg; + EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass); + + // Erase the original `memory.fill`. + MI.eraseFromParent(); + + // Test if `Len` is zero. + BuildMI(BB, DL, TII.get(Eqz), EqzReg).add(NoKillLen); + + // Insert a new `memory.copy`. + BuildMI(TrueMBB, DL, TII.get(MemoryFill)).add(Mem).add(Dst).add(Val).add(Len); + + // Create the CFG triangle. + BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(DoneMBB).addReg(EqzReg); + BuildMI(TrueMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB); + + return DoneMBB; +} + static MachineBasicBlock * LowerCallResults(MachineInstr &CallResults, DebugLoc DL, MachineBasicBlock *BB, const WebAssemblySubtarget *Subtarget, @@ -725,6 +857,14 @@ MachineBasicBlock *WebAssemblyTargetLowering::EmitInstrWithCustomInserter( case WebAssembly::FP_TO_UINT_I64_F64: return LowerFPToInt(MI, DL, BB, TII, true, true, true, WebAssembly::I64_TRUNC_U_F64); + case WebAssembly::MEMCPY_A32: + return LowerMemcpy(MI, DL, BB, TII, false); + case WebAssembly::MEMCPY_A64: + return LowerMemcpy(MI, DL, BB, TII, true); + case WebAssembly::MEMSET_A32: + return LowerMemset(MI, DL, BB, TII, false); + case WebAssembly::MEMSET_A64: + return LowerMemset(MI, DL, BB, TII, true); case WebAssembly::CALL_RESULTS: case WebAssembly::RET_CALL_RESULTS: return LowerCallResults(MI, DL, BB, Subtarget, TII); diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td index 7aeae54d95a8c..de79f2d44cd32 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td +++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td @@ -21,16 +21,33 @@ multiclass BULK_I, SDTCisInt<1>, SDTCisPtrTy<2>, SDTCisPtrTy<3>, SDTCisInt<4>] +>; +def wasm_memory_copy : SDNode<"WebAssemblyISD::MEMORY_COPY", wasm_memory_copy_t, + [SDNPHasChain, SDNPMayLoad, SDNPMayStore]>; + +// memory.copy with a branch to avoid trapping def wasm_memcpy_t : SDTypeProfile<0, 5, [SDTCisInt<0>, SDTCisInt<1>, SDTCisPtrTy<2>, SDTCisPtrTy<3>, SDTCisInt<4>] >; -def wasm_memcpy : SDNode<"WebAssemblyISD::MEMORY_COPY", wasm_memcpy_t, +def wasm_memcpy : SDNode<"WebAssemblyISD::MEMCPY", wasm_memcpy_t, [SDNPHasChain, SDNPMayLoad, SDNPMayStore]>; +// memory.fill (may trap on empty ranges) +def wasm_memory_fill_t : SDTypeProfile<0, 4, + [SDTCisInt<0>, SDTCisPtrTy<1>, SDTCisInt<2>, SDTCisInt<3>] +>; +def wasm_memory_fill : SDNode<"WebAssemblyISD::MEMORY_FILL", wasm_memory_fill_t, + [SDNPHasChain, SDNPMayStore]>; + +// memory.fill with a branch to avoid trapping def wasm_memset_t : SDTypeProfile<0, 4, [SDTCisInt<0>, SDTCisPtrTy<1>, SDTCisInt<2>, SDTCisInt<3>] >; -def wasm_memset : SDNode<"WebAssemblyISD::MEMORY_FILL", wasm_memset_t, +def wasm_memset : SDNode<"WebAssemblyISD::MEMSET", wasm_memset_t, [SDNPHasChain, SDNPMayStore]>; multiclass BulkMemoryOps { @@ -51,25 +68,83 @@ defm DATA_DROP : [], "data.drop\t$seg", "data.drop\t$seg", 0x09>; +} + +defm : BulkMemoryOps; +defm : BulkMemoryOps; + +// Define copy/fill manually instead of using the `BulkMemoryOps` multiclass +// because when a multiclass defines opcodes, it gives them anonymous names +// and we need opcodes with names so that we can handle them with custom code. + let mayLoad = 1, mayStore = 1 in -defm MEMORY_COPY_A#B : +defm MEMORY_COPY_A32 : BULK_I<(outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx, - rc:$dst, rc:$src, rc:$len), + I32:$dst, I32:$src, I32:$len), (outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx), - [(wasm_memcpy (i32 imm:$src_idx), (i32 imm:$dst_idx), - rc:$dst, rc:$src, rc:$len + [(wasm_memory_copy (i32 imm:$src_idx), (i32 imm:$dst_idx), + I32:$dst, I32:$src, I32:$len )], "memory.copy\t$src_idx, $dst_idx, $dst, $src, $len", "memory.copy\t$src_idx, $dst_idx", 0x0a>; let mayStore = 1 in -defm MEMORY_FILL_A#B : - BULK_I<(outs), (ins i32imm_op:$idx, rc:$dst, I32:$value, rc:$size), +defm MEMORY_FILL_A32 : + BULK_I<(outs), (ins i32imm_op:$idx, I32:$dst, I32:$value, I32:$size), (outs), (ins i32imm_op:$idx), - [(wasm_memset (i32 imm:$idx), rc:$dst, I32:$value, rc:$size)], + [(wasm_memory_fill (i32 imm:$idx), I32:$dst, I32:$value, I32:$size)], "memory.fill\t$idx, $dst, $value, $size", "memory.fill\t$idx", 0x0b>; -} -defm : BulkMemoryOps; -defm : BulkMemoryOps; +let mayLoad = 1, mayStore = 1 in +defm MEMORY_COPY_A64 : + BULK_I<(outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx, + I64:$dst, I64:$src, I64:$len), + (outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx), + [(wasm_memory_copy (i32 imm:$src_idx), (i32 imm:$dst_idx), + I64:$dst, I64:$src, I64:$len + )], + "memory.copy\t$src_idx, $dst_idx, $dst, $src, $len", + "memory.copy\t$src_idx, $dst_idx", 0x0a>; + +let mayStore = 1 in +defm MEMORY_FILL_A64 : + BULK_I<(outs), (ins i32imm_op:$idx, I64:$dst, I32:$value, I64:$size), + (outs), (ins i32imm_op:$idx), + [(wasm_memory_fill (i32 imm:$idx), I64:$dst, I32:$value, I64:$size)], + "memory.fill\t$idx, $dst, $value, $size", + "memory.fill\t$idx", 0x0b>; + +let usesCustomInserter = 1, isCodeGenOnly = 1, mayLoad = 1, mayStore = 1 in +defm MEMCPY_A32 : I<(outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx, + I32:$dst, I32:$src, I32:$len), + (outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx), + [(wasm_memcpy (i32 imm:$src_idx), (i32 imm:$dst_idx), + I32:$dst, I32:$src, I32:$len + )], + "", "", 0>, + Requires<[HasBulkMemory]>; + +let usesCustomInserter = 1, isCodeGenOnly = 1, mayStore = 1 in +defm MEMSET_A32 : I<(outs), (ins i32imm_op:$idx, I32:$dst, I32:$value, I32:$size), + (outs), (ins i32imm_op:$idx), + [(wasm_memset (i32 imm:$idx), I32:$dst, I32:$value, I32:$size)], + "", "", 0>, + Requires<[HasBulkMemory]>; + +let usesCustomInserter = 1, isCodeGenOnly = 1, mayLoad = 1, mayStore = 1 in +defm MEMCPY_A64 : I<(outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx, + I64:$dst, I64:$src, I64:$len), + (outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx), + [(wasm_memcpy (i32 imm:$src_idx), (i32 imm:$dst_idx), + I64:$dst, I64:$src, I64:$len + )], + "", "", 0>, + Requires<[HasBulkMemory]>; + +let usesCustomInserter = 1, isCodeGenOnly = 1, mayStore = 1 in +defm MEMSET_A64 : I<(outs), (ins i32imm_op:$idx, I64:$dst, I32:$value, I64:$size), + (outs), (ins i32imm_op:$idx), + [(wasm_memset (i32 imm:$idx), I64:$dst, I32:$value, I64:$size)], + "", "", 0>, + Requires<[HasBulkMemory]>; diff --git a/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp index 74af4c8873f73..d51bfeb6d8592 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp @@ -28,9 +28,13 @@ SDValue WebAssemblySelectionDAGInfo::EmitTargetCodeForMemcpy( SDValue MemIdx = DAG.getConstant(0, DL, MVT::i32); auto LenMVT = ST.hasAddr64() ? MVT::i64 : MVT::i32; - return DAG.getNode(WebAssemblyISD::MEMORY_COPY, DL, MVT::Other, - {Chain, MemIdx, MemIdx, Dst, Src, - DAG.getZExtOrTrunc(Size, DL, LenMVT)}); + + // Use `MEMCPY` here instead of `MEMORY_COPY` because `memory.copy` traps + // if the pointers are invalid even if the length is zero. `MEMCPY` gets + // extra code to handle this in the way that LLVM IR expects. + return DAG.getNode( + WebAssemblyISD::MEMCPY, DL, MVT::Other, + {Chain, MemIdx, MemIdx, Dst, Src, DAG.getZExtOrTrunc(Size, DL, LenMVT)}); } SDValue WebAssemblySelectionDAGInfo::EmitTargetCodeForMemmove( @@ -52,8 +56,13 @@ SDValue WebAssemblySelectionDAGInfo::EmitTargetCodeForMemset( SDValue MemIdx = DAG.getConstant(0, DL, MVT::i32); auto LenMVT = ST.hasAddr64() ? MVT::i64 : MVT::i32; + + // Use `MEMSET` here instead of `MEMORY_FILL` because `memory.fill` traps + // if the pointers are invalid even if the length is zero. `MEMSET` gets + // extra code to handle this in the way that LLVM IR expects. + // // Only low byte matters for val argument, so anyext the i8 - return DAG.getNode(WebAssemblyISD::MEMORY_FILL, DL, MVT::Other, Chain, MemIdx, - Dst, DAG.getAnyExtOrTrunc(Val, DL, MVT::i32), + return DAG.getNode(WebAssemblyISD::MEMSET, DL, MVT::Other, Chain, MemIdx, Dst, + DAG.getAnyExtOrTrunc(Val, DL, MVT::i32), DAG.getZExtOrTrunc(Size, DL, LenMVT)); } diff --git a/llvm/test/CodeGen/WebAssembly/bulk-memory.ll b/llvm/test/CodeGen/WebAssembly/bulk-memory.ll index dc29dc81c13ec..ae170d757a305 100644 --- a/llvm/test/CodeGen/WebAssembly/bulk-memory.ll +++ b/llvm/test/CodeGen/WebAssembly/bulk-memory.ll @@ -17,7 +17,12 @@ declare void @llvm.memset.p0.i32(ptr, i8, i32, i1) ; CHECK-LABEL: memcpy_i8: ; NO-BULK-MEM-NOT: memory.copy ; BULK-MEM-NEXT: .functype memcpy_i8 (i32, i32, i32) -> () +; BULK-MEM-NEXT: block +; BULK-MEM-NEXT: i32.eqz $push0=, $2 +; BULK-MEM-NEXT: br_if 0, $pop0 ; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $2 +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memcpy_i8(ptr %dest, ptr %src, i8 zeroext %len) { call void @llvm.memcpy.p0.p0.i8(ptr %dest, ptr %src, i8 %len, i1 0) @@ -27,7 +32,12 @@ define void @memcpy_i8(ptr %dest, ptr %src, i8 zeroext %len) { ; CHECK-LABEL: memmove_i8: ; NO-BULK-MEM-NOT: memory.copy ; BULK-MEM-NEXT: .functype memmove_i8 (i32, i32, i32) -> () +; BULK-MEM-NEXT: block +; BULK-MEM-NEXT: i32.eqz $push0=, $2 +; BULK-MEM-NEXT: br_if 0, $pop0 ; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $2 +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memmove_i8(ptr %dest, ptr %src, i8 zeroext %len) { call void @llvm.memmove.p0.p0.i8(ptr %dest, ptr %src, i8 %len, i1 0) @@ -37,7 +47,12 @@ define void @memmove_i8(ptr %dest, ptr %src, i8 zeroext %len) { ; CHECK-LABEL: memset_i8: ; NO-BULK-MEM-NOT: memory.fill ; BULK-MEM-NEXT: .functype memset_i8 (i32, i32, i32) -> () +; BULK-MEM-NEXT: block +; BULK-MEM-NEXT: i32.eqz $push0=, $2 +; BULK-MEM-NEXT: br_if 0, $pop0 ; BULK-MEM-NEXT: memory.fill 0, $0, $1, $2 +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memset_i8(ptr %dest, i8 %val, i8 zeroext %len) { call void @llvm.memset.p0.i8(ptr %dest, i8 %val, i8 %len, i1 0) @@ -47,7 +62,12 @@ define void @memset_i8(ptr %dest, i8 %val, i8 zeroext %len) { ; CHECK-LABEL: memcpy_i32: ; NO-BULK-MEM-NOT: memory.copy ; BULK-MEM-NEXT: .functype memcpy_i32 (i32, i32, i32) -> () +; BULK-MEM-NEXT: block +; BULK-MEM-NEXT: i32.eqz $push0=, $2 +; BULK-MEM-NEXT: br_if 0, $pop0 ; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $2 +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memcpy_i32(ptr %dest, ptr %src, i32 %len) { call void @llvm.memcpy.p0.p0.i32(ptr %dest, ptr %src, i32 %len, i1 0) @@ -57,7 +77,12 @@ define void @memcpy_i32(ptr %dest, ptr %src, i32 %len) { ; CHECK-LABEL: memmove_i32: ; NO-BULK-MEM-NOT: memory.copy ; BULK-MEM-NEXT: .functype memmove_i32 (i32, i32, i32) -> () +; BULK-MEM-NEXT: block +; BULK-MEM-NEXT: i32.eqz $push0=, $2 +; BULK-MEM-NEXT: br_if 0, $pop0 ; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $2 +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memmove_i32(ptr %dest, ptr %src, i32 %len) { call void @llvm.memmove.p0.p0.i32(ptr %dest, ptr %src, i32 %len, i1 0) @@ -67,7 +92,12 @@ define void @memmove_i32(ptr %dest, ptr %src, i32 %len) { ; CHECK-LABEL: memset_i32: ; NO-BULK-MEM-NOT: memory.fill ; BULK-MEM-NEXT: .functype memset_i32 (i32, i32, i32) -> () +; BULK-MEM-NEXT: block +; BULK-MEM-NEXT: i32.eqz $push0=, $2 +; BULK-MEM-NEXT: br_if 0, $pop0 ; BULK-MEM-NEXT: memory.fill 0, $0, $1, $2 +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memset_i32(ptr %dest, i8 %val, i32 %len) { call void @llvm.memset.p0.i32(ptr %dest, i8 %val, i32 %len, i1 0) @@ -107,8 +137,14 @@ define void @memset_1(ptr %dest, i8 %val) { ; CHECK-LABEL: memcpy_1024: ; NO-BULK-MEM-NOT: memory.copy ; BULK-MEM-NEXT: .functype memcpy_1024 (i32, i32) -> () +; BULK-MEM-NEXT: block ; BULK-MEM-NEXT: i32.const $push[[L0:[0-9]+]]=, 1024 -; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop[[L0]] +; BULK-MEM-NEXT: i32.eqz $push[[L1:[0-9]+]]=, $pop[[L0]] +; BULK-MEM-NEXT: br_if 0, $pop[[L1]] +; BULK-MEM-NEXT: i32.const $push[[L2:[0-9]+]]=, 1024 +; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop[[L2]] +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memcpy_1024(ptr %dest, ptr %src) { call void @llvm.memcpy.p0.p0.i32(ptr %dest, ptr %src, i32 1024, i1 0) @@ -118,8 +154,14 @@ define void @memcpy_1024(ptr %dest, ptr %src) { ; CHECK-LABEL: memmove_1024: ; NO-BULK-MEM-NOT: memory.copy ; BULK-MEM-NEXT: .functype memmove_1024 (i32, i32) -> () +; BULK-MEM-NEXT: block ; BULK-MEM-NEXT: i32.const $push[[L0:[0-9]+]]=, 1024 -; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop[[L0]] +; BULK-MEM-NEXT: i32.eqz $push[[L1:[0-9]+]]=, $pop[[L0]] +; BULK-MEM-NEXT: br_if 0, $pop[[L1]] +; BULK-MEM-NEXT: i32.const $push[[L2:[0-9]+]]=, 1024 +; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop[[L2]] +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memmove_1024(ptr %dest, ptr %src) { call void @llvm.memmove.p0.p0.i32(ptr %dest, ptr %src, i32 1024, i1 0) @@ -129,8 +171,14 @@ define void @memmove_1024(ptr %dest, ptr %src) { ; CHECK-LABEL: memset_1024: ; NO-BULK-MEM-NOT: memory.fill ; BULK-MEM-NEXT: .functype memset_1024 (i32, i32) -> () +; BULK-MEM-NEXT: block ; BULK-MEM-NEXT: i32.const $push[[L0:[0-9]+]]=, 1024 -; BULK-MEM-NEXT: memory.fill 0, $0, $1, $pop[[L0]] +; BULK-MEM-NEXT: i32.eqz $push[[L1:[0-9]+]]=, $pop[[L0]] +; BULK-MEM-NEXT: br_if 0, $pop[[L1]] +; BULK-MEM-NEXT: i32.const $push[[L2:[0-9]+]]=, 1024 +; BULK-MEM-NEXT: memory.fill 0, $0, $1, $pop[[L2]] +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memset_1024(ptr %dest, i8 %val) { call void @llvm.memset.p0.i32(ptr %dest, i8 %val, i32 1024, i1 0) @@ -153,11 +201,17 @@ define void @memset_1024(ptr %dest, i8 %val) { ; BULK-MEM-NEXT: .functype memcpy_alloca_src (i32) -> () ; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer ; BULK-MEM-NEXT: i32.const $push[[L1:[0-9]+]]=, 112 -; BULK-MEM-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]] -; BULK-MEM-NEXT: i32.const $push[[L3:[0-9]+]]=, 12 -; BULK-MEM-NEXT: i32.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]] -; BULK-MEM-NEXT: i32.const $push[[L5:[0-9]+]]=, 100 -; BULK-MEM-NEXT: memory.copy 0, 0, $0, $pop[[L4]], $pop[[L5]] +; BULK-MEM-NEXT: i32.sub $[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]] +; BULK-MEM-NEXT: block +; BULK-MEM-NEXT: i32.const $push[[L3:[0-9]+]]=, 100 +; BULK-MEM-NEXT: i32.eqz $push[[L4:[0-9]+]]=, $pop[[L3]] +; BULK-MEM-NEXT: br_if 0, $pop[[L4]] +; BULK-MEM-NEXT: i32.const $push[[L5:[0-9]+]]=, 12 +; BULK-MEM-NEXT: i32.add $push[[L6:[0-9]+]]=, $[[L2]], $pop[[L5]] +; BULK-MEM-NEXT: i32.const $push[[L7:[0-9]+]]=, 100 +; BULK-MEM-NEXT: memory.copy 0, 0, $0, $pop[[L6]], $pop[[L7]] +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memcpy_alloca_src(ptr %dst) { %a = alloca [100 x i8] @@ -170,11 +224,17 @@ define void @memcpy_alloca_src(ptr %dst) { ; BULK-MEM-NEXT: .functype memcpy_alloca_dst (i32) -> () ; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer ; BULK-MEM-NEXT: i32.const $push[[L1:[0-9]+]]=, 112 -; BULK-MEM-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]] -; BULK-MEM-NEXT: i32.const $push[[L3:[0-9]+]]=, 12 -; BULK-MEM-NEXT: i32.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]] -; BULK-MEM-NEXT: i32.const $push[[L5:[0-9]+]]=, 100 -; BULK-MEM-NEXT: memory.copy 0, 0, $pop[[L4]], $0, $pop[[L5]] +; BULK-MEM-NEXT: i32.sub $[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]] +; BULK-MEM-NEXT: block +; BULK-MEM-NEXT: i32.const $push[[L3:[0-9]+]]=, 100 +; BULK-MEM-NEXT: i32.eqz $push[[L4:[0-9]+]]=, $pop[[L3]] +; BULK-MEM-NEXT: br_if 0, $pop[[L4]] +; BULK-MEM-NEXT: i32.const $push[[L5:[0-9]+]]=, 12 +; BULK-MEM-NEXT: i32.add $push[[L6:[0-9]+]]=, $[[L2]], $pop[[L5]] +; BULK-MEM-NEXT: i32.const $push[[L7:[0-9]+]]=, 100 +; BULK-MEM-NEXT: memory.copy 0, 0, $pop[[L6]], $0, $pop[[L7]] +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memcpy_alloca_dst(ptr %src) { %a = alloca [100 x i8] @@ -187,11 +247,17 @@ define void @memcpy_alloca_dst(ptr %src) { ; BULK-MEM-NEXT: .functype memset_alloca (i32) -> () ; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer ; BULK-MEM-NEXT: i32.const $push[[L1:[0-9]+]]=, 112 -; BULK-MEM-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]] -; BULK-MEM-NEXT: i32.const $push[[L3:[0-9]+]]=, 12 -; BULK-MEM-NEXT: i32.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]] -; BULK-MEM-NEXT: i32.const $push[[L5:[0-9]+]]=, 100 -; BULK-MEM-NEXT: memory.fill 0, $pop[[L4]], $0, $pop[[L5]] +; BULK-MEM-NEXT: i32.sub $1=, $pop[[L0]], $pop[[L1]] +; BULK-MEM-NEXT: block +; BULK-MEM-NEXT: i32.const $push[[L2:[0-9]+]]=, 100 +; BULK-MEM-NEXT: i32.eqz $push[[L3:[0-9]+]]=, $pop[[L2]] +; BULK-MEM-NEXT: br_if 0, $pop[[L3]] +; BULK-MEM-NEXT: i32.const $push[[L4:[0-9]+]]=, 12 +; BULK-MEM-NEXT: i32.add $push[[L5:[0-9]+]]=, $1, $pop[[L4]] +; BULK-MEM-NEXT: i32.const $push[[L6:[0-9]+]]=, 100 +; BULK-MEM-NEXT: memory.fill 0, $pop[[L5]], $0, $pop[[L6]] +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memset_alloca(i8 %val) { %a = alloca [100 x i8] diff --git a/llvm/test/CodeGen/WebAssembly/bulk-memory64.ll b/llvm/test/CodeGen/WebAssembly/bulk-memory64.ll index 8ee5f6314381c..0cf8493a995f9 100644 --- a/llvm/test/CodeGen/WebAssembly/bulk-memory64.ll +++ b/llvm/test/CodeGen/WebAssembly/bulk-memory64.ll @@ -17,8 +17,14 @@ declare void @llvm.memset.p0.i64(ptr, i8, i64, i1) ; CHECK-LABEL: memcpy_i8: ; NO-BULK-MEM-NOT: memory.copy ; BULK-MEM-NEXT: .functype memcpy_i8 (i64, i64, i32) -> () -; BULK-MEM-NEXT: i64.extend_i32_u $push0=, $2 -; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop0 +; BULK-MEM-NEXT: block +; BULK-MEM-NEXT: i64.extend_i32_u $push[[L0:[0-9]+]]=, $2 +; BULK-MEM-NEXT: local.tee $push[[L1:[0-9]+]]=, $3=, $pop[[L0]] +; BULK-MEM-NEXT: i64.eqz $push0=, $pop[[L1]] +; BULK-MEM-NEXT: br_if 0, $pop0 +; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $3 +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memcpy_i8(ptr %dest, ptr %src, i8 zeroext %len) { call void @llvm.memcpy.p0.p0.i8(ptr %dest, ptr %src, i8 %len, i1 0) @@ -28,8 +34,14 @@ define void @memcpy_i8(ptr %dest, ptr %src, i8 zeroext %len) { ; CHECK-LABEL: memmove_i8: ; NO-BULK-MEM-NOT: memory.copy ; BULK-MEM-NEXT: .functype memmove_i8 (i64, i64, i32) -> () -; BULK-MEM-NEXT: i64.extend_i32_u $push0=, $2 -; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop0 +; BULK-MEM-NEXT: block +; BULK-MEM-NEXT: i64.extend_i32_u $push[[L0:[0-9]+]]=, $2 +; BULK-MEM-NEXT: local.tee $push[[L1:[0-9]+]]=, $3=, $pop[[L0]] +; BULK-MEM-NEXT: i64.eqz $push0=, $pop[[L1]] +; BULK-MEM-NEXT: br_if 0, $pop0 +; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $3 +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memmove_i8(ptr %dest, ptr %src, i8 zeroext %len) { call void @llvm.memmove.p0.p0.i8(ptr %dest, ptr %src, i8 %len, i1 0) @@ -39,8 +51,14 @@ define void @memmove_i8(ptr %dest, ptr %src, i8 zeroext %len) { ; CHECK-LABEL: memset_i8: ; NO-BULK-MEM-NOT: memory.fill ; BULK-MEM-NEXT: .functype memset_i8 (i64, i32, i32) -> () -; BULK-MEM-NEXT: i64.extend_i32_u $push0=, $2 -; BULK-MEM-NEXT: memory.fill 0, $0, $1, $pop0 +; BULK-MEM-NEXT: block +; BULK-MEM-NEXT: i64.extend_i32_u $push[[L0:[0-9]+]]=, $2 +; BULK-MEM-NEXT: local.tee $push[[L1:[0-9]+]]=, $3=, $pop[[L0]] +; BULK-MEM-NEXT: i64.eqz $push0=, $pop[[L1]] +; BULK-MEM-NEXT: br_if 0, $pop0 +; BULK-MEM-NEXT: memory.fill 0, $0, $1, $3 +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memset_i8(ptr %dest, i8 %val, i8 zeroext %len) { call void @llvm.memset.p0.i8(ptr %dest, i8 %val, i8 %len, i1 0) @@ -50,7 +68,12 @@ define void @memset_i8(ptr %dest, i8 %val, i8 zeroext %len) { ; CHECK-LABEL: memcpy_i32: ; NO-BULK-MEM-NOT: memory.copy ; BULK-MEM-NEXT: .functype memcpy_i32 (i64, i64, i64) -> () -; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $2 +; BULK-MEM-NEXT: block +; BULK-MEM-NEXT: i64.eqz $push0=, $2 +; BULK-MEM-NEXT: br_if 0, $pop0 +; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $2 +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memcpy_i32(ptr %dest, ptr %src, i64 %len) { call void @llvm.memcpy.p0.p0.i64(ptr %dest, ptr %src, i64 %len, i1 0) @@ -60,7 +83,12 @@ define void @memcpy_i32(ptr %dest, ptr %src, i64 %len) { ; CHECK-LABEL: memmove_i32: ; NO-BULK-MEM-NOT: memory.copy ; BULK-MEM-NEXT: .functype memmove_i32 (i64, i64, i64) -> () -; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $2 +; BULK-MEM-NEXT: block +; BULK-MEM-NEXT: i64.eqz $push0=, $2 +; BULK-MEM-NEXT: br_if 0, $pop0 +; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $2 +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memmove_i32(ptr %dest, ptr %src, i64 %len) { call void @llvm.memmove.p0.p0.i64(ptr %dest, ptr %src, i64 %len, i1 0) @@ -70,7 +98,12 @@ define void @memmove_i32(ptr %dest, ptr %src, i64 %len) { ; CHECK-LABEL: memset_i32: ; NO-BULK-MEM-NOT: memory.fill ; BULK-MEM-NEXT: .functype memset_i32 (i64, i32, i64) -> () -; BULK-MEM-NEXT: memory.fill 0, $0, $1, $2 +; BULK-MEM-NEXT: block +; BULK-MEM-NEXT: i64.eqz $push0=, $2 +; BULK-MEM-NEXT: br_if 0, $pop0 +; BULK-MEM-NEXT: memory.fill 0, $0, $1, $2 +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memset_i32(ptr %dest, i8 %val, i64 %len) { call void @llvm.memset.p0.i64(ptr %dest, i8 %val, i64 %len, i1 0) @@ -110,8 +143,14 @@ define void @memset_1(ptr %dest, i8 %val) { ; CHECK-LABEL: memcpy_1024: ; NO-BULK-MEM-NOT: memory.copy ; BULK-MEM-NEXT: .functype memcpy_1024 (i64, i64) -> () -; BULK-MEM-NEXT: i64.const $push[[L0:[0-9]+]]=, 1024 -; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop[[L0]] +; BULK-MEM-NEXT: block +; BULK-MEM-NEXT: i64.const $push[[L1:[0-9]+]]=, 1024 +; BULK-MEM-NEXT: i64.eqz $push0=, $pop[[L1]] +; BULK-MEM-NEXT: br_if 0, $pop0 +; BULK-MEM-NEXT: i64.const $push[[L0:[0-9]+]]=, 1024 +; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop[[L0]] +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memcpy_1024(ptr %dest, ptr %src) { call void @llvm.memcpy.p0.p0.i64(ptr %dest, ptr %src, i64 1024, i1 0) @@ -121,8 +160,14 @@ define void @memcpy_1024(ptr %dest, ptr %src) { ; CHECK-LABEL: memmove_1024: ; NO-BULK-MEM-NOT: memory.copy ; BULK-MEM-NEXT: .functype memmove_1024 (i64, i64) -> () -; BULK-MEM-NEXT: i64.const $push[[L0:[0-9]+]]=, 1024 -; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop[[L0]] +; BULK-MEM-NEXT: block +; BULK-MEM-NEXT: i64.const $push[[L1:[0-9]+]]=, 1024 +; BULK-MEM-NEXT: i64.eqz $push0=, $pop[[L1]] +; BULK-MEM-NEXT: br_if 0, $pop0 +; BULK-MEM-NEXT: i64.const $push[[L0:[0-9]+]]=, 1024 +; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop[[L0]] +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memmove_1024(ptr %dest, ptr %src) { call void @llvm.memmove.p0.p0.i64(ptr %dest, ptr %src, i64 1024, i1 0) @@ -132,8 +177,14 @@ define void @memmove_1024(ptr %dest, ptr %src) { ; CHECK-LABEL: memset_1024: ; NO-BULK-MEM-NOT: memory.fill ; BULK-MEM-NEXT: .functype memset_1024 (i64, i32) -> () -; BULK-MEM-NEXT: i64.const $push[[L0:[0-9]+]]=, 1024 -; BULK-MEM-NEXT: memory.fill 0, $0, $1, $pop[[L0]] +; BULK-MEM-NEXT: block +; BULK-MEM-NEXT: i64.const $push[[L1:[0-9]+]]=, 1024 +; BULK-MEM-NEXT: i64.eqz $push0=, $pop[[L1]] +; BULK-MEM-NEXT: br_if 0, $pop0 +; BULK-MEM-NEXT: i64.const $push[[L0:[0-9]+]]=, 1024 +; BULK-MEM-NEXT: memory.fill 0, $0, $1, $pop[[L0]] +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memset_1024(ptr %dest, i8 %val) { call void @llvm.memset.p0.i64(ptr %dest, i8 %val, i64 1024, i1 0) @@ -154,13 +205,19 @@ define void @memset_1024(ptr %dest, i8 %val) { ; CHECK-LABEL: memcpy_alloca_src: ; NO-BULK-MEM-NOT: memory.copy ; BULK-MEM-NEXT: .functype memcpy_alloca_src (i64) -> () -; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer -; BULK-MEM-NEXT: i64.const $push[[L1:[0-9]+]]=, 112 -; BULK-MEM-NEXT: i64.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]] -; BULK-MEM-NEXT: i64.const $push[[L3:[0-9]+]]=, 12 -; BULK-MEM-NEXT: i64.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]] -; BULK-MEM-NEXT: i64.const $push[[L5:[0-9]+]]=, 100 -; BULK-MEM-NEXT: memory.copy 0, 0, $0, $pop[[L4]], $pop[[L5]] +; BULK-MEM-NEXT: global.get $push[[L1:[0-9]+]]=, __stack_pointer +; BULK-MEM-NEXT: i64.const $push[[L0:[0-9]+]]=, 112 +; BULK-MEM-NEXT: i64.sub $[[L2:[0-9]+]]=, $pop[[L1]], $pop[[L0]] +; BULK-MEM-NEXT: block +; BULK-MEM-NEXT: i64.const $push[[L3:[0-9]+]]=, 100 +; BULK-MEM-NEXT: i64.eqz $push[[L4:[0-9]+]]=, $pop[[L3]] +; BULK-MEM-NEXT: br_if 0, $pop[[L4]] +; BULK-MEM-NEXT: i64.const $push[[L5:[0-9]+]]=, 12 +; BULK-MEM-NEXT: i64.add $push[[L6:[0-9]+]]=, $[[L2]], $pop[[L5]] +; BULK-MEM-NEXT: i64.const $push[[L7:[0-9]+]]=, 100 +; BULK-MEM-NEXT: memory.copy 0, 0, $0, $pop[[L6]], $pop[[L7]] +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memcpy_alloca_src(ptr %dst) { %a = alloca [100 x i8] @@ -171,13 +228,19 @@ define void @memcpy_alloca_src(ptr %dst) { ; CHECK-LABEL: memcpy_alloca_dst: ; NO-BULK-MEM-NOT: memory.copy ; BULK-MEM-NEXT: .functype memcpy_alloca_dst (i64) -> () -; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer -; BULK-MEM-NEXT: i64.const $push[[L1:[0-9]+]]=, 112 -; BULK-MEM-NEXT: i64.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]] -; BULK-MEM-NEXT: i64.const $push[[L3:[0-9]+]]=, 12 -; BULK-MEM-NEXT: i64.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]] -; BULK-MEM-NEXT: i64.const $push[[L5:[0-9]+]]=, 100 -; BULK-MEM-NEXT: memory.copy 0, 0, $pop[[L4]], $0, $pop[[L5]] +; BULK-MEM-NEXT: global.get $push[[L1:[0-9]+]]=, __stack_pointer +; BULK-MEM-NEXT: i64.const $push[[L0:[0-9]+]]=, 112 +; BULK-MEM-NEXT: i64.sub $[[L2:[0-9]+]]=, $pop[[L1]], $pop[[L0]] +; BULK-MEM-NEXT: block +; BULK-MEM-NEXT: i64.const $push[[L3:[0-9]+]]=, 100 +; BULK-MEM-NEXT: i64.eqz $push[[L4:[0-9]+]]=, $pop[[L3]] +; BULK-MEM-NEXT: br_if 0, $pop[[L4]] +; BULK-MEM-NEXT: i64.const $push[[L5:[0-9]+]]=, 12 +; BULK-MEM-NEXT: i64.add $push[[L6:[0-9]+]]=, $[[L2]], $pop[[L5]] +; BULK-MEM-NEXT: i64.const $push[[L7:[0-9]+]]=, 100 +; BULK-MEM-NEXT: memory.copy 0, 0, $pop[[L6]], $0, $pop[[L7]] +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memcpy_alloca_dst(ptr %src) { %a = alloca [100 x i8] @@ -188,13 +251,19 @@ define void @memcpy_alloca_dst(ptr %src) { ; CHECK-LABEL: memset_alloca: ; NO-BULK-MEM-NOT: memory.fill ; BULK-MEM-NEXT: .functype memset_alloca (i32) -> () -; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer -; BULK-MEM-NEXT: i64.const $push[[L1:[0-9]+]]=, 112 -; BULK-MEM-NEXT: i64.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]] -; BULK-MEM-NEXT: i64.const $push[[L3:[0-9]+]]=, 12 -; BULK-MEM-NEXT: i64.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]] -; BULK-MEM-NEXT: i64.const $push[[L5:[0-9]+]]=, 100 -; BULK-MEM-NEXT: memory.fill 0, $pop[[L4]], $0, $pop[[L5]] +; BULK-MEM-NEXT: global.get $push[[L1:[0-9]+]]=, __stack_pointer +; BULK-MEM-NEXT: i64.const $push[[L0:[0-9]+]]=, 112 +; BULK-MEM-NEXT: i64.sub $1=, $pop[[L1]], $pop[[L0]] +; BULK-MEM-NEXT: block +; BULK-MEM-NEXT: i64.const $push[[L2:[0-9]+]]=, 100 +; BULK-MEM-NEXT: i64.eqz $push[[L3:[0-9]+]]=, $pop[[L2]] +; BULK-MEM-NEXT: br_if 0, $pop[[L3]] +; BULK-MEM-NEXT: i64.const $push[[L4:[0-9]+]]=, 12 +; BULK-MEM-NEXT: i64.add $push[[L5:[0-9]+]]=, $1, $pop[[L4]] +; BULK-MEM-NEXT: i64.const $push[[L6:[0-9]+]]=, 100 +; BULK-MEM-NEXT: memory.fill 0, $pop[[L5]], $0, $pop[[L6]] +; BULK-MEM-NEXT: .LBB{{.*}}: +; BULK-MEM-NEXT: end_block ; BULK-MEM-NEXT: return define void @memset_alloca(i8 %val) { %a = alloca [100 x i8] From f667122302eddc5366378569a641d029d29428b3 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Thu, 17 Oct 2024 07:04:10 -0700 Subject: [PATCH 2/9] Define the bulk-memory instructions using a multiclass. --- .../WebAssembly/WebAssemblyInstrBulkMemory.td | 93 +++++++------------ 1 file changed, 33 insertions(+), 60 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td index de79f2d44cd32..4dc0c930b8f68 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td +++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td @@ -50,10 +50,13 @@ def wasm_memset_t : SDTypeProfile<0, 4, def wasm_memset : SDNode<"WebAssemblyISD::MEMSET", wasm_memset_t, [SDNPHasChain, SDNPMayStore]>; +// A multiclass for defining Wasm's raw bulk-memory `memory.*` instructions. +// `memory.copy` and `memory.fill` have Wasm's behavior rather than +// `memcpy`/`memset` behavior. multiclass BulkMemoryOps { let mayStore = 1, hasSideEffects = 1 in -defm MEMORY_INIT_A#B : +defm INIT_A#B : BULK_I<(outs), (ins i32imm_op:$seg, i32imm_op:$idx, rc:$dest, I32:$offset, I32:$size), @@ -62,89 +65,59 @@ defm MEMORY_INIT_A#B : "memory.init\t$seg, $idx, $dest, $offset, $size", "memory.init\t$seg, $idx", 0x08>; -let hasSideEffects = 1 in -defm DATA_DROP : - BULK_I<(outs), (ins i32imm_op:$seg), (outs), (ins i32imm_op:$seg), - [], - "data.drop\t$seg", "data.drop\t$seg", 0x09>; - -} - -defm : BulkMemoryOps; -defm : BulkMemoryOps; - -// Define copy/fill manually instead of using the `BulkMemoryOps` multiclass -// because when a multiclass defines opcodes, it gives them anonymous names -// and we need opcodes with names so that we can handle them with custom code. - let mayLoad = 1, mayStore = 1 in -defm MEMORY_COPY_A32 : +defm COPY_A#B : BULK_I<(outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx, - I32:$dst, I32:$src, I32:$len), + rc:$dst, rc:$src, rc:$len), (outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx), - [(wasm_memory_copy (i32 imm:$src_idx), (i32 imm:$dst_idx), - I32:$dst, I32:$src, I32:$len + [(wasm_memcpy (i32 imm:$src_idx), (i32 imm:$dst_idx), + rc:$dst, rc:$src, rc:$len )], "memory.copy\t$src_idx, $dst_idx, $dst, $src, $len", "memory.copy\t$src_idx, $dst_idx", 0x0a>; let mayStore = 1 in -defm MEMORY_FILL_A32 : - BULK_I<(outs), (ins i32imm_op:$idx, I32:$dst, I32:$value, I32:$size), +defm FILL_A#B : + BULK_I<(outs), (ins i32imm_op:$idx, rc:$dst, I32:$value, rc:$size), (outs), (ins i32imm_op:$idx), - [(wasm_memory_fill (i32 imm:$idx), I32:$dst, I32:$value, I32:$size)], + [(wasm_memset (i32 imm:$idx), rc:$dst, I32:$value, rc:$size)], "memory.fill\t$idx, $dst, $value, $size", "memory.fill\t$idx", 0x0b>; +} -let mayLoad = 1, mayStore = 1 in -defm MEMORY_COPY_A64 : - BULK_I<(outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx, - I64:$dst, I64:$src, I64:$len), - (outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx), - [(wasm_memory_copy (i32 imm:$src_idx), (i32 imm:$dst_idx), - I64:$dst, I64:$src, I64:$len - )], - "memory.copy\t$src_idx, $dst_idx, $dst, $src, $len", - "memory.copy\t$src_idx, $dst_idx", 0x0a>; +defm MEMORY_ : BulkMemoryOps; +defm MEMORY_ : BulkMemoryOps; -let mayStore = 1 in -defm MEMORY_FILL_A64 : - BULK_I<(outs), (ins i32imm_op:$idx, I64:$dst, I32:$value, I64:$size), - (outs), (ins i32imm_op:$idx), - [(wasm_memory_fill (i32 imm:$idx), I64:$dst, I32:$value, I64:$size)], - "memory.fill\t$idx, $dst, $value, $size", - "memory.fill\t$idx", 0x0b>; +// A multiclass for defining `memcpy`/`memset` pseudo instructions. These have +// the behavior the rest of LLVM CodeGen expects, and we lower them into code +// sequences that include the Wasm `memory.fill` and `memory.copy` instructions +// using custom inserters, because they introduce new control flow. +multiclass BulkMemOps { let usesCustomInserter = 1, isCodeGenOnly = 1, mayLoad = 1, mayStore = 1 in -defm MEMCPY_A32 : I<(outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx, - I32:$dst, I32:$src, I32:$len), +defm CPY_A#B : I<(outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx, + rc:$dst, rc:$src, rc:$len), (outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx), [(wasm_memcpy (i32 imm:$src_idx), (i32 imm:$dst_idx), - I32:$dst, I32:$src, I32:$len + rc:$dst, rc:$src, rc:$len )], "", "", 0>, Requires<[HasBulkMemory]>; let usesCustomInserter = 1, isCodeGenOnly = 1, mayStore = 1 in -defm MEMSET_A32 : I<(outs), (ins i32imm_op:$idx, I32:$dst, I32:$value, I32:$size), +defm SET_A#B : I<(outs), (ins i32imm_op:$idx, rc:$dst, I32:$value, rc:$size), (outs), (ins i32imm_op:$idx), - [(wasm_memset (i32 imm:$idx), I32:$dst, I32:$value, I32:$size)], + [(wasm_memset (i32 imm:$idx), rc:$dst, I32:$value, rc:$size)], "", "", 0>, Requires<[HasBulkMemory]>; -let usesCustomInserter = 1, isCodeGenOnly = 1, mayLoad = 1, mayStore = 1 in -defm MEMCPY_A64 : I<(outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx, - I64:$dst, I64:$src, I64:$len), - (outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx), - [(wasm_memcpy (i32 imm:$src_idx), (i32 imm:$dst_idx), - I64:$dst, I64:$src, I64:$len - )], - "", "", 0>, - Requires<[HasBulkMemory]>; +} -let usesCustomInserter = 1, isCodeGenOnly = 1, mayStore = 1 in -defm MEMSET_A64 : I<(outs), (ins i32imm_op:$idx, I64:$dst, I32:$value, I64:$size), - (outs), (ins i32imm_op:$idx), - [(wasm_memset (i32 imm:$idx), I64:$dst, I32:$value, I64:$size)], - "", "", 0>, - Requires<[HasBulkMemory]>; +defm MEM : BulkMemOps; +defm MEM : BulkMemOps; + +let hasSideEffects = 1 in +defm DATA_DROP : + BULK_I<(outs), (ins i32imm_op:$seg), (outs), (ins i32imm_op:$seg), + [], + "data.drop\t$seg", "data.drop\t$seg", 0x09>; From ac72747eccc6cd0d0f969bbf650138b2020939fb Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Thu, 17 Oct 2024 07:11:25 -0700 Subject: [PATCH 3/9] Factor out common types. --- .../WebAssembly/WebAssemblyInstrBulkMemory.td | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td index 4dc0c930b8f68..ce4d2cab8ff01 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td +++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td @@ -22,32 +22,27 @@ multiclass BULK_I, SDTCisInt<1>, SDTCisPtrTy<2>, SDTCisPtrTy<3>, SDTCisInt<4>] >; -def wasm_memory_copy : SDNode<"WebAssemblyISD::MEMORY_COPY", wasm_memory_copy_t, +def wasm_memsetlike_t : SDTypeProfile<0, 4, + [SDTCisInt<0>, SDTCisPtrTy<1>, SDTCisInt<2>, SDTCisInt<3>] +>; + +// memory.copy (may trap on empty ranges) +def wasm_memory_copy : SDNode<"WebAssemblyISD::MEMORY_COPY", wasm_memcpylike_t, [SDNPHasChain, SDNPMayLoad, SDNPMayStore]>; // memory.copy with a branch to avoid trapping -def wasm_memcpy_t : SDTypeProfile<0, 5, - [SDTCisInt<0>, SDTCisInt<1>, SDTCisPtrTy<2>, SDTCisPtrTy<3>, SDTCisInt<4>] ->; -def wasm_memcpy : SDNode<"WebAssemblyISD::MEMCPY", wasm_memcpy_t, +def wasm_memcpy : SDNode<"WebAssemblyISD::MEMCPY", wasm_memcpylike_t, [SDNPHasChain, SDNPMayLoad, SDNPMayStore]>; // memory.fill (may trap on empty ranges) -def wasm_memory_fill_t : SDTypeProfile<0, 4, - [SDTCisInt<0>, SDTCisPtrTy<1>, SDTCisInt<2>, SDTCisInt<3>] ->; -def wasm_memory_fill : SDNode<"WebAssemblyISD::MEMORY_FILL", wasm_memory_fill_t, +def wasm_memory_fill : SDNode<"WebAssemblyISD::MEMORY_FILL", wasm_memsetlike_t, [SDNPHasChain, SDNPMayStore]>; // memory.fill with a branch to avoid trapping -def wasm_memset_t : SDTypeProfile<0, 4, - [SDTCisInt<0>, SDTCisPtrTy<1>, SDTCisInt<2>, SDTCisInt<3>] ->; -def wasm_memset : SDNode<"WebAssemblyISD::MEMSET", wasm_memset_t, +def wasm_memset : SDNode<"WebAssemblyISD::MEMSET", wasm_memsetlike_t, [SDNPHasChain, SDNPMayStore]>; // A multiclass for defining Wasm's raw bulk-memory `memory.*` instructions. From 67a59c9f7701e578e244d75e90e42376b6a32a84 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Tue, 22 Oct 2024 16:42:26 -0700 Subject: [PATCH 4/9] Don't, like, say, "like". --- .../Target/WebAssembly/WebAssemblyInstrBulkMemory.td | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td index ce4d2cab8ff01..fbb5e04d5906f 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td +++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td @@ -22,27 +22,27 @@ multiclass BULK_I, SDTCisInt<1>, SDTCisPtrTy<2>, SDTCisPtrTy<3>, SDTCisInt<4>] >; -def wasm_memsetlike_t : SDTypeProfile<0, 4, +def wasm_memset_t : SDTypeProfile<0, 4, [SDTCisInt<0>, SDTCisPtrTy<1>, SDTCisInt<2>, SDTCisInt<3>] >; // memory.copy (may trap on empty ranges) -def wasm_memory_copy : SDNode<"WebAssemblyISD::MEMORY_COPY", wasm_memcpylike_t, +def wasm_memory_copy : SDNode<"WebAssemblyISD::MEMORY_COPY", wasm_memcpy_t, [SDNPHasChain, SDNPMayLoad, SDNPMayStore]>; // memory.copy with a branch to avoid trapping -def wasm_memcpy : SDNode<"WebAssemblyISD::MEMCPY", wasm_memcpylike_t, +def wasm_memcpy : SDNode<"WebAssemblyISD::MEMCPY", wasm_memcpy_t, [SDNPHasChain, SDNPMayLoad, SDNPMayStore]>; // memory.fill (may trap on empty ranges) -def wasm_memory_fill : SDNode<"WebAssemblyISD::MEMORY_FILL", wasm_memsetlike_t, +def wasm_memory_fill : SDNode<"WebAssemblyISD::MEMORY_FILL", wasm_memset_t, [SDNPHasChain, SDNPMayStore]>; // memory.fill with a branch to avoid trapping -def wasm_memset : SDNode<"WebAssemblyISD::MEMSET", wasm_memsetlike_t, +def wasm_memset : SDNode<"WebAssemblyISD::MEMSET", wasm_memset_t, [SDNPHasChain, SDNPMayStore]>; // A multiclass for defining Wasm's raw bulk-memory `memory.*` instructions. From 62974b86324dad56d89849c0ab4f50cbf6daac83 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Tue, 22 Oct 2024 18:06:01 -0700 Subject: [PATCH 5/9] Remove `WebAssemblyISD::MEMORY_COPY` and add comments. --- llvm/lib/Target/WebAssembly/WebAssemblyISD.def | 7 ++++--- .../WebAssembly/WebAssemblyInstrBulkMemory.td | 14 ++++---------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def index 90c373e59d187..ebe705bb54b9b 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def +++ b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def @@ -42,8 +42,6 @@ HANDLE_NODETYPE(PROMOTE_LOW) HANDLE_NODETYPE(TRUNC_SAT_ZERO_S) HANDLE_NODETYPE(TRUNC_SAT_ZERO_U) HANDLE_NODETYPE(DEMOTE_ZERO) -HANDLE_NODETYPE(MEMORY_COPY) -HANDLE_NODETYPE(MEMORY_FILL) HANDLE_NODETYPE(I64_ADD128) HANDLE_NODETYPE(I64_SUB128) HANDLE_NODETYPE(I64_MUL_WIDE_S) @@ -55,6 +53,9 @@ HANDLE_MEM_NODETYPE(GLOBAL_SET) HANDLE_MEM_NODETYPE(TABLE_GET) HANDLE_MEM_NODETYPE(TABLE_SET) -// Bulk memory instructions that require branching to handle empty ranges. +// Bulk memory instructions. These follow LLVM's expected semantics of +// supporting out-of-bounds pointers if the length is zero, by insertig +// a branch around Wasm's `memory.copy` and `memory.fill`, which would +// otherwise trap. HANDLE_NODETYPE(MEMCPY) HANDLE_NODETYPE(MEMSET) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td index fbb5e04d5906f..b3f3ab67241c7 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td +++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td @@ -29,19 +29,13 @@ def wasm_memset_t : SDTypeProfile<0, 4, [SDTCisInt<0>, SDTCisPtrTy<1>, SDTCisInt<2>, SDTCisInt<3>] >; -// memory.copy (may trap on empty ranges) -def wasm_memory_copy : SDNode<"WebAssemblyISD::MEMORY_COPY", wasm_memcpy_t, - [SDNPHasChain, SDNPMayLoad, SDNPMayStore]>; - -// memory.copy with a branch to avoid trapping +// memory.copy with a branch to avoid trapping in the case of out-of-bounds +// pointers with empty ranges. def wasm_memcpy : SDNode<"WebAssemblyISD::MEMCPY", wasm_memcpy_t, [SDNPHasChain, SDNPMayLoad, SDNPMayStore]>; -// memory.fill (may trap on empty ranges) -def wasm_memory_fill : SDNode<"WebAssemblyISD::MEMORY_FILL", wasm_memset_t, - [SDNPHasChain, SDNPMayStore]>; - -// memory.fill with a branch to avoid trapping +// memory.fill with a branch to avoid trapping in the case of out-of-bounds +// pointers with empty ranges. def wasm_memset : SDNode<"WebAssemblyISD::MEMSET", wasm_memset_t, [SDNPHasChain, SDNPMayStore]>; From fa5ee9394bf9b00b9fcc9399d95c8b9f5fe7afe4 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Wed, 23 Oct 2024 11:45:25 -0700 Subject: [PATCH 6/9] Fix copy+pasto in the memcpy patterns. --- llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td index b3f3ab67241c7..792365128f3ab 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td +++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td @@ -59,9 +59,7 @@ defm COPY_A#B : BULK_I<(outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx, rc:$dst, rc:$src, rc:$len), (outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx), - [(wasm_memcpy (i32 imm:$src_idx), (i32 imm:$dst_idx), - rc:$dst, rc:$src, rc:$len - )], + [], "memory.copy\t$src_idx, $dst_idx, $dst, $src, $len", "memory.copy\t$src_idx, $dst_idx", 0x0a>; @@ -69,7 +67,7 @@ let mayStore = 1 in defm FILL_A#B : BULK_I<(outs), (ins i32imm_op:$idx, rc:$dst, I32:$value, rc:$size), (outs), (ins i32imm_op:$idx), - [(wasm_memset (i32 imm:$idx), rc:$dst, I32:$value, rc:$size)], + [], "memory.fill\t$idx, $dst, $value, $size", "memory.fill\t$idx", 0x0b>; } From bc3d440fde55c1f97e9c96a62d6019dc330605fb Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Thu, 24 Oct 2024 07:55:02 -0700 Subject: [PATCH 7/9] Update llvm/lib/Target/WebAssembly/WebAssemblyISD.def Co-authored-by: Heejin Ahn --- llvm/lib/Target/WebAssembly/WebAssemblyISD.def | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def index ebe705bb54b9b..3502c47016c6b 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def +++ b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def @@ -54,7 +54,7 @@ HANDLE_MEM_NODETYPE(TABLE_GET) HANDLE_MEM_NODETYPE(TABLE_SET) // Bulk memory instructions. These follow LLVM's expected semantics of -// supporting out-of-bounds pointers if the length is zero, by insertig +// supporting out-of-bounds pointers if the length is zero, by inserting // a branch around Wasm's `memory.copy` and `memory.fill`, which would // otherwise trap. HANDLE_NODETYPE(MEMCPY) From 86b9f6fbe27bb8607c8ef22497ab8d6e7739559b Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Thu, 24 Oct 2024 07:55:12 -0700 Subject: [PATCH 8/9] Update llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td Co-authored-by: Heejin Ahn --- .../Target/WebAssembly/WebAssemblyInstrBulkMemory.td | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td index 792365128f3ab..1ba2691c6ed46 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td +++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td @@ -83,12 +83,12 @@ multiclass BulkMemOps { let usesCustomInserter = 1, isCodeGenOnly = 1, mayLoad = 1, mayStore = 1 in defm CPY_A#B : I<(outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx, - rc:$dst, rc:$src, rc:$len), - (outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx), - [(wasm_memcpy (i32 imm:$src_idx), (i32 imm:$dst_idx), - rc:$dst, rc:$src, rc:$len - )], - "", "", 0>, + rc:$dst, rc:$src, rc:$len), + (outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx), + [(wasm_memcpy (i32 imm:$src_idx), (i32 imm:$dst_idx), + rc:$dst, rc:$src, rc:$len + )], + "", "", 0>, Requires<[HasBulkMemory]>; let usesCustomInserter = 1, isCodeGenOnly = 1, mayStore = 1 in From 63f5bb517b271a06ee8dab825ffa145aa230a4c6 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Thu, 24 Oct 2024 07:55:21 -0700 Subject: [PATCH 9/9] Update llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td Co-authored-by: Heejin Ahn --- llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td index 1ba2691c6ed46..0772afb039f82 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td +++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td @@ -93,10 +93,10 @@ defm CPY_A#B : I<(outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx, let usesCustomInserter = 1, isCodeGenOnly = 1, mayStore = 1 in defm SET_A#B : I<(outs), (ins i32imm_op:$idx, rc:$dst, I32:$value, rc:$size), - (outs), (ins i32imm_op:$idx), - [(wasm_memset (i32 imm:$idx), rc:$dst, I32:$value, rc:$size)], - "", "", 0>, - Requires<[HasBulkMemory]>; + (outs), (ins i32imm_op:$idx), + [(wasm_memset (i32 imm:$idx), rc:$dst, I32:$value, rc:$size)], + "", "", 0>, + Requires<[HasBulkMemory]>; }