Division and Remainder Instruction Optimization on AArch64

src/machine/asm/execute_aarch64.S

@@ -466,58 +466,38 @@ ckb_vm_x64_execute:
   DECODE_R
   ldr RS1, REGISTER_ADDRESS(RS1)
   ldr RS2, REGISTER_ADDRESS(RS2)
+  sdiv TEMP1, RS1, RS2
   cmp RS2, 0
-  bne .div_branch2
-  mov RS1, UINT64_MAX
-  b .div_branch3
-.div_branch2:
-  sdiv RS1, RS1, RS2
-.div_branch3:
+  csinv RS1, TEMP1, xzr, ne
   WRITE_RD(RS1)
   NEXT_INST
 .CKB_VM_ASM_LABEL_OP_DIVU:
   DECODE_R
+  ldr RS1, REGISTER_ADDRESS(RS1)
   ldr RS2, REGISTER_ADDRESS(RS2)
+  udiv TEMP1, RS1, RS2
   cmp RS2, 0
-  bne .divu_branch1
-  mov TEMP1, UINT64_MAX
-  WRITE_RD(TEMP1)
-  NEXT_INST
-.divu_branch1:
-  ldr TEMP1, REGISTER_ADDRESS(RS1)
-  udiv TEMP1, TEMP1, RS2
+  csinv TEMP1, TEMP1, xzr, ne
   WRITE_RD(TEMP1)
   NEXT_INST
 .CKB_VM_ASM_LABEL_OP_DIVUW:
   DECODE_R
+  ldr RS1, REGISTER_ADDRESS(RS1)
   ldr RS2, REGISTER_ADDRESS(RS2)
-  mov RS2w, RS2w
-  cmp RS2, 0
-  bne .divuw_branch1
-  mov TEMP1, UINT64_MAX
-  WRITE_RD(TEMP1)
-  NEXT_INST
-.divuw_branch1:
-  ldr TEMP1, REGISTER_ADDRESS(RS1)
-  mov TEMP1w, TEMP1w
-  udiv TEMP1, TEMP1, RS2
+  udiv TEMP1w, RS1w, RS2w
   sxtw TEMP1, TEMP1w
+  cmp RS2w, 0
+  csinv TEMP1, TEMP1, xzr, ne
   WRITE_RD(TEMP1)
   NEXT_INST
 .CKB_VM_ASM_LABEL_OP_DIVW:
   DECODE_R
   ldr RS1, REGISTER_ADDRESS(RS1)
-  sxtw RS1, RS1w
   ldr RS2, REGISTER_ADDRESS(RS2)
-  sxtw RS2, RS2w
-  cmp RS2, 0
-  bne .divw_branch2
-  mov RS1, UINT64_MAX
-  b .divw_branch3
-.divw_branch2:
-  sdiv RS1, RS1, RS2
-.divw_branch3:
-  sxtw RS1, RS1w
+  sdiv TEMP1w, RS1w, RS2w
+  sxtw TEMP1, TEMP1w
+  cmp RS2w, 0
+  csinv RS1, TEMP1, xzr, ne
   WRITE_RD(RS1)
   NEXT_INST
 .CKB_VM_ASM_LABEL_OP_EBREAK:
@@ -753,59 +733,44 @@ ckb_vm_x64_execute:
   DECODE_R
   ldr RS1, REGISTER_ADDRESS(RS1)
   ldr RS2, REGISTER_ADDRESS(RS2)
-  cmp RS2, 0
-  bne .rem_branch2
-  b .rem_branch3
-.rem_branch2:
   sdiv TEMP1, RS1, RS2
-  msub RS1, TEMP1, RS2, RS1
-.rem_branch3:
+  msub TEMP2, TEMP1, RS2, RS1
+  cmp RS2, 0
+  csel RS1, RS1, TEMP2, eq
   WRITE_RD(RS1)
   NEXT_INST
 .CKB_VM_ASM_LABEL_OP_REMU:
   DECODE_R
   ldr RS1, REGISTER_ADDRESS(RS1)
   ldr RS2, REGISTER_ADDRESS(RS2)
-  cmp RS2, 0
-  bne .remu_branch2
-  WRITE_RD(RS1)
-  NEXT_INST
-.remu_branch2:
   udiv TEMP1, RS1, RS2
   msub TEMP2, TEMP1, RS2, RS1
+  cmp RS2, 0
+  csel TEMP2, RS1, TEMP2, eq
   WRITE_RD(TEMP2)
   NEXT_INST
 .CKB_VM_ASM_LABEL_OP_REMUW:
   DECODE_R
   ldr RS1, REGISTER_ADDRESS(RS1)
   ldr RS2, REGISTER_ADDRESS(RS2)
-  mov RS2w, RS2w
-  cmp RS2, 0
-  bne .remuw_branch2
-  sxtw RS1, RS1w
-  WRITE_RD(RS1)
-  NEXT_INST
-.remuw_branch2:
-  mov RS1w, RS1w
-  udiv TEMP1, RS1, RS2
-  msub TEMP2, TEMP1, RS2, RS1
+  udiv TEMP1w, RS1w, RS2w
+  msub TEMP2w, TEMP1w, RS2w, RS1w
   sxtw TEMP2, TEMP2w
+  sxtw RS1, RS1w
+  cmp RS2w, 0
+  csel TEMP2, RS1, TEMP2, eq
   WRITE_RD(TEMP2)
   NEXT_INST
 .CKB_VM_ASM_LABEL_OP_REMW:
   DECODE_R
   ldr RS1, REGISTER_ADDRESS(RS1)
-  sxtw RS1, RS1w
   ldr RS2, REGISTER_ADDRESS(RS2)
-  sxtw RS2, RS2w
-  cmp RS2, 0
-  bne .remw_branch2
-  b .remw_branch3
-.remw_branch2:
-  sdiv TEMP1, RS1, RS2
-  msub RS1, TEMP1, RS2, RS1
-.remw_branch3:
   sxtw RS1, RS1w
+  sdiv TEMP1w, RS1w, RS2w
+  msub TEMP2w, TEMP1w, RS2w, RS1w
+  sxtw TEMP2, TEMP2w
+  cmp RS2w, 0
+  csel RS1, RS1, TEMP2, eq
   WRITE_RD(RS1)
   NEXT_INST
 .CKB_VM_ASM_LABEL_OP_SB:

tests/programs/_build_all_native.sh

@@ -76,6 +76,7 @@ gcc_compile mop_wide_divide
 asm_link mop_wide_mul_zero
 gcc_compile mop_wide_multiply
 asm_link mulw mulw64
+gcc_compile division division64
 asm_link nop nop64
 asm_link nop_loop
 # SKIP: op_rvc_slli_crash_32

tests/programs/division.c

@@ -0,0 +1,129 @@
+#include <stdint.h>
+
+static int failed = 0;
+
+static void check_div(int64_t a, int64_t b, int64_t expected) {
+    int64_t result;
+    asm volatile("div %0, %1, %2" : "=r"(result) : "r"(a), "r"(b));
+    if (result != expected) {
+        failed = 1;
+    }
+}
+
+static void check_divu(uint64_t a, uint64_t b, uint64_t expected) {
+    uint64_t result;
+    asm volatile("divu %0, %1, %2" : "=r"(result) : "r"(a), "r"(b));
+    if (result != expected) {
+        failed = 1;
+    }
+}
+
+static void check_divw(int32_t a, int32_t b, int64_t expected) {
+    int64_t result;
+    asm volatile("divw %0, %1, %2" : "=r"(result) : "r"((int64_t)a), "r"((int64_t)b));
+    if (result != expected) {
+        failed = 1;
+    }
+}
+
+static void check_divuw(uint32_t a, uint32_t b, int64_t expected) {
+    int64_t result;
+    asm volatile("divuw %0, %1, %2" : "=r"(result) : "r"((int64_t)a), "r"((int64_t)b));
+    if (result != expected) {
+        failed = 1;
+    }
+}
+
+static void check_rem(int64_t a, int64_t b, int64_t expected) {
+    int64_t result;
+    asm volatile("rem %0, %1, %2" : "=r"(result) : "r"(a), "r"(b));
+    if (result != expected) {
+        failed = 1;
+    }
+}
+
+static void check_remu(uint64_t a, uint64_t b, uint64_t expected) {
+    uint64_t result;
+    asm volatile("remu %0, %1, %2" : "=r"(result) : "r"(a), "r"(b));
+    if (result != expected) {
+        failed = 1;
+    }
+}
+
+static void check_remw(int32_t a, int32_t b, int64_t expected) {
+    int64_t result;
+    asm volatile("remw %0, %1, %2" : "=r"(result) : "r"((int64_t)a), "r"((int64_t)b));
+    if (result != expected) {
+        failed = 1;
+    }
+}
+
+static void check_remuw(uint32_t a, uint32_t b, int64_t expected) {
+    int64_t result;
+    asm volatile("remuw %0, %1, %2" : "=r"(result) : "r"((int64_t)a), "r"((int64_t)b));
+    if (result != expected) {
+        failed = 1;
+    }
+}
+
+int main() {
+    /* div/divu */
+    check_div(10, 3, 3);
+    check_div(-10, 3, -3);
+    check_div(10, -3, -3);
+    check_div(-10, -3, 3);
+    check_div(42, 0, -1);
+    check_div(INT64_MIN, -1, INT64_MIN);
+
+    check_divu(10, 3, 3);
+    check_divu(0, 5, 0);
+    check_divu(42, 0, UINT64_MAX);
+    check_divu(0x8000000000000000ULL, 2, 0x4000000000000000ULL);
+
+    /* divw/divuw */
+    check_divw(10, 3, 3);
+    check_divw(-10, 3, -3);
+    check_divw(10, -3, -3);
+    check_divw(-10, -3, 3);
+    check_divw(42, 0, -1);
+    check_divw(-1, 0, -1);
+    check_divw(INT32_MIN, -1, (int64_t)INT32_MIN);
+    check_divw(INT32_MIN + 1, 1, (int64_t)(INT32_MIN + 1));
+
+    check_divuw(10, 3, 3);
+    check_divuw(0, 5, 0);
+    check_divuw(42, 0, -1);
+    check_divuw(0x80000000u, 2, (int64_t)(int32_t)0x40000000u);
+    check_divuw(0x80000000u, 1, (int64_t)INT32_MIN);
+    check_divuw(UINT32_MAX, 1, -1LL);
+    check_divuw(6, 2, 3);
+
+    /* rem/remu */
+    check_rem(10, 3, 1);
+    check_rem(-10, 3, -1);
+    check_rem(10, -3, 1);
+    check_rem(-10, -3, -1);
+    check_rem(42, 0, 42);
+    check_rem(-42, 0, -42);
+    check_rem(INT64_MIN, -1, 0);
+
+    check_remu(10, 3, 1);
+    check_remu(0, 5, 0);
+    check_remu(42, 0, 42);
+    check_remu(UINT64_MAX, 0, UINT64_MAX);
+
+    /* remw/remuw */
+    check_remw(10, 3, 1);
+    check_remw(-10, 3, -1);
+    check_remw(42, 0, 42);
+    check_remw(INT32_MIN, -1, 0);
+    check_remw(INT32_MIN, 0, (int64_t)INT32_MIN);
+
+    check_remuw(10, 3, 1);
+    check_remuw(42, 0, 42);
+    check_remuw(0x80000000u, 0, (int64_t)INT32_MIN);
+    check_remuw(UINT32_MAX, 0, -1LL);
+    check_remuw(0x80000000u, UINT32_MAX, (int64_t)INT32_MIN);
+
+    return failed;
+}

tests/test_asm.rs

@@ -199,6 +199,20 @@ pub fn test_misaligned_jump64() {
     assert!(result.is_ok());
 }
 
+#[test]
+pub fn test_division64() {
+    let buffer = fs::read("tests/programs/division64").unwrap().into();
+    let asm_core = <AsmCoreMachine as SupportMachine>::new(ISA_IMC, VERSION0, u64::MAX);
+    let core = AsmDefaultMachineBuilder::new(asm_core).build();
+    let mut machine = AsmMachine::new(core);
+    machine
+        .load_program(&buffer, [Ok("division64".into())].into_iter())
+        .unwrap();
+    let result = machine.run();
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), 0);
+}
+
 #[test]
 pub fn test_mulw64() {
     let buffer = fs::read("tests/programs/mulw64").unwrap().into();

tests/test_misc.rs

@@ -128,6 +128,14 @@ pub fn test_misaligned_jump64() {
     assert!(result.is_ok());
 }
 
+#[test]
+pub fn test_division64() {
+    let buffer = fs::read("tests/programs/division64").unwrap().into();
+    let result = run::<u64, SparseMemory<u64>>(&buffer, &vec!["division64".into()]);
+    assert!(result.is_ok());
+    assert_eq!(result.unwrap(), 0);
+}
+
 #[test]
 pub fn test_mulw64() {
     let buffer = fs::read("tests/programs/mulw64").unwrap().into();