[vm/compiler] Various 64-bit operator improvements.

Rationale:
Improves the slow path of 64-bit REM/TRUNCDIV on
X64 and ARM64. Also introduces 64-bit negate operator,
which was needed to expose negative contants
(viz. x / -3 was represented as x / - (3) first).
The negate operator is not recognized in AOT mode yet,
since shifts by out-of-range constants should learn
how to throw rather than deopt....


#33967
flutter/flutter#19677

Change-Id: I7d81c9b1c72d99e8c4018f68c0501c7b599e073f
Reviewed-on: https://dart-review.googlesource.com/68280
Commit-Queue: Aart Bik <[email protected]>
Reviewed-by: Alexander Markov <[email protected]>
Reviewed-by: Vyacheslav Egorov <[email protected]>

Author: aartbik

runtime/vm/compiler/aot/aot_call_specializer.cc

@@ -565,7 +565,6 @@ bool AotCallSpecializer::TryOptimizeStaticCallUsingStaticTypes(
         }
         break;
       }
-      // TODO(dartbug.com/30480): Enable 64-bit integer shifts (SHL, SHR).
       case Token::kBIT_OR:
       case Token::kBIT_XOR:
       case Token::kBIT_AND:
@@ -608,7 +607,6 @@ bool AotCallSpecializer::TryOptimizeStaticCallUsingStaticTypes(
         }
         break;
       }
-
       case Token::kSHL:
       case Token::kSHR: {
         Value* left_value = instr->PushArgumentAt(receiver_index)->value();
@@ -622,7 +620,6 @@ bool AotCallSpecializer::TryOptimizeStaticCallUsingStaticTypes(
         }
         break;
       }
-
       default:
         break;
     }

runtime/vm/compiler/backend/il.h

@@ -6076,9 +6076,13 @@ class UnaryUint32OpInstr : public UnaryIntegerOpInstr {
 
 class UnaryInt64OpInstr : public UnaryIntegerOpInstr {
  public:
-  UnaryInt64OpInstr(Token::Kind op_kind, Value* value, intptr_t deopt_id)
-      : UnaryIntegerOpInstr(op_kind, value, deopt_id) {
-    ASSERT(op_kind == Token::kBIT_NOT);
+  UnaryInt64OpInstr(Token::Kind op_kind,
+                    Value* value,
+                    intptr_t deopt_id,
+                    SpeculativeMode speculative_mode = kGuardInputs)
+      : UnaryIntegerOpInstr(op_kind, value, deopt_id),
+        speculative_mode_(speculative_mode) {
+    ASSERT(op_kind == Token::kBIT_NOT || op_kind == Token::kNEGATE);
   }
 
   virtual bool ComputeCanDeoptimize() const { return false; }
@@ -6092,9 +6096,17 @@ class UnaryInt64OpInstr : public UnaryIntegerOpInstr {
     return kUnboxedInt64;
   }
 
+  virtual bool AttributesEqual(Instruction* other) const {
+    return UnaryIntegerOpInstr::AttributesEqual(other) &&
+           (speculative_mode() == other->speculative_mode());
+  }
+
+  virtual SpeculativeMode speculative_mode() const { return speculative_mode_; }
+
   DECLARE_INSTRUCTION(UnaryInt64Op)
 
  private:
+  const SpeculativeMode speculative_mode_;
   DISALLOW_COPY_AND_ASSIGN(UnaryInt64OpInstr);
 };
 

runtime/vm/compiler/backend/il_arm64.cc

@@ -5047,7 +5047,8 @@ class Int64DivideSlowPath : public ThrowErrorSlowPathCode {
   static const intptr_t kNumberOfArguments = 0;
 
   Int64DivideSlowPath(BinaryInt64OpInstr* instruction,
-                      Register right,
+                      Register divisor,
+                      Range* divisor_range,
                       Register tmp,
                       Register out,
                       intptr_t try_index)
@@ -5056,36 +5057,63 @@ class Int64DivideSlowPath : public ThrowErrorSlowPathCode {
                                kNumberOfArguments,
                                try_index),
         is_mod_(instruction->op_kind() == Token::kMOD),
-        right_(right),
+        divisor_(divisor),
+        divisor_range_(divisor_range),
         tmp_(tmp),
         out_(out),
         adjust_sign_label_() {}
 
   void EmitNativeCode(FlowGraphCompiler* compiler) override {
-    // Main entry throws, use code of superclass.
-    ThrowErrorSlowPathCode::EmitNativeCode(compiler);
-    // Adjust modulo for negative sign.
-    // if (right < 0)
-    //   out -= right;
+    // Handle modulo/division by zero, if needed. Use superclass code.
+    if (has_divide_by_zero()) {
+      ThrowErrorSlowPathCode::EmitNativeCode(compiler);
+    } else {
+      __ Bind(entry_label());  // not used, but keeps destructor happy
+      if (Assembler::EmittingComments()) {
+        __ Comment("slow path %s operation (no throw)", name());
+      }
+    }
+    // Adjust modulo for negative sign, optimized for known ranges.
+    // if (divisor < 0)
+    //   out -= divisor;
     // else
-    //   out += right;
-    if (is_mod_) {
+    //   out += divisor;
+    if (has_adjust_sign()) {
       __ Bind(adjust_sign_label());
-      __ CompareRegisters(right_, ZR);
-      __ sub(tmp_, out_, Operand(right_));
-      __ add(out_, out_, Operand(right_));
-      __ csel(out_, tmp_, out_, LT);
+      if (RangeUtils::Overlaps(divisor_range_, -1, 1)) {
+        // General case.
+        __ CompareRegisters(divisor_, ZR);
+        __ sub(tmp_, out_, Operand(divisor_));
+        __ add(out_, out_, Operand(divisor_));
+        __ csel(out_, tmp_, out_, LT);
+      } else if (divisor_range_->IsPositive()) {
+        // Always positive.
+        __ add(out_, out_, Operand(divisor_));
+      } else {
+        // Always negative.
+        __ sub(out_, out_, Operand(divisor_));
+      }
       __ b(exit_label());
     }
   }
 
   const char* name() override { return "int64 divide"; }
 
-  Label* adjust_sign_label() { return &adjust_sign_label_; }
+  bool has_divide_by_zero() { return RangeUtils::CanBeZero(divisor_range_); }
+
+  bool has_adjust_sign() { return is_mod_; }
+
+  bool is_needed() { return has_divide_by_zero() || has_adjust_sign(); }
+
+  Label* adjust_sign_label() {
+    ASSERT(has_adjust_sign());
+    return &adjust_sign_label_;
+  }
 
  private:
   bool is_mod_;
-  Register right_;
+  Register divisor_;
+  Range* divisor_range_;
   Register tmp_;
   Register out_;
   Label adjust_sign_label_;
@@ -5100,14 +5128,16 @@ static void EmitInt64ModTruncDiv(FlowGraphCompiler* compiler,
                                  Register out) {
   ASSERT(op_kind == Token::kMOD || op_kind == Token::kTRUNCDIV);
 
-  // Set up a slow path.
+  // Prepare a slow path.
+  Range* right_range = instruction->right()->definition()->range();
   Int64DivideSlowPath* slow_path = new (Z) Int64DivideSlowPath(
-      instruction, right, tmp, out, compiler->CurrentTryIndex());
-  compiler->AddSlowPathCode(slow_path);
+      instruction, right, right_range, tmp, out, compiler->CurrentTryIndex());
 
   // Handle modulo/division by zero exception on slow path.
-  __ CompareRegisters(right, ZR);
-  __ b(slow_path->entry_label(), EQ);
+  if (slow_path->has_divide_by_zero()) {
+    __ CompareRegisters(right, ZR);
+    __ b(slow_path->entry_label(), EQ);
+  }
 
   // Perform actual operation
   //   out = left % right
@@ -5124,7 +5154,10 @@ static void EmitInt64ModTruncDiv(FlowGraphCompiler* compiler,
     __ sdiv(out, left, right);
   }
 
-  __ Bind(slow_path->exit_label());
+  if (slow_path->is_needed()) {
+    __ Bind(slow_path->exit_label());
+    compiler->AddSlowPathCode(slow_path);
+  }
 }
 
 LocationSummary* BinaryInt64OpInstr::MakeLocationSummary(Zone* zone,
@@ -5571,11 +5604,18 @@ LocationSummary* UnaryInt64OpInstr::MakeLocationSummary(Zone* zone,
 }
 
 void UnaryInt64OpInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
-  ASSERT(op_kind() == Token::kBIT_NOT);
   const Register left = locs()->in(0).reg();
   const Register out = locs()->out(0).reg();
-  ASSERT(out == left);
-  __ mvn(out, left);
+  switch (op_kind()) {
+    case Token::kBIT_NOT:
+      __ mvn(out, left);
+      break;
+    case Token::kNEGATE:
+      __ sub(out, ZR, Operand(left));
+      break;
+    default:
+      UNREACHABLE();
+  }
 }
 
 CompileType BinaryUint32OpInstr::ComputeType() const {

runtime/vm/compiler/backend/il_x64.cc

@@ -5201,54 +5201,96 @@ class Int64DivideSlowPath : public ThrowErrorSlowPathCode {
   static const intptr_t kNumberOfArguments = 0;
 
   Int64DivideSlowPath(BinaryInt64OpInstr* instruction,
-                      Register right,
+                      Register divisor,
+                      Range* divisor_range,
                       intptr_t try_index)
       : ThrowErrorSlowPathCode(instruction,
                                kIntegerDivisionByZeroExceptionRuntimeEntry,
                                kNumberOfArguments,
                                try_index),
         is_mod_(instruction->op_kind() == Token::kMOD),
-        right_(right),
+        divisor_(divisor),
+        divisor_range_(divisor_range),
         div_by_minus_one_label_(),
         adjust_sign_label_() {}
 
   void EmitNativeCode(FlowGraphCompiler* compiler) override {
-    // Main entry throws, use code of superclass.
-    ThrowErrorSlowPathCode::EmitNativeCode(compiler);
-    // Handle modulo/division by minus one.
-    __ Bind(div_by_minus_one_label());
-    if (is_mod_) {
-      __ xorq(RDX, RDX);  // x % -1 =  0
+    // Handle modulo/division by zero, if needed. Use superclass code.
+    if (has_divide_by_zero()) {
+      ThrowErrorSlowPathCode::EmitNativeCode(compiler);
     } else {
-      __ negq(RAX);  // x / -1 = -x
+      __ Bind(entry_label());  // not used, but keeps destructor happy
+      if (Assembler::EmittingComments()) {
+        __ Comment("slow path %s operation (no throw)", name());
+      }
     }
-    __ jmp(exit_label());
-    // Adjust modulo for negative sign.
-    // if (right < 0)
-    //   out -= right;
+    // Handle modulo/division by minus one, if needed.
+    // Note: an exact -1 divisor is best optimized prior to codegen.
+    if (has_divide_by_minus_one()) {
+      __ Bind(div_by_minus_one_label());
+      if (is_mod_) {
+        __ xorq(RDX, RDX);  // x % -1 =  0
+      } else {
+        __ negq(RAX);  // x / -1 = -x
+      }
+      __ jmp(exit_label());
+    }
+    // Adjust modulo for negative sign, optimized for known ranges.
+    // if (divisor < 0)
+    //   out -= divisor;
     // else
-    //   out += right;
-    if (is_mod_) {
-      Label subtract;
+    //   out += divisor;
+    if (has_adjust_sign()) {
       __ Bind(adjust_sign_label());
-      __ testq(right_, right_);
-      __ j(LESS, &subtract, Assembler::kNearJump);
-      __ addq(RDX, right_);
-      __ jmp(exit_label());
-      __ Bind(&subtract);
-      __ subq(RDX, right_);
+      if (RangeUtils::Overlaps(divisor_range_, -1, 1)) {
+        // General case.
+        Label subtract;
+        __ testq(divisor_, divisor_);
+        __ j(LESS, &subtract, Assembler::kNearJump);
+        __ addq(RDX, divisor_);
+        __ jmp(exit_label());
+        __ Bind(&subtract);
+        __ subq(RDX, divisor_);
+      } else if (divisor_range_->IsPositive()) {
+        // Always positive.
+        __ addq(RDX, divisor_);
+      } else {
+        // Always negative.
+        __ subq(RDX, divisor_);
+      }
       __ jmp(exit_label());
     }
   }
 
   const char* name() override { return "int64 divide"; }
 
-  Label* div_by_minus_one_label() { return &div_by_minus_one_label_; }
-  Label* adjust_sign_label() { return &adjust_sign_label_; }
+  bool has_divide_by_zero() { return RangeUtils::CanBeZero(divisor_range_); }
+
+  bool has_divide_by_minus_one() {
+    return RangeUtils::Overlaps(divisor_range_, -1, -1);
+  }
+
+  bool has_adjust_sign() { return is_mod_; }
+
+  bool is_needed() {
+    return has_divide_by_zero() || has_divide_by_minus_one() ||
+           has_adjust_sign();
+  }
+
+  Label* div_by_minus_one_label() {
+    ASSERT(has_divide_by_minus_one());
+    return &div_by_minus_one_label_;
+  }
+
+  Label* adjust_sign_label() {
+    ASSERT(has_adjust_sign());
+    return &adjust_sign_label_;
+  }
 
  private:
   bool is_mod_;
-  Register right_;
+  Register divisor_;
+  Range* divisor_range_;
   Label div_by_minus_one_label_;
   Label adjust_sign_label_;
 };
@@ -5262,19 +5304,23 @@ static void EmitInt64ModTruncDiv(FlowGraphCompiler* compiler,
                                  Register out) {
   ASSERT(op_kind == Token::kMOD || op_kind == Token::kTRUNCDIV);
 
-  // Set up a slow path.
-  Int64DivideSlowPath* slow_path = new (Z)
-      Int64DivideSlowPath(instruction, right, compiler->CurrentTryIndex());
-  compiler->AddSlowPathCode(slow_path);
+  // Prepare a slow path.
+  Range* right_range = instruction->right()->definition()->range();
+  Int64DivideSlowPath* slow_path = new (Z) Int64DivideSlowPath(
+      instruction, right, right_range, compiler->CurrentTryIndex());
 
   // Handle modulo/division by zero exception on slow path.
-  __ testq(right, right);
-  __ j(EQUAL, slow_path->entry_label());
+  if (slow_path->has_divide_by_zero()) {
+    __ testq(right, right);
+    __ j(EQUAL, slow_path->entry_label());
+  }
 
   // Handle modulo/division by minus one explicitly on slow path
   // (to avoid arithmetic exception on 0x8000000000000000 / -1).
-  __ cmpq(right, Immediate(-1));
-  __ j(EQUAL, slow_path->div_by_minus_one_label());
+  if (slow_path->has_divide_by_minus_one()) {
+    __ cmpq(right, Immediate(-1));
+    __ j(EQUAL, slow_path->div_by_minus_one_label());
+  }
 
   // Perform actual operation
   //   out = left % right
@@ -5294,7 +5340,11 @@ static void EmitInt64ModTruncDiv(FlowGraphCompiler* compiler,
     ASSERT(out == RAX);
     ASSERT(tmp == RDX);
   }
-  __ Bind(slow_path->exit_label());
+
+  if (slow_path->is_needed()) {
+    __ Bind(slow_path->exit_label());
+    compiler->AddSlowPathCode(slow_path);
+  }
 }
 
 template <typename OperandType>
@@ -5393,11 +5443,19 @@ LocationSummary* UnaryInt64OpInstr::MakeLocationSummary(Zone* zone,
 }
 
 void UnaryInt64OpInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
-  ASSERT(op_kind() == Token::kBIT_NOT);
   const Register left = locs()->in(0).reg();
   const Register out = locs()->out(0).reg();
   ASSERT(out == left);
-  __ notq(left);
+  switch (op_kind()) {
+    case Token::kBIT_NOT:
+      __ notq(left);
+      break;
+    case Token::kNEGATE:
+      __ negq(left);
+      break;
+    default:
+      UNREACHABLE();
+  }
 }
 
 static void EmitShiftInt64ByConstant(FlowGraphCompiler* compiler,