Implement trackpad gestures in engine

Author: moffatman

lib/ui/platform_dispatcher.dart

@@ -303,7 +303,7 @@ class PlatformDispatcher {
   //  * pointer_data.cc
   //  * pointer.dart
   //  * AndroidTouchProcessor.java
-  static const int _kPointerDataFieldCount = 29;
+  static const int _kPointerDataFieldCount = 35;
 
   static PointerDataPacket _unpackPointerDataPacket(ByteData packet) {
     const int kStride = Int64List.bytesPerElement;
@@ -343,6 +343,12 @@ class PlatformDispatcher {
         platformData: packet.getInt64(kStride * offset++, _kFakeHostEndian),
         scrollDeltaX: packet.getFloat64(kStride * offset++, _kFakeHostEndian),
         scrollDeltaY: packet.getFloat64(kStride * offset++, _kFakeHostEndian),
+        panX: packet.getFloat64(kStride * offset++, _kFakeHostEndian),
+        panY: packet.getFloat64(kStride * offset++, _kFakeHostEndian),
+        panDeltaX: packet.getFloat64(kStride * offset++, _kFakeHostEndian),
+        panDeltaY: packet.getFloat64(kStride * offset++, _kFakeHostEndian),
+        scale: packet.getFloat64(kStride * offset++, _kFakeHostEndian),
+        angle: packet.getFloat64(kStride * offset++, _kFakeHostEndian),
       ));
       assert(offset == (i + 1) * _kPointerDataFieldCount);
     }

lib/ui/pointer.dart

@@ -36,6 +36,15 @@ enum PointerChange {
 
   /// The pointer has stopped making contact with the device.
   up,
+
+  /// An offscreen interaction gesture has started (such as trackpad gesture)
+  flowStart,
+
+  /// An offscreen interaction has changed
+  flowUpdate,
+
+  /// An offscreen interaction has ended
+  flowEnd,
 }
 
 /// The kind of pointer device.
@@ -101,6 +110,12 @@ class PointerData {
     this.platformData = 0,
     this.scrollDeltaX = 0.0,
     this.scrollDeltaY = 0.0,
+    this.panX = 0.0,
+    this.panY = 0.0,
+    this.panDeltaX = 0.0,
+    this.panDeltaY = 0.0,
+    this.scale = 0.0,
+    this.angle = 0.0,
   });
 
   /// Unique identifier that ties the [PointerEvent] to embedder event created it.
@@ -265,6 +280,36 @@ class PointerData {
   /// The amount to scroll in the y direction, in physical pixels.
   final double scrollDeltaY;
 
+  /// For events with change of PointerChange.flowUpdate:
+  ///
+  /// The current panning magnitude of the gesture in the x direction, in physical pixels.
+  final double panX;
+
+  /// For events with change of PointerChange.flowUpdate:
+  ///
+  /// The current panning magnitude of the gesture in the x direction, in physical pixels.
+  final double panY;
+
+  /// For events with change of PointerChange.flowUpdate:
+  ///
+  /// The difference in panning of the gesture in the x direction since the latest flowUpdate event, in physical pixels.
+  final double panDeltaX;
+
+  /// For events with change of PointerChange.flowUpdate:
+  ///
+  /// The difference in panning of the gesture in the y direction since the last flowUpdate event, in physical pixels.
+  final double panDeltaY;
+
+  /// For events with change of PointerChange.flowUpdate:
+  ///
+  /// The current scale of the gesture (unitless), with 1.0 as the initial scale.
+  final double scale;
+
+  /// For events with change of PointerChange.flowUpdate:
+  ///
+  /// The current angle of the gesture in radians, with 0.0 as the initial angle.
+  final double angle;
+
   @override
   String toString() => 'PointerData(x: $physicalX, y: $physicalY)';
 
@@ -298,7 +343,13 @@ class PointerData {
              'tilt: $tilt, '
              'platformData: $platformData, '
              'scrollDeltaX: $scrollDeltaX, '
-             'scrollDeltaY: $scrollDeltaY'
+             'scrollDeltaY: $scrollDeltaY, '
+             'panX: $panX, '
+             'panY: $panY, '
+             'panDeltaX: $panDeltaX, '
+             'panDeltaY: $panDeltaY, '
+             'scale: $scale, '
+             'angle: $angle'
            ')';
   }
 }

lib/ui/window/pointer_data.h

@@ -10,7 +10,7 @@
 namespace flutter {
 
 // If this value changes, update the pointer data unpacking code in hooks.dart.
-static constexpr int kPointerDataFieldCount = 29;
+static constexpr int kPointerDataFieldCount = 35;
 static constexpr int kBytesPerField = sizeof(int64_t);
 // Must match the button constants in events.dart.
 enum PointerButtonMouse : int64_t {
@@ -42,6 +42,9 @@ struct alignas(8) PointerData {
     kDown,
     kMove,
     kUp,
+    kFlowStart,
+    kFlowUpdate,
+    kFlowEnd,
   };
 
   // Must match the PointerDeviceKind enum in pointer.dart.
@@ -53,10 +56,7 @@ struct alignas(8) PointerData {
   };
 
   // Must match the PointerSignalKind enum in pointer.dart.
-  enum class SignalKind : int64_t {
-    kNone,
-    kScroll,
-  };
+  enum class SignalKind : int64_t { kNone, kScroll };
 
   int64_t embedder_id;
   int64_t time_stamp;
@@ -87,6 +87,12 @@ struct alignas(8) PointerData {
   int64_t platformData;
   double scroll_delta_x;
   double scroll_delta_y;
+  double pan_x;
+  double pan_y;
+  double pan_delta_x;
+  double pan_delta_y;
+  double scale;
+  double angle;
 
   void Clear();
 };

lib/ui/window/pointer_data_packet_converter.cc

@@ -4,6 +4,7 @@
 
 #include "flutter/lib/ui/window/pointer_data_packet_converter.h"
 
+#include <cmath>
 #include <cstring>
 
 #include "flutter/fml/logging.h"
@@ -207,6 +208,88 @@ void PointerDataPacketConverter::ConvertPointerData(
         converted_pointers.push_back(pointer_data);
         break;
       }
+      case PointerData::Change::kFlowStart: {
+        // Makes sure we have an existing pointer
+        auto iter = states_.find(pointer_data.device);
+        PointerState state;
+        if (iter == states_.end()) {
+          // Synthesizes add event if the pointer is not previously added.
+          PointerData synthesized_add_event = pointer_data;
+          synthesized_add_event.change = PointerData::Change::kAdd;
+          synthesized_add_event.synthesized = 1;
+          synthesized_add_event.buttons = 0;
+          state = EnsurePointerState(synthesized_add_event);
+          converted_pointers.push_back(synthesized_add_event);
+        } else {
+          state = iter->second;
+        }
+        FML_DCHECK(!state.is_down);
+        FML_DCHECK(!state.is_flow_active);
+        if (LocationNeedsUpdate(pointer_data, state)) {
+          // Synthesizes a hover event if the location does not match.
+          PointerData synthesized_hover_event = pointer_data;
+          synthesized_hover_event.change = PointerData::Change::kHover;
+          synthesized_hover_event.synthesized = 1;
+          synthesized_hover_event.buttons = 0;
+
+          UpdateDeltaAndState(synthesized_hover_event, state);
+          converted_pointers.push_back(synthesized_hover_event);
+        }
+
+        UpdatePointerIdentifier(pointer_data, state, true);
+        state.is_flow_active = true;
+        state.pan_x = 0;
+        state.pan_y = 0;
+        state.scale = 1;
+        state.angle = 0;
+        states_[pointer_data.device] = state;
+        converted_pointers.push_back(pointer_data);
+        break;
+      }
+      case PointerData::Change::kFlowUpdate: {
+        // Makes sure we have an existing pointer in flow_active state
+        auto iter = states_.find(pointer_data.device);
+        FML_DCHECK(iter != states_.end());
+        PointerState state = iter->second;
+        FML_DCHECK(!state.is_down);
+        FML_DCHECK(state.is_flow_active);
+
+        UpdatePointerIdentifier(pointer_data, state, false);
+        UpdateDeltaAndState(pointer_data, state);
+
+        converted_pointers.push_back(pointer_data);
+        break;
+      }
+      case PointerData::Change::kFlowEnd: {
+        // Makes sure we have an existing pointer in flow_active state
+        auto iter = states_.find(pointer_data.device);
+        FML_DCHECK(iter != states_.end());
+        PointerState state = iter->second;
+        FML_DCHECK(state.is_flow_active);
+
+        UpdatePointerIdentifier(pointer_data, state, false);
+
+        if (LocationNeedsUpdate(pointer_data, state)) {
+          // Synthesizes an update event if the location does not match.
+          PointerData synthesized_move_event = pointer_data;
+          synthesized_move_event.change = PointerData::Change::kFlowUpdate;
+          synthesized_move_event.pan_x = state.pan_x;
+          synthesized_move_event.pan_y = state.pan_y;
+          synthesized_move_event.pan_delta_x = 0;
+          synthesized_move_event.pan_delta_y = 0;
+          synthesized_move_event.scale = state.scale;
+          synthesized_move_event.angle = state.angle;
+          synthesized_move_event.synthesized = 1;
+
+          UpdateDeltaAndState(synthesized_move_event, state);
+          converted_pointers.push_back(synthesized_move_event);
+        }
+
+        state.is_flow_active = false;
+        states_[pointer_data.device] = state;
+        converted_pointers.push_back(pointer_data);
+        break;
+      }
       default: {
         converted_pointers.push_back(pointer_data);
         break;
@@ -261,8 +344,11 @@ PointerState PointerDataPacketConverter::EnsurePointerState(
   PointerState state;
   state.pointer_identifier = 0;
   state.is_down = false;
+  state.is_flow_active = false;
   state.physical_x = pointer_data.physical_x;
   state.physical_y = pointer_data.physical_y;
+  state.pan_x = pointer_data.pan_x;
+  state.pan_y = pointer_data.pan_y;
   states_[pointer_data.device] = state;
   return state;
 }
@@ -271,8 +357,14 @@ void PointerDataPacketConverter::UpdateDeltaAndState(PointerData& pointer_data,
                                                      PointerState& state) {
   pointer_data.physical_delta_x = pointer_data.physical_x - state.physical_x;
   pointer_data.physical_delta_y = pointer_data.physical_y - state.physical_y;
+  pointer_data.pan_delta_x = pointer_data.pan_x - state.pan_x;
+  pointer_data.pan_delta_y = pointer_data.pan_y - state.pan_y;
   state.physical_x = pointer_data.physical_x;
   state.physical_y = pointer_data.physical_y;
+  state.pan_x = pointer_data.pan_x;
+  state.pan_y = pointer_data.pan_y;
+  state.scale = pointer_data.scale;
+  state.angle = pointer_data.angle;
   states_[pointer_data.device] = state;
 }
 

lib/ui/window/pointer_data_packet_converter.h

@@ -30,8 +30,13 @@ namespace flutter {
 struct PointerState {
   int64_t pointer_identifier;
   bool is_down;
+  bool is_flow_active;
   double physical_x;
   double physical_y;
+  double pan_x;
+  double pan_y;
+  double scale;
+  double angle;
   int64_t buttons;
 };
 

lib/ui/window/pointer_data_packet_converter_unittests.cc

@@ -86,6 +86,51 @@ void CreateSimulatedMousePointerData(PointerData& data,  // NOLINT
   data.scroll_delta_y = scroll_delta_y;
 }
 
+void CreateSimulatedTrackpadGestureData(PointerData& data,  // NOLINT
+                                        PointerData::Change change,
+                                        int64_t device,
+                                        double dx,
+                                        double dy,
+                                        double pan_x,
+                                        double pan_y,
+                                        double scale,
+                                        double angle) {
+  data.time_stamp = 0;
+  data.change = change;
+  data.kind = PointerData::DeviceKind::kMouse;
+  data.signal_kind = PointerData::SignalKind::kNone;
+  data.device = device;
+  data.pointer_identifier = 0;
+  data.physical_x = dx;
+  data.physical_y = dy;
+  data.physical_delta_x = 0.0;
+  data.physical_delta_y = 0.0;
+  data.buttons = 0;
+  data.obscured = 0;
+  data.synthesized = 0;
+  data.pressure = 0.0;
+  data.pressure_min = 0.0;
+  data.pressure_max = 0.0;
+  data.distance = 0.0;
+  data.distance_max = 0.0;
+  data.size = 0.0;
+  data.radius_major = 0.0;
+  data.radius_minor = 0.0;
+  data.radius_min = 0.0;
+  data.radius_max = 0.0;
+  data.orientation = 0.0;
+  data.tilt = 0.0;
+  data.platformData = 0;
+  data.scroll_delta_x = 0.0;
+  data.scroll_delta_y = 0.0;
+  data.pan_x = pan_x;
+  data.pan_y = pan_y;
+  data.pan_delta_x = 0.0;
+  data.pan_delta_y = 0.0;
+  data.scale = scale;
+  data.angle = angle;
+}
+
 void UnpackPointerPacket(std::vector<PointerData>& output,  // NOLINT
                          std::unique_ptr<PointerDataPacket> packet) {
   size_t kBytesPerPointerData = kPointerDataFieldCount * kBytesPerField;
@@ -599,5 +644,56 @@ TEST(PointerDataPacketConverterTest, CanConvetScroll) {
   ASSERT_EQ(result[6].scroll_delta_y, 0.0);
 }
 
+TEST(PointerDataPacketConverterTest, CanConvertTrackpadGesture) {
+  PointerDataPacketConverter converter;
+  auto packet = std::make_unique<PointerDataPacket>(3);
+  PointerData data;
+  CreateSimulatedTrackpadGestureData(data, PointerData::Change::kFlowStart, 0,
+                                     0.0, 0.0, 0.0, 0.0, 1.0, 0.0);
+  packet->SetPointerData(0, data);
+  CreateSimulatedTrackpadGestureData(data, PointerData::Change::kFlowUpdate, 0,
+                                     0.0, 0.0, 3.0, 4.0, 1.0, 0.0);
+  packet->SetPointerData(1, data);
+  CreateSimulatedTrackpadGestureData(data, PointerData::Change::kFlowEnd, 0,
+                                     0.0, 0.0, 0.0, 0.0, 1.0, 0.0);
+  packet->SetPointerData(2, data);
+  auto converted_packet = converter.Convert(std::move(packet));
+
+  std::vector<PointerData> result;
+  UnpackPointerPacket(result, std::move(converted_packet));
+
+  ASSERT_EQ(result.size(), (size_t)4);
+  ASSERT_EQ(result[0].change, PointerData::Change::kAdd);
+  ASSERT_EQ(result[0].device, 0);
+  ASSERT_EQ(result[0].synthesized, 1);
+
+  ASSERT_EQ(result[1].change, PointerData::Change::kFlowStart);
+  ASSERT_EQ(result[1].signal_kind, PointerData::SignalKind::kNone);
+  ASSERT_EQ(result[1].device, 0);
+  ASSERT_EQ(result[1].physical_x, 0.0);
+  ASSERT_EQ(result[1].physical_y, 0.0);
+  ASSERT_EQ(result[1].synthesized, 0);
+
+  ASSERT_EQ(result[2].change, PointerData::Change::kFlowUpdate);
+  ASSERT_EQ(result[2].signal_kind, PointerData::SignalKind::kNone);
+  ASSERT_EQ(result[2].device, 0);
+  ASSERT_EQ(result[2].physical_x, 0.0);
+  ASSERT_EQ(result[2].physical_y, 0.0);
+  ASSERT_EQ(result[2].pan_x, 3.0);
+  ASSERT_EQ(result[2].pan_y, 4.0);
+  ASSERT_EQ(result[2].pan_delta_x, 3.0);
+  ASSERT_EQ(result[2].pan_delta_y, 4.0);
+  ASSERT_EQ(result[2].scale, 1.0);
+  ASSERT_EQ(result[2].angle, 0.0);
+  ASSERT_EQ(result[2].synthesized, 0);
+
+  ASSERT_EQ(result[3].change, PointerData::Change::kFlowEnd);
+  ASSERT_EQ(result[3].signal_kind, PointerData::SignalKind::kNone);
+  ASSERT_EQ(result[3].device, 0);
+  ASSERT_EQ(result[3].physical_x, 0.0);
+  ASSERT_EQ(result[3].physical_y, 0.0);
+  ASSERT_EQ(result[3].synthesized, 0);
+}
+
 }  // namespace testing
 }  // namespace flutter