[fuchsia][a11y] Set explicit hit regions per compositor layer on GFX.

Author: abrush21

shell/platform/fuchsia/flutter/gfx_external_view_embedder.cc

@@ -10,6 +10,7 @@
 
 #include <algorithm>  // For std::clamp
 
+#include "flutter/fml/logging.h"
 #include "flutter/fml/trace_event.h"
 #include "third_party/skia/include/core/SkPicture.h"
 #include "third_party/skia/include/core/SkSurface.h"
@@ -169,8 +170,9 @@ void GfxExternalViewEmbedder::PrerollCompositeEmbeddedView(
   zx_handle_t handle = static_cast<zx_handle_t>(view_id);
   FML_CHECK(frame_layers_.count(handle) == 0);
 
-  frame_layers_.emplace(std::make_pair(EmbedderLayerId{handle},
-                                       EmbedderLayer(frame_size_, *params)));
+  frame_layers_.emplace(std::make_pair(
+      EmbedderLayerId{handle},
+      EmbedderLayer(frame_size_, *params, flutter::RTreeFactory())));
   frame_composition_order_.push_back(handle);
 }
 
@@ -200,8 +202,9 @@ void GfxExternalViewEmbedder::BeginFrame(
   frame_dpr_ = device_pixel_ratio;
 
   // Create the root layer.
-  frame_layers_.emplace(
-      std::make_pair(kRootLayerId, EmbedderLayer(frame_size, std::nullopt)));
+  frame_layers_.emplace(std::make_pair(
+      kRootLayerId,
+      EmbedderLayer(frame_size, std::nullopt, flutter::RTreeFactory())));
   frame_composition_order_.push_back(kRootLayerId);
 
   // Set up the input interceptor at the top of the scene, if applicable.
@@ -271,6 +274,19 @@ void GfxExternalViewEmbedder::SubmitFrame(
     }
   }
 
+  // Finish recording SkPictures.
+  {
+    TRACE_EVENT0("flutter", "FinishRecordingPictures");
+
+    for (const auto& surface_index : frame_surface_indices) {
+      const auto& layer = frame_layers_.find(surface_index.first);
+      FML_CHECK(layer != frame_layers_.end());
+      layer->second.picture =
+          layer->second.recorder->finishRecordingAsPicture();
+      FML_CHECK(layer->second.picture != nullptr);
+    }
+  }
+
   // Submit layers and platform views to Scenic in composition order.
   {
     TRACE_EVENT0("flutter", "SubmitLayers");
@@ -437,10 +453,18 @@ void GfxExternalViewEmbedder::SubmitFrame(
         FML_CHECK(scenic_layer_index <= scenic_layers_.size());
         if (scenic_layer_index == scenic_layers_.size()) {
           ScenicLayer new_layer{
-              .shape_node = scenic::ShapeNode(session_->get()),
-              .material = scenic::Material(session_->get()),
+              .layer_node = scenic::EntityNode(session_->get()),
+              .image =
+                  ScenicImage{
+                      .shape_node = scenic::ShapeNode(session_->get()),
+                      .material = scenic::Material(session_->get()),
+                  },
+              // We'll set hit regions later.
+              .hit_regions = {},
           };
-          new_layer.shape_node.SetMaterial(new_layer.material);
+          new_layer.layer_node.SetLabel("Flutter::Layer");
+          new_layer.layer_node.AddChild(new_layer.image.shape_node);
+          new_layer.image.shape_node.SetMaterial(new_layer.image.material);
           scenic_layers_.emplace_back(std::move(new_layer));
         }
 
@@ -491,25 +515,50 @@ void GfxExternalViewEmbedder::SubmitFrame(
             embedded_views_height;
         auto& scenic_layer = scenic_layers_[scenic_layer_index];
         auto& scenic_rect = found_rects->second[rect_index];
-        scenic_layer.shape_node.SetLabel("Flutter::Layer");
-        scenic_layer.shape_node.SetShape(scenic_rect);
-        scenic_layer.shape_node.SetTranslation(
+        auto& image = scenic_layer.image;
+        image.shape_node.SetLabel("Flutter::Layer::Image");
+        image.shape_node.SetShape(scenic_rect);
+        image.shape_node.SetTranslation(
             layer->second.surface_size.width() * 0.5f,
             layer->second.surface_size.height() * 0.5f, -layer_elevation);
-        scenic_layer.material.SetColor(SK_AlphaOPAQUE, SK_AlphaOPAQUE,
-                                       SK_AlphaOPAQUE, layer_opacity);
-        scenic_layer.material.SetTexture(surface_for_layer->GetImageId());
-
-        // Only the first (i.e. the bottom-most) layer should receive input.
-        // TODO: Workaround for invisible overlays stealing input. Remove when
-        // the underlying bug is fixed.
-        const fuchsia::ui::gfx::HitTestBehavior layer_hit_test_behavior =
-            first_layer ? fuchsia::ui::gfx::HitTestBehavior::kDefault
-                        : fuchsia::ui::gfx::HitTestBehavior::kSuppress;
-        scenic_layer.shape_node.SetHitTestBehavior(layer_hit_test_behavior);
+        image.material.SetColor(SK_AlphaOPAQUE, SK_AlphaOPAQUE, SK_AlphaOPAQUE,
+                                layer_opacity);
+        image.material.SetTexture(surface_for_layer->GetImageId());
+
+        // We'll set hit regions expliclty on a separate ShapeNode, so the image
+        // itself should be unhittable and semantically invisible.
+        image.shape_node.SetHitTestBehavior(
+            fuchsia::ui::gfx::HitTestBehavior::kSuppress);
+        image.shape_node.SetSemanticVisibility(false);
 
         // Attach the ScenicLayer to the main scene graph.
-        layer_tree_node_.AddChild(scenic_layer.shape_node);
+        layer_tree_node_.AddChild(scenic_layer.layer_node);
+
+        // Compute the set of non-overlapping set of bounding boxes for the
+        // painted content in this layer.
+        {
+          FML_CHECK(layer->second.rtree);
+          std::list<SkRect> intersection_rects =
+              layer->second.rtree->searchNonOverlappingDrawnRects(
+                  SkRect::Make(layer->second.surface_size));
+
+          // SkRect joined_rect = SkRect::MakeEmpty();
+          for (const SkRect& rect : intersection_rects) {
+            auto paint_bounds =
+                scenic::Rectangle(session_->get(), rect.width(), rect.height());
+            auto hit_region = scenic::ShapeNode(session_->get());
+            hit_region.SetLabel("Flutter::Layer::HitRegion");
+            hit_region.SetShape(paint_bounds);
+            hit_region.SetTranslation(rect.centerX(), rect.centerY(),
+                                      -layer_elevation);
+            hit_region.SetHitTestBehavior(
+                fuchsia::ui::gfx::HitTestBehavior::kDefault);
+            hit_region.SetSemanticVisibility(true);
+
+            scenic_layer.layer_node.AddChild(hit_region);
+            scenic_layer.hit_regions.push_back(std::move(hit_region));
+          }
+        }
       }
 
       // Reset for the next pass:
@@ -527,7 +576,11 @@ void GfxExternalViewEmbedder::SubmitFrame(
     session_->Present();
   }
 
-  // Render the recorded SkPictures into the surfaces.
+  // Flush pending skia operations.
+  // NOTE: This operation MUST occur AFTER the `Present() ` call above. We
+  // pipeline the Skia rendering work with scenic IPC, and scenic will delay
+  // internally until Skia is finished. So, doing this work before calling
+  // `Present()` would adversely affect performance.
   {
     TRACE_EVENT0("flutter", "RasterizeSurfaces");
 
@@ -548,13 +601,10 @@ void GfxExternalViewEmbedder::SubmitFrame(
 
       const auto& layer = frame_layers_.find(surface_index.first);
       FML_CHECK(layer != frame_layers_.end());
-      sk_sp<SkPicture> picture =
-          layer->second.recorder->finishRecordingAsPicture();
-      FML_CHECK(picture != nullptr);
 
       canvas->setMatrix(SkMatrix::I());
       canvas->clear(SK_ColorTRANSPARENT);
-      canvas->drawPicture(picture);
+      canvas->drawPicture(layer->second.picture);
       canvas->flush();
     }
   }
@@ -636,7 +686,16 @@ void GfxExternalViewEmbedder::Reset() {
 
   // Clear images on all layers so they aren't cached unnecessarily.
   for (auto& layer : scenic_layers_) {
-    layer.material.SetTexture(0);
+    layer.image.material.SetTexture(0);
+
+    // Detach hit regions; otherwise, they may persist across frames
+    // incorrectly.
+    for (auto& hit_region : layer.hit_regions) {
+      hit_region.Detach();
+    }
+
+    // Remove cached hit regions so that we don't recreate stale ones.
+    layer.hit_regions.clear();
   }
 }
 

shell/platform/fuchsia/flutter/gfx_external_view_embedder.h

@@ -17,6 +17,7 @@
 #include <vector>
 
 #include "flutter/flow/embedded_views.h"
+#include "flutter/flow/rtree.h"
 #include "flutter/fml/logging.h"
 #include "flutter/fml/macros.h"
 #include "flutter/shell/common/canvas_spy.h"
@@ -138,18 +139,23 @@ class GfxExternalViewEmbedder final : public flutter::ExternalViewEmbedder {
 
   struct EmbedderLayer {
     EmbedderLayer(const SkISize& frame_size,
-                  std::optional<flutter::EmbeddedViewParams> view_params)
-        : embedded_view_params(std::move(view_params)),
+                  std::optional<flutter::EmbeddedViewParams> view_params,
+                  flutter::RTreeFactory rtree_factory)
+        : rtree(rtree_factory.getInstance()),
+          embedded_view_params(std::move(view_params)),
           recorder(std::make_unique<SkPictureRecorder>()),
           canvas_spy(std::make_unique<flutter::CanvasSpy>(
-              recorder->beginRecording(frame_size.width(),
-                                       frame_size.height()))),
-          surface_size(frame_size) {}
+              recorder->beginRecording(SkRect::Make(frame_size),
+                                       &rtree_factory))),
+          surface_size(frame_size),
+          picture(nullptr) {}
 
+    sk_sp<flutter::RTree> rtree;
     std::optional<flutter::EmbeddedViewParams> embedded_view_params;
     std::unique_ptr<SkPictureRecorder> recorder;
     std::unique_ptr<flutter::CanvasSpy> canvas_spy;
     SkISize surface_size;
+    sk_sp<SkPicture> picture;
   };
   using EmbedderLayerId = std::optional<uint32_t>;
   constexpr static EmbedderLayerId kRootLayerId = EmbedderLayerId{};
@@ -172,11 +178,23 @@ class GfxExternalViewEmbedder final : public flutter::ExternalViewEmbedder {
     bool pending_focusable = true;
   };
 
-  struct ScenicLayer {
+  struct ScenicImage {
     scenic::ShapeNode shape_node;
     scenic::Material material;
   };
 
+  struct ScenicLayer {
+    // Root of the subtree containing the scenic resources for this layer.
+    scenic::EntityNode layer_node;
+
+    // Scenic resources used to render this layer's image.
+    ScenicImage image;
+
+    // Scenic resources that specify which parts of this layer are responsive
+    // to input.
+    std::vector<scenic::ShapeNode> hit_regions;
+  };
+
   std::shared_ptr<GfxSessionConnection> session_;
   std::shared_ptr<SurfaceProducer> surface_producer_;