Enhance image_filter_layer caching to filter a cached child (flutter#17175)

Author: flar

flow/layers/container_layer.cc

@@ -160,4 +160,36 @@ void ContainerLayer::UpdateSceneChildren(SceneUpdateContext& context) {
 
 #endif  // defined(OS_FUCHSIA)
 
+MergedContainerLayer::MergedContainerLayer() {
+  // Ensure the layer has only one direct child.
+  //
+  // Any children will actually be added as children of this empty
+  // ContainerLayer which can be accessed via ::GetContainerLayer().
+  // If only one child is ever added to this layer then that child
+  // will become the layer returned from ::GetCacheableChild().
+  // If multiple child layers are added, then this implicit container
+  // child becomes the cacheable child, but at the potential cost of
+  // not being as stable in the raster cache from frame to frame.
+  ContainerLayer::Add(std::make_shared<ContainerLayer>());
+}
+
+void MergedContainerLayer::Add(std::shared_ptr<Layer> layer) {
+  GetChildContainer()->Add(std::move(layer));
+}
+
+ContainerLayer* MergedContainerLayer::GetChildContainer() const {
+  FML_DCHECK(layers().size() == 1);
+
+  return static_cast<ContainerLayer*>(layers()[0].get());
+}
+
+Layer* MergedContainerLayer::GetCacheableChild() const {
+  ContainerLayer* child_container = GetChildContainer();
+  if (child_container->layers().size() == 1) {
+    return child_container->layers()[0].get();
+  }
+
+  return child_container;
+}
+
 }  // namespace flutter

flow/layers/container_layer.h

@@ -35,9 +35,6 @@ class ContainerLayer : public Layer {
   void UpdateSceneChildren(SceneUpdateContext& context);
 #endif  // defined(OS_FUCHSIA)
 
-  // For OpacityLayer to restructure to have a single child.
-  void ClearChildren() { layers_.clear(); }
-
   // Try to prepare the raster cache for a given layer.
   //
   // The raster cache would fail if either of the followings is true:
@@ -58,6 +55,81 @@ class ContainerLayer : public Layer {
   FML_DISALLOW_COPY_AND_ASSIGN(ContainerLayer);
 };
 
+//------------------------------------------------------------------------------
+/// Some ContainerLayer objects perform a rendering operation or filter on
+/// the rendered output of their children. Often that operation is changed
+/// slightly from frame to frame as part of an animation. During such an
+/// animation, the children can be cached if they are stable to avoid having
+/// to render them on every frame. Even if the children are not stable,
+/// rendering them into the raster cache during a Preroll operation will save
+/// an extra change of rendering surface during the Paint phase as compared
+/// to using the SaveLayer that would otherwise be needed with no caching.
+///
+/// Typically the Flutter Widget objects that lead to the creation of these
+/// layers will try to enforce only a single child Widget by their design.
+/// Unfortunately, the process of turning Widgets eventually into engine
+/// layers is not a 1:1 process so this layer might end up with multiple
+/// child layers even if the Widget only had a single child Widget.
+///
+/// When such a layer goes to cache the output of its children, it will
+/// need to supply a single layer to the cache mechanism since the raster
+/// cache uses a layer unique_id() as part of the cache key. If this layer
+/// ended up with multiple children, then it must first collect them into
+/// one layer for the cache mechanism. In order to provide a single layer
+/// for all of the children, this utility class will implicitly collect
+/// the children into a secondary ContainerLayer called the child container.
+///
+/// A by-product of creating a hidden child container, though, is that the
+/// child container is created new every time this layer is created with
+/// different properties, such as during an animation. In that scenario,
+/// it would be best to cache the single real child of this layer if it
+/// is unique and if it is stable from frame to frame. To facilitate this
+/// optimal caching strategy, this class implements two accessor methods
+/// to be used for different purposes:
+///
+/// When the layer needs to recurse to perform some operation on its children,
+/// it can call GetChildContainer() to return the hidden container containing
+/// all of the real children.
+///
+/// When the layer wants to cache the rendered contents of its children, it
+/// should call GetCacheableChild() for best performance. This method may
+/// end up returning the same layer as GetChildContainer(), but only if the
+/// conditions for optimal caching of a single child are not met.
+///
+class MergedContainerLayer : public ContainerLayer {
+ public:
+  MergedContainerLayer();
+
+  void Add(std::shared_ptr<Layer> layer) override;
+
+ protected:
+  /**
+   * @brief Returns the ContainerLayer used to hold all of the children of the
+   * MergedContainerLayer. Note that this may not be the best layer to use
+   * for caching the children.
+   *
+   * @see GetCacheableChild()
+   * @return the ContainerLayer child used to hold the children
+   */
+  ContainerLayer* GetChildContainer() const;
+
+  /**
+   * @brief Returns the best choice for a Layer object that can be used
+   * in RasterCache operations to cache the children.
+   *
+   * The returned Layer must represent all children and try to remain stable
+   * if the MergedContainerLayer is reconstructed in subsequent frames of
+   * the scene.
+   *
+   * @see GetChildContainer()
+   * @return the best candidate Layer for caching the children
+   */
+  Layer* GetCacheableChild() const;
+
+ private:
+  FML_DISALLOW_COPY_AND_ASSIGN(MergedContainerLayer);
+};
+
 }  // namespace flutter
 
 #endif  // FLUTTER_FLOW_LAYERS_CONTAINER_LAYER_H_

flow/layers/container_layer_unittests.cc

@@ -198,5 +198,75 @@ TEST_F(ContainerLayerTest, NeedsSystemComposite) {
                                                child_path2, child_paint2}}}));
 }
 
+TEST_F(ContainerLayerTest, MergedOneChild) {
+  SkPath child_path;
+  child_path.addRect(5.0f, 6.0f, 20.5f, 21.5f);
+  SkPaint child_paint(SkColors::kGreen);
+  SkMatrix initial_transform = SkMatrix::Translate(-0.5f, -0.5f);
+
+  auto mock_layer = std::make_shared<MockLayer>(child_path, child_paint);
+  auto layer = std::make_shared<MergedContainerLayer>();
+  layer->Add(mock_layer);
+
+  layer->Preroll(preroll_context(), initial_transform);
+  EXPECT_FALSE(preroll_context()->has_platform_view);
+  EXPECT_EQ(mock_layer->paint_bounds(), child_path.getBounds());
+  EXPECT_EQ(layer->paint_bounds(), child_path.getBounds());
+  EXPECT_TRUE(mock_layer->needs_painting());
+  EXPECT_TRUE(layer->needs_painting());
+  EXPECT_FALSE(mock_layer->needs_system_composite());
+  EXPECT_FALSE(layer->needs_system_composite());
+  EXPECT_EQ(mock_layer->parent_matrix(), initial_transform);
+  EXPECT_EQ(mock_layer->parent_cull_rect(), kGiantRect);
+
+  layer->Paint(paint_context());
+  EXPECT_EQ(mock_canvas().draw_calls(),
+            std::vector({MockCanvas::DrawCall{
+                0, MockCanvas::DrawPathData{child_path, child_paint}}}));
+}
+
+TEST_F(ContainerLayerTest, MergedMultipleChildren) {
+  SkPath child_path1;
+  child_path1.addRect(5.0f, 6.0f, 20.5f, 21.5f);
+  SkPath child_path2;
+  child_path2.addRect(58.0f, 2.0f, 16.5f, 14.5f);
+  SkPaint child_paint1(SkColors::kGray);
+  SkPaint child_paint2(SkColors::kGreen);
+  SkMatrix initial_transform = SkMatrix::Translate(-0.5f, -0.5f);
+
+  auto mock_layer1 = std::make_shared<MockLayer>(child_path1, child_paint1);
+  auto mock_layer2 = std::make_shared<MockLayer>(child_path2, child_paint2);
+  auto layer = std::make_shared<MergedContainerLayer>();
+  layer->Add(mock_layer1);
+  layer->Add(mock_layer2);
+
+  SkRect expected_total_bounds = child_path1.getBounds();
+  expected_total_bounds.join(child_path2.getBounds());
+  layer->Preroll(preroll_context(), initial_transform);
+  EXPECT_FALSE(preroll_context()->has_platform_view);
+  EXPECT_EQ(mock_layer1->paint_bounds(), child_path1.getBounds());
+  EXPECT_EQ(mock_layer2->paint_bounds(), child_path2.getBounds());
+  EXPECT_EQ(layer->paint_bounds(), expected_total_bounds);
+  EXPECT_TRUE(mock_layer1->needs_painting());
+  EXPECT_TRUE(mock_layer2->needs_painting());
+  EXPECT_TRUE(layer->needs_painting());
+  EXPECT_FALSE(mock_layer1->needs_system_composite());
+  EXPECT_FALSE(mock_layer2->needs_system_composite());
+  EXPECT_FALSE(layer->needs_system_composite());
+  EXPECT_EQ(mock_layer1->parent_matrix(), initial_transform);
+  EXPECT_EQ(mock_layer2->parent_matrix(), initial_transform);
+  EXPECT_EQ(mock_layer1->parent_cull_rect(), kGiantRect);
+  EXPECT_EQ(mock_layer2->parent_cull_rect(),
+            kGiantRect);  // Siblings are independent
+
+  layer->Paint(paint_context());
+  EXPECT_EQ(
+      mock_canvas().draw_calls(),
+      std::vector({MockCanvas::DrawCall{
+                       0, MockCanvas::DrawPathData{child_path1, child_paint1}},
+                   MockCanvas::DrawCall{0, MockCanvas::DrawPathData{
+                                               child_path2, child_paint2}}}));
+}
+
 }  // namespace testing
 }  // namespace flutter

flow/layers/image_filter_layer.cc

@@ -7,7 +7,9 @@
 namespace flutter {
 
 ImageFilterLayer::ImageFilterLayer(sk_sp<SkImageFilter> filter)
-    : filter_(std::move(filter)) {}
+    : filter_(std::move(filter)),
+      transformed_filter_(nullptr),
+      render_count_(1) {}
 
 void ImageFilterLayer::Preroll(PrerollContext* context,
                                const SkMatrix& matrix) {
@@ -16,39 +18,77 @@ void ImageFilterLayer::Preroll(PrerollContext* context,
   Layer::AutoPrerollSaveLayerState save =
       Layer::AutoPrerollSaveLayerState::Create(context);
 
-  child_paint_bounds_ = SkRect::MakeEmpty();
-  PrerollChildren(context, matrix, &child_paint_bounds_);
+  SkRect child_bounds = SkRect::MakeEmpty();
+  PrerollChildren(context, matrix, &child_bounds);
   if (filter_) {
-    const SkIRect filter_input_bounds = child_paint_bounds_.roundOut();
+    const SkIRect filter_input_bounds = child_bounds.roundOut();
     SkIRect filter_output_bounds =
         filter_->filterBounds(filter_input_bounds, SkMatrix::I(),
                               SkImageFilter::kForward_MapDirection);
-    set_paint_bounds(SkRect::Make(filter_output_bounds));
-  } else {
-    set_paint_bounds(child_paint_bounds_);
+    child_bounds = SkRect::Make(filter_output_bounds);
   }
+  set_paint_bounds(child_bounds);
+
+  transformed_filter_ = nullptr;
+  if (render_count_ >= kMinimumRendersBeforeCachingFilterLayer) {
+    // We have rendered this same ImageFilterLayer object enough
+    // times to consider its properties and children to be stable
+    // from frame to frame so we try to cache the layer itself
+    // for maximum performance.
+    TryToPrepareRasterCache(context, this, matrix);
+  } else {
+    // This ImageFilterLayer is not yet considered stable so we
+    // increment the count to measure how many times it has been
+    // seen from frame to frame.
+    render_count_++;
 
-  TryToPrepareRasterCache(context, this, matrix);
+    // Now we will try to pre-render the children into the cache.
+    // To apply the filter to pre-rendered children, we must first
+    // modify the filter to be aware of the transform under which
+    // the cached bitmap was produced. Some SkImageFilter
+    // instances can do this operation on some transforms and some
+    // (filters or transforms) cannot. We can only cache the children
+    // and apply the filter on the fly if this operation succeeds.
+    transformed_filter_ = filter_->makeWithLocalMatrix(matrix);
+    if (transformed_filter_) {
+      // With a modified SkImageFilter we can now try to cache the
+      // children to avoid their rendering costs if they remain
+      // stable between frames and also avoiding a rendering surface
+      // switch during the Paint phase even if they are not stable.
+      // This benefit is seen most during animations.
+      TryToPrepareRasterCache(context, GetCacheableChild(), matrix);
+    }
+  }
 }
 
 void ImageFilterLayer::Paint(PaintContext& context) const {
   TRACE_EVENT0("flutter", "ImageFilterLayer::Paint");
   FML_DCHECK(needs_painting());
 
-  if (context.raster_cache &&
-      context.raster_cache->Draw(this, *context.leaf_nodes_canvas)) {
-    return;
+  if (context.raster_cache) {
+    if (context.raster_cache->Draw(this, *context.leaf_nodes_canvas)) {
+      return;
+    }
+    if (transformed_filter_) {
+      SkPaint paint;
+      paint.setImageFilter(transformed_filter_);
+
+      if (context.raster_cache->Draw(GetCacheableChild(),
+                                     *context.leaf_nodes_canvas, &paint)) {
+        return;
+      }
+    }
   }
 
   SkPaint paint;
   paint.setImageFilter(filter_);
 
   // Normally a save_layer is sized to the current layer bounds, but in this
   // case the bounds of the child may not be the same as the filtered version
-  // so we use the child_paint_bounds_ which were snapshotted from the
-  // Preroll on the children before we adjusted them based on the filter.
-  Layer::AutoSaveLayer save_layer =
-      Layer::AutoSaveLayer::Create(context, child_paint_bounds_, &paint);
+  // so we use the bounds of the child container which do not include any
+  // modifications that the filter might apply.
+  Layer::AutoSaveLayer save_layer = Layer::AutoSaveLayer::Create(
+      context, GetChildContainer()->paint_bounds(), &paint);
   PaintChildren(context);
 }
 

flow/layers/image_filter_layer.h

@@ -11,7 +11,7 @@
 
 namespace flutter {
 
-class ImageFilterLayer : public ContainerLayer {
+class ImageFilterLayer : public MergedContainerLayer {
  public:
   ImageFilterLayer(sk_sp<SkImageFilter> filter);
 
@@ -20,8 +20,25 @@ class ImageFilterLayer : public ContainerLayer {
   void Paint(PaintContext& context) const override;
 
  private:
+  // The ImageFilterLayer might cache the filtered output of this layer
+  // if the layer remains stable (if it is not animating for instance).
+  // If the ImageFilterLayer is not the same between rendered frames,
+  // though, it will cache its children instead and filter their cached
+  // output on the fly.
+  // Caching just the children saves the time to render them and also
+  // avoids a rendering surface switch to draw them.
+  // Caching the layer itself avoids all of that and additionally avoids
+  // the cost of applying the filter, but can be worse than caching the
+  // children if the filter itself is not stable from frame to frame.
+  // This constant controls how many times we will Preroll and Paint this
+  // same ImageFilterLayer before we consider the layer and filter to be
+  // stable enough to switch from caching the children to caching the
+  // filtered output of this layer.
+  static constexpr int kMinimumRendersBeforeCachingFilterLayer = 3;
+
   sk_sp<SkImageFilter> filter_;
-  SkRect child_paint_bounds_;
+  sk_sp<SkImageFilter> transformed_filter_;
+  int render_count_;
 
   FML_DISALLOW_COPY_AND_ASSIGN(ImageFilterLayer);
 };