Fix hybrid composition bugs (flutter#19325)

Author: Emmanuel Garcia

shell/platform/android/external_view_embedder/external_view_embedder.cc

@@ -26,12 +26,12 @@ void AndroidExternalViewEmbedder::PrerollCompositeEmbeddedView(
                "AndroidExternalViewEmbedder::PrerollCompositeEmbeddedView");
 
   auto rtree_factory = RTreeFactory();
-  view_rtrees_.insert({view_id, rtree_factory.getInstance()});
+  view_rtrees_.insert_or_assign(view_id, rtree_factory.getInstance());
 
   auto picture_recorder = std::make_unique<SkPictureRecorder>();
   picture_recorder->beginRecording(SkRect::Make(frame_size_), &rtree_factory);
 
-  picture_recorders_.insert({view_id, std::move(picture_recorder)});
+  picture_recorders_.insert_or_assign(view_id, std::move(picture_recorder));
   composition_order_.push_back(view_id);
   // Update params only if they changed.
   if (view_params_.count(view_id) == 1 &&
@@ -96,6 +96,8 @@ bool AndroidExternalViewEmbedder::SubmitFrame(
         picture_recorders_.at(view_id)->finishRecordingAsPicture();
     FML_CHECK(picture);
     pictures.insert({view_id, picture});
+    overlay_layers.insert({view_id, {}});
+
     sk_sp<RTree> rtree = view_rtrees_.at(view_id);
     // Determinate if Flutter UI intersects with any of the previous
     // platform views stacked by z position.
@@ -124,20 +126,17 @@ bool AndroidExternalViewEmbedder::SubmitFrame(
         intersection_rects.push_back(joined_rect);
       }
       for (SkRect& intersection_rect : intersection_rects) {
-        // Get the intersection rect between the current rect
-        // and the platform view rect.
-        // joined_rect.intersect(platform_view_rect);
         // Subpixels in the platform may not align with the canvas subpixels.
         //
         // To workaround it, round the floating point bounds and make the rect
         // slighly larger. For example, {0.3, 0.5, 3.1, 4.7} becomes {0, 0, 4,
         // 5}.
         intersection_rect.set(intersection_rect.roundOut());
+        overlay_layers.at(view_id).push_back(intersection_rect);
         // Clip the background canvas, so it doesn't contain any of the pixels
         // drawn on the overlay layer.
         background_canvas->clipRect(intersection_rect, SkClipOp::kDifference);
       }
-      overlay_layers.insert({current_view_id, intersection_rects});
     }
     background_canvas->drawPicture(pictures.at(view_id));
   }

shell/platform/android/io/flutter/embedding/android/FlutterImageView.java

@@ -41,13 +41,33 @@ public class FlutterImageView extends View implements RenderSurface {
   @Nullable private Bitmap currentBitmap;
   @Nullable private FlutterRenderer flutterRenderer;
 
+  public enum SurfaceKind {
+    /** Displays the background canvas. */
+    background,
+
+    /** Displays the overlay surface canvas. */
+    overlay,
+  }
+
+  /** The kind of surface. */
+  private SurfaceKind kind;
+
+  /**
+   * The number of images acquired from the current {@link android.media.ImageReader} that are
+   * waiting to be painted. This counter is decreased after calling {@link
+   * android.media.Image#close()}.
+   */
+  private int pendingImages = 0;
+
   /**
    * Constructs a {@code FlutterImageView} with an {@link android.media.ImageReader} that provides
    * the Flutter UI.
    */
-  public FlutterImageView(@NonNull Context context, @NonNull ImageReader imageReader) {
+  public FlutterImageView(
+      @NonNull Context context, @NonNull ImageReader imageReader, SurfaceKind kind) {
     super(context, null);
     this.imageReader = imageReader;
+    this.kind = kind;
   }
 
   @Nullable
@@ -62,29 +82,56 @@ public FlutterRenderer getAttachedRenderer() {
    */
   @Override
   public void attachToRenderer(@NonNull FlutterRenderer flutterRenderer) {
-    if (this.flutterRenderer != null) {
-      this.flutterRenderer.stopRenderingToSurface();
-    }
-
     this.flutterRenderer = flutterRenderer;
-    flutterRenderer.startRenderingToSurface(imageReader.getSurface());
+    switch (kind) {
+      case background:
+        flutterRenderer.swapSurface(imageReader.getSurface());
+        break;
+      case overlay:
+        // Don't do anything as this is done by the handler of
+        // `FlutterJNI#createOverlaySurface()` in the native side.
+        break;
+    }
   }
 
   /**
    * Invoked by the owner of this {@code FlutterImageView} when it no longer wants to render a
    * Flutter UI to this {@code FlutterImageView}.
    */
   public void detachFromRenderer() {
-    if (flutterRenderer != null) {
-      flutterRenderer.stopRenderingToSurface();
-      flutterRenderer = null;
+    switch (kind) {
+      case background:
+        // TODO: Swap the surface back to the original one.
+        // https://github.com/flutter/flutter/issues/58291
+        break;
+      case overlay:
+        // TODO: Handle this in the native side.
+        // https://github.com/flutter/flutter/issues/59904
+        break;
     }
   }
 
+  public void pause() {
+    // Not supported.
+  }
+
   /** Acquires the next image to be drawn to the {@link android.graphics.Canvas}. */
   @TargetApi(19)
   public void acquireLatestImage() {
-    nextImage = imageReader.acquireLatestImage();
+    // There's no guarantee that the image will be closed before the next call to
+    // `acquireLatestImage()`. For example, the device may not produce new frames if
+    // it's in sleep mode, so the calls to `invalidate()` will be queued up
+    // until the device produces a new frame.
+    //
+    // While the engine will also stop producing frames, there is a race condition.
+    //
+    // To avoid exceptions, check if a new image can be acquired.
+    if (pendingImages < imageReader.getMaxImages()) {
+      nextImage = imageReader.acquireLatestImage();
+      if (nextImage != null) {
+        pendingImages++;
+      }
+    }
     invalidate();
   }
 
@@ -94,6 +141,7 @@ protected void onDraw(Canvas canvas) {
     if (nextImage != null) {
       if (currentImage != null) {
         currentImage.close();
+        pendingImages--;
       }
       currentImage = nextImage;
       nextImage = null;

shell/platform/android/io/flutter/embedding/android/FlutterSurfaceView.java

@@ -200,7 +200,7 @@ public void detachFromRenderer() {
       // Make the SurfaceView invisible to avoid showing a black rectangle.
       setAlpha(0.0f);
 
-      this.flutterRenderer.removeIsDisplayingFlutterUiListener(flutterUiDisplayListener);
+      flutterRenderer.removeIsDisplayingFlutterUiListener(flutterUiDisplayListener);
 
       flutterRenderer = null;
       isAttachedToFlutterRenderer = false;
@@ -209,6 +209,21 @@ public void detachFromRenderer() {
     }
   }
 
+  /**
+   * Invoked by the owner of this {@code FlutterSurfaceView} when it should pause rendering Flutter
+   * UI to this {@code FlutterSurfaceView}.
+   */
+  public void pause() {
+    if (flutterRenderer != null) {
+      // Don't remove the `flutterUiDisplayListener` as `onFlutterUiDisplayed()` will make
+      // the `FlutterSurfaceView` visible.
+      flutterRenderer = null;
+      isAttachedToFlutterRenderer = false;
+    } else {
+      Log.w(TAG, "pause() invoked when no FlutterRenderer was attached.");
+    }
+  }
+
   // FlutterRenderer and getSurfaceTexture() must both be non-null.
   private void connectSurfaceToRenderer() {
     if (flutterRenderer == null || getHolder() == null) {

shell/platform/android/io/flutter/embedding/android/FlutterTextureView.java

@@ -173,6 +173,19 @@ public void detachFromRenderer() {
     }
   }
 
+  /**
+   * Invoked by the owner of this {@code FlutterTextureView} when it should pause rendering Flutter
+   * UI to this {@code FlutterTextureView}.
+   */
+  public void pause() {
+    if (flutterRenderer != null) {
+      flutterRenderer = null;
+      isAttachedToFlutterRenderer = false;
+    } else {
+      Log.w(TAG, "pause() invoked when no FlutterRenderer was attached.");
+    }
+  }
+
   // FlutterRenderer and getSurfaceTexture() must both be non-null.
   private void connectSurfaceToRenderer() {
     if (flutterRenderer == null || getSurfaceTexture() == null) {

shell/platform/android/io/flutter/embedding/android/FlutterView.java

@@ -943,16 +943,15 @@ public void detachFromFlutterEngine() {
   }
 
   public void convertToImageView() {
-    renderSurface.detachFromRenderer();
+    renderSurface.pause();
 
     ImageReader imageReader = PlatformViewsController.createImageReader(getWidth(), getHeight());
-    flutterImageView = new FlutterImageView(getContext(), imageReader);
+    flutterImageView =
+        new FlutterImageView(getContext(), imageReader, FlutterImageView.SurfaceKind.background);
     renderSurface = flutterImageView;
     if (flutterEngine != null) {
       renderSurface.attachToRenderer(flutterEngine.getRenderer());
     }
-
-    removeAllViews();
     addView(flutterImageView);
   }
 

shell/platform/android/io/flutter/embedding/engine/FlutterJNI.java

@@ -313,6 +313,23 @@ public void onSurfaceCreated(@NonNull Surface surface) {
 
   private native void nativeSurfaceCreated(long nativePlatformViewId, @NonNull Surface surface);
 
+  /**
+   * In hybrid composition, call this method when the {@link Surface} has changed.
+   *
+   * <p>In hybrid composition, the root surfaces changes from {@link
+   * android.view.SurfaceHolder#getSurface()} to {@link android.media.ImageReader#getSurface()} when
+   * a platform view is in the current frame.
+   */
+  @UiThread
+  public void onSurfaceWindowChanged(@NonNull Surface surface) {
+    ensureRunningOnMainThread();
+    ensureAttachedToNative();
+    nativeSurfaceWindowChanged(nativePlatformViewId, surface);
+  }
+
+  private native void nativeSurfaceWindowChanged(
+      long nativePlatformViewId, @NonNull Surface surface);
+
   /**
    * Call this method when the {@link Surface} changes that was previously registered with {@link
    * #onSurfaceCreated(Surface)}.

shell/platform/android/io/flutter/embedding/engine/renderer/FlutterRenderer.java

@@ -183,6 +183,18 @@ public void startRenderingToSurface(@NonNull Surface surface) {
     flutterJNI.onSurfaceCreated(surface);
   }
 
+  /**
+   * Swaps the {@link Surface} used to render the current frame.
+   *
+   * <p>In hybrid composition, the root surfaces changes from {@link
+   * android.view.SurfaceHolder#getSurface()} to {@link android.media.ImageReader#getSurface()} when
+   * a platform view is in the current frame.
+   */
+  public void swapSurface(@NonNull Surface surface) {
+    this.surface = surface;
+    flutterJNI.onSurfaceWindowChanged(surface);
+  }
+
   /**
    * Notifies Flutter that a {@code surface} previously registered with {@link
    * #startRenderingToSurface(Surface)} has changed size to the given {@code width} and {@code

shell/platform/android/io/flutter/embedding/engine/renderer/RenderSurface.java

@@ -53,4 +53,11 @@ public interface RenderSurface {
    * connected {@code FlutterRenderer}.
    */
   void detachFromRenderer();
+
+  /**
+   * Instructs this {@code RenderSurface} to stop forwarding {@code Surface} notifications to the
+   * {@code FlutterRenderer} that was previously connected with {@link
+   * #attachToRenderer(FlutterRenderer)}.
+   */
+  void pause();
 }

shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java

@@ -54,6 +54,7 @@ public class PlatformViewsController implements PlatformViewsAccessibilityDelega
   private Context context;
 
   // The View currently rendering the Flutter UI associated with these platform views.
+  // TODO(egarciad): Investigate if this can be downcasted to `FlutterView`.
   private View flutterView;
 
   // The texture registry maintaining the textures into which the embedded views will be rendered.
@@ -557,20 +558,23 @@ private void flushAllViews() {
     vdControllers.clear();
   }
 
+  private void initializeRootImageViewIfNeeded() {
+    if (!flutterViewConvertedToImageView) {
+      ((FlutterView) flutterView).convertToImageView();
+      flutterViewConvertedToImageView = true;
+    }
+  }
+
   public void onDisplayPlatformView(int viewId, int x, int y, int width, int height) {
+    initializeRootImageViewIfNeeded();
     // TODO: Implement this method. https://github.com/flutter/flutter/issues/58288
   }
 
   public void onDisplayOverlaySurface(int id, int x, int y, int width, int height) {
-    FlutterView flutterView = (FlutterView) this.flutterView;
-    if (!flutterViewConvertedToImageView) {
-      flutterView.convertToImageView();
-      flutterViewConvertedToImageView = true;
-    }
-
+    initializeRootImageViewIfNeeded();
     FlutterImageView overlayView = overlayLayerViews.get(id);
     if (overlayView.getParent() == null) {
-      flutterView.addView(overlayView);
+      ((FlutterView) flutterView).addView(overlayView);
     }
 
     FrameLayout.LayoutParams layoutParams = new FrameLayout.LayoutParams((int) width, (int) height);
@@ -579,7 +583,6 @@ public void onDisplayOverlaySurface(int id, int x, int y, int width, int height)
     overlayView.setLayoutParams(layoutParams);
     overlayView.setVisibility(View.VISIBLE);
     overlayView.bringToFront();
-
     currentFrameUsedOverlayLayerIds.add(id);
   }
 
@@ -599,8 +602,7 @@ public void onEndFrame() {
     }
 
     if (flutterViewConvertedToImageView) {
-      FlutterView flutterView = (FlutterView) this.flutterView;
-      flutterView.acquireLatestImageViewFrame();
+      ((FlutterView) flutterView).acquireLatestImageViewFrame();
     }
   }
 
@@ -621,7 +623,9 @@ public static ImageReader createImageReader(int width, int height) {
   @TargetApi(19)
   public FlutterOverlaySurface createOverlaySurface() {
     ImageReader imageReader = createImageReader(flutterView.getWidth(), flutterView.getHeight());
-    FlutterImageView imageView = new FlutterImageView(flutterView.getContext(), imageReader);
+    FlutterImageView imageView =
+        new FlutterImageView(
+            flutterView.getContext(), imageReader, FlutterImageView.SurfaceKind.overlay);
     int id = nextOverlayLayerId++;
     overlayLayerViews.put(id, imageView);
 

shell/platform/android/platform_view_android.cc

@@ -104,6 +104,22 @@ void PlatformViewAndroid::NotifyCreated(
   PlatformView::NotifyCreated();
 }
 
+void PlatformViewAndroid::NotifySurfaceWindowChanged(
+    fml::RefPtr<AndroidNativeWindow> native_window) {
+  if (android_surface_) {
+    fml::AutoResetWaitableEvent latch;
+    fml::TaskRunner::RunNowOrPostTask(
+        task_runners_.GetRasterTaskRunner(),
+        [&latch, surface = android_surface_.get(),
+         native_window = std::move(native_window)]() {
+          surface->TeardownOnScreenContext();
+          surface->SetNativeWindow(native_window);
+          latch.Signal();
+        });
+    latch.Wait();
+  }
+}
+
 void PlatformViewAndroid::NotifyDestroyed() {
   PlatformView::NotifyDestroyed();