Fix click outside and scrim drawing for WINDOW layers

compose/ui/ui/src/desktopMain/kotlin/androidx/compose/ui/awt/Utils.desktop.kt

@@ -17,10 +17,13 @@
 package androidx.compose.ui.awt
 
 import androidx.compose.ui.graphics.Color
+import androidx.compose.ui.unit.Density
+import androidx.compose.ui.unit.IntRect
 import java.awt.Component
-import java.awt.Transparency
+import java.awt.Rectangle
 import javax.swing.JComponent
-import org.jetbrains.skiko.ClipRectangle
+import kotlin.math.ceil
+import kotlin.math.floor
 import org.jetbrains.skiko.GraphicsApi
 import org.jetbrains.skiko.OS
 import org.jetbrains.skiko.hostOs
@@ -36,6 +39,18 @@ internal fun Component.isParentOf(component: Component?): Boolean {
     return false
 }
 
+internal fun IntRect.toAwtRectangle(density: Density): Rectangle {
+    val left = floor(left / density.density).toInt()
+    val top = floor(top / density.density).toInt()
+    val right = ceil(right / density.density).toInt()
+    val bottom = ceil(bottom / density.density).toInt()
+    val width = right - left
+    val height = bottom - top
+    return Rectangle(
+        left, top, width, height
+    )
+}
+
 internal fun Color.toAwtColor() = java.awt.Color(red, green, blue, alpha)
 
 internal fun getTransparentWindowBackground(

compose/ui/ui/src/desktopMain/kotlin/androidx/compose/ui/scene/ComposeContainer.desktop.kt

@@ -16,8 +16,8 @@
 
 package androidx.compose.ui.scene
 
-import java.awt.event.MouseEvent as AwtMouseEvent
 import java.awt.event.KeyEvent as AwtKeyEvent
+import java.awt.event.MouseEvent as AwtMouseEvent
 import androidx.compose.runtime.Composable
 import androidx.compose.runtime.CompositionContext
 import androidx.compose.runtime.CompositionLocalProvider
@@ -36,6 +36,7 @@ import androidx.compose.ui.skiko.OverlaySkikoViewDecorator
 import androidx.compose.ui.unit.Density
 import androidx.compose.ui.unit.LayoutDirection
 import androidx.compose.ui.util.fastForEach
+import androidx.compose.ui.util.fastForEachReversed
 import androidx.compose.ui.window.Dialog
 import androidx.compose.ui.window.Popup
 import androidx.compose.ui.window.WindowExceptionHandler
@@ -123,6 +124,7 @@ internal class ComposeContainer(
         },
         eventFilter = AwtEventFilters(
             OnlyValidPrimaryMouseButtonFilter,
+            DetectEventOutsideLayer(),
             FocusableLayerEventFilter()
         ),
         coroutineContext = coroutineContext,
@@ -197,7 +199,9 @@ internal class ComposeContainer(
      * Callback to let layers draw overlay on main [mediator].
      */
     private fun onRenderOverlay(canvas: Canvas, width: Int, height: Int) {
-        layers.fastForEach { it.onRenderOverlay(canvas, width, height) }
+        layers.fastForEach {
+            it.onRenderOverlay(canvas, width, height, windowContext.isWindowTransparent)
+        }
     }
 
     fun onChangeWindowTransparency(value: Boolean) {
@@ -312,6 +316,7 @@ internal class ComposeContainer(
             LayerType.OnWindow -> WindowComposeSceneLayer(
                 composeContainer = this,
                 skiaLayerAnalytics = skiaLayerAnalytics,
+                transparent = true, // TODO: Consider allowing opaque window layers
                 density = density,
                 layoutDirection = layoutDirection,
                 focusable = focusable,
@@ -329,12 +334,55 @@ internal class ComposeContainer(
         }
     }
 
+    /**
+     * Generates a sequence of layers that are positioned above the given layer in the layers list.
+     *
+     * @param layer the layer to find layers above
+     * @return a sequence of layers positioned above the given layer
+     */
+    fun layersAbove(layer: DesktopComposeSceneLayer) = sequence {
+        var isAbove = false
+        for (i in layers) {
+            if (i == layer) {
+                isAbove = true
+            } else if (isAbove) {
+                yield(i)
+            }
+        }
+    }
+
+    /**
+     * Notify layers about change in layers list. Required for additional invalidation and
+     * re-drawing if needed.
+     *
+     * @param layer the layer that triggered the change
+     */
+    private fun onLayersChange(layer: DesktopComposeSceneLayer) {
+        layers.fastForEach {
+            if (it != layer) {
+                it.onLayersChange()
+            }
+        }
+    }
+
+    /**
+     * Attaches a [DesktopComposeSceneLayer] to the list of layers.
+     *
+     * @param layer the layer to attach
+     */
     fun attachLayer(layer: DesktopComposeSceneLayer) {
         layers.add(layer)
+        onLayersChange(layer)
     }
 
+    /**
+     * Detaches a [DesktopComposeSceneLayer] from the list of layers.
+     *
+     * @param layer the layer to detach
+     */
     fun detachLayer(layer: DesktopComposeSceneLayer) {
         layers.remove(layer)
+        onLayersChange(layer)
     }
 
     fun createComposeSceneContext(platformContext: PlatformContext): ComposeSceneContext =
@@ -363,6 +411,22 @@ internal class ComposeContainer(
         }
     }
 
+    /**
+     * Detect and trigger [DesktopComposeSceneLayer.onMouseEventOutside] if event happened below
+     * focused layer.
+     */
+    private inner class DetectEventOutsideLayer : AwtEventFilter {
+        override fun shouldSendMouseEvent(event: AwtMouseEvent): Boolean {
+            layers.fastForEachReversed {
+                it.onMouseEventOutside(event)
+                if (it.focusable) {
+                    return true
+                }
+            }
+            return true
+        }
+    }
+
     private inner class FocusableLayerEventFilter : AwtEventFilter {
         private val noFocusableLayers get() = layers.all { !it.focusable }
 

compose/ui/ui/src/desktopMain/kotlin/androidx/compose/ui/scene/DesktopComposeSceneLayer.desktop.kt

@@ -16,6 +16,21 @@
 
 package androidx.compose.ui.scene
 
+import androidx.annotation.CallSuper
+import androidx.compose.runtime.Composable
+import androidx.compose.runtime.CompositionLocalContext
+import androidx.compose.ui.awt.AwtEventFilter
+import androidx.compose.ui.awt.AwtEventFilters
+import androidx.compose.ui.awt.OnlyValidPrimaryMouseButtonFilter
+import androidx.compose.ui.awt.toAwtRectangle
+import androidx.compose.ui.input.pointer.PointerEventType
+import androidx.compose.ui.unit.Density
+import androidx.compose.ui.unit.IntOffset
+import androidx.compose.ui.unit.LayoutDirection
+import androidx.compose.ui.util.fastForEachReversed
+import java.awt.event.KeyEvent
+import java.awt.event.MouseEvent
+import javax.swing.SwingUtilities
 import org.jetbrains.skia.Canvas
 
 /**
@@ -24,7 +39,67 @@ import org.jetbrains.skia.Canvas
  * @see SwingComposeSceneLayer
  * @see WindowComposeSceneLayer
  */
-internal abstract class DesktopComposeSceneLayer : ComposeSceneLayer {
+internal abstract class DesktopComposeSceneLayer(
+    protected val composeContainer: ComposeContainer,
+    density: Density,
+    layoutDirection: LayoutDirection,
+) : ComposeSceneLayer {
+    protected val windowContainer get() = composeContainer.windowContainer
+    protected val layersAbove get() = composeContainer.layersAbove(this)
+    protected val eventFilter get() = AwtEventFilters(
+        OnlyValidPrimaryMouseButtonFilter,
+        DetectEventOutsideLayer(),
+        FocusableLayerEventFilter()
+    )
+
+    protected abstract val mediator: ComposeSceneMediator?
+
+    private var outsidePointerCallback: ((eventType: PointerEventType) -> Unit)? = null
+    private var isClosed = false
+
+    final override var density: Density = density
+        set(value) {
+            field = value
+            mediator?.onChangeDensity(value)
+        }
+
+    final override var layoutDirection: LayoutDirection = layoutDirection
+        set(value) {
+            field = value
+            mediator?.onChangeLayoutDirection(value)
+        }
+
+    final override var compositionLocalContext: CompositionLocalContext?
+        get() = mediator?.compositionLocalContext
+        set(value) { mediator?.compositionLocalContext = value }
+
+    @CallSuper
+    override fun close() {
+        isClosed = true
+    }
+
+    final override fun setContent(content: @Composable () -> Unit) {
+        mediator?.setContent(content)
+    }
+
+    final override fun setKeyEventListener(
+        onPreviewKeyEvent: ((androidx.compose.ui.input.key.KeyEvent) -> Boolean)?,
+        onKeyEvent: ((androidx.compose.ui.input.key.KeyEvent) -> Boolean)?
+    ) {
+        mediator?.setKeyEventListeners(
+            onPreviewKeyEvent = onPreviewKeyEvent ?: { false },
+            onKeyEvent = onKeyEvent ?: { false }
+        )
+    }
+
+    final override fun setOutsidePointerEventListener(
+        onOutsidePointerEvent: ((eventType: PointerEventType) -> Unit)?
+    ) {
+        outsidePointerCallback = onOutsidePointerEvent
+    }
+
+    override fun calculateLocalPosition(positionInWindow: IntOffset) =
+        positionInWindow // [ComposeScene] is equal to [windowContainer] for the layer.
 
     /**
      * Called when the focus of the window containing main Compose view has changed.
@@ -45,12 +120,72 @@ internal abstract class DesktopComposeSceneLayer : ComposeSceneLayer {
     }
 
     /**
-     * Renders the overlay on the main Compose view canvas.
+     * Called when the layers in [composeContainer] have changed.
+     */
+    open fun onLayersChange() {
+    }
+
+    /**
+     * Renders an overlay on the canvas.
+     *
+     * @param canvas the canvas to render on
+     * @param width the width of the overlay
+     * @param height the height of the overlay
+     * @param transparent a flag indicating whether [canvas] is transparent
+     */
+    open fun onRenderOverlay(canvas: Canvas, width: Int, height: Int, transparent: Boolean) {
+    }
+
+    /**
+     * This method is called when a mouse event occurs outside of this layer.
      *
-     * @param canvas the canvas of the main Compose view
-     * @param width the width of the canvas
-     * @param height the height of the canvas
+     * @param event the mouse event
+     */
+    fun onMouseEventOutside(event: MouseEvent) {
+        if (isClosed || !event.isMainAction() || inBounds(event)) {
+            return
+        }
+        val eventType = when (event.id) {
+            MouseEvent.MOUSE_PRESSED -> PointerEventType.Press
+            MouseEvent.MOUSE_RELEASED -> PointerEventType.Release
+            else -> return
+        }
+        outsidePointerCallback?.invoke(eventType)
+    }
+
+    private fun inBounds(event: MouseEvent): Boolean {
+        val point = if (event.component != windowContainer) {
+            SwingUtilities.convertPoint(event.component, event.point, windowContainer)
+        } else {
+            event.point
+        }
+        return boundsInWindow.toAwtRectangle(density).contains(point)
+    }
+
+    /**
+     * Detect and trigger [DesktopComposeSceneLayer.onMouseEventOutside] if event happened below
+     * focused layer.
      */
-    open fun onRenderOverlay(canvas: Canvas, width: Int, height: Int) {
+    private inner class DetectEventOutsideLayer : AwtEventFilter {
+        override fun shouldSendMouseEvent(event: MouseEvent): Boolean {
+            layersAbove.toList().fastForEachReversed {
+                it.onMouseEventOutside(event)
+                if (it.focusable) {
+                    return true
+                }
+            }
+            return true
+        }
+    }
+
+    private inner class FocusableLayerEventFilter : AwtEventFilter {
+        private val noFocusableLayersAbove: Boolean
+            get() = layersAbove.all { !it.focusable }
+
+        override fun shouldSendMouseEvent(event: MouseEvent): Boolean = noFocusableLayersAbove
+        override fun shouldSendKeyEvent(event: KeyEvent): Boolean = focusable && noFocusableLayersAbove
     }
 }
+
+private fun MouseEvent.isMainAction() =
+    button == MouseEvent.BUTTON1