From b303bd63f13561f8cb2c88acc31a16980bc17f35 Mon Sep 17 00:00:00 2001 From: nayuta Date: Mon, 19 Dec 2022 16:38:25 +0800 Subject: [PATCH 1/8] intersect --- .../android/external_view_embedder/external_view_embedder.cc | 4 ++-- .../darwin/ios/framework/Source/FlutterPlatformViews.mm | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/shell/platform/android/external_view_embedder/external_view_embedder.cc b/shell/platform/android/external_view_embedder/external_view_embedder.cc index c1054c78b9ee6..b588b2f3d822f 100644 --- a/shell/platform/android/external_view_embedder/external_view_embedder.cc +++ b/shell/platform/android/external_view_embedder/external_view_embedder.cc @@ -145,8 +145,8 @@ void AndroidExternalViewEmbedder::SubmitFrame( // // In this case, the rects are merged into a single one that is the union // of all the rects. - for (const SkRect& rect : intersection_rects) { - joined_rect.join(rect); + for (SkRect rect : intersection_rects) { + rect.intersect(current_view_rect); } } if (!joined_rect.isEmpty()) { diff --git a/shell/platform/darwin/ios/framework/Source/FlutterPlatformViews.mm b/shell/platform/darwin/ios/framework/Source/FlutterPlatformViews.mm index 12bdf28269d67..4c5468b8ed9ae 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterPlatformViews.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterPlatformViews.mm @@ -674,7 +674,8 @@ static bool ClipBoundsContainsPlatformViewBoundingRect(const SkRect& clip_bounds // https://github.com/flutter/flutter/issues/52510 if (allocation_size > kMaxLayerAllocations) { SkRect joined_rect; - for (const SkRect& rect : intersection_rects) { + for (SkRect rect : intersection_rects) { + rect.intersect(platform_view_rect); joined_rect.join(rect); } // Replace the rects in the intersection rects list for a single rect that is From 08f0127c59c759109da2f5e8595af0349454005b Mon Sep 17 00:00:00 2001 From: nayuta Date: Mon, 19 Dec 2022 16:52:48 +0800 Subject: [PATCH 2/8] add --- .../android/external_view_embedder/external_view_embedder.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/shell/platform/android/external_view_embedder/external_view_embedder.cc b/shell/platform/android/external_view_embedder/external_view_embedder.cc index b588b2f3d822f..83c3a6a9aef2c 100644 --- a/shell/platform/android/external_view_embedder/external_view_embedder.cc +++ b/shell/platform/android/external_view_embedder/external_view_embedder.cc @@ -147,6 +147,7 @@ void AndroidExternalViewEmbedder::SubmitFrame( // of all the rects. for (SkRect rect : intersection_rects) { rect.intersect(current_view_rect); + joined_rect.join(rect); } } if (!joined_rect.isEmpty()) { From 1adf801f3a2efeab5f5d8e2b89f0a1aeb94ebb1e Mon Sep 17 00:00:00 2001 From: nayuta Date: Mon, 19 Dec 2022 18:00:44 +0800 Subject: [PATCH 3/8] test fix --- .../external_view_embedder_unittests.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/shell/platform/android/external_view_embedder/external_view_embedder_unittests.cc b/shell/platform/android/external_view_embedder/external_view_embedder_unittests.cc index b174cd92703f9..648b04b48d1e8 100644 --- a/shell/platform/android/external_view_embedder/external_view_embedder_unittests.cc +++ b/shell/platform/android/external_view_embedder/external_view_embedder_unittests.cc @@ -425,7 +425,7 @@ TEST(AndroidExternalViewEmbedder, SubmitFrame) { 0, 150, 150, 300, 300, 300, 300, stack1)); // The JNI call to display the overlay surface. EXPECT_CALL(*jni_mock, - FlutterViewDisplayOverlaySurface(0, 50, 50, 200, 200)); + FlutterViewDisplayOverlaySurface(0, 50, 50, 150, 150)); auto did_submit_frame = false; auto surface_frame = std::make_unique( @@ -491,7 +491,7 @@ TEST(AndroidExternalViewEmbedder, SubmitFrame) { 0, 150, 150, 300, 300, 300, 300, stack1)); // The JNI call to display the overlay surface. EXPECT_CALL(*jni_mock, - FlutterViewDisplayOverlaySurface(0, 50, 50, 200, 200)); + FlutterViewDisplayOverlaySurface(0, 50, 50, 150, 150)); auto did_submit_frame = false; auto surface_frame = std::make_unique( @@ -776,7 +776,7 @@ TEST(AndroidExternalViewEmbedder, DestroyOverlayLayersOnSizeChange) { EXPECT_CALL(*jni_mock, FlutterViewOnDisplayPlatformView(0, 0, 0, 200, 200, 300, 300, stack1)); EXPECT_CALL(*jni_mock, - FlutterViewDisplayOverlaySurface(0, 50, 50, 200, 200)); + FlutterViewDisplayOverlaySurface(0, 50, 50, 150, 150)); SurfaceFrame::FramebufferInfo framebuffer_info; auto surface_frame = std::make_unique( @@ -865,7 +865,7 @@ TEST(AndroidExternalViewEmbedder, DoesNotDestroyOverlayLayersOnSizeChange) { EXPECT_CALL(*jni_mock, FlutterViewOnDisplayPlatformView(0, 0, 0, 200, 200, 300, 300, stack1)); EXPECT_CALL(*jni_mock, - FlutterViewDisplayOverlaySurface(0, 50, 50, 200, 200)); + FlutterViewDisplayOverlaySurface(0, 50, 50, 150, 150)); auto surface_frame = std::make_unique( SkSurface::MakeNull(1000, 1000), framebuffer_info, From fc288eb4fedaf14aee39dfb7f3052140907a8f8f Mon Sep 17 00:00:00 2001 From: nayuta Date: Mon, 19 Dec 2022 18:43:33 +0800 Subject: [PATCH 4/8] fix test --- .../external_view_embedder_unittests.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shell/platform/android/external_view_embedder/external_view_embedder_unittests.cc b/shell/platform/android/external_view_embedder/external_view_embedder_unittests.cc index 648b04b48d1e8..0411b0cbbac38 100644 --- a/shell/platform/android/external_view_embedder/external_view_embedder_unittests.cc +++ b/shell/platform/android/external_view_embedder/external_view_embedder_unittests.cc @@ -425,7 +425,7 @@ TEST(AndroidExternalViewEmbedder, SubmitFrame) { 0, 150, 150, 300, 300, 300, 300, stack1)); // The JNI call to display the overlay surface. EXPECT_CALL(*jni_mock, - FlutterViewDisplayOverlaySurface(0, 50, 50, 150, 150)); + FlutterViewDisplayOverlaySurface(0, 150, 150, 100, 100)); auto did_submit_frame = false; auto surface_frame = std::make_unique( @@ -491,7 +491,7 @@ TEST(AndroidExternalViewEmbedder, SubmitFrame) { 0, 150, 150, 300, 300, 300, 300, stack1)); // The JNI call to display the overlay surface. EXPECT_CALL(*jni_mock, - FlutterViewDisplayOverlaySurface(0, 50, 50, 150, 150)); + FlutterViewDisplayOverlaySurface(0, 150, 150, 100, 100)); auto did_submit_frame = false; auto surface_frame = std::make_unique( From dd5b92f734b4aba159b2a218bfc1420ba9d06747 Mon Sep 17 00:00:00 2001 From: nayuta Date: Tue, 20 Dec 2022 08:10:29 +0800 Subject: [PATCH 5/8] modify --- .../external_view_embedder/external_view_embedder.cc | 7 +++++-- .../darwin/ios/framework/Source/FlutterPlatformViews.mm | 3 +-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/shell/platform/android/external_view_embedder/external_view_embedder.cc b/shell/platform/android/external_view_embedder/external_view_embedder.cc index 83c3a6a9aef2c..1fa879b0fbea0 100644 --- a/shell/platform/android/external_view_embedder/external_view_embedder.cc +++ b/shell/platform/android/external_view_embedder/external_view_embedder.cc @@ -145,10 +145,13 @@ void AndroidExternalViewEmbedder::SubmitFrame( // // In this case, the rects are merged into a single one that is the union // of all the rects. - for (SkRect rect : intersection_rects) { - rect.intersect(current_view_rect); + for (const SkRect& rect : intersection_rects) { joined_rect.join(rect); } + + // Get the intersection rect between the current rect + // and the platform view rect. + joined_rect.intersect(current_view_rect) } if (!joined_rect.isEmpty()) { // Subpixels in the platform may not align with the canvas subpixels. diff --git a/shell/platform/darwin/ios/framework/Source/FlutterPlatformViews.mm b/shell/platform/darwin/ios/framework/Source/FlutterPlatformViews.mm index 4c5468b8ed9ae..12bdf28269d67 100644 --- a/shell/platform/darwin/ios/framework/Source/FlutterPlatformViews.mm +++ b/shell/platform/darwin/ios/framework/Source/FlutterPlatformViews.mm @@ -674,8 +674,7 @@ static bool ClipBoundsContainsPlatformViewBoundingRect(const SkRect& clip_bounds // https://github.com/flutter/flutter/issues/52510 if (allocation_size > kMaxLayerAllocations) { SkRect joined_rect; - for (SkRect rect : intersection_rects) { - rect.intersect(platform_view_rect); + for (const SkRect& rect : intersection_rects) { joined_rect.join(rect); } // Replace the rects in the intersection rects list for a single rect that is From 79406def7de3627c7072c024a7de849b49c083d2 Mon Sep 17 00:00:00 2001 From: nayuta Date: Tue, 20 Dec 2022 08:23:05 +0800 Subject: [PATCH 6/8] typo --- .../android/external_view_embedder/external_view_embedder.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell/platform/android/external_view_embedder/external_view_embedder.cc b/shell/platform/android/external_view_embedder/external_view_embedder.cc index 1fa879b0fbea0..4377074c8d27f 100644 --- a/shell/platform/android/external_view_embedder/external_view_embedder.cc +++ b/shell/platform/android/external_view_embedder/external_view_embedder.cc @@ -151,7 +151,7 @@ void AndroidExternalViewEmbedder::SubmitFrame( // Get the intersection rect between the current rect // and the platform view rect. - joined_rect.intersect(current_view_rect) + joined_rect.intersect(current_view_rect); } if (!joined_rect.isEmpty()) { // Subpixels in the platform may not align with the canvas subpixels. From ebca48fce91ebd0066b9b8a7f8a81ccecebfce48 Mon Sep 17 00:00:00 2001 From: nayuta Date: Tue, 20 Dec 2022 16:27:12 +0800 Subject: [PATCH 7/8] modify --- .../external_view_embedder.cc | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/shell/platform/android/external_view_embedder/external_view_embedder.cc b/shell/platform/android/external_view_embedder/external_view_embedder.cc index 4377074c8d27f..b8d20e3f1578c 100644 --- a/shell/platform/android/external_view_embedder/external_view_embedder.cc +++ b/shell/platform/android/external_view_embedder/external_view_embedder.cc @@ -126,7 +126,7 @@ void AndroidExternalViewEmbedder::SubmitFrame( slice->end_recording(); - SkRect joined_rect = SkRect::MakeEmpty(); + SkRect full_joined_rect = SkRect::MakeEmpty(); // Determinate if Flutter UI intersects with any of the previous // platform views stacked by z position. @@ -137,6 +137,8 @@ void AndroidExternalViewEmbedder::SubmitFrame( for (ssize_t j = i; j >= 0; j--) { int64_t current_view_id = composition_order_[j]; SkRect current_view_rect = GetViewRect(current_view_id); + // The rect above the `current_view_rect` + SkRect partial_joined_rect = SkRect::MakeEmpty(); // Each rect corresponds to a native view that renders Flutter UI. std::list intersection_rects = slice->searchNonOverlappingDrawnRects(current_view_rect); @@ -146,25 +148,26 @@ void AndroidExternalViewEmbedder::SubmitFrame( // In this case, the rects are merged into a single one that is the union // of all the rects. for (const SkRect& rect : intersection_rects) { - joined_rect.join(rect); + partial_joined_rect.join(rect); } - - // Get the intersection rect between the current rect - // and the platform view rect. - joined_rect.intersect(current_view_rect); + // Get the intersection rect with the `current_view_rect`, + partial_joined_rect.intersect(current_view_rect); + // Join the `partial_joined_rect` into `full_joined_rect` to get the rect + // above the current `slice` + full_joined_rect.join(partial_joined_rect); } - if (!joined_rect.isEmpty()) { + if (!full_joined_rect.isEmpty()) { // Subpixels in the platform may not align with the canvas subpixels. // // To workaround it, round the floating point bounds and make the rect // slightly larger. // // For example, {0.3, 0.5, 3.1, 4.7} becomes {0, 0, 4, 5}. - joined_rect.set(joined_rect.roundOut()); - overlay_layers.insert({view_id, joined_rect}); + full_joined_rect.set(full_joined_rect.roundOut()); + overlay_layers.insert({view_id, full_joined_rect}); // Clip the background canvas, so it doesn't contain any of the pixels // drawn on the overlay layer. - background_canvas->clipRect(joined_rect, SkClipOp::kDifference); + background_canvas->clipRect(full_joined_rect, SkClipOp::kDifference); } if (background_builder) { slice->render_into(background_builder); From d876848f2478548f8304357917ec1418adc46979 Mon Sep 17 00:00:00 2001 From: nayuta Date: Mon, 26 Dec 2022 10:23:40 +0800 Subject: [PATCH 8/8] test --- .../external_view_embedder_unittests.cc | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/shell/platform/android/external_view_embedder/external_view_embedder_unittests.cc b/shell/platform/android/external_view_embedder/external_view_embedder_unittests.cc index 0411b0cbbac38..f05db8e6e8071 100644 --- a/shell/platform/android/external_view_embedder/external_view_embedder_unittests.cc +++ b/shell/platform/android/external_view_embedder/external_view_embedder_unittests.cc @@ -516,6 +516,108 @@ TEST(AndroidExternalViewEmbedder, SubmitFrame) { } } +TEST(AndroidExternalViewEmbedder, OverlayCoverTwoPlatformViews) { + // In this test we will simulate two Android views appearing on the screen + // with a rect intersecting both of them + + auto jni_mock = std::make_shared(); + auto android_context = + std::make_shared(AndroidRenderingAPI::kSoftware); + + auto window = fml::MakeRefCounted(nullptr); + auto gr_context = GrDirectContext::MakeMock(nullptr); + auto frame_size = SkISize::Make(1000, 1000); + SurfaceFrame::FramebufferInfo framebuffer_info; + auto surface_factory = std::make_shared( + [&android_context, gr_context, window, frame_size, framebuffer_info]() { + auto surface_frame_1 = std::make_unique( + SkSurface::MakeNull(1000, 1000), framebuffer_info, + [](const SurfaceFrame& surface_frame, SkCanvas* canvas) { + return true; + }, + /*frame_size=*/SkISize::Make(800, 600)); + + auto surface_mock = std::make_unique(); + EXPECT_CALL(*surface_mock, AcquireFrame(frame_size)) + .Times(1 /* frames */) + .WillOnce(Return(ByMove(std::move(surface_frame_1)))); + + auto android_surface_mock = + std::make_unique(android_context); + EXPECT_CALL(*android_surface_mock, IsValid()).WillOnce(Return(true)); + + EXPECT_CALL(*android_surface_mock, CreateGPUSurface(gr_context.get())) + .WillOnce(Return(ByMove(std::move(surface_mock)))); + + EXPECT_CALL(*android_surface_mock, SetNativeWindow(window)); + return android_surface_mock; + }); + auto embedder = std::make_unique( + *android_context, jni_mock, surface_factory, GetTaskRunnersForFixture()); + + auto raster_thread_merger = GetThreadMergerFromPlatformThread(); + + EXPECT_CALL(*jni_mock, FlutterViewBeginFrame()); + embedder->BeginFrame(frame_size, nullptr, 1.5, raster_thread_merger); + + { + // Add first Android view. + SkMatrix matrix = SkMatrix::Translate(100, 100); + MutatorsStack stack; + embedder->PrerollCompositeEmbeddedView( + 0, std::make_unique(matrix, SkSize::Make(100, 100), + stack)); + // The JNI call to display the Android view. + EXPECT_CALL(*jni_mock, FlutterViewOnDisplayPlatformView( + 0, 100, 100, 100, 100, 150, 150, stack)); + } + + { + // Add second Android view. + SkMatrix matrix = SkMatrix::Translate(300, 100); + MutatorsStack stack; + embedder->PrerollCompositeEmbeddedView( + 1, std::make_unique(matrix, SkSize::Make(100, 100), + stack)); + // The JNI call to display the Android view. + EXPECT_CALL(*jni_mock, FlutterViewOnDisplayPlatformView( + 1, 300, 100, 100, 100, 150, 150, stack)); + } + auto rect_paint = SkPaint(); + rect_paint.setColor(SkColors::kCyan); + rect_paint.setStyle(SkPaint::Style::kFill_Style); + + // This simulates Flutter UI that intersects with the two Android views. + // Since we will compute the intersection for each android view in turn, and + // finally merge The final size of the overlay will be smaller than the + // width and height of the rect. + embedder->CompositeEmbeddedView(1).canvas->drawRect( + SkRect::MakeXYWH(150, 50, 200, 200), rect_paint); + + EXPECT_CALL(*jni_mock, FlutterViewCreateOverlaySurface()) + .WillRepeatedly([&]() { + return std::make_unique( + 1, window); + }); + + // The JNI call to display the overlay surface. + EXPECT_CALL(*jni_mock, + FlutterViewDisplayOverlaySurface(1, 150, 100, 200, 100)) + .Times(1); + + auto surface_frame = std::make_unique( + SkSurface::MakeNull(1000, 1000), framebuffer_info, + [](const SurfaceFrame& surface_frame, SkCanvas* canvas) mutable { + return true; + }, + /*frame_size=*/SkISize::Make(800, 600)); + + embedder->SubmitFrame(gr_context.get(), std::move(surface_frame)); + + EXPECT_CALL(*jni_mock, FlutterViewEndFrame()); + embedder->EndFrame(/*should_resubmit_frame=*/false, raster_thread_merger); +} + TEST(AndroidExternalViewEmbedder, SubmitFrameOverlayComposition) { auto jni_mock = std::make_shared(); auto android_context =