From 2eee0c1b704ff9389756204325a7d298366ab4d3 Mon Sep 17 00:00:00 2001 From: Gray Mackall <34871572+gmackall@users.noreply.github.com> Date: Tue, 14 May 2024 16:07:53 -0700 Subject: [PATCH 1/4] Fix non-vd android platform view input event offsets (#52532) Fixes https://github.com/flutter/flutter/issues/146570, which tracks a regression from https://github.com/flutter/engine/pull/49268 regarding platform view inputs in some specific cases. This PR translates the input event location to be the same as the location we calculated before https://github.com/flutter/engine/pull/49268, returning to the previous behavior, while maintaining the input event's verified status (I checked this manually with the `InputManager`). Tested manually with the reproduction in the linked issue. [C++, Objective-C, Java style guides]: https://github.com/flutter/engine/blob/main/CONTRIBUTING.md#style --- .../systemchannels/PlatformViewsChannel.java | 7 +- .../platform/PlatformViewsController.java | 41 ++++++--- .../platform/PlatformViewsControllerTest.java | 91 ++++++++++++++----- 3 files changed, 102 insertions(+), 37 deletions(-) diff --git a/shell/platform/android/io/flutter/embedding/engine/systemchannels/PlatformViewsChannel.java b/shell/platform/android/io/flutter/embedding/engine/systemchannels/PlatformViewsChannel.java index df58b0badfdc5..780ed5261cf1b 100644 --- a/shell/platform/android/io/flutter/embedding/engine/systemchannels/PlatformViewsChannel.java +++ b/shell/platform/android/io/flutter/embedding/engine/systemchannels/PlatformViewsChannel.java @@ -501,7 +501,12 @@ public static class PlatformViewTouch { public final int action; /** The number of pointers (e.g, fingers) involved in the touch event. */ public final int pointerCount; - /** Properties for each pointer, encoded in a raw format. */ + /** + * Properties for each pointer, encoded in a raw format. + * Expected to be formatted as a List[List[Integer]], where each inner list has two items: + * - An id, at index 0, corresponding to {@link android.view.MotionEvent.PointerProperties#id} + * - A tool type, at index 1, corresponding to {@link android.view.MotionEvent.PointerProperties#toolType}. + * */ @NonNull public final Object rawPointerPropertiesList; /** Coordinates for each pointer, encoded in a raw format. */ @NonNull public final Object rawPointerCoords; diff --git a/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java b/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java index 093be1566cb4e..9c78a3f030135 100644 --- a/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java +++ b/shell/platform/android/io/flutter/plugin/platform/PlatformViewsController.java @@ -4,8 +4,6 @@ package io.flutter.plugin.platform; -import static android.view.MotionEvent.PointerCoords; -import static android.view.MotionEvent.PointerProperties; import static io.flutter.Build.API_LEVELS; import android.annotation.TargetApi; @@ -668,6 +666,25 @@ public long configureForTextureLayerComposition( return textureId; } + /** + * Translates an original touch event to have the same locations as the ones that Flutter + * calculates (because original + flutter's - original = flutter's). + * + * @param originalEvent The saved original input event. + * @param pointerCoords The coordinates that Flutter thinks the touch is happening at. + */ + private static void translateMotionEvent( + MotionEvent originalEvent, PointerCoords[] pointerCoords) { + if (pointerCoords.length < 1) { + return; + } + + float xOffset = pointerCoords[0].x - originalEvent.getX(); + float yOffset = pointerCoords[0].y - originalEvent.getY(); + + originalEvent.offsetLocation(xOffset, yOffset); + } + @VisibleForTesting public MotionEvent toMotionEvent( float density, PlatformViewsChannel.PlatformViewTouch touch, boolean usingVirtualDiplay) { @@ -675,25 +692,27 @@ public MotionEvent toMotionEvent( MotionEventTracker.MotionEventId.from(touch.motionEventId); MotionEvent trackedEvent = motionEventTracker.pop(motionEventId); + // Pointer coordinates in the tracked events are global to FlutterView + // The framework converts them to be local to a widget, given that + // motion events operate on local coords, we need to replace these in the tracked + // event with their local counterparts. + // Compute this early so it can be used as input to translateNonVirtualDisplayMotionEvent. + PointerCoords[] pointerCoords = + parsePointerCoordsList(touch.rawPointerCoords, density) + .toArray(new PointerCoords[touch.pointerCount]); + if (!usingVirtualDiplay && trackedEvent != null) { - // We have the original event, deliver it as it will pass the verifiable + // We have the original event, deliver it after offsetting as it will pass the verifiable // input check. + translateMotionEvent(trackedEvent, pointerCoords); return trackedEvent; } // We are in virtual display mode or don't have a reference to the original MotionEvent. // In this case we manually recreate a MotionEvent to be delivered. This MotionEvent // will fail the verifiable input check. - - // Pointer coordinates in the tracked events are global to FlutterView - // framework converts them to be local to a widget, given that - // motion events operate on local coords, we need to replace these in the tracked - // event with their local counterparts. PointerProperties[] pointerProperties = parsePointerPropertiesList(touch.rawPointerPropertiesList) .toArray(new PointerProperties[touch.pointerCount]); - PointerCoords[] pointerCoords = - parsePointerCoordsList(touch.rawPointerCoords, density) - .toArray(new PointerCoords[touch.pointerCount]); // TODO (kaushikiska) : warn that we are potentially using an untracked // event in the platform views. diff --git a/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java b/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java index 0b69e6a801880..76fdffc8f23ab 100644 --- a/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java +++ b/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java @@ -361,34 +361,48 @@ public void itUsesActionEventTypeFromFrameworkEventAsActionChanged() { assertNotEquals(resolvedEvent.getAction(), frameWorkTouch.action); } - @Ignore - @Test - public void itUsesActionEventTypeFromMotionEventForHybridPlatformViews() { - MotionEventTracker motionEventTracker = MotionEventTracker.getInstance(); - PlatformViewsController platformViewsController = new PlatformViewsController(); - - MotionEvent original = - MotionEvent.obtain( - 100, // downTime - 100, // eventTime - 1, // action - 0, // x - 0, // y - 0 // metaState - ); - - // track an event that will later get passed to us from framework + private MotionEvent makePlatformViewTouchAndInvokeToMotionEvent( + PlatformViewsController platformViewsController, + MotionEventTracker motionEventTracker, + MotionEvent original, + boolean usingVirtualDisplays) { MotionEventTracker.MotionEventId motionEventId = motionEventTracker.track(original); - PlatformViewTouch frameWorkTouch = + // Construct a PlatformViewTouch.rawPointerPropertiesList by doing the inverse of + // PlatformViewsController.parsePointerPropertiesList. + List> pointerProperties = + Arrays.asList( + Arrays.asList( + original.getPointerId(0), + original.getToolType(0) + ) + ); + // Construct a PlatformViewTouch.rawPointerCoords by doing the inverse of + // PlatformViewsController.parsePointerCoordsList. + List> pointerCoordinates = + Arrays.asList( + Arrays.asList( + (double) original.getOrientation(), + (double) original.getPressure(), + (double) original.getSize(), + (double) original.getToolMajor(), + (double) original.getToolMinor(), + (double) original.getTouchMajor(), + (double) original.getTouchMinor(), + (double) original.getX(), + (double) original.getY() + ) + ); + // Make a platform view touch from the motion event. + PlatformViewTouch frameWorkTouchNonVd = new PlatformViewTouch( 0, // viewId original.getDownTime(), original.getEventTime(), - 2, // action + original.getAction(), 1, // pointerCount - Arrays.asList(Arrays.asList(0, 0)), // pointer properties - Arrays.asList(Arrays.asList(0., 1., 2., 3., 4., 5., 6., 7., 8.)), // pointer coords + pointerProperties, // pointer properties + pointerCoordinates, // pointer coords original.getMetaState(), original.getButtonState(), original.getXPrecision(), @@ -399,11 +413,38 @@ public void itUsesActionEventTypeFromMotionEventForHybridPlatformViews() { original.getFlags(), motionEventId.getId()); - MotionEvent resolvedEvent = - platformViewsController.toMotionEvent( - /*density=*/ 1, frameWorkTouch, /*usingVirtualDisplay=*/ false); + return platformViewsController.toMotionEvent( + 1, // density + frameWorkTouchNonVd, + usingVirtualDisplays); + } - assertEquals(resolvedEvent.getAction(), frameWorkTouch.action); + @Test + public void toMotionEvent_returnsSameCoordsForVdAndNonVd() { + MotionEventTracker motionEventTracker = MotionEventTracker.getInstance(); + PlatformViewsController platformViewsController = new PlatformViewsController(); + + MotionEvent original = + MotionEvent.obtain( + 10, // downTime + 10, // eventTime + 261, // action + 1, // x + 1, // y + 0 // metaState + ); + + MotionEvent resolvedNonVdEvent = + makePlatformViewTouchAndInvokeToMotionEvent( + platformViewsController, motionEventTracker, original, false); + + MotionEvent resolvedVdEvent = + makePlatformViewTouchAndInvokeToMotionEvent( + platformViewsController, motionEventTracker, original, true); + + assertEquals(resolvedVdEvent.getEventTime(), resolvedNonVdEvent.getEventTime()); + assertEquals(resolvedVdEvent.getX(), resolvedNonVdEvent.getX(), 0.001f); + assertEquals(resolvedVdEvent.getY(), resolvedNonVdEvent.getY(), 0.001f); } @Test From d23685f8e71e4fdf25597a62cc70630da279ff9c Mon Sep 17 00:00:00 2001 From: Gray Mackall <34871572+gmackall@users.noreply.github.com> Date: Wed, 29 May 2024 12:19:22 -0700 Subject: [PATCH 2/4] manually apply format to PlatformViewsChannel.java --- .../engine/systemchannels/PlatformViewsChannel.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/shell/platform/android/io/flutter/embedding/engine/systemchannels/PlatformViewsChannel.java b/shell/platform/android/io/flutter/embedding/engine/systemchannels/PlatformViewsChannel.java index 780ed5261cf1b..109721300cfe4 100644 --- a/shell/platform/android/io/flutter/embedding/engine/systemchannels/PlatformViewsChannel.java +++ b/shell/platform/android/io/flutter/embedding/engine/systemchannels/PlatformViewsChannel.java @@ -502,11 +502,11 @@ public static class PlatformViewTouch { /** The number of pointers (e.g, fingers) involved in the touch event. */ public final int pointerCount; /** - * Properties for each pointer, encoded in a raw format. - * Expected to be formatted as a List[List[Integer]], where each inner list has two items: - * - An id, at index 0, corresponding to {@link android.view.MotionEvent.PointerProperties#id} - * - A tool type, at index 1, corresponding to {@link android.view.MotionEvent.PointerProperties#toolType}. - * */ + * Properties for each pointer, encoded in a raw format. Expected to be formatted as a + * List[List[Integer]], where each inner list has two items: - An id, at index 0, corresponding + * to {@link android.view.MotionEvent.PointerProperties#id} - A tool type, at index 1, + * corresponding to {@link android.view.MotionEvent.PointerProperties#toolType}. + */ @NonNull public final Object rawPointerPropertiesList; /** Coordinates for each pointer, encoded in a raw format. */ @NonNull public final Object rawPointerCoords; From 9c3ed601ddd01b5e2937e9b7c2d78fe27e047287 Mon Sep 17 00:00:00 2001 From: Gray Mackall <34871572+gmackall@users.noreply.github.com> Date: Wed, 29 May 2024 12:23:22 -0700 Subject: [PATCH 3/4] manually apply format to PlatformViewsControllerTest.java --- .../platform/PlatformViewsControllerTest.java | 30 +++++++------------ 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java b/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java index 76fdffc8f23ab..4a95ec41e6dee 100644 --- a/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java +++ b/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java @@ -371,28 +371,20 @@ private MotionEvent makePlatformViewTouchAndInvokeToMotionEvent( // Construct a PlatformViewTouch.rawPointerPropertiesList by doing the inverse of // PlatformViewsController.parsePointerPropertiesList. List> pointerProperties = - Arrays.asList( - Arrays.asList( - original.getPointerId(0), - original.getToolType(0) - ) - ); + Arrays.asList(Arrays.asList(original.getPointerId(0), original.getToolType(0))); // Construct a PlatformViewTouch.rawPointerCoords by doing the inverse of // PlatformViewsController.parsePointerCoordsList. List> pointerCoordinates = - Arrays.asList( - Arrays.asList( - (double) original.getOrientation(), - (double) original.getPressure(), - (double) original.getSize(), - (double) original.getToolMajor(), - (double) original.getToolMinor(), - (double) original.getTouchMajor(), - (double) original.getTouchMinor(), - (double) original.getX(), - (double) original.getY() - ) - ); + Arrays.asList(Arrays.asList( + (double) original.getOrientation(), + (double) original.getPressure(), + (double) original.getSize(), + (double) original.getToolMajor(), + (double) original.getToolMinor(), + (double) original.getTouchMajor(), + (double) original.getTouchMinor(), + (double) original.getX(), + (double) original.getY())); // Make a platform view touch from the motion event. PlatformViewTouch frameWorkTouchNonVd = new PlatformViewTouch( From b8e6af00d15edc80b950fc2d9341c4bb62fa9764 Mon Sep 17 00:00:00 2001 From: Gray Mackall <34871572+gmackall@users.noreply.github.com> Date: Wed, 29 May 2024 13:32:28 -0700 Subject: [PATCH 4/4] manually apply format to PlatformViewsControllerTest.java (correctly this time) --- .../flutter/plugin/platform/PlatformViewsControllerTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java b/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java index 4a95ec41e6dee..0ecc09838bc33 100644 --- a/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java +++ b/shell/platform/android/test/io/flutter/plugin/platform/PlatformViewsControllerTest.java @@ -375,7 +375,8 @@ private MotionEvent makePlatformViewTouchAndInvokeToMotionEvent( // Construct a PlatformViewTouch.rawPointerCoords by doing the inverse of // PlatformViewsController.parsePointerCoordsList. List> pointerCoordinates = - Arrays.asList(Arrays.asList( + Arrays.asList( + Arrays.asList( (double) original.getOrientation(), (double) original.getPressure(), (double) original.getSize(),