Semantics framework updates

lib/ui/semantics.dart

@@ -26,6 +26,7 @@ class SemanticsAction {
   static const int _kPasteIndex = 1 << 14;
   static const int _kDidGainAccessibilityFocusIndex = 1 << 15;
   static const int _kDidLoseAccessibilityFocusIndex = 1 << 16;
+  static const int _kDismissIndex = 1 << 17;
 
   /// The numerical value for this action.
   ///
@@ -146,6 +147,14 @@ class SemanticsAction {
   /// Accessibility focus and input focus can be held by two different nodes!
   static const SemanticsAction didLoseAccessibilityFocus = const SemanticsAction._(_kDidLoseAccessibilityFocusIndex);
 
+  /// A request that the node should be dismissed.
+  ///
+  /// A Snackbar, for example, may have a dismiss action to indicate to the user
+  /// that it can removed after it is no longer relevant.  on Android, TalkBack
+  /// announces this after reading the label. On iOS, VoiceOver users can
+  /// perform a standard gesture to dismiss it.
+  static const SemanticsAction dismiss = const SemanticsAction._(_kDismissIndex);
+
   /// The possible semantics actions.
   ///
   /// The map's key is the [index] of the action and the value is the action
@@ -168,6 +177,7 @@ class SemanticsAction {
     _kPasteIndex: paste,
     _kDidGainAccessibilityFocusIndex: didGainAccessibilityFocus,
     _kDidLoseAccessibilityFocusIndex: didLoseAccessibilityFocus,
+    _kDismissIndex: dismiss,
   };
 
   @override
@@ -207,6 +217,8 @@ class SemanticsAction {
         return 'SemanticsAction.didGainAccessibilityFocus';
       case _kDidLoseAccessibilityFocusIndex:
         return 'SemanticsAction.didLoseAccessibilityFocus';
+      case _kDismissIndex:
+        return 'SemanticsAction.dismiss';
     }
     return null;
   }
@@ -228,6 +240,8 @@ class SemanticsFlag {
   static const int _kScopesRouteIndex= 1 << 11;
   static const int _kNamesRouteIndex = 1 << 12;
   static const int _kIsHiddenIndex = 1 << 13;
+  static const int _kIsImageIndex = 1 << 14;
+  static const int _kIsLiveRegionIndex = 1 << 15;
 
   const SemanticsFlag._(this.index);
 
@@ -366,6 +380,28 @@ class SemanticsFlag {
   /// used to implement accessibility scrolling on iOS.
   static const SemanticsFlag isHidden = const SemanticsFlag._(_kIsHiddenIndex);
 
+  /// Whether the semantics node represents an image.
+  ///
+  /// Platforms have special behavior for images.  TalkBack will inform the user
+  /// the labeled node is an image. iOS may use the image flag to avoid
+  /// inverting their color when using smart invert.
+  static const SemanticsFlag isImage = const SemanticsFlag._(_kIsImageIndex);
+
+  /// Whether the semantics node is a live region.
+  /// 
+  /// A live region indicates that updates to semantics node are important. Platforms 
+  /// are free to use this information to make polite updates to the user to inform 
+  /// them of this.
+  /// 
+  /// On Android, TalkBack will make a polite announcement of the first and
+  /// subsequent updates to the label of this node. This flag is not currently
+  /// supported on iOS.
+  /// 
+  /// An example of a live region is a [SnackBar] widget. When it appears
+  /// on the screen it may be difficult to focus to read the value. A live
+  /// region causes a polite announcement to be generated automatically.
+  static const SemanticsFlag isLiveRegion = const SemanticsFlag._(_kIsLiveRegionIndex);
+
   /// The possible semantics flags.
   ///
   /// The map's key is the [index] of the flag and the value is the flag itself.
@@ -384,6 +420,8 @@ class SemanticsFlag {
     _kScopesRouteIndex: scopesRoute,
     _kNamesRouteIndex: namesRoute,
     _kIsHiddenIndex: isHidden,
+    _kIsImageIndex: isImage,
+    _kIsLiveRegionIndex: isLiveRegion,
   };
 
   @override
@@ -417,6 +455,10 @@ class SemanticsFlag {
         return 'SemanticsFlag.namesRoute';
       case _kIsHiddenIndex:
         return 'SemanticsFlag.isHidden';
+      case _kIsImageIndex:
+        return 'SemanticsFlag.isImage';
+      case _kIsLiveRegionIndex:
+        return 'SemanticsFlag.isLiveRegion';
     }
     return null;
   }

lib/ui/semantics/semantics_node.h

@@ -35,6 +35,7 @@ enum class SemanticsAction : int32_t {
   kPaste = 1 << 14,
   kDidGainAccessibilityFocus = 1 << 15,
   kDidLoseAccessibilityFocus = 1 << 16,
+  kDismiss = 1 << 18,
 };
 
 const int kScrollableSemanticsActions =
@@ -59,6 +60,8 @@ enum class SemanticsFlags : int32_t {
   kScopesRoute = 1 << 11,
   kNamesRoute = 1 << 12,
   kIsHidden = 1 << 13,
+  kIsImage = 1 << 14,
+  kIsLiveRegion = 1 << 15,
 };
 
 struct SemanticsNode {

shell/platform/android/io/flutter/view/AccessibilityBridge.java

@@ -8,6 +8,7 @@
 import android.opengl.Matrix;
 import android.os.Build;
 import android.os.Bundle;
+import android.content.Context;
 import android.util.Log;
 import android.view.View;
 import android.view.accessibility.AccessibilityEvent;
@@ -41,13 +42,15 @@ class AccessibilityBridge extends AccessibilityNodeProvider implements BasicMess
     private static final int ROOT_NODE_ID = 0;
 
     private Map<Integer, SemanticsObject> mObjects;
+    private Map<Integer, String> mLiveRegions;
     private final FlutterView mOwner;
     private boolean mAccessibilityEnabled = false;
     private SemanticsObject mA11yFocusedObject;
     private SemanticsObject mInputFocusedObject;
     private SemanticsObject mHoveredObject;
     private int previousRouteId = ROOT_NODE_ID;
     private List<Integer> previousRoutes;
+    private String mPackageName;
 
     private final BasicMessageChannel<Object> mFlutterAccessibilityChannel;
 
@@ -68,7 +71,8 @@ enum Action {
         CUT(1 << 13),
         PASTE(1 << 14),
         DID_GAIN_ACCESSIBILITY_FOCUS(1 << 15),
-        DID_LOSE_ACCESSIBILITY_FOCUS(1 << 16);
+        DID_LOSE_ACCESSIBILITY_FOCUS(1 << 16),
+        DISMISS(1 << 17);
 
         Action(int value) {
             this.value = value;
@@ -91,7 +95,9 @@ enum Flag {
         IS_OBSCURED(1 << 10),
         SCOPES_ROUTE(1 << 11),
         NAMES_ROUTE(1 << 12),
-        IS_HIDDEN(1 << 13);
+        IS_HIDDEN(1 << 13),
+        IS_IMAGE(1 << 14),
+        IS_LIVE_REGION(1 << 15);
 
         Flag(int value) {
             this.value = value;
@@ -104,9 +110,11 @@ enum Flag {
         assert owner != null;
         mOwner = owner;
         mObjects = new HashMap<Integer, SemanticsObject>();
+        mLiveRegions = new HashMap<Integer, String>();
         previousRoutes = new ArrayList<>();
         mFlutterAccessibilityChannel = new BasicMessageChannel<>(owner, "flutter/accessibility",
             StandardMessageCodec.INSTANCE);
+        mPackageName = owner.getContext().getPackageName();
     }
 
     void setAccessibilityEnabled(boolean accessibilityEnabled) {
@@ -182,6 +190,16 @@ public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualViewId) {
         if (object.hasFlag(Flag.IS_BUTTON)) {
           result.setClassName("android.widget.Button");
         }
+        if (object.hasFlag(Flag.IS_IMAGE)) {
+          result.setClassName("android.widget.ImageView");
+          // conform to the expected id from TalkBack's CustomLabelManager.
+          // talkback/src/main/java/labeling/CustomLabelManager.java#L525
+          result.setViewIdResourceName(mPackageName + ":id/" + Integer.toString(virtualViewId));
+        }
+        if (object.hasAction(Action.DISMISS)) {
+          result.setDismissable(true);
+          result.addAction(AccessibilityNodeInfo.ACTION_DISMISS);
+        }
 
         if (object.parent != null) {
             assert object.id > ROOT_NODE_ID;
@@ -230,6 +248,7 @@ public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualViewId) {
             }
         }
         if (object.hasAction(Action.INCREASE) || object.hasAction(Action.DECREASE)) {
+            // TODO(jonahwilliams): support AccessibilityAction.ACTION_SET_PROGRESS once SDK is updated.
             result.setClassName("android.widget.SeekBar");
             if (object.hasAction(Action.INCREASE)) {
                 result.addAction(AccessibilityNodeInfo.ACTION_SCROLL_FORWARD);
@@ -238,7 +257,12 @@ public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualViewId) {
                 result.addAction(AccessibilityNodeInfo.ACTION_SCROLL_BACKWARD);
             }
         }
-
+        if (object.hasFlag(Flag.IS_LIVE_REGION)) {
+            result.setLiveRegion(View.ACCESSIBILITY_LIVE_REGION_POLITE);
+        } else if (object.hadFlag(Flag.IS_LIVE_REGION)) {
+            mLiveRegions.remove(object.id);
+        }
+
         boolean hasCheckedState = object.hasFlag(Flag.HAS_CHECKED_STATE);
         result.setCheckable(hasCheckedState);
         if (hasCheckedState) {
@@ -390,6 +414,9 @@ public boolean performAction(int virtualViewId, int action, Bundle arguments) {
                 mOwner.dispatchSemanticsAction(virtualViewId, Action.PASTE);
                 return true;
             }
+            case AccessibilityNodeInfo.ACTION_DISMISS: {
+                mOwner.dispatchSemanticsAction(virtualViewId, Action.DISMISS);
+            }
         }
         return false;
     }
@@ -598,6 +625,14 @@ void updateSemantics(ByteBuffer buffer, String[] strings) {
                     sendAccessibilityEvent(selectionEvent);
                 }
             }
+            if (object.hasFlag(Flag.IS_LIVE_REGION)) {
+                String priorLabelHint = mLiveRegions.get(object.id);
+                String labelHint = object.getLabelHint();
+                mLiveRegions.put(object.id, labelHint);
+                if (priorLabelHint == null || !priorLabelHint.equals(labelHint)) {
+                    sendAccessibilityEvent(object.id, AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED);
+                }
+            }
         }
     }
 
@@ -705,6 +740,7 @@ private void createWindowChangeEvent(SemanticsObject route) {
     private void willRemoveSemanticsObject(SemanticsObject object) {
         assert mObjects.containsKey(object.id);
         assert mObjects.get(object.id) == object;
+        mLiveRegions.remove(object.id);
         object.parent = null;
         if (mA11yFocusedObject == object) {
             sendAccessibilityEvent(mA11yFocusedObject.id, AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED);
@@ -1061,6 +1097,20 @@ private float max(float a, float b, float c, float d) {
             return Math.max(a, Math.max(b, Math.max(c, d)));
         }
 
+        private String getLabelHint() {
+            StringBuilder sb = new StringBuilder();
+            if (label != null) {
+                sb.append(label);
+            }
+            if (sb.length() > 0) {
+                sb.append(", ");
+            }
+            if (hint != null) {
+                sb.append(hint);
+            }
+            return sb.length() > 0 ? sb.toString() : null;
+        }
+
         private String getValueLabelHint() {
             StringBuilder sb = new StringBuilder();
             String[] array = { value, label, hint };

shell/platform/darwin/ios/framework/Source/accessibility_bridge.mm

@@ -294,6 +294,13 @@ - (BOOL)accessibilityScroll:(UIAccessibilityScrollDirection)direction {
   return YES;
 }
 
+- (BOOL)accessibilityPerformEscape {
+  if (![self node].HasAction(blink::SemanticsAction::kDismiss))
+    return NO;
+  [self bridge] -> DispatchSemanticsAction([self uid], blink::SemanticsAction::kDismiss);
+  return YES;
+}
+
 #pragma mark UIAccessibilityFocus overrides
 
 - (void)accessibilityElementDidBecomeFocused {
@@ -356,6 +363,12 @@ - (UIAccessibilityTraits)accessibilityTraits {
   if ([self node].HasFlag(blink::SemanticsFlags::kIsHeader)) {
     traits |= UIAccessibilityTraitHeader;
   }
+  if ([self node].HasFlag(blink::SemanticsFlags::kIsImage)) {
+    traits |= UIAccessibilityTraitImage;
+  }
+  if ([self node].HasFlag(blink::SemanticsFlags::kIsLiveRegion)) {
+    traits |= UIAccessibilityTraitUpdatesFrequently;
+  }
   return traits;
 }