Accessibility fixes (#709)

* Show/hide accessibility overlay on scroll based on several visibility factors

* Improvements to accessibility overlay anchor view tracking

* Increase recursion limit and check for null children when walking the node tree

* Cleanup

* Hide overlay when expanding status (notification) bar

* use .Any() instead of .Count()

src/Android/Accessibility/AccessibilityHelpers.cs

@@ -120,7 +120,7 @@ namespace Bit.Droid.Accessibility
                 if(addressNode != null)
                 {
                     uri = ExtractUri(uri, addressNode, browser);
-                    addressNode.Dispose();
+                    addressNode.Recycle();
                 }
                 else
                 {
@@ -217,7 +217,7 @@ namespace Bit.Droid.Accessibility
                 nodes = new NodeList();
             }
             var dispose = disposeIfUnused;
-            if(n != null && recursionDepth < 50)
+            if(n != null && recursionDepth < 100)
             {
                 var add = n.WindowId == e.WindowId &&
                     !(n.ViewIdResourceName?.StartsWith(SystemUiPackage) ?? false) &&
@@ -231,7 +231,11 @@ namespace Bit.Droid.Accessibility
                 for(var i = 0; i < n.ChildCount; i++)
                 {
                     var childNode = n.GetChild(i);
-                    if(i > 100)
+                    if(childNode == null)
+                    {
+                        continue;
+                    }
+                    else if(i > 100)
                     {
                         Android.Util.Log.Info(BitwardenTag, "Too many child iterations.");
                         break;
@@ -248,7 +252,7 @@ namespace Bit.Droid.Accessibility
             }
             if(dispose)
             {
-                n?.Dispose();
+                n?.Recycle();
             }
             return nodes;
         }
@@ -282,21 +286,23 @@ namespace Bit.Droid.Accessibility
         {
             var allEditTexts = GetWindowNodes(root, e, n => EditText(n), false);
             var usernameEditText = GetUsernameEditTextIfPasswordExists(allEditTexts);
+
+            var isUsernameEditText = false;
             if(usernameEditText != null)
             {
-                var isUsernameEditText = IsSameNode(usernameEditText, e.Source);
+                isUsernameEditText = IsSameNode(usernameEditText, e.Source);
-                allEditTexts.Dispose();
+                usernameEditText.Recycle();
-                usernameEditText = null;
-                return isUsernameEditText;
             }
-            return false;
+            allEditTexts.Dispose();
+
+            return isUsernameEditText;
         }
 
-        public static bool IsSameNode(AccessibilityNodeInfo info1, AccessibilityNodeInfo info2)
+        public static bool IsSameNode(AccessibilityNodeInfo node1, AccessibilityNodeInfo node2)
         {
-            if(info1 != null && info2 != null) 
+            if(node1 != null && node2 != null)
             {
-                return info1.Equals(info2) || info1.GetHashCode() == info2.GetHashCode();
+                return node1.Equals(node2) || node1.GetHashCode() == node2.GetHashCode();
             }
             return false;
         }
@@ -350,38 +356,117 @@ namespace Bit.Droid.Accessibility
             return layoutParams;
         }
 
-        public static Point GetOverlayAnchorPosition(AccessibilityNodeInfo root, AccessibilityNodeInfo anchorView)
+        public static Point GetOverlayAnchorPosition(AccessibilityNodeInfo root, AccessibilityNodeInfo anchorView,
+            int rootRectHeight = 0)
         {
-            var rootRect = new Rect();
+            if(rootRectHeight == 0)
-            root.GetBoundsInScreen(rootRect);
+            {
-            var rootRectHeight = rootRect.Height();
+                rootRectHeight = GetNodeHeight(root);
+            }
 
             var anchorViewRect = new Rect();
             anchorView.GetBoundsInScreen(anchorViewRect);
             var anchorViewRectLeft = anchorViewRect.Left;
             var anchorViewRectTop = anchorViewRect.Top;
+            anchorViewRect.Dispose();
 
-            var navBarHeight = GetNavigationBarHeight();
+            var calculatedTop = rootRectHeight - anchorViewRectTop - GetNavigationBarHeight();
-            var calculatedTop = rootRectHeight - anchorViewRectTop - navBarHeight;
             return new Point(anchorViewRectLeft, calculatedTop);
         }
 
-        public static Point GetOverlayAnchorPosition(int nodeHash, AccessibilityNodeInfo root, AccessibilityEvent e)
+        public static Point GetOverlayAnchorPosition(AccessibilityNodeInfo anchorNode, AccessibilityNodeInfo root,
+            IEnumerable<AccessibilityWindowInfo> windows)
         {
             Point point = null;
-            var allEditTexts = GetWindowNodes(root, e, n => EditText(n), false);
+            if(anchorNode != null)
-            foreach(var node in allEditTexts)
             {
-                if(node.GetHashCode() == nodeHash)
+                anchorNode.Refresh(); // update node's info since this is still a reference from an older event
+                if(!anchorNode.VisibleToUser)
                 {
-                    point = GetOverlayAnchorPosition(root, node);
+                    return new Point(-1, -1);
-                    break;
+                }
+
+                // node.VisibleToUser doesn't always give us exactly what we want, so attempt to tighten up the range
+                // of visibility
+                var rootNodeHeight = GetNodeHeight(root);
+                var limitLowY = 0;
+                var limitHighY = rootNodeHeight - GetNodeHeight(anchorNode);
+                if(windows != null)
+                {
+                    if(IsStatusBarExpanded(windows))
+                    {
+                        return new Point(-1, -1);
+                    }
+                    Rect inputWindowRect = GetInputMethodWindowRect(windows);
+                    if(inputWindowRect != null)
+                    {
+                        limitLowY += inputWindowRect.Height();
+                        if(Build.VERSION.SdkInt >= BuildVersionCodes.Q)
+                        {
+                            limitLowY += GetNavigationBarHeight() + GetStatusBarHeight();
+                        }
+                        inputWindowRect.Dispose();
+                    }
+                }
+
+                point = GetOverlayAnchorPosition(root, anchorNode, rootNodeHeight);
+
+                if(point.Y < limitLowY || point.Y > limitHighY)
+                {
+                    point.X = -1;
+                    point.Y = -1;
                 }
             }
-            allEditTexts.Dispose();
             return point;
         }
 
+        public static bool IsStatusBarExpanded(IEnumerable<AccessibilityWindowInfo> windows)
+        {
+            if(windows != null && windows.Any())
+            {
+                var isSystemWindowsOnly = true;
+                foreach(var window in windows)
+                {
+                    if(window.Type != AccessibilityWindowType.System)
+                    {
+                        isSystemWindowsOnly = false;
+                        break;
+                    }
+                }
+                return isSystemWindowsOnly;
+            }
+            return false;
+        }
+
+        public static Rect GetInputMethodWindowRect(IEnumerable<AccessibilityWindowInfo> windows)
+        {
+            Rect windowRect = null;
+            if(windows != null)
+            {
+                foreach(var window in windows)
+                {
+                    if(window.Type == AccessibilityWindowType.InputMethod)
+                    {
+                        windowRect = new Rect();
+                        window.GetBoundsInScreen(windowRect);
+                        window.Recycle();
+                        break;
+                    }
+                    window.Recycle();
+                }
+            }
+            return windowRect;
+        }
+
+        public static int GetNodeHeight(AccessibilityNodeInfo node)
+        {
+            var nodeRect = new Rect();
+            node.GetBoundsInScreen(nodeRect);
+            var nodeRectHeight = nodeRect.Height();
+            nodeRect.Dispose();
+            return nodeRectHeight;
+        }
+
         private static int GetStatusBarHeight()
         {
             return GetSystemResourceDimenPx("status_bar_height");

src/Android/Accessibility/AccessibilityService.cs

@@ -15,6 +15,7 @@ using Bit.App.Resources;
 using Bit.Core;
 using Bit.Core.Abstractions;
 using Bit.Core.Utilities;
+using Java.Util;
 
 namespace Bit.Droid.Accessibility
 {
@@ -27,17 +28,19 @@ namespace Bit.Droid.Accessibility
         private const string BitwardenPackage = "com.x8bit.bitwarden";
         private const string BitwardenWebsite = "vault.bitwarden.com";
 
-        private string _lastNotificationUri = null;
+        private AccessibilityNodeInfo _anchorNode = null;
+        private int _lastAnchorX, _lastAnchorY = 0;
+        private static bool _overlayAnchorObserverRunning = false;
+        private IWindowManager _windowManager = null;
+        private LinearLayout _overlayView = null;
+        private long _lastAutoFillTime = 0;
+        private Java.Lang.Runnable _overlayAnchorObserverRunnable = null;
+        private Handler _handler = new Handler(Looper.MainLooper);
 
         private HashSet<string> _launcherPackageNames = null;
         private DateTime? _lastLauncherSetBuilt = null;
         private TimeSpan _rebuildLauncherSpan = TimeSpan.FromHours(1);
 
-        private IWindowManager _windowManager = null;
-        private LinearLayout _overlayView = null;
-        private int _anchorViewHash = 0;
-        private int _lastAnchorX, _lastAnchorY = 0;
-
         public override void OnAccessibilityEvent(AccessibilityEvent e)
         {
             try
@@ -54,106 +57,95 @@ namespace Bit.Droid.Accessibility
 
                 if(SkipPackage(e?.PackageName))
                 {
-                    CancelOverlayPrompt();
+                    if(e?.PackageName != "com.android.systemui")
-                    return;
+                    {
-                }
+                        CancelOverlayPrompt();
-
+                    }
-                var root = RootInActiveWindow;
-                if(root == null || root.PackageName != e.PackageName)
-                {
                     return;
                 }
 
                 // AccessibilityHelpers.PrintTestData(root, e);
 
+                AccessibilityNodeInfo root = null;
+
                 switch(e.EventType)
                 {
                     case EventTypes.ViewFocused:
                     case EventTypes.ViewClicked:
-                    case EventTypes.ViewScrolled:
-                        var isKnownBroswer = AccessibilityHelpers.SupportedBrowsers.ContainsKey(root.PackageName);
-                        if(e.EventType == EventTypes.ViewClicked && isKnownBroswer)
-                        {
-                            break;
-                        }
                         if(e.Source == null || e.PackageName == BitwardenPackage)
                         {
                             CancelOverlayPrompt();
+                            e.Recycle();
                             break;
                         }
-                        if(e.EventType == EventTypes.ViewScrolled)
+
+                        root = RootInActiveWindow;
+                        if(root == null || root.PackageName != e.PackageName)
                         {
-                            AdjustOverlayForScroll(root, e);
+                            e.Recycle();
                             break;
                         }
+                        var isKnownBroswer = AccessibilityHelpers.SupportedBrowsers.ContainsKey(root.PackageName);
+                        if(e.EventType == EventTypes.ViewClicked && isKnownBroswer)
+                        {
+                            e.Recycle();
+                            break;
+                        }
+                        if(!(e.Source?.Password ?? false) && !AccessibilityHelpers.IsUsernameEditText(root, e))
+                        {
+                            CancelOverlayPrompt();
+                            e.Recycle();
+                            break;
+                        }
+                        if(ScanAndAutofill(root, e))
+                        {
+                            CancelOverlayPrompt();
+                            e.Recycle();
+                        }
                         else
                         {
-                            var isUsernameEditText1 = AccessibilityHelpers.IsUsernameEditText(root, e);
+                            OverlayPromptToAutofill(root, e);
-                            var isPasswordEditText1 = e.Source?.Password ?? false;
-                            if(!isUsernameEditText1 && !isPasswordEditText1)
-                            {
-                                CancelOverlayPrompt();
-                                break;
-                            }
-                            if(ScanAndAutofill(root, e))
-                            {
-                                CancelOverlayPrompt();
-                            }
-                            else
-                            {
-                                OverlayPromptToAutofill(root, e);
-                            }
                         }
                         break;
                     case EventTypes.WindowContentChanged:
                     case EventTypes.WindowStateChanged:
-                        var isUsernameEditText2 = AccessibilityHelpers.IsUsernameEditText(root, e);
+                        if(AccessibilityHelpers.LastCredentials == null)
-                        var isPasswordEditText2 = e.Source?.Password ?? false;
-                        if(e.Source == null || isUsernameEditText2 || isPasswordEditText2)
                         {
+                            e.Recycle();
                             break;
                         }
-                        else if(AccessibilityHelpers.LastCredentials == null)
-                        {
-                            if(string.IsNullOrWhiteSpace(_lastNotificationUri))
-                            {
-                                CancelOverlayPrompt();
-                                break;
-                            }
-                            var uri = AccessibilityHelpers.GetUri(root);
-                            if(uri != null && uri != _lastNotificationUri)
-                            {
-                                CancelOverlayPrompt();
-                            }
-                            else if(uri != null && uri.StartsWith(Constants.AndroidAppProtocol))
-                            {
-                                CancelOverlayPrompt();
-                            }
-                            break;
-                        }
-
                         if(e.PackageName == BitwardenPackage)
                         {
                             CancelOverlayPrompt();
+                            e.Recycle();
                             break;
                         }
 
+                        root = RootInActiveWindow;
+                        if(root == null || root.PackageName != e.PackageName)
+                        {
+                            e.Recycle();
+                            break;
+                        }
                         if(ScanAndAutofill(root, e))
                         {
                             CancelOverlayPrompt();
                         }
+                        e.Recycle();
                         break;
                     default:
                         break;
                 }
 
-                root.Dispose();
+                if(root != null)
-                e.Dispose();
+                {
+                    root.Recycle();
+                }
             }
             // Suppress exceptions so that service doesn't crash.
             catch(Exception ex)
             {
-                System.Diagnostics.Debug.WriteLine(">>> Exception: " + ex.StackTrace);
+                System.Diagnostics.Debug.WriteLine(">>> {0}: {1}", ex.GetType(), ex.StackTrace);
             }
         }
 
@@ -175,8 +167,8 @@ namespace Bit.Droid.Accessibility
                     {
                         AccessibilityHelpers.GetNodesAndFill(root, e, passwordNodes);
                         filled = true;
+                        _lastAutoFillTime = Java.Lang.JavaSystem.CurrentTimeMillis();
                     }
-
                 }
                 AccessibilityHelpers.LastCredentials = null;
             }
@@ -194,19 +186,23 @@ namespace Bit.Droid.Accessibility
 
         private void CancelOverlayPrompt()
         {
-            if(_windowManager == null || _overlayView == null)
+            _overlayAnchorObserverRunning = false;
+
+            if(_windowManager != null && _overlayView != null)
             {
-                return;
+                _windowManager.RemoveViewImmediate(_overlayView);
+                System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Removed");
             }
 
-            _windowManager.RemoveViewImmediate(_overlayView);
-            System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Removed");
-
             _overlayView = null;
-            _anchorViewHash = 0;
-            _lastNotificationUri = null;
             _lastAnchorX = 0;
             _lastAnchorY = 0;
+
+            if(_anchorNode != null)
+            {
+                _anchorNode.Recycle();
+                _anchorNode = null;
+            }
         }
 
         private void OverlayPromptToAutofill(AccessibilityNodeInfo root, AccessibilityEvent e)
@@ -215,12 +211,25 @@ namespace Bit.Droid.Accessibility
             {
                 System.Diagnostics.Debug.WriteLine(">>> Overlay Permission not granted");
                 Toast.MakeText(this, AppResources.AccessibilityOverlayPermissionAlert, ToastLength.Long).Show();
+                e.Recycle();
+                return;
+            }
+
+            if(_overlayView != null || _anchorNode != null || _overlayAnchorObserverRunning)
+            {
+                CancelOverlayPrompt();
+            }
+
+            if(Java.Lang.JavaSystem.CurrentTimeMillis() - _lastAutoFillTime < 1000)
+            {
+                e.Recycle();
                 return;
             }
 
             var uri = AccessibilityHelpers.GetUri(root);
             if(string.IsNullOrWhiteSpace(uri))
             {
+                e.Recycle();
                 return;
             }
 
@@ -234,54 +243,85 @@ namespace Bit.Droid.Accessibility
                 _windowManager = GetSystemService(WindowService).JavaCast<IWindowManager>();
             }
 
-            if(_overlayView == null)
+            var intent = new Intent(this, typeof(AccessibilityActivity));
+            intent.PutExtra("uri", uri);
+            intent.SetFlags(ActivityFlags.NewTask | ActivityFlags.SingleTop | ActivityFlags.ClearTop);
+
+            _overlayView = AccessibilityHelpers.GetOverlayView(this);
+            _overlayView.Click += (sender, eventArgs) =>
             {
-                var intent = new Intent(this, typeof(AccessibilityActivity));
+                CancelOverlayPrompt();
-                intent.PutExtra("uri", uri);
+                StartActivity(intent);
-                intent.SetFlags(ActivityFlags.NewTask | ActivityFlags.SingleTop | ActivityFlags.ClearTop);
+            };
 
-                _overlayView = AccessibilityHelpers.GetOverlayView(this);
+            _anchorNode = e.Source;
-                _overlayView.Click += (sender, eventArgs) =>
-                {
-                    CancelOverlayPrompt();
-                    StartActivity(intent);
-                };
-
-                _lastNotificationUri = uri;
-
-                _windowManager.AddView(_overlayView, layoutParams);
-
-                System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Added at X:{0} Y:{1}",
-                    layoutParams.X, layoutParams.Y);
-            }
-            else
-            {
-                _windowManager.UpdateViewLayout(_overlayView, layoutParams);
-
-                System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Updated to X:{0} Y:{1}",
-                    layoutParams.X, layoutParams.Y);
-            }
-
-            _anchorViewHash = e.Source.GetHashCode();
             _lastAnchorX = anchorPosition.X;
             _lastAnchorY = anchorPosition.Y;
+
+            _windowManager.AddView(_overlayView, layoutParams);
+
+            System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Added at X:{0} Y:{1}",
+                layoutParams.X, layoutParams.Y);
+
+            StartOverlayAnchorObserver();
         }
 
-        private void AdjustOverlayForScroll(AccessibilityNodeInfo root, AccessibilityEvent e)
+        private void StartOverlayAnchorObserver()
         {
-            if(_overlayView == null || _anchorViewHash <= 0)
+            if(_overlayAnchorObserverRunning)
             {
                 return;
             }
+            _overlayAnchorObserverRunning = true;
+
+            _overlayAnchorObserverRunnable = new Java.Lang.Runnable(() =>
+            {
+                if(_overlayAnchorObserverRunning)
+                {
+                    AdjustOverlayForScroll();
+                    _handler.PostDelayed(_overlayAnchorObserverRunnable, 250);
+                }
+            });
+
+            _handler.PostDelayed(_overlayAnchorObserverRunnable, 250);
+        }
+
+        private void AdjustOverlayForScroll()
+        {
+            if(_overlayView == null || _anchorNode == null)
+            {
+                CancelOverlayPrompt();
+                return;
+            }
+
+            var root = RootInActiveWindow;
+            IEnumerable<AccessibilityWindowInfo> windows = null;
+            if(Build.VERSION.SdkInt > BuildVersionCodes.Kitkat)
+            {
+                windows = Windows;
+            }
+            var anchorPosition = AccessibilityHelpers.GetOverlayAnchorPosition(_anchorNode, root, windows);
+            root.Recycle();
 
-            var anchorPosition = AccessibilityHelpers.GetOverlayAnchorPosition(_anchorViewHash, root, e);
             if(anchorPosition == null)
             {
+                CancelOverlayPrompt();
                 return;
             }
-
+            else if(anchorPosition.X == -1 && anchorPosition.Y == -1)
-            if(anchorPosition.X == _lastAnchorX && anchorPosition.Y == _lastAnchorY)
             {
+                if(_overlayView.Visibility != ViewStates.Gone)
+                {
+                    _overlayView.Visibility = ViewStates.Gone;
+                }
+                return;
+            }
+            else if(anchorPosition.X == _lastAnchorX && anchorPosition.Y == _lastAnchorY)
+            {
+                if(_overlayView.Visibility != ViewStates.Visible)
+                {
+                    _overlayView.Visibility = ViewStates.Visible;
+                }
                 return;
             }
 
@@ -289,11 +329,16 @@ namespace Bit.Droid.Accessibility
             layoutParams.X = anchorPosition.X;
             layoutParams.Y = anchorPosition.Y;
 
-            _windowManager.UpdateViewLayout(_overlayView, layoutParams);
-
             _lastAnchorX = anchorPosition.X;
             _lastAnchorY = anchorPosition.Y;
 
+            _windowManager.UpdateViewLayout(_overlayView, layoutParams);
+
+            if(_overlayView.Visibility != ViewStates.Visible)
+            {
+                _overlayView.Visibility = ViewStates.Visible;
+            }
+
             System.Diagnostics.Debug.WriteLine(">>> Accessibility Overlay View Updated to X:{0} Y:{1}",
                     layoutParams.X, layoutParams.Y);
         }

src/Android/Resources/xml/accessibilityservice.xml

@@ -2,8 +2,8 @@
 <accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
     android:summary="@string/AutoFillServiceSummary"
     android:description="@string/AutoFillServiceDescription"
-    android:accessibilityEventTypes="typeWindowStateChanged|typeWindowContentChanged|typeViewFocused|typeViewClicked|typeViewScrolled"
+    android:accessibilityEventTypes="typeWindowStateChanged|typeWindowContentChanged|typeViewFocused|typeViewClicked"
     android:accessibilityFeedbackType="feedbackGeneric"
-    android:accessibilityFlags="flagReportViewIds"
+    android:accessibilityFlags="flagReportViewIds|flagRetrieveInteractiveWindows"
     android:notificationTimeout="100"
     android:canRetrieveWindowContent="true"/>