Abstract vision framework with included support for focus, navigator …

…object and browse mode caret highlighting (PR #9064)

Foundations of framework to enable:

- #7857: Making the screen black (i.e. a screen curtain) while NVDA is active, mainly for privacy reasons
- #971: Visual highlight of focus, review or browse mode caret location
- Basic screen magnification facility within NVDA.

This pr intends to lay the base of a vision framework that can be used to implement such functionality in the core of NVDA. Though there is no GUI in the current pull request, the framework is functional.

source/NVDAObjects/__init__.py

@@ -25,6 +25,7 @@
 import appModuleHandler
 import treeInterceptorHandler
 import braille
+import vision
 import globalPluginHandler
 import brailleInput
 import locationHelper
@@ -991,14 +992,15 @@ def event_mouseMove(self,x,y):
 		else:
 			speechWasCanceled=False
 		self._mouseEntered=True
+		vision.handler.handleMouseMove(self, x, y)
 		try:
 			info=self.makeTextInfo(locationHelper.Point(x,y))
 		except NotImplementedError:
 			info=NVDAObjectTextInfo(self,textInfos.POSITION_FIRST)
 		except LookupError:
 			return
 		if config.conf["reviewCursor"]["followMouse"]:
-			api.setReviewPosition(info)
+			api.setReviewPosition(info, isCaret=True)
 		info.expand(info.unit_mouseChunk)
 		oldInfo=getattr(self,'_lastMouseTextInfoObject',None)
 		self._lastMouseTextInfoObject=info
@@ -1018,6 +1020,7 @@ def event_stateChange(self):
 		if self is api.getFocusObject():
 			speech.speakObjectProperties(self,states=True, reason=controlTypes.REASON_CHANGE)
 		braille.handler.handleUpdate(self)
+		vision.handler.handleUpdate(self, property="states")
 
 	def event_focusEntered(self):
 		if self.role in (controlTypes.ROLE_MENUBAR,controlTypes.ROLE_POPUPMENU,controlTypes.ROLE_MENUITEM):
@@ -1033,6 +1036,7 @@ def event_gainFocus(self):
 		self.reportFocus()
 		braille.handler.handleGainFocus(self)
 		brailleInput.handler.handleGainFocus(self)
+		vision.handler.handleGainFocus(self)
 
 	def event_loseFocus(self):
 		# Forget the word currently being typed as focus is moving to a new control. 
@@ -1044,6 +1048,7 @@ def event_foreground(self):
 		L{event_focusEntered} or L{event_gainFocus} will be called for this object, so this method should not speak/braille the object, etc.
 		"""
 		speech.cancelSpeech()
+		vision.handler.handleForeground(self)
 
 	def event_becomeNavigatorObject(self, isFocus=False):
 		"""Called when this object becomes the navigator object.
@@ -1052,29 +1057,35 @@ def event_becomeNavigatorObject(self, isFocus=False):
 		"""
 		# When the navigator object follows the focus and braille is auto tethered to review,
 		# we should not update braille with the new review position as a tether to focus is due.
-		if braille.handler.shouldAutoTether and isFocus:
-			return
-		braille.handler.handleReviewMove(shouldAutoTether=not isFocus)
+		if not (braille.handler.shouldAutoTether and isFocus):
+			braille.handler.handleReviewMove(shouldAutoTether=not isFocus)
+		vision.handler.handleReviewMove(
+			context=vision.constants.Context.FOCUS if isFocus else vision.constants.Context.NAVIGATOR
+		)
 
 	def event_valueChange(self):
 		if self is api.getFocusObject():
 			speech.speakObjectProperties(self, value=True, reason=controlTypes.REASON_CHANGE)
 		braille.handler.handleUpdate(self)
+		vision.handler.handleUpdate(self, property="value")
 
 	def event_nameChange(self):
 		if self is api.getFocusObject():
 			speech.speakObjectProperties(self, name=True, reason=controlTypes.REASON_CHANGE)
 		braille.handler.handleUpdate(self)
+		vision.handler.handleUpdate(self, property="name")
 
 	def event_descriptionChange(self):
 		if self is api.getFocusObject():
 			speech.speakObjectProperties(self, description=True, reason=controlTypes.REASON_CHANGE)
 		braille.handler.handleUpdate(self)
+		vision.handler.handleUpdate(self, property="description")
 
 	def event_caret(self):
 		if self is api.getFocusObject() and not eventHandler.isPendingEvents("gainFocus"):
 			braille.handler.handleCaretMove(self)
 			brailleInput.handler.handleCaretMove(self)
+			vision.handler.handleCaretMove(self)
 			review.handleCaretMove(self)
 
 	def _get_flatReviewPosition(self):

source/NVDAObjects/window/excel.py

@@ -36,6 +36,7 @@
 import browseMode
 import inputCore
 import ctypes
+import vision
 
 excel2010VersionMajor=14
 
@@ -609,14 +610,14 @@ def getCellAddress(cell, external=False,format=xlA1):
 			text=_("{start} through {end}").format(start=textList[0], end=textList[1])
 		return text
 
-	def _getDropdown(self):
+	def _getDropdown(self, selection=None):
 		w=winUser.getAncestor(self.windowHandle,winUser.GA_ROOT)
 		if not w:
 			log.debugWarning("Could not get ancestor window (GA_ROOT)")
 			return
 		obj=Window(windowHandle=w,chooseBestAPI=False)
 		if not obj:
-			log.debugWarning("Could not instnaciate NVDAObject for ancestor window")
+			log.debugWarning("Could not instanciate NVDAObject for ancestor window")
 			return
 		threadID=obj.windowThreadID
 		while not eventHandler.isPendingEvents("gainFocus"):
@@ -626,6 +627,10 @@ def _getDropdown(self):
 				return
 			if obj.windowClassName=='EXCEL:':
 				break
+		if selection:
+			# If we are getting a dropdown for a selection,
+			# we want the selection to be presented as the direct ancestor of the dropdown.
+			obj.parent = selection
 		return obj
 
 	def _getSelection(self):
@@ -665,16 +670,19 @@ class Excel7Window(ExcelBase):
 	def _get_excelWindowObject(self):
 		return self.excelWindowObjectFromWindow(self.windowHandle)
 
-	def event_gainFocus(self):
+	def _get_focusRedirect(self):
 		selection=self._getSelection()
-		dropdown=self._getDropdown()
+		dropdown = self._getDropdown(selection=selection)
 		if dropdown:
-			if selection:
-				dropdown.parent=selection
-			eventHandler.executeEvent('gainFocus',dropdown)
-			return
+			return dropdown
 		if selection:
-			eventHandler.executeEvent('gainFocus',selection)
+			return selection
+
+	def event_caret(self):
+		# This object never gains focus, so normally, caret updates would be ignored.
+		# However, we need to tell the vision handler that a caret move has occured on this object,
+		# in order for a magnifier or highlighter to be positioned correctly.
+		vision.handler.handleCaretMove(self)
 
 class ExcelWorksheet(ExcelBase):
 

source/api.py

@@ -21,8 +21,11 @@
 import controlTypes
 import eventHandler
 import braille
+import vision
 import watchdog
 import appModuleHandler
+import cursorManager
+from typing import Any
 
 #User functions
 
@@ -177,22 +180,36 @@ def getReviewPosition():
 		globalVars.reviewPosition,globalVars.reviewPositionObj=review.getPositionForCurrentMode(obj)
 		return globalVars.reviewPosition
 
-def setReviewPosition(reviewPosition,clearNavigatorObject=True,isCaret=False):
+
+def setReviewPosition(
+		reviewPosition,
+		clearNavigatorObject=True,
+		isCaret=False,
+		isMouse=False
+):
 	"""Sets a TextInfo instance as the review position.
 	@param clearNavigatorObject: if  true, It sets the current navigator object to C{None}.
 		In that case, the next time the navigator object is asked for it fetches it from the review position.
 	@type clearNavigatorObject: bool
 	@param isCaret: Whether the review position is changed due to caret following.
 	@type isCaret: bool
+	@param isMouse: Whether the review position is changed due to mouse following.
+	@type isMouse: bool
 	"""
 	globalVars.reviewPosition=reviewPosition.copy()
 	globalVars.reviewPositionObj=reviewPosition.obj
 	if clearNavigatorObject: globalVars.navigatorObject=None
 	# When the review cursor follows the caret and braille is auto tethered to review,
 	# we should not update braille with the new review position as a tether to focus is due.
-	if braille.handler.shouldAutoTether and isCaret:
-		return
-	braille.handler.handleReviewMove(shouldAutoTether=not isCaret)
+	if not (braille.handler.shouldAutoTether and isCaret):
+		braille.handler.handleReviewMove(shouldAutoTether=not isCaret)
+	if isCaret:
+		visionContext = vision.constants.Context.CARET
+	elif isMouse:
+		visionContext = vision.constants.Context.MOUSE
+	else:
+		visionContext = vision.constants.Context.REVIEW
+	vision.handler.handleReviewMove(context=visionContext)
 
 def getNavigatorObject():
 	"""Gets the current navigator object. Navigator objects can be used to navigate around the operating system (with the number pad) with out moving the focus. If the navigator object is not set, it fetches it from the review position. 
@@ -346,6 +363,34 @@ def filterFileName(name):
 		name=name.replace(c,'_')
 	return name
 
+
+def isNVDAObject(obj: Any) -> bool:
+	"""Returns whether the supplied object is a L{NVDAObjects.NVDAObject}"""
+	return isinstance(obj, NVDAObjects.NVDAObject)
+
+
+def isCursorManager(obj: Any) -> bool:
+	"""Returns whether the supplied object is a L{cursorManager.CursorManager}"""
+	return isinstance(obj, cursorManager.CursorManager)
+
+
+def isTreeInterceptor(obj: Any) -> bool:
+	"""Returns whether the supplied object is a L{treeInterceptorHandler.TreeInterceptor}"""
+	return isinstance(obj, treeInterceptorHandler.TreeInterceptor)
+
+
+def isObjectInActiveTreeInterceptor(obj: NVDAObjects.NVDAObject) -> bool:
+	"""Returns whether the supplied L{NVDAObjects.NVDAObject} is
+	in an active L{treeInterceptorHandler.TreeInterceptor},
+	i.e. a tree interceptor that is not in pass through mode.
+	"""
+	return bool(
+		isinstance(obj, NVDAObjects.NVDAObject)
+		and obj.treeInterceptor
+		and not obj.treeInterceptor.passThrough
+	)
+
+
 def getCaretObject():
 	"""Gets the object which contains the caret.
 	This is normally the focus object.

source/browseMode.py

@@ -27,6 +27,7 @@
 import config
 import textInfos
 import braille
+import vision
 import speech
 import sayAllHandler
 import treeInterceptorHandler
@@ -1503,6 +1504,9 @@ def event_gainFocus(self, obj, nextHandler):
 				speech.speakTextInfo(focusInfo,reason=controlTypes.REASON_FOCUS)
 				# However, we still want to update the speech property cache so that property changes will be spoken properly.
 				speech.speakObject(obj,controlTypes.REASON_ONLYCACHE)
+				# As we do not call nextHandler which would trigger the vision framework to handle gain focus,
+				# we need to call it manually here.
+				vision.handler.handleGainFocus(obj)
 			else:
 				# Although we are going to speak the object rather than textInfo content, we still need to silently speak the textInfo content so that the textInfo speech cache is updated correctly.
 				# Not doing this would cause  later browseMode speaking to either not speak controlFields it had entered, or speak controlField exits after having already exited.
@@ -1518,9 +1522,13 @@ def event_gainFocus(self, obj, nextHandler):
 				# This focus change was caused by a virtual caret movement, so don't speak the focused node to avoid double speaking.
 				# However, we still want to update the speech property cache so that property changes will be spoken properly.
 				speech.speakObject(obj,controlTypes.REASON_ONLYCACHE)
-				if (
-					not config.conf["virtualBuffers"]["autoFocusFocusableElements"]
-					and self._lastFocusableObj
+				if config.conf["virtualBuffers"]["autoFocusFocusableElements"]:
+					# As we do not call nextHandler which would trigger the vision framework to handle gain focus,
+					# we need to call it manually here.
+					# Note: this is usually called after the caret movement.
+					vision.handler.handleGainFocus(obj)
+				elif (
+					self._lastFocusableObj
 					and obj == self._lastFocusableObj
 					and obj is not self._lastFocusableObj
 				):

source/colors.py

@@ -64,6 +64,20 @@ def fromString(cls,s):
 					return RGB(r,g,b)
 		raise ValueError("invalid RGB string: %s"%s)
 
+	def toCOLORREF(self) -> COLORREF:
+		"""Returns a COLORREF ctypes instance
+		"""
+		return COLORREF(self.red & 0xff | ((self.green & 0xff) << 8) | ((self.blue & 0xff) << 16))
+
+	def toGDIPlusARGB(self, alpha: int = 255) -> int:
+		"""Creates a GDI+ compatible ARGB color, using the specified alpha for the alpha component.
+		@param alpha: The alpha part of the ARGB color,
+			0 is fully transparent and 255 is fully opaque.
+			Defaults to 255 (opaque).
+		@type alpha: int
+		"""
+		return (alpha << 24) | (self.red << 16) | (self.green << 8) | self.blue
+
 	@property
 	def name(self):
 		foundName=RGBToNamesCache.get(self,None)

source/compoundDocuments.py

@@ -17,6 +17,7 @@
 import api
 import config
 import review
+import vision
 from logHandler import log
 from locationHelper import RectLTWH
 
@@ -464,6 +465,7 @@ def event_treeInterceptor_gainFocus(self):
 	def event_caret(self, obj, nextHandler):
 		self.detectPossibleSelectionChange()
 		braille.handler.handleCaretMove(self)
+		vision.handler.handleCaretMove(self)
 		caret = self.makeTextInfo(textInfos.POSITION_CARET)
 		review.handleCaretMove(caret)
 

source/config/__init__.py

@@ -135,7 +135,15 @@ def getSystemConfigPath():
 			pass
 	return None
 
-SCRATCH_PAD_ONLY_DIRS = ('appModules','brailleDisplayDrivers','globalPlugins','synthDrivers')
+
+SCRATCH_PAD_ONLY_DIRS = (
+	'appModules',
+	'brailleDisplayDrivers',
+	'globalPlugins',
+	'synthDrivers',
+	'visionEnhancementProviders',
+)
+
 
 def getScratchpadDir(ensureExists=False):
 	""" Returns the path where custom appModules, globalPlugins and drivers can be placed while being developed."""

source/config/configSpec.py

@@ -71,6 +71,13 @@
 	[[__many__]]
 		port = string(default="")
 
+# Vision enhancement provider settings
+[vision]
+	providers = string_list(=default=list())
+
+	# Vision enhancement provider settings
+	[[__many__]]
+
 # Presentation settings
 [presentation]
 		reportKeyboardShortcuts = boolean(default=true)
@@ -212,6 +219,7 @@
 	gui = boolean(default=false)
 	louis = boolean(default=false)
 	timeSinceInput = boolean(default=false)
+	vision = boolean(default=false)
 
 [uwpOcr]
 	language = string(default="")