Support UI Automation custom annotations such as notes in MS Excel

nvdaHelper/local/UIAUtils.cpp

@@ -18,3 +18,13 @@ PROPERTYID registerUIAProperty(GUID* guid, LPCWSTR programmaticName, UIAutomatio
 	registrar->Release();
 	return propertyId;
 }
+
+int registerUIAAnnotationType(GUID* guid) {
+	if(!guid) {
+		LOG_DEBUGWARNING(L"NULL GUID given");
+		return 0;
+	}
+	winrt::Windows::UI::UIAutomation::Core::CoreAutomationRegistrar registrar {};
+	auto res = registrar.RegisterAnnotationType(*guid);
+	return res.LocalId;
+}

nvdaHelper/local/UIAUtils.h

@@ -1,9 +1,15 @@
 #ifndef NVDAHELPERLOCAL_UIAUTILS_H
 #define NVDAHELPERLOCAL_UIAUTILS_H
 
+// The following header included to allow winrt::guid to be converted to GUID
+#include <unknwn.h>
+
+#include <winrt/windows.ui.uiautomation.core.h>
 #include <uiAutomationCore.h>
 
 PROPERTYID registerUIAProperty(GUID* guid, LPCWSTR programmaticName, UIAutomationType propertyType);
+int registerUIAAnnotationType(GUID* guid);
+
 
 #endif
 

nvdaHelper/local/nvdaHelperLocal.def

@@ -55,6 +55,7 @@ EXPORTS
 	calculateCharacterOffsets
 	findWindowWithClassInThread
 	registerUIAProperty
+	registerUIAAnnotationType
 	dllImportTableHooks_hookSingle
 	dllImportTableHooks_unhookSingle
 	audioDucking_shouldDelay

readme.md

@@ -68,7 +68,7 @@ The following dependencies need to be installed on your system:
 				* Desktop development with C++
 			* Then in the Installation details tree view, under Desktop for C++, Optional, ensure the following are selected:
 				* MSVC v142 - VS 2019 C++ x64/x86 build tools
-				* Windows 10 SDK (10.0.19041.0)
+				* Windows 11 SDK (10.0.22000.0)
 				* C++ ATL for v142 build tools (x86 & x64)
 				* C++ Clang tools for Windows
 		* On the Individual components tab, ensure the following items are selected:

source/NVDAObjects/UIA/__init__.py

@@ -17,6 +17,7 @@
 import languageHandler
 import UIAHandler
 import _UIACustomProps
+import _UIACustomAnnotations
 import globalVars
 import eventHandler
 import controlTypes
@@ -159,7 +160,12 @@ def _getFormatFieldAtRange(self,textRange,formatConfig,ignoreMixedValues=False):
 		formatField=textInfos.FormatField()
 		if not isinstance(textRange,UIAHandler.IUIAutomationTextRange):
 			raise ValueError("%s is not a text range"%textRange)
-		fetchAnnotationTypes=formatConfig["reportSpellingErrors"] or formatConfig["reportComments"] or formatConfig["reportRevisions"]
+		fetchAnnotationTypes = (
+			formatConfig["reportSpellingErrors"]
+			or formatConfig["reportComments"]
+			or formatConfig["reportRevisions"]
+			or formatConfig["reportBookmarks"]
+		)
 		try:
 			textRange=textRange.QueryInterface(UIAHandler.IUIAutomationTextRange3)
 		except (COMError,AttributeError):
@@ -291,13 +297,22 @@ def _getFormatFieldAtRange(self,textRange,formatConfig,ignoreMixedValues=False):
 				if UIAHandler.AnnotationType_GrammarError in annotationTypes:
 					formatField["invalid-grammar"]=True
 			if formatConfig["reportComments"]:
-				if UIAHandler.AnnotationType_Comment in annotationTypes:
-					formatField["comment"]=True
+				cats = self.obj._UIACustomAnnotationTypes
+				if cats.microsoftWord_draftComment.id and cats.microsoftWord_draftComment.id in annotationTypes:
+					formatField["comment"] = textInfos.CommentType.DRAFT
+				elif cats.microsoftWord_resolvedComment.id and cats.microsoftWord_resolvedComment.id in annotationTypes:
+					formatField["comment"] = textInfos.CommentType.RESOLVED
+				elif UIAHandler.AnnotationType_Comment in annotationTypes:
+					formatField["comment"] = True
 			if formatConfig["reportRevisions"]:
 				if UIAHandler.AnnotationType_InsertionChange in annotationTypes:
 					formatField["revision-insertion"]=True
 				elif UIAHandler.AnnotationType_DeletionChange in annotationTypes:
 					formatField["revision-deletion"]=True
+			if formatConfig["reportBookmarks"]:
+				cats = self.obj._UIACustomAnnotationTypes
+				if cats.microsoftWord_bookmark.id and cats.microsoftWord_bookmark.id in annotationTypes:
+					formatField["bookmark"] = True
 		cultureVal=fetcher.getValue(UIAHandler.UIA_CultureAttributeId,ignoreMixedValues=ignoreMixedValues)
 		if cultureVal and isinstance(cultureVal,int):
 			try:
@@ -903,6 +918,7 @@ def updateSelection(self):
 
 class UIA(Window):
 	_UIACustomProps = _UIACustomProps.CustomPropertiesCommon.get()
+	_UIACustomAnnotationTypes = _UIACustomAnnotations.CustomAnnotationTypesCommon.get()
 
 	shouldAllowDuplicateUIAFocusEvent = False
 

source/NVDAObjects/UIA/excel.py

@@ -4,6 +4,8 @@
 # Copyright (C) 2018-2021 NV Access Limited, Leonard de Ruijter
 
 from typing import Optional, Tuple
+from comtypes import COMError
+import winVersion
 import UIAHandler
 import _UIAHandler
 import _UIAConstants
@@ -16,6 +18,9 @@
 from _UIACustomProps import (
 	CustomPropertyInfo,
 )
+from _UIACustomAnnotations import (
+	CustomAnnotationTypeInfo,
+)
 from comtypes import GUID
 from scriptHandler import script
 import ui
@@ -90,10 +95,39 @@ def __init__(self):
 		)
 
 
+class ExcelCustomAnnotationTypes:
+	""" UIA 'custom annotation types' specific to Excel.
+	Once registered, all subsequent registrations will return the same ID value.
+	This class should be used as a singleton via ExcelCustomAnnotationTypes.get()
+	to prevent unnecessary work by repeatedly interacting with UIA.
+	"""
+	#: Singleton instance
+	_instance: "Optional[ExcelCustomAnnotationTypes]" = None
+
+	@classmethod
+	def get(cls) -> "ExcelCustomAnnotationTypes":
+		"""Get the singleton instance or initialise it.
+		"""
+		if cls._instance is None:
+			cls._instance = cls()
+		return cls._instance
+
+	def __init__(self):
+		#  Available custom Annotations list at https://docs.microsoft.com/en-us/office/uia/excel/excelannotations
+		# Note annotation:
+		# Represents an old-style comment (now known as a note)
+		# which contains non-threaded plain text content.
+		self.note = CustomAnnotationTypeInfo(
+			guid=GUID("{4E863D9A-F502-4A67-808F-9E711702D05E}"),
+		)
+
+
 class ExcelObject(UIA):
 	"""Common base class for all Excel UIA objects
 	"""
 	_UIAExcelCustomProps = ExcelCustomProperties.get()
+	_UIAExcelCustomAnnotationTypes = ExcelCustomAnnotationTypes.get()
+
 
 
 class ExcelCell(ExcelObject):
@@ -367,6 +401,15 @@ def _get_states(self):
 			states.add(controlTypes.State.HASFORMULA)
 		if self._getUIACacheablePropertyValue(self._UIAExcelCustomProps.hasDataValidationDropdown.id):
 			states.add(controlTypes.State.HASPOPUP)
+		if winVersion.getWinVer() >= winVersion.WIN11:
+			try:
+				annotationTypes = self._getUIACacheablePropertyValue(UIAHandler.UIA_AnnotationTypesPropertyId)
+			except COMError:
+				# annotationTypes cannot be fetched on older Operating Systems such as Windows 7.
+				annotationTypes = None
+			if annotationTypes:
+				if self._UIAExcelCustomAnnotationTypes.note.id in annotationTypes:
+					states.add(controlTypes.State.HASNOTE)
 		return states
 
 	def _get_cellCoordsText(self):
@@ -421,28 +464,39 @@ def _get_cellCoordsText(self):
 		description=_("Reports the note or comment thread on the current cell"),
 		gesture="kb:NVDA+alt+c")
 	def script_reportComment(self, gesture):
+		if winVersion.getWinVer() >= winVersion.WIN11:
+			noteElement = self.UIAAnnotationObjects.get(self._UIAExcelCustomAnnotationTypes.note.id)
+			if noteElement:
+				name = noteElement.CurrentName
+				desc = noteElement.GetCurrentPropertyValue(UIAHandler.UIA_FullDescriptionPropertyId)
+				# Translators: a note on a cell in Microsoft excel.
+				text = _("{name}: {desc}").format(name=name, desc=desc)
+				ui.message(text)
+			else:
+				# Translators: message when a cell in Excel contains no note
+				ui.message(_("No note on this cell"))
 		commentsElement = self.UIAAnnotationObjects.get(UIAHandler.AnnotationType_Comment)
 		if commentsElement:
 			comment = commentsElement.GetCurrentPropertyValue(UIAHandler.UIA_FullDescriptionPropertyId)
 			author = commentsElement.GetCurrentPropertyValue(UIAHandler.UIA_AnnotationAuthorPropertyId)
 			numReplies = commentsElement.GetCurrentPropertyValue(self._UIAExcelCustomProps.commentReplyCount.id)
 			if numReplies == 0:
 				# Translators: a comment on a cell in Microsoft excel.
-				text = _("{comment}  by {author}").format(
+				text = _("Comment thread: {comment}  by {author}").format(
 					comment=comment,
 					author=author
 				)
 			else:
 				# Translators: a comment on a cell in Microsoft excel.
-				text = _("{comment}  by {author} with {numReplies} replies").format(
+				text = _("Comment thread: {comment}  by {author} with {numReplies} replies").format(
 					comment=comment,
 					author=author,
 					numReplies=numReplies
 				)
 			ui.message(text)
 		else:
-			# Translators: A message in Excel when there is no note
-			ui.message(_("No note or comment thread on this cell"))
+			# Translators: A message in Excel when there is no comment thread
+			ui.message(_("No comment thread on this cell"))
 
 
 class ExcelWorksheet(ExcelObject):

source/NVDAObjects/UIA/wordDocument.py

@@ -32,6 +32,7 @@
 #: the non-printable unicode character that represents the end of cell or end of row mark in Microsoft Word
 END_OF_ROW_MARK = '\x07'
 
+
 class ElementsListDialog(browseMode.ElementsListDialog):
 
 	ELEMENT_TYPES=(browseMode.ElementsListDialog.ELEMENT_TYPES[0],browseMode.ElementsListDialog.ELEMENT_TYPES[1],
@@ -80,7 +81,12 @@ def getCommentInfoFromPosition(position):
 		UIAElement=UIAElement.buildUpdatedCache(UIAHandler.handler.baseCacheRequest)
 		typeID = UIAElement.GetCurrentPropertyValue(UIAHandler.UIA_AnnotationAnnotationTypeIdPropertyId)
 		# Use Annotation Type Comment if available
-		if typeID == UIAHandler.AnnotationType_Comment:
+		cats = position.obj._UIACustomAnnotationTypes
+		if (
+			typeID == UIAHandler.AnnotationType_Comment
+			or (typeID and typeID == cats.microsoftWord_draftComment)
+			or (typeID and typeID == cats.microsoftWord_resolvedComment)
+		):
 			comment = UIAElement.GetCurrentPropertyValue(UIAHandler.UIA_NamePropertyId)
 			author = UIAElement.GetCurrentPropertyValue(UIAHandler.UIA_AnnotationAuthorPropertyId)
 			date = UIAElement.GetCurrentPropertyValue(UIAHandler.UIA_AnnotationDateTimePropertyId)

source/_UIACustomAnnotations.py

@@ -0,0 +1,89 @@
+# A part of NonVisual Desktop Access (NVDA)
+# Copyright (C) 2021 NV Access Limited
+# This file is covered by the GNU General Public License.
+# See the file COPYING for more details.
+
+from dataclasses import (
+	dataclass,
+	field,
+)
+from typing import Optional
+
+from comtypes import (
+	GUID,
+	byref,
+)
+import winVersion
+
+
+"""
+This module provides helpers and a common format to define UIA custom annotation types.
+The common custom annotation types are defined here.
+Custom annotation types specific to an application should be defined within a NVDAObjects/UIA
+submodule specific to that application, E.G. 'NVDAObjects/UIA/excel.py'
+
+UIA originally had hard coded 'static' ID's for annotation types.
+For an example see 'AnnotationType_SpellingError' in
+`source/comInterfaces/_944DE083_8FB8_45CF_BCB7_C477ACB2F897_0_1_0.py`
+imported via `UIAutomationClient.py`.
+When a new annotation type was added the UIA spec had to be updated.
+Now a mechanism is in place to allow applications to register "custom annotation types".
+This relies on both the UIA server application and the UIA client application sharing a known
+GUID for the annotation type.
+"""
+
+
+@dataclass
+class CustomAnnotationTypeInfo:
+	"""Holds information about a CustomAnnotationType
+	This makes it easy to define custom annotation types to be loaded.
+	"""
+	guid: GUID
+	id: int = field(init=False)
+
+	def __post_init__(self) -> None:
+		""" The id field must be initialised at runtime.
+		A GUID uniquely identifies a custom annotation, but the UIA system relies on integer IDs.
+		Any application (clients or providers) can register a custom annotation type, subsequent applications
+		will get the same id for a given GUID.
+		Registering custom annotations is only supported on Windows 11 and above.
+		For any lesser version, id will be 0.
+
+		"""
+		if winVersion.getWinVer() >= winVersion.WIN11:
+			import NVDAHelper
+			self.id = NVDAHelper.localLib.registerUIAAnnotationType(
+				byref(self.guid),
+			)
+		else:
+			self.id = 0
+
+
+class CustomAnnotationTypesCommon:
+	"""UIA 'custom annotation types' common to all applications.
+	Once registered, all subsequent registrations will return the same ID value.
+	This class should be used as a singleton via CustomAnnotationTypesCommon.get()
+	to prevent unnecessary work by repeatedly interacting with UIA.
+	"""
+	#: Singleton instance
+	_instance: "Optional[CustomAnnotationTypesCommon]" = None
+
+	@classmethod
+	def get(cls) -> "CustomAnnotationTypesCommon":
+		"""Get the singleton instance or initialise it.
+		"""
+		if cls._instance is None:
+			cls._instance = cls()
+		return cls._instance
+
+	def __init__(self):
+		# Registration of Custom annotation types used across multiple applications or frameworks should go here.
+		self.microsoftWord_resolvedComment = CustomAnnotationTypeInfo(
+			guid=GUID("{A015030C-5B44-4EAC-B0CC-21BA35DE6D07}"),
+		)
+		self.microsoftWord_draftComment = CustomAnnotationTypeInfo(
+			guid=GUID("{26BAEBC6-591E-4116-BBCF-E9A7996CD169}"),
+		)
+		self.microsoftWord_bookmark = CustomAnnotationTypeInfo(
+			guid=GUID("{25330951-A372-4DB9-A88A-85137AD008D2}"),
+		)

source/braille.py

@@ -834,6 +834,29 @@ def getFormatFieldBraille(field, fieldCache, isAtStart, formatConfig):
 		oldLink=fieldCache.get("link")
 		if link and link != oldLink:
 			textList.append(roleLabels[controlTypes.Role.LINK])
+	if formatConfig["reportComments"]:
+		comment = field.get("comment")
+		oldComment = fieldCache.get("comment") if fieldCache is not None else None
+		if (comment or oldComment is not None) and comment != oldComment:
+			if comment:
+				if comment is textInfos.CommentType.DRAFT:
+					# Translators: Brailled when text contains a draft comment.
+					text = _("drft cmnt")
+				elif comment is textInfos.CommentType.RESOLVED:
+					# Translators: Brailled when text contains a resolved comment.
+					text = _("rslvd cmnt")
+				else:  # generic
+					# Translators: Brailled when text contains a generic comment.
+					text = _("cmnt")
+				textList.append(text)
+	if formatConfig["reportBookmarks"]:
+		bookmark = field.get("bookmark")
+		oldBookmark = fieldCache.get("bookmark") if fieldCache is not None else None
+		if (bookmark or oldBookmark is not None) and bookmark != oldBookmark:
+			if bookmark:
+				# Translators: brailled when text contains a bookmark
+				text = _("bkmk")
+				textList.append(text)
 	fieldCache.clear()
 	fieldCache.update(field)
 	return TEXT_SEPARATOR.join([x for x in textList if x])

source/config/configSpec.py

@@ -203,6 +203,7 @@
 	reportLinks = boolean(default=true)
 	reportGraphics = boolean(default=True)
 	reportComments = boolean(default=true)
+	reportBookmarks = boolean(default=true)
 	reportLists = boolean(default=true)
 	reportHeadings = boolean(default=true)
 	reportBlockQuotes = boolean(default=true)

source/controlTypes/state.py

@@ -71,6 +71,7 @@ def negativeDisplayString(self) -> str:
 	OVERFLOWING = 0x10000000000
 	UNLOCKED = 0x20000000000
 	HAS_ARIA_DETAILS = 0x40000000000
+	HASNOTE = 0x80000000000
 
 
 STATES_SORTED = frozenset([State.SORTED, State.SORTED_ASCENDING, State.SORTED_DESCENDING])
@@ -160,6 +161,8 @@ def negativeDisplayString(self) -> str:
 	# Translators: a state that denotes that the object is unlocked (such as an unlocked cell in a protected
 	# Excel spreadsheet).
 	State.UNLOCKED: _("unlocked"),
+	# Translators: a state that denotes the existance of a note.
+	State.HASNOTE: _("has note"),
 }