Skip to content

Commit

Permalink
MS Word with UIA: use custom extensions to support line number, layou…
Browse files Browse the repository at this point in the history
…t column number and section number (#13283)

Supersedes pr #13149

Summary of the issue:
The following error is in the log when viewing annotations in the NVDA elements list:
```
ERROR - queueHandler.flushQueue (19:47:20.811) - MainThread (9476):
Error in func ElementsListDialog.initElementType
Traceback (most recent call last):
  File "queueHandler.pyc", line 55, in flushQueue
  File "browseMode.pyc", line 1054, in initElementType
  File "browseMode.pyc", line 1074, in filter
  File "NVDAObjects\UIA\wordDocument.pyc", line 139, in label
  File "NVDAObjects\UIA\wordDocument.pyc", line 95, in getCommentInfoFromPosition
AttributeError: 'WordBrowseModeDocument' object has no attribute '_UIACustomAnnotationTypes'
```
This is because getCommentInfoFromPosition tries to fetch custom annotation types off its obj property, which it incorrectly expects to be a UIA NVDAObject. But in this case it is a TreeInterceptor.
PR #13149 was opened to address this by specifically creating a UIA NVDAObject rather than getting it from the position TextInfo. However, Further investigation has found that it is actually not necessary to even support custom annotation types (draft comment, resolved comment) in getCommentInfoFromPosition as:
1. Quicknav / Elements List iteration was never updated to specifically jump to / list those custom comment types. See CommentUIATextInfoQuickNavItem.wantedAttribValues
2. Although the IUIAutomationTextRange implementation in MS Word does return these custom comment types in GetAttributeValue when given UIA_AnnotationTypesAttributeId, the elements returned with UIA_AnnotationObjectsAttributeId only ever return the standard Comment annotation type via their UIA_AnnotationAnnotationTypeId property. In fact for draft comments, no annotation objects are returned at all.
In short, supporting custom comment annotation types in getCommentInfoFromPosition was not necessary in the first place.

Description of how this pull request fixes the issue:
Remove support for custom comment annotation types from getCommentInfoFromPosition. It again only supports the standard comment annotation type.
  • Loading branch information
michaelDCurran committed Feb 16, 2022
1 parent 21a61e5 commit a3ed8b8
Show file tree
Hide file tree
Showing 15 changed files with 547 additions and 4 deletions.
6 changes: 6 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -30,3 +30,9 @@
[submodule "include/Detours"]
path = include/detours
url = https://github.com/microsoft/Detours/
[submodule "include/microsoft-ui-uiautomation"]
path = include/microsoft-ui-uiautomation
url = https://github.com/michaeldcurran/microsoft-ui-uiautomation
[submodule "include/wil"]
path = include/wil
url = https://github.com/microsoft/wil
1 change: 1 addition & 0 deletions include/microsoft-ui-uiautomation
1 change: 1 addition & 0 deletions include/wil
Submodule wil added at 323400
163 changes: 163 additions & 0 deletions nvdaHelper/UIARemote/UIARemote.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
/*
This file is a part of the NVDA project.
URL: http://www.nvaccess.org/
Copyright 2021-2022 NV Access Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License version 2.0, as published by
the Free Software Foundation.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
This license can be found at:
http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
*/

#include <memory>
#include <functional>
#include <string>
#include <windows.h>
#include <atlsafe.h>
#include <atlcomcli.h>
#include <roapi.h>
#include <winstring.h>
#include <UIAutomation.h>
#include <UiaOperationAbstraction/UiaOperationAbstraction.h>
#include <UiaOperationAbstraction/SafeArrayUtil.h>
#include <common/log.h>
#include <winrt/microsoft.ui.uiautomation.h>

using namespace UiaOperationAbstraction;

#include "remoteLog.h"

wchar_t dllDirectory[MAX_PATH];

// Several custom extension GUIDs specific to Microsoft Word
winrt::guid guid_msWord_extendedTextRangePattern{ 0x93514122, 0xff04, 0x4b2c, { 0xa4, 0xad, 0x4a, 0xb0, 0x45, 0x87, 0xc1, 0x29 } };
winrt::guid guid_msWord_getCustomAttributeValue{ 0x81aca91, 0x32f2, 0x46f0, { 0x9f, 0xb9, 0x1, 0x70, 0x38, 0xbc, 0x45, 0xf8 } };

bool _isInitialized {false};

// Fetches a custom attribute value from a range of text in Microsoft Word.
// this function uses the UI automation Operation Abstraction API to call the Microsoft Word specific custom extension
extern "C" __declspec(dllexport) bool __stdcall msWord_getCustomAttributeValue(IUIAutomationElement* docElementArg, IUIAutomationTextRange* pTextRangeArg, int customAttribIDArg, VARIANT* pCustomAttribValueArg) {
if(!_isInitialized) {
LOG_ERROR(L"UIARemote not initialized!");
return false;
}
try {
auto scope=UiaOperationScope::StartNew();
RemoteableLogger logger{scope};
// Here starts declaritive code which will be executed remotely
logger<<L"Remoting msWord_getCustomAttributeValue"<<endl;
UiaBool isExtensionSupported{false};
UiaElement docElement{docElementArg};
UiaTextRange textRange{pTextRangeArg};
UiaInt customAttribID{customAttribIDArg};
UiaVariant customAttribValue;
scope.If(
/* condition */ docElement.IsExtensionSupported(guid_msWord_extendedTextRangePattern),
/* body */ [&]() {
logger<<L"guid_msWord_extendedTextRangePattern is supported extension"<<endl;
UiaElement patternElement{nullptr};
docElement.CallExtension(guid_msWord_extendedTextRangePattern, patternElement);
scope.If(
/* condition */ patternElement,
/* body */ [&]() {
logger<<L"Got custom pattern element "<<endl;
scope.If(
/* condition */ patternElement.IsExtensionSupported(guid_msWord_getCustomAttributeValue),
/* body */ [&]() {
isExtensionSupported = true;
logger<<L"guid_msWord_getCustomAttributeValue extension supported on pattern"<<endl;
patternElement.CallExtension(guid_msWord_getCustomAttributeValue, textRange, customAttribID, customAttribValue);
logger<<L"Called guid_msWord_getCustomAttributeValue extention"<<endl;
},
/* else */ [&]() {
logger<<L"No guid_msWord_getCustomAttributeValue extension supported"<<endl;
}
);
},
/* else */ [&]() {
logger<<L"Could not fetch guid_msWord_extendedTextRangePattern pattern"<<endl;
}
);
},
/* else */ [&]() {
logger<<L"No guid_msWord_extendedTextRangePattern extension supported"<<endl;
}
);
// Request that certain variables be made available locally after execution remotely
scope.BindResult(isExtensionSupported, customAttribValue);
// Actually execute the remote code
auto res = scope.ResolveHr();
if(res != S_OK) {
LOG_ERROR(L"Error in scope.Resolve: code "<<res);
return false;
}
logger.dumpLog();
// We are back to local again
if(isExtensionSupported) {
if(customAttribValue.IsInt()) {
pCustomAttribValueArg->vt = VT_I4;
pCustomAttribValueArg->lVal = customAttribValue.AsInt();
return true;
} else if(customAttribValue.IsString()) {
pCustomAttribValueArg->vt = VT_BSTR;
pCustomAttribValueArg->bstrVal = customAttribValue.AsString().get();
return true;
} else {
LOG_ERROR(L"Unknown data type");
return false;
}
} else {
LOG_DEBUG(L"Extension not supported");
}
} catch (std::exception& e) {
auto wideWhat = stringToWstring(e.what());
LOG_ERROR(L"msWord_getCustomAttributeValue exception: "<<wideWhat);
} catch(...) {
LOG_ERROR(L"msWord_getCustomAttributeValue exception: unknown");
}
return false;
}

// Registers and initializes the Microsoft-ui-UIAutomation remote operations library.
extern "C" __declspec(dllexport) bool __stdcall initialize(bool doRemote, IUIAutomation* client) {
std::wstring manifestPath = dllDirectory;
manifestPath += L"\\Microsoft.UI.UIAutomation.dll.manifest";
ACTCTX actCtx{};
actCtx.cbSize=sizeof(actCtx);
actCtx.lpSource = L"Microsoft.UI.UIAutomation.dll.manifest";
actCtx.lpAssemblyDirectory = dllDirectory;
actCtx.dwFlags = ACTCTX_FLAG_ASSEMBLY_DIRECTORY_VALID;
HANDLE hActCtx=CreateActCtx(&actCtx);
if(hActCtx == nullptr) {
LOG_ERROR(L"Could not create activation context for "<<manifestPath);
return false;
}
ULONG_PTR actCtxCookie;
if(!ActivateActCtx(hActCtx,&actCtxCookie)) {
LOG_ERROR(L"Error activating activation context for "<<manifestPath);
ReleaseActCtx(hActCtx);
return false;
}
LOG_INFO(L"Registered "<<manifestPath);
if(!winrt::get_activation_factory<winrt::Microsoft::UI::UIAutomation::AutomationRemoteOperation>()) {
LOG_ERROR(L"Unable to get Microsoft.UI.UIAutomation activation factory");
return false;
}
LOG_INFO(L"Microsoft.UI.UIAutomation is available");
UiaOperationAbstraction::Initialize(doRemote,client);
_isInitialized = true;
return true;
}

BOOL WINAPI DllMain(HINSTANCE hModule,DWORD reason,LPVOID lpReserved) {
if(reason==DLL_PROCESS_ATTACH) {
GetModuleFileName(hModule,dllDirectory,MAX_PATH);
PathRemoveFileSpec(dllDirectory);
}
return true;
}

77 changes: 77 additions & 0 deletions nvdaHelper/UIARemote/remoteLog.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
/*
This file is a part of the NVDA project.
URL: http://www.nvaccess.org/
Copyright 2021-2022 NV Access Limited
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License version 2.0, as published by
the Free Software Foundation.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
This license can be found at:
http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
*/

#pragma once

// Converts a utf8 encoded string into a utf16 encoded wstring
std::wstring stringToWstring(const std::string& from) {
int wideLen = MultiByteToWideChar(CP_UTF8, 0, from.c_str(), from.length(), nullptr, 0);
std::wstring wideBuf (wideLen, L'\0');
MultiByteToWideChar(CP_UTF8, 0, from.c_str(), from.length(), wideBuf.data(), wideLen);
return wideBuf;
}

const std::wstring endl{L"\n"};

class RemoteableLogger;

// A class for logging messages from within a remote ops call.
// Push messages to the object with << just like an ostream.
// Currently standard strings, UiaStrings, and UiaInt instances are supported.
// After remote execution is complete, call dumpLog to write the content to our standard logging framework.
class RemoteableLogger {
public:

RemoteableLogger(UiaOperationScope& scope): _log{} {
scope.BindResult(_log);
}

RemoteableLogger& operator <<(UiaInt& message) {
_log.Append(message.Stringify());
return *this;
}

RemoteableLogger& operator <<(UiaString& message) {
_log.Append(message);
return *this;
}

RemoteableLogger& operator <<(const std::wstring message) {
_log.Append(message);
return *this;
}

void dumpLog() {
assert(!UiaOperationAbstraction::ShouldUseRemoteApi());
std::wstring messageBlock{L"Dump log start:\n"};
try {
// locally, a UiaArray is a shared_ptr to a vector of will_shared_bstr
const std::vector<wil::shared_bstr>& v = *_log;
for(const auto& message: v) {
messageBlock+=message.get();
}
} catch (std::exception& e) {
auto wideWhat = stringToWstring(e.what());
messageBlock += L"dumpLog exception: ";
messageBlock += wideWhat;
messageBlock += L"\n";
}
messageBlock+=L"Dump log end";
LOG_DEBUG(messageBlock);
}

private:
UiaArray<UiaString> _log;

};
41 changes: 41 additions & 0 deletions nvdaHelper/UIARemote/sconscript
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
###
#This file is a part of the NVDA project.
#URL: http://www.nvda-project.org/
#Copyright 2021 NV Access Limited.
#This program is free software: you can redistribute it and/or modify
#it under the terms of the GNU General Public License version 2.0, as published by
#the Free Software Foundation.
#This program is distributed in the hope that it will be useful,
#but WITHOUT ANY WARRANTY; without even the implied warranty of
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#This license can be found at:
#http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
###

Import([
'env',
'localLib',
'MSUIA_lib_outDir',
'MSUIA_include_outDir',
])

env = env.Clone()
env.Append(CPPPATH=Dir('#include/wil/include'))
env.Append(CPPPATH=MSUIA_include_outDir)
env.Append(CCFLAGS='/MD')

UIARemoteLib=env.SharedLibrary(
target="UIARemote",
source=[
env['projectResFile'],
"UIARemote.cpp",
],
LIBS=[
"runtimeobject",
"UIAutomationCore",
localLib[2],
MSUIA_lib_outDir.File('UiaOperationAbstraction.lib'),
],
)

Return('UIARemoteLib')
7 changes: 7 additions & 0 deletions nvdaHelper/archBuild_sconscript
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,13 @@ if TARGET_ARCH=='x86':
if signExec:
env.AddPostAction(win10localLib[0],[signExec])
env.Install(libInstallDir,win10localLib)
MSUIA_lib_outDir,MSUIA_include_outDir = thirdPartyEnv.SConscript('microsoft-ui-uiautomation/sconscript')
Export('MSUIA_lib_outDir')
Export('MSUIA_include_outDir')
UIARemoteLib=env.SConscript('UIARemote/sconscript')
if signExec:
env.AddPostAction(UIARemoteLib[0],[signExec])
env.Install(libInstallDir,UIARemoteLib)

clientLib=env.SConscript('client/sconscript')
Export('clientLib')
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<assembly manifestVersion="1.0" xmlns="urn:schemas-microsoft-com:asm.v1">
<assemblyIdentity version="1.0.0.0" name="UIARemote"/>

<file name="Microsoft.UI.UIAutomation.dll">
<activatableClass
name="Microsoft.UI.UIAutomation.AutomationRemoteOperation"
threadingModel="both"
xmlns="urn:schemas-microsoft-com:winrt.v1" />
</file>

</assembly>
Loading

0 comments on commit a3ed8b8

Please sign in to comment.