Collect XHarness diagnostics data and upload to App Insights (#7842)

src/Microsoft.DotNet.Helix/Sdk/Microsoft.DotNet.Helix.Sdk.csproj

@@ -32,6 +32,9 @@
   </ItemGroup>
 
   <ItemGroup>
+    <EmbeddedResource Include="tools\xharness-runner\xharness-event-reporter.py">
+      <CopyToOutputDirectory>Never</CopyToOutputDirectory>
+    </EmbeddedResource>
     <EmbeddedResource Include="tools\xharness-runner\xharness-helix-job.apple.sh">
       <CopyToOutputDirectory>Never</CopyToOutputDirectory>
     </EmbeddedResource>

src/Microsoft.DotNet.Helix/Sdk/XharnessTaskBase.cs

@@ -25,6 +25,7 @@ public class MetadataName
 
         private const string ScriptNamespace = "tools.xharness_runner.";
         private const string CustomCommandsScript = "command";
+        private const string DiagnosticsScript = "xharness-event-reporter.py";
 
         /// <summary>
         /// Extra arguments that will be passed to the iOS/Android/... app that is being run
@@ -173,6 +174,11 @@ await zipArchiveManager.AddResourceFileToArchive<XHarnessTaskBase>(
                     payloadScript);
             }
 
+            await zipArchiveManager.AddResourceFileToArchive<XHarnessTaskBase>(
+                outputZipPath,
+                ScriptNamespace + DiagnosticsScript,
+                DiagnosticsScript);
+
             await zipArchiveManager.AddContentToArchive(
                 outputZipPath,
                 CustomCommandsScript + (isPosix ? ".sh" : ".ps1"),

src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/XHarnessRunner.targets

@@ -60,6 +60,9 @@
       <HelixPreCommands>$(HelixPreCommands);export XHARNESS_DISABLE_COLORED_OUTPUT=true</HelixPreCommands>
       <HelixPreCommands>$(HelixPreCommands);export XHARNESS_LOG_WITH_TIMESTAMPS=true</HelixPreCommands>
       <HelixPreCommands>$(HelixPreCommands);export XHARNESS_CLI_PATH=$HELIX_CORRELATION_PAYLOAD/microsoft.dotnet.xharness.cli/$(_XHarnessPackageVersion)/tools/net6.0/any/Microsoft.DotNet.XHarness.CLI.dll</HelixPreCommands>
+
+      <HelixPreCommands Condition=" '$(EnableXHarnessTelemetry)' == 'true' ">$(HelixPreCommands);export XHARNESS_DIAGNOSTICS_PATH=$HELIX_WORKITEM_ROOT/diagnostics.json</HelixPreCommands>
+      <HelixPostCommands Condition=" '$(EnableXHarnessTelemetry)' == 'true' ">"$HELIX_PYTHONPATH" "$HELIX_WORKITEM_PAYLOAD/xharness-event-reporter.py";$(HelixPostCommands)</HelixPostCommands>
     </PropertyGroup>
 
     <PropertyGroup Condition="!$(IsPosixShell)">
@@ -68,6 +71,9 @@
       <HelixPreCommands>$(HelixPreCommands);set XHARNESS_LOG_WITH_TIMESTAMPS=true</HelixPreCommands>
       <HelixPreCommands>$(HelixPreCommands);set XHARNESS_CLI_PATH=%HELIX_CORRELATION_PAYLOAD%\microsoft.dotnet.xharness.cli\$(_XHarnessPackageVersion)\tools\net6.0\any\Microsoft.DotNet.XHarness.CLI.dll</HelixPreCommands>
       <HelixPreCommands>$(HelixPreCommands);doskey xharness="dotnet exec %XHARNESS_CLI_PATH%"</HelixPreCommands>
+
+      <HelixPreCommands Condition=" '$(EnableXHarnessTelemetry)' == 'true' ">$(HelixPreCommands);set XHARNESS_DIAGNOSTICS_PATH=%HELIX_WORKITEM_ROOT%\diagnostics.json</HelixPreCommands>
+      <HelixPostCommands Condition=" '$(EnableXHarnessTelemetry)' == 'true' ">"%HELIX_PYTHONPATH%" "%HELIX_WORKITEM_PAYLOAD%\xharness-event-reporter.py";$(HelixPostCommands)</HelixPostCommands>
     </PropertyGroup>
   </Target>
 

src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-event-reporter.py

@@ -0,0 +1,44 @@
+import getopt
+import json
+import os
+import sys
+
+from helix.appinsights import app_insights
+
+opts, args = getopt.gnu_getopt(sys.argv[1:], 'd:', ['diagnostics-data='])
+opt_dict = dict(opts)
+
+diagnostics_file = None
+
+if '--data' in opt_dict:
+    diagnostics_file = opt_dict['--data']
+elif '-d' in opt_dict:
+    diagnostics_file = opt_dict['-d']
+else:
+    diagnostics_file = os.getenv('XHARNESS_DIAGNOSTICS_PATH')
+
+if not diagnostics_file:
+    print('ERROR: Expected path to the diagnostics JSON file generated by XHarness')
+    exit(1)
+
+if not os.path.isfile(diagnostics_file):
+    print(f"WARNING: Diagnostics file not found at `{diagnostics_file}`")
+    exit(2)
+
+# The JSON should be an array of objects (one per each executed XHarness command)
+operations = json.load(open(diagnostics_file))
+
+for operation in operations:
+    custom_dimensions = dict()
+
+    custom_dimensions['command'] = operation['command']
+    custom_dimensions['platform'] = operation['platform']
+
+    if 'target' in operation:
+        if 'targetOS' in operation:
+            custom_dimensions['target'] = operation['target'] + '_' + operation['targetOS']
+        else:
+            custom_dimensions['target'] = operation['target']
+
+    app_insights.send_metric('XHarnessOperation', operation['exitCode'], properties=custom_dimensions)
+    app_insights.send_metric('XHarnessOperationDuration', operation['duration'], properties=custom_dimensions)

src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.android.ps1

@@ -32,9 +32,13 @@ function xharness() {
     dotnet exec $Env:XHARNESS_CLI_PATH @args
 }
 
+$ErrorActionPreference="Continue"
+
 # Act out the actual commands
 . "$PSScriptRoot\command.ps1"
 
+$ErrorActionPreference="Continue"
+
 $exit_code=$LASTEXITCODE
 
 $retry=$false
@@ -44,7 +48,6 @@ switch ($exit_code)
 {
     # ADB_DEVICE_ENUMERATION_FAILURE
     85 {
-        $ErrorActionPreference="Continue"
         Write-Error "Encountered ADB_DEVICE_ENUMERATION_FAILURE. This is typically not a failure of the work item. We will run it again and reboot this computer to help its devices"
         Write-Error "If this occurs repeatedly, please check for architectural mismatch, e.g. sending x86 or x86_64 APKs to an arm64_v8a-only queue."
         $retry=$true
@@ -54,7 +57,6 @@ switch ($exit_code)
 
     # PACKAGE_INSTALLATION_FAILURE
     78 {
-        $ErrorActionPreference="Continue"
         Write-Error "Encountered PACKAGE_INSTALLATION_FAILURE. This is typically not a failure of the work item. We will try it again on another Helix agent"
         Write-Error "If this occurs repeatedly, please check for architectural mismatch, e.g. requesting installation on arm64_v8a-only queue for x86 or x86_64 APKs."
         $retry=$true

src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-helix-job.apple.sh

@@ -27,13 +27,16 @@ set -x
 
 # It is important we call the script via `launchctl asuser` in order to be able to spawn
 # the simulator which needs to run in a user session with GUI rendering capabilities.
+# The problem with launchctl is that the spawned process won't share environment variables
+# so we have to pass all of them as parameters are set them again.
 chmod +x xharness-runner.apple.sh
 helix_runner_uid=$(id -u)
 sudo launchctl asuser "$helix_runner_uid" sh ./xharness-runner.apple.sh \
     $forwarded_args                                                     \
     --app "$HELIX_WORKITEM_ROOT/$app"                                   \
     --xharness-cli-path "$XHARNESS_CLI_PATH"                            \
     --output-directory "$HELIX_WORKITEM_UPLOAD_ROOT"                    \
+    --diagnostics-path "$XHARNESS_DIAGNOSTICS_PATH"                     \
 
 exit_code=$?
 

src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/xharness-runner.apple.sh

@@ -29,6 +29,10 @@ while [[ $# -gt 0 ]]; do
         output_directory="$2"
         shift
         ;;
+      --diagnostics-path)
+        export XHARNESS_DIAGNOSTICS_PATH="$2"
+        shift
+        ;;
       --target)
         target="$2"
         shift

tests/UnitTests.XHarness.Common.props

@@ -18,6 +18,7 @@
     <EnableXUnitReporter>true</EnableXUnitReporter>
     <EnableAzurePipelinesReporter>true</EnableAzurePipelinesReporter>
     <HelixBaseUri>https://helix.dot.net</HelixBaseUri>
+    <EnableXHarnessTelemetry>true</EnableXHarnessTelemetry>
   </PropertyGroup>
 
   <!-- For non-ci local runs -->