diff --git a/scripts/vstest-codecoverage2.runsettings b/scripts/vstest-codecoverage2.runsettings
new file mode 100644
index 0000000000..8a70b1f9ed
--- /dev/null
+++ b/scripts/vstest-codecoverage2.runsettings
@@ -0,0 +1,31 @@
+
+
+
+
+
+
+
+
+
+ .*CodeCoverage.exe$
+
+
+
+
+ True
+ True
+ True
+ False
+
+
+
+ .*TestSign.*
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs
new file mode 100644
index 0000000000..5401266b1c
--- /dev/null
+++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs
@@ -0,0 +1,79 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.TestPlatform.AcceptanceTests
+{
+ using System;
+ using System.Diagnostics;
+ using System.IO;
+ using System.Xml;
+
+ using Microsoft.TestPlatform.TestUtilities;
+ using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+ public class CodeCoverageAcceptanceTestBase : AcceptanceTestBase
+ {
+ /*
+ * Below value is just safe coverage result for which all tests are passing.
+ * Inspecting this value gives us confidence that there is no big drop in overall coverage.
+ */
+ protected const double ExpectedMinimalModuleCoverage = 30.0;
+
+ protected string GetCodeCoveragePath()
+ {
+ return Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "Microsoft.CodeCoverage");
+ }
+
+ protected string GetCodeCoverageExePath()
+ {
+ return Path.Combine(this.GetCodeCoveragePath(), "CodeCoverage", "CodeCoverage.exe");
+ }
+
+ protected XmlNode GetModuleNode(XmlNode node, string name)
+ {
+ return this.GetNode(node, "module", name);
+ }
+
+ protected XmlNode GetNode(XmlNode node, string type, string name)
+ {
+ return node.SelectSingleNode($"//{type}[@name='{name}']");
+ }
+
+ protected XmlDocument GetXmlCoverage(string coverageResult)
+ {
+ var codeCoverageExe = this.GetCodeCoverageExePath();
+ var output = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString() + ".xml");
+
+ var watch = new Stopwatch();
+
+ Console.WriteLine($"Starting {codeCoverageExe}");
+ watch.Start();
+ var analyze = Process.Start(new ProcessStartInfo
+ {
+ FileName = codeCoverageExe,
+ Arguments = $"analyze /include_skipped_functions /include_skipped_modules /output:\"{output}\" \"{coverageResult}\"",
+ RedirectStandardOutput = true,
+ UseShellExecute = false
+ });
+
+ string analysisOutput = analyze.StandardOutput.ReadToEnd();
+
+ analyze.WaitForExit();
+ watch.Stop();
+ Console.WriteLine($"Total execution time: {watch.Elapsed.Duration()}");
+
+ Assert.IsTrue(0 == analyze.ExitCode, $"Code Coverage analyze failed: {analysisOutput}");
+
+ XmlDocument coverage = new XmlDocument();
+ coverage.Load(output);
+ return coverage;
+ }
+
+ protected void AssertCoverage(XmlNode node, double expectedCoverage)
+ {
+ var coverage = double.Parse(node.Attributes["block_coverage"].Value);
+ Console.WriteLine($"Checking coverage for {node.Name} {node.Attributes["name"].Value}. Expected at least: {expectedCoverage}. Result: {coverage}");
+ Assert.IsTrue(coverage > expectedCoverage, $"Coverage check failed for {node.Name} {node.Attributes["name"].Value}. Expected at least: {expectedCoverage}. Found: {coverage}");
+ }
+ }
+}
diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs
index 38f8c73a85..8dba9d84f6 100644
--- a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs
+++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs
@@ -5,19 +5,42 @@ namespace Microsoft.TestPlatform.AcceptanceTests
{
using System;
using System.IO;
- using System.Linq;
using System.Xml;
+
using Microsoft.TestPlatform.TestUtilities;
using Microsoft.VisualStudio.TestTools.UnitTesting;
-#if NET451
- using VisualStudio.Coverage.Analysis;
-#endif
+
+ internal struct TestParameters
+ {
+ public enum SettingsType
+ {
+ None = 0,
+ Default = 1,
+ Custom = 2
+ }
+
+ public string AssemblyName { get; set; }
+
+ public string TargetPlatform { get; set; }
+
+ public SettingsType RunSettingsType { get; set; }
+
+ public string RunSettingsPath { get; set; }
+
+ public int ExpectedPassedTests { get; set; }
+
+ public int ExpectedSkippedTests { get; set; }
+
+ public int ExpectedFailedTests { get; set; }
+
+ public bool CheckSkipped { get; set; }
+ }
[TestClass]
- public class CodeCoverageTests : AcceptanceTestBase
+ public class CodeCoverageTests : CodeCoverageAcceptanceTestBase
{
private readonly string resultsDirectory;
- private readonly string assemblyName = "SimpleTestProject.dll";
+
public CodeCoverageTests()
{
this.resultsDirectory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
@@ -28,7 +51,18 @@ public CodeCoverageTests()
[NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
public void CollectCodeCoverageWithCollectOptionForx86(RunnerInfo runnerInfo)
{
- this.CollectCodeCoverage(runnerInfo, "x86", withRunsettings: false);
+ var parameters = new TestParameters()
+ {
+ AssemblyName = "SimpleTestProject.dll",
+ TargetPlatform = "x86",
+ RunSettingsPath = string.Empty,
+ RunSettingsType = TestParameters.SettingsType.None,
+ ExpectedPassedTests = 1,
+ ExpectedSkippedTests = 1,
+ ExpectedFailedTests = 1
+ };
+
+ this.CollectCodeCoverage(runnerInfo, parameters);
}
[TestMethod]
@@ -36,7 +70,18 @@ public void CollectCodeCoverageWithCollectOptionForx86(RunnerInfo runnerInfo)
[NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
public void CollectCodeCoverageWithCollectOptionForx64(RunnerInfo runnerInfo)
{
- this.CollectCodeCoverage(runnerInfo, "x64", withRunsettings: false);
+ var parameters = new TestParameters()
+ {
+ AssemblyName = "SimpleTestProject.dll",
+ TargetPlatform = "x64",
+ RunSettingsPath = string.Empty,
+ RunSettingsType = TestParameters.SettingsType.None,
+ ExpectedPassedTests = 1,
+ ExpectedSkippedTests = 1,
+ ExpectedFailedTests = 1
+ };
+
+ this.CollectCodeCoverage(runnerInfo, parameters);
}
[TestMethod]
@@ -44,7 +89,18 @@ public void CollectCodeCoverageWithCollectOptionForx64(RunnerInfo runnerInfo)
[NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
public void CollectCodeCoverageX86WithRunSettings(RunnerInfo runnerInfo)
{
- this.CollectCodeCoverage(runnerInfo, "x86", withRunsettings: true);
+ var parameters = new TestParameters()
+ {
+ AssemblyName = "SimpleTestProject.dll",
+ TargetPlatform = "x86",
+ RunSettingsPath = string.Empty,
+ RunSettingsType = TestParameters.SettingsType.Default,
+ ExpectedPassedTests = 1,
+ ExpectedSkippedTests = 1,
+ ExpectedFailedTests = 1
+ };
+
+ this.CollectCodeCoverage(runnerInfo, parameters);
}
[TestMethod]
@@ -52,36 +108,98 @@ public void CollectCodeCoverageX86WithRunSettings(RunnerInfo runnerInfo)
[NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
public void CollectCodeCoverageX64WithRunSettings(RunnerInfo runnerInfo)
{
- this.CollectCodeCoverage(runnerInfo, "x64", withRunsettings: true);
+ var parameters = new TestParameters()
+ {
+ AssemblyName = "SimpleTestProject.dll",
+ TargetPlatform = "x64",
+ RunSettingsPath = string.Empty,
+ RunSettingsType = TestParameters.SettingsType.Default,
+ ExpectedPassedTests = 1,
+ ExpectedSkippedTests = 1,
+ ExpectedFailedTests = 1
+ };
+
+ this.CollectCodeCoverage(runnerInfo, parameters);
}
- private void CollectCodeCoverage(RunnerInfo runnerInfo, string targetPlatform, bool withRunsettings)
+ [TestMethod]
+ [NetFullTargetFrameworkDataSource(useDesktopRunner: false)]
+ [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
+ public void CodeCoverageShouldAvoidExclusionsX86(RunnerInfo runnerInfo)
+ {
+ var parameters = new TestParameters()
+ {
+ AssemblyName = "CodeCoverageTest.dll",
+ TargetPlatform = "x86",
+ RunSettingsPath = Path.Combine(
+ IntegrationTestEnvironment.TestPlatformRootDirectory,
+ @"scripts\vstest-codecoverage2.runsettings"),
+ RunSettingsType = TestParameters.SettingsType.Custom,
+ ExpectedPassedTests = 3,
+ ExpectedSkippedTests = 0,
+ ExpectedFailedTests = 0,
+ CheckSkipped = true
+ };
+
+ this.CollectCodeCoverage(runnerInfo, parameters);
+ }
+
+ [TestMethod]
+ [NetFullTargetFrameworkDataSource(useDesktopRunner: false)]
+ [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
+ public void CodeCoverageShouldAvoidExclusionsX64(RunnerInfo runnerInfo)
+ {
+ var parameters = new TestParameters()
+ {
+ AssemblyName = "CodeCoverageTest.dll",
+ TargetPlatform = "x64",
+ RunSettingsPath = Path.Combine(
+ IntegrationTestEnvironment.TestPlatformRootDirectory,
+ @"scripts\vstest-codecoverage2.runsettings"),
+ RunSettingsType = TestParameters.SettingsType.Custom,
+ ExpectedPassedTests = 3,
+ ExpectedSkippedTests = 0,
+ ExpectedFailedTests = 0,
+ CheckSkipped = true
+ };
+
+ this.CollectCodeCoverage(runnerInfo, parameters);
+ }
+
+ private void CollectCodeCoverage(RunnerInfo runnerInfo, TestParameters testParameters)
{
AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
- var arguments = CreateArguments(runnerInfo, targetPlatform, withRunsettings, out var trxFilePath);
+ var arguments = this.CreateArguments(runnerInfo, testParameters, out var trxFilePath);
this.InvokeVsTest(arguments);
- this.ValidateSummaryStatus(1, 1, 1);
+ this.ValidateSummaryStatus(
+ testParameters.ExpectedPassedTests,
+ testParameters.ExpectedSkippedTests,
+ testParameters.ExpectedFailedTests);
var actualCoverageFile = CodeCoverageTests.GetCoverageFileNameFromTrx(trxFilePath, resultsDirectory);
Console.WriteLine($@"Coverage file: {actualCoverageFile} Results directory: {resultsDirectory} trxfile: {trxFilePath}");
Assert.IsTrue(File.Exists(actualCoverageFile), "Coverage file not found: {0}", actualCoverageFile);
- // Microsoft.VisualStudio.Coverage.Analysis assembly not available for .NET Core.
-#if NET451
- this.ValidateCoverageData(actualCoverageFile);
-#endif
+ var coverageDocument = this.GetXmlCoverage(actualCoverageFile);
+ if (testParameters.CheckSkipped)
+ {
+ this.AssertSkippedMethod(coverageDocument);
+ }
+
+ this.ValidateCoverageData(coverageDocument, testParameters.AssemblyName);
+
Directory.Delete(this.resultsDirectory, true);
}
- private string CreateArguments(RunnerInfo runnerInfo, string targetPlatform, bool withRunsettings,
+ private string CreateArguments(
+ RunnerInfo runnerInfo,
+ TestParameters testParameters,
out string trxFilePath)
{
- var assemblyPaths = this.GetAssetFullPath(assemblyName);
- string runSettings = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory,
- @"scripts\vstest-codecoverage.runsettings");
+ var assemblyPaths = this.GetAssetFullPath(testParameters.AssemblyName);
string traceDataCollectorDir = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory,
$@"src\DataCollectors\TraceDataCollector\bin\{IntegrationTestEnvironment.BuildConfiguration}\netstandard2.0");
@@ -91,62 +209,80 @@ private string CreateArguments(RunnerInfo runnerInfo, string targetPlatform, boo
this.FrameworkArgValue, runnerInfo.InIsolationValue);
arguments = string.Concat(arguments, $" /ResultsDirectory:{resultsDirectory}", $" /Diag:{diagFileName}",
$" /TestAdapterPath:{traceDataCollectorDir}");
- arguments = string.Concat(arguments, $" /Platform:{targetPlatform}");
+ arguments = string.Concat(arguments, $" /Platform:{testParameters.TargetPlatform}");
trxFilePath = Path.Combine(this.resultsDirectory, Guid.NewGuid() + ".trx");
arguments = string.Concat(arguments, " /logger:trx;logfilename=" + trxFilePath);
- if (withRunsettings)
- {
- arguments = string.Concat(arguments, $" /settings:{runSettings}");
- }
- else
+ var defaultRunSettingsPath = Path.Combine(
+ IntegrationTestEnvironment.TestPlatformRootDirectory,
+ @"scripts\vstest-codecoverage.runsettings");
+
+ var runSettings = string.Empty;
+ switch (testParameters.RunSettingsType)
{
- // With /collect:"Code Coverage" option.
- arguments = string.Concat(arguments, $" /collect:\"Code Coverage\"");
+ case TestParameters.SettingsType.None:
+ runSettings = $" /collect:\"Code Coverage\"";
+ break;
+ case TestParameters.SettingsType.Default:
+ runSettings = $" /settings:{defaultRunSettingsPath}";
+ break;
+ case TestParameters.SettingsType.Custom:
+ runSettings = $" /settings:{testParameters.RunSettingsPath}";
+ break;
}
+ arguments = string.Concat(arguments, runSettings);
+
return arguments;
}
-#if NET451
- private void ValidateCoverageData(string coverageFile)
+ private void AssertSkippedMethod(XmlDocument document)
{
- using (var converageInfo = CoverageInfo.CreateFromFile(coverageFile))
- {
- CoverageDS coverageDs = converageInfo.BuildDataSet();
- AssertModuleCoverageCollected(coverageDs);
- AssertSourceFileName(coverageDs);
- }
+ var module = this.GetModuleNode(document.DocumentElement, "codecoveragetest.dll");
+ Assert.IsNotNull(module);
+
+ var coverage = double.Parse(module.Attributes["block_coverage"].Value);
+ Assert.IsTrue(coverage > CodeCoverageAcceptanceTestBase.ExpectedMinimalModuleCoverage);
+
+ var testSignFunction = this.GetNode(module, "skipped_function", "TestSign()");
+ Assert.IsNotNull(testSignFunction);
+ Assert.AreEqual("name_excluded", testSignFunction.Attributes["reason"].Value);
+
+ var skippedTestMethod = this.GetNode(module, "skipped_function", "__CxxPureMSILEntry_Test()");
+ Assert.IsNotNull(skippedTestMethod);
+ Assert.AreEqual("name_excluded", skippedTestMethod.Attributes["reason"].Value);
+
+ var testAbsFunction = this.GetNode(module, "function", "TestAbs()");
+ Assert.IsNotNull(testAbsFunction);
}
- private static void AssertSourceFileName(CoverageDS coverageDS)
+ private void ValidateCoverageData(XmlDocument document, string moduleName)
{
- var sourceFileNames = from sourceFilePath in coverageDS.GetSourceFiles()
- select Path.GetFileName(sourceFilePath);
- var expectedFileName = "UnitTest1.cs";
- CollectionAssert.Contains(
- sourceFileNames.ToArray(),
- expectedFileName,
- $"Code Coverage not collected for file: {expectedFileName}");
+ var module = this.GetModuleNode(document.DocumentElement, moduleName.ToLower());
+ Assert.IsNotNull(module);
+
+ this.AssertCoverage(module, CodeCoverageAcceptanceTestBase.ExpectedMinimalModuleCoverage);
+ this.AssertSourceFileName(module);
}
- private void AssertModuleCoverageCollected(CoverageDS coverageDS)
+ private void AssertSourceFileName(XmlNode module)
{
- var moduleFound = false;
- for (int i = 0; i < coverageDS.Module.Count; i++)
+ const string ExpectedFileName = "UnitTest1.cs";
+
+ var found = false;
+ var sourcesNode = module.SelectSingleNode("./source_files");
+ foreach (XmlNode node in sourcesNode.ChildNodes)
{
- var module = coverageDS.Module[i];
- if (module.ModuleName.Equals(assemblyName, StringComparison.OrdinalIgnoreCase))
+ if (node.Attributes["path"].Value.Contains(ExpectedFileName))
{
- moduleFound = true;
+ found = true;
break;
}
}
- Assert.IsTrue(moduleFound, $"Code coverage not collected for module: {assemblyName}");
+ Assert.IsTrue(found);
}
-#endif
private static string GetCoverageFileNameFromTrx(string trxFilePath, string resultsDirectory)
{
@@ -179,27 +315,5 @@ private static string GetCoverageFileNameFromTrx(string trxFilePath, string resu
return Path.Combine(resultsDirectory, deploymentDir, "In", fileName);
}
}
-
- private bool SkipIfRuningInCI(string message)
- {
- // Setting Console.ForegroundColor to newColor which will be used to determine whether
- // test command output is redirecting to file or writing to console.
- // If command output is redirecting to file, then Console.ForegroundColor can't be modified.
- // So that tests which assert Console.ForegroundColor should not run.
- var previousColor = Console.ForegroundColor;
- var newColor = previousColor == ConsoleColor.Gray
- ? ConsoleColor.Black
- : ConsoleColor.Blue;
- Console.ForegroundColor = newColor;
- if (Console.ForegroundColor != newColor)
- {
- Console.ForegroundColor = previousColor;
- Assert.Inconclusive(message);
- }
-
- Console.ForegroundColor = previousColor;
-
- return false;
- }
}
}
\ No newline at end of file
diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj
index 928f62332f..dbc75999c4 100644
--- a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj
+++ b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj
@@ -35,5 +35,8 @@
+
+
+
diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs
index 81c1fe51ae..1185e8bd40 100644
--- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs
+++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs
@@ -1,14 +1,8 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
+// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests
{
- using Castle.Core.Internal;
- using Microsoft.TestPlatform.TestUtilities;
- using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces;
- using Microsoft.VisualStudio.TestPlatform.Common.Telemetry;
- using Microsoft.VisualStudio.TestPlatform.ObjectModel;
- using Microsoft.VisualStudio.TestTools.UnitTesting;
using System;
using System.Collections.Generic;
using System.Diagnostics;
@@ -16,20 +10,17 @@ namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
- using System.Xml;
- ///
- /// The Test run attachments processing tests using VsTestConsoleWrapper API's
- ///
- [TestClass]
- public class CodeCoverageTests : AcceptanceTestBase
- {
- /*
- * Below value is just safe coverage result for which all tests are passing.
- * Inspecting this value gives us confidence that there is no big drop in overall coverage.
- */
- private const double ExpectedMinimalModuleCoverage = 30.0;
+ using Microsoft.TestPlatform.TestUtilities;
+ using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces;
+ using Microsoft.VisualStudio.TestPlatform.Common.Telemetry;
+ using Microsoft.VisualStudio.TestPlatform.ObjectModel;
+ using Microsoft.VisualStudio.TestTools.UnitTesting;
+ using Castle.Core.Internal;
+
+ public class CodeCoverageTests : CodeCoverageAcceptanceTestBase
+ {
private IVsTestConsoleWrapper vstestConsoleWrapper;
private RunEventHandler runEventHandler;
private TestRunAttachmentsProcessingEventHandler testRunAttachmentsProcessingEventHandler;
@@ -99,7 +90,7 @@ public async Task TestRunWithCodeCoverageAndAttachmentsProcessing(RunnerInfo run
this.vstestConsoleWrapper.RunTests(this.GetTestAssemblies().Take(1), this.GetCodeCoverageRunSettings(1), this.runEventHandler);
this.vstestConsoleWrapper.RunTests(this.GetTestAssemblies().Skip(1), this.GetCodeCoverageRunSettings(1), this.runEventHandler);
-
+
Assert.AreEqual(6, this.runEventHandler.TestResults.Count);
Assert.AreEqual(2, this.runEventHandler.Attachments.Count);
@@ -125,7 +116,7 @@ public async Task TestRunWithCodeCoverageAndAttachmentsProcessing(RunnerInfo run
if (testRunAttachmentsProcessingEventHandler.ProgressArgs.Count == 2)
{
Assert.AreEqual(i == 0 ? 50 : 100, progressArgs.CurrentAttachmentProcessorProgress);
- }
+ }
}
Assert.AreEqual("Completed", testRunAttachmentsProcessingEventHandler.CompleteArgs.Metrics[TelemetryDataConstants.AttachmentsProcessingState]);
@@ -174,7 +165,7 @@ public async Task TestRunWithCodeCoverageAndAttachmentsProcessingNoMetrics(Runne
if (testRunAttachmentsProcessingEventHandler.ProgressArgs.Count == 2)
{
Assert.AreEqual(i == 0 ? 50 : 100, progressArgs.CurrentAttachmentProcessorProgress);
- }
+ }
}
Assert.IsTrue(testRunAttachmentsProcessingEventHandler.CompleteArgs.Metrics.IsNullOrEmpty());
@@ -221,7 +212,7 @@ public async Task TestRunWithCodeCoverageAndAttachmentsProcessingModuleDuplicate
if (testRunAttachmentsProcessingEventHandler.ProgressArgs.Count == 3)
{
Assert.AreEqual(i == 0 ? 33 : i == 1 ? 66 : 100, progressArgs.CurrentAttachmentProcessorProgress);
- }
+ }
}
Assert.AreEqual("Completed", testRunAttachmentsProcessingEventHandler.CompleteArgs.Metrics[TelemetryDataConstants.AttachmentsProcessingState]);
@@ -251,9 +242,9 @@ public async Task TestRunWithCodeCoverageAndAttachmentsProcessingCancelled(Runne
List attachments = Enumerable.Range(0, 1000).Select(i => this.runEventHandler.Attachments.First()).ToList();
CancellationTokenSource cts = new CancellationTokenSource();
-
+
Task attachmentsProcessing = this.vstestConsoleWrapper.ProcessTestRunAttachmentsAsync(attachments, null, true, true, testRunAttachmentsProcessingEventHandler, cts.Token);
-
+
while (true)
{
try
@@ -291,7 +282,7 @@ public async Task TestRunWithCodeCoverageAndAttachmentsProcessingCancelled(Runne
if (i == 0)
{
Assert.AreEqual(0, progressArgs.CurrentAttachmentProcessorProgress);
- }
+ }
}
Assert.AreEqual("Canceled", testRunAttachmentsProcessingEventHandler.CompleteArgs.Metrics[TelemetryDataConstants.AttachmentsProcessingState]);
@@ -343,13 +334,13 @@ private IList GetProjects()
/// Default RunSettings
///
///
- public string GetCodeCoverageRunSettings(int cpuCount)
+ private string GetCodeCoverageRunSettings(int cpuCount)
{
string runSettingsXml = $@"
{FrameworkArgValue}
- {GetCodeCoveragePath()}
+ {this.GetCodeCoveragePath()}
{cpuCount}
@@ -381,63 +372,14 @@ private void AssertCoverageResults(IList attachments)
{
if (attachments.Count == 1)
{
- var xmlCoverage = GetXmlCoverage(attachments.First());
+ var xmlCoverage = this.GetXmlCoverage(attachments.First().Attachments.First().Uri.LocalPath);
- foreach (var project in GetProjects())
+ foreach (var project in this.GetProjects())
{
- var moduleNode = GetModuleNode(xmlCoverage.DocumentElement, project.ToLower());
- AssertCoverage(moduleNode, ExpectedMinimalModuleCoverage);
+ var moduleNode = this.GetModuleNode(xmlCoverage.DocumentElement, project.ToLower());
+ this.AssertCoverage(moduleNode, CodeCoverageAcceptanceTestBase.ExpectedMinimalModuleCoverage);
}
}
}
-
- private XmlDocument GetXmlCoverage(AttachmentSet attachment)
- {
- string output = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString() + ".xml");
-
- var analyze = Process.Start(new ProcessStartInfo
- {
- FileName = GetCodeCoverageExePath(),
- Arguments = $"analyze /include_skipped_functions /include_skipped_modules /output:\"{output}\" \"{attachment.Attachments.First().Uri.LocalPath}\"",
- RedirectStandardOutput = true,
- UseShellExecute = false
- });
-
- string analysisOutput = analyze.StandardOutput.ReadToEnd();
-
- analyze.WaitForExit();
- Assert.IsTrue(0 == analyze.ExitCode, $"Code Coverage analyze failed: {analysisOutput}");
-
- XmlDocument coverage = new XmlDocument();
- coverage.Load(output);
- return coverage;
- }
-
- private string GetCodeCoveragePath()
- {
- return Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "Microsoft.CodeCoverage");
- }
-
- private string GetCodeCoverageExePath()
- {
- return Path.Combine(GetCodeCoveragePath(), "CodeCoverage", "CodeCoverage.exe");
- }
-
- private XmlNode GetModuleNode(XmlNode node, string name)
- {
- return GetNode(node, "module", name);
- }
-
- private XmlNode GetNode(XmlNode node, string type, string name)
- {
- return node.SelectSingleNode($"//{type}[@name='{name}']");
- }
-
- private void AssertCoverage(XmlNode node, double expectedCoverage)
- {
- var coverage = double.Parse(node.Attributes["block_coverage"].Value);
- Console.WriteLine($"Checking coverage for {node.Name} {node.Attributes["name"].Value}. Expected at least: {expectedCoverage}. Result: {coverage}");
- Assert.IsTrue(coverage > expectedCoverage, $"Coverage check failed for {node.Name} {node.Attributes["name"].Value}. Expected at least: {expectedCoverage}. Found: {coverage}");
- }
}
-}
\ No newline at end of file
+}
diff --git a/test/TestAssets/CodeCoverageTest/CodeCoverageTest.csproj b/test/TestAssets/CodeCoverageTest/CodeCoverageTest.csproj
new file mode 100644
index 0000000000..26a8b2276f
--- /dev/null
+++ b/test/TestAssets/CodeCoverageTest/CodeCoverageTest.csproj
@@ -0,0 +1,32 @@
+
+
+
+
+
+
+
+
+ CodeCoverageTest
+ netcoreapp2.1;net451
+
+
+
+
+ $(MSTestFrameworkVersion)
+
+
+ $(MSTestAdapterVersion)
+
+
+ $(NETTestSdkVersion)
+
+
+
+
+
+
+
+
+
+
+
diff --git a/test/TestAssets/CodeCoverageTest/Logic.cs b/test/TestAssets/CodeCoverageTest/Logic.cs
new file mode 100644
index 0000000000..a6481b04af
--- /dev/null
+++ b/test/TestAssets/CodeCoverageTest/Logic.cs
@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CodeCoverageTest
+{
+ public class Logic
+ {
+ public Logic()
+ {
+ }
+
+ public int Abs(int x)
+ {
+ return (x < 0) ? (-x) : (x);
+ }
+
+ public int Sign(int x)
+ {
+ if (x < 0) return -1;
+ if (x > 0) return 1;
+
+ return 0;
+ }
+ }
+}
diff --git a/test/TestAssets/CodeCoverageTest/UnitTest1.cs b/test/TestAssets/CodeCoverageTest/UnitTest1.cs
new file mode 100644
index 0000000000..071d4dfe93
--- /dev/null
+++ b/test/TestAssets/CodeCoverageTest/UnitTest1.cs
@@ -0,0 +1,38 @@
+using System;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace CodeCoverageTest
+{
+ [TestClass]
+ public class UnitTest1
+ {
+ private Logic logic;
+
+ public UnitTest1()
+ {
+ this.logic = new Logic();
+ }
+
+ [TestMethod]
+ public void TestAbs()
+ {
+ Assert.AreEqual(logic.Abs(0), 0);
+ Assert.AreEqual(logic.Abs(-5), 5);
+ Assert.AreEqual(logic.Abs(7), 7);
+ }
+
+ [TestMethod]
+ public void TestSign()
+ {
+ Assert.AreEqual(logic.Sign(0), 0);
+ Assert.AreEqual(logic.Sign(-5), -1);
+ Assert.AreEqual(logic.Sign(7), 1);
+ }
+
+ [TestMethod]
+ public void __CxxPureMSILEntry_Test()
+ {
+ Assert.AreEqual(logic.Abs(0), 0);
+ }
+ }
+}
diff --git a/test/TestAssets/TestAssets.sln/TestAssets.sln b/test/TestAssets/TestAssets.sln/TestAssets.sln
index e8e695f290..325d36db55 100644
--- a/test/TestAssets/TestAssets.sln/TestAssets.sln
+++ b/test/TestAssets/TestAssets.sln/TestAssets.sln
@@ -69,8 +69,11 @@ EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SimpleClassLibrary", "..\SimpleClassLibrary\SimpleClassLibrary.csproj", "{F37144D7-2C6D-42AF-9E85-EF10E5244A7B}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SelfContainedAppTestProject", "..\SelfContainedAppTestProject\SelfContainedAppTestProject.csproj", "{7F85E9D0-7BCE-431D-B468-4F6104E52DFA}"
+EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CoverletCoverageTestProject", "..\CoverletCoverageTestProject\CoverletCoverageTestProject.csproj", "{0D4D59D7-C52F-4858-A220-EAC7484E3827}"
EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodeCoverageTest", "..\CodeCoverageTest\CodeCoverageTest.csproj", "{23790570-66D2-4CD5-9CD0-F8787C5B61BF}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -465,6 +468,18 @@ Global
{0D4D59D7-C52F-4858-A220-EAC7484E3827}.Release|x64.Build.0 = Release|Any CPU
{0D4D59D7-C52F-4858-A220-EAC7484E3827}.Release|x86.ActiveCfg = Release|Any CPU
{0D4D59D7-C52F-4858-A220-EAC7484E3827}.Release|x86.Build.0 = Release|Any CPU
+ {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Debug|x64.ActiveCfg = Debug|Any CPU
+ {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Debug|x64.Build.0 = Debug|Any CPU
+ {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Debug|x86.Build.0 = Debug|Any CPU
+ {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Release|Any CPU.Build.0 = Release|Any CPU
+ {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Release|x64.ActiveCfg = Release|Any CPU
+ {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Release|x64.Build.0 = Release|Any CPU
+ {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Release|x86.ActiveCfg = Release|Any CPU
+ {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE