diff --git a/scripts/vstest-codecoverage2.runsettings b/scripts/vstest-codecoverage2.runsettings new file mode 100644 index 0000000000..8a70b1f9ed --- /dev/null +++ b/scripts/vstest-codecoverage2.runsettings @@ -0,0 +1,31 @@ + + + + + + + + + + .*CodeCoverage.exe$ + + + + + True + True + True + False + + + + .*TestSign.* + + + + + + + + + \ No newline at end of file diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs new file mode 100644 index 0000000000..5401266b1c --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.AcceptanceTests +{ + using System; + using System.Diagnostics; + using System.IO; + using System.Xml; + + using Microsoft.TestPlatform.TestUtilities; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + public class CodeCoverageAcceptanceTestBase : AcceptanceTestBase + { + /* + * Below value is just safe coverage result for which all tests are passing. + * Inspecting this value gives us confidence that there is no big drop in overall coverage. + */ + protected const double ExpectedMinimalModuleCoverage = 30.0; + + protected string GetCodeCoveragePath() + { + return Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "Microsoft.CodeCoverage"); + } + + protected string GetCodeCoverageExePath() + { + return Path.Combine(this.GetCodeCoveragePath(), "CodeCoverage", "CodeCoverage.exe"); + } + + protected XmlNode GetModuleNode(XmlNode node, string name) + { + return this.GetNode(node, "module", name); + } + + protected XmlNode GetNode(XmlNode node, string type, string name) + { + return node.SelectSingleNode($"//{type}[@name='{name}']"); + } + + protected XmlDocument GetXmlCoverage(string coverageResult) + { + var codeCoverageExe = this.GetCodeCoverageExePath(); + var output = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString() + ".xml"); + + var watch = new Stopwatch(); + + Console.WriteLine($"Starting {codeCoverageExe}"); + watch.Start(); + var analyze = Process.Start(new ProcessStartInfo + { + FileName = codeCoverageExe, + Arguments = $"analyze /include_skipped_functions /include_skipped_modules /output:\"{output}\" \"{coverageResult}\"", + RedirectStandardOutput = true, + UseShellExecute = false + }); + + string analysisOutput = analyze.StandardOutput.ReadToEnd(); + + analyze.WaitForExit(); + watch.Stop(); + Console.WriteLine($"Total execution time: {watch.Elapsed.Duration()}"); + + Assert.IsTrue(0 == analyze.ExitCode, $"Code Coverage analyze failed: {analysisOutput}"); + + XmlDocument coverage = new XmlDocument(); + coverage.Load(output); + return coverage; + } + + protected void AssertCoverage(XmlNode node, double expectedCoverage) + { + var coverage = double.Parse(node.Attributes["block_coverage"].Value); + Console.WriteLine($"Checking coverage for {node.Name} {node.Attributes["name"].Value}. Expected at least: {expectedCoverage}. Result: {coverage}"); + Assert.IsTrue(coverage > expectedCoverage, $"Coverage check failed for {node.Name} {node.Attributes["name"].Value}. Expected at least: {expectedCoverage}. Found: {coverage}"); + } + } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs index 38f8c73a85..8dba9d84f6 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs @@ -5,19 +5,42 @@ namespace Microsoft.TestPlatform.AcceptanceTests { using System; using System.IO; - using System.Linq; using System.Xml; + using Microsoft.TestPlatform.TestUtilities; using Microsoft.VisualStudio.TestTools.UnitTesting; -#if NET451 - using VisualStudio.Coverage.Analysis; -#endif + + internal struct TestParameters + { + public enum SettingsType + { + None = 0, + Default = 1, + Custom = 2 + } + + public string AssemblyName { get; set; } + + public string TargetPlatform { get; set; } + + public SettingsType RunSettingsType { get; set; } + + public string RunSettingsPath { get; set; } + + public int ExpectedPassedTests { get; set; } + + public int ExpectedSkippedTests { get; set; } + + public int ExpectedFailedTests { get; set; } + + public bool CheckSkipped { get; set; } + } [TestClass] - public class CodeCoverageTests : AcceptanceTestBase + public class CodeCoverageTests : CodeCoverageAcceptanceTestBase { private readonly string resultsDirectory; - private readonly string assemblyName = "SimpleTestProject.dll"; + public CodeCoverageTests() { this.resultsDirectory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); @@ -28,7 +51,18 @@ public CodeCoverageTests() [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)] public void CollectCodeCoverageWithCollectOptionForx86(RunnerInfo runnerInfo) { - this.CollectCodeCoverage(runnerInfo, "x86", withRunsettings: false); + var parameters = new TestParameters() + { + AssemblyName = "SimpleTestProject.dll", + TargetPlatform = "x86", + RunSettingsPath = string.Empty, + RunSettingsType = TestParameters.SettingsType.None, + ExpectedPassedTests = 1, + ExpectedSkippedTests = 1, + ExpectedFailedTests = 1 + }; + + this.CollectCodeCoverage(runnerInfo, parameters); } [TestMethod] @@ -36,7 +70,18 @@ public void CollectCodeCoverageWithCollectOptionForx86(RunnerInfo runnerInfo) [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)] public void CollectCodeCoverageWithCollectOptionForx64(RunnerInfo runnerInfo) { - this.CollectCodeCoverage(runnerInfo, "x64", withRunsettings: false); + var parameters = new TestParameters() + { + AssemblyName = "SimpleTestProject.dll", + TargetPlatform = "x64", + RunSettingsPath = string.Empty, + RunSettingsType = TestParameters.SettingsType.None, + ExpectedPassedTests = 1, + ExpectedSkippedTests = 1, + ExpectedFailedTests = 1 + }; + + this.CollectCodeCoverage(runnerInfo, parameters); } [TestMethod] @@ -44,7 +89,18 @@ public void CollectCodeCoverageWithCollectOptionForx64(RunnerInfo runnerInfo) [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)] public void CollectCodeCoverageX86WithRunSettings(RunnerInfo runnerInfo) { - this.CollectCodeCoverage(runnerInfo, "x86", withRunsettings: true); + var parameters = new TestParameters() + { + AssemblyName = "SimpleTestProject.dll", + TargetPlatform = "x86", + RunSettingsPath = string.Empty, + RunSettingsType = TestParameters.SettingsType.Default, + ExpectedPassedTests = 1, + ExpectedSkippedTests = 1, + ExpectedFailedTests = 1 + }; + + this.CollectCodeCoverage(runnerInfo, parameters); } [TestMethod] @@ -52,36 +108,98 @@ public void CollectCodeCoverageX86WithRunSettings(RunnerInfo runnerInfo) [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)] public void CollectCodeCoverageX64WithRunSettings(RunnerInfo runnerInfo) { - this.CollectCodeCoverage(runnerInfo, "x64", withRunsettings: true); + var parameters = new TestParameters() + { + AssemblyName = "SimpleTestProject.dll", + TargetPlatform = "x64", + RunSettingsPath = string.Empty, + RunSettingsType = TestParameters.SettingsType.Default, + ExpectedPassedTests = 1, + ExpectedSkippedTests = 1, + ExpectedFailedTests = 1 + }; + + this.CollectCodeCoverage(runnerInfo, parameters); } - private void CollectCodeCoverage(RunnerInfo runnerInfo, string targetPlatform, bool withRunsettings) + [TestMethod] + [NetFullTargetFrameworkDataSource(useDesktopRunner: false)] + [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)] + public void CodeCoverageShouldAvoidExclusionsX86(RunnerInfo runnerInfo) + { + var parameters = new TestParameters() + { + AssemblyName = "CodeCoverageTest.dll", + TargetPlatform = "x86", + RunSettingsPath = Path.Combine( + IntegrationTestEnvironment.TestPlatformRootDirectory, + @"scripts\vstest-codecoverage2.runsettings"), + RunSettingsType = TestParameters.SettingsType.Custom, + ExpectedPassedTests = 3, + ExpectedSkippedTests = 0, + ExpectedFailedTests = 0, + CheckSkipped = true + }; + + this.CollectCodeCoverage(runnerInfo, parameters); + } + + [TestMethod] + [NetFullTargetFrameworkDataSource(useDesktopRunner: false)] + [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)] + public void CodeCoverageShouldAvoidExclusionsX64(RunnerInfo runnerInfo) + { + var parameters = new TestParameters() + { + AssemblyName = "CodeCoverageTest.dll", + TargetPlatform = "x64", + RunSettingsPath = Path.Combine( + IntegrationTestEnvironment.TestPlatformRootDirectory, + @"scripts\vstest-codecoverage2.runsettings"), + RunSettingsType = TestParameters.SettingsType.Custom, + ExpectedPassedTests = 3, + ExpectedSkippedTests = 0, + ExpectedFailedTests = 0, + CheckSkipped = true + }; + + this.CollectCodeCoverage(runnerInfo, parameters); + } + + private void CollectCodeCoverage(RunnerInfo runnerInfo, TestParameters testParameters) { AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo); - var arguments = CreateArguments(runnerInfo, targetPlatform, withRunsettings, out var trxFilePath); + var arguments = this.CreateArguments(runnerInfo, testParameters, out var trxFilePath); this.InvokeVsTest(arguments); - this.ValidateSummaryStatus(1, 1, 1); + this.ValidateSummaryStatus( + testParameters.ExpectedPassedTests, + testParameters.ExpectedSkippedTests, + testParameters.ExpectedFailedTests); var actualCoverageFile = CodeCoverageTests.GetCoverageFileNameFromTrx(trxFilePath, resultsDirectory); Console.WriteLine($@"Coverage file: {actualCoverageFile} Results directory: {resultsDirectory} trxfile: {trxFilePath}"); Assert.IsTrue(File.Exists(actualCoverageFile), "Coverage file not found: {0}", actualCoverageFile); - // Microsoft.VisualStudio.Coverage.Analysis assembly not available for .NET Core. -#if NET451 - this.ValidateCoverageData(actualCoverageFile); -#endif + var coverageDocument = this.GetXmlCoverage(actualCoverageFile); + if (testParameters.CheckSkipped) + { + this.AssertSkippedMethod(coverageDocument); + } + + this.ValidateCoverageData(coverageDocument, testParameters.AssemblyName); + Directory.Delete(this.resultsDirectory, true); } - private string CreateArguments(RunnerInfo runnerInfo, string targetPlatform, bool withRunsettings, + private string CreateArguments( + RunnerInfo runnerInfo, + TestParameters testParameters, out string trxFilePath) { - var assemblyPaths = this.GetAssetFullPath(assemblyName); - string runSettings = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, - @"scripts\vstest-codecoverage.runsettings"); + var assemblyPaths = this.GetAssetFullPath(testParameters.AssemblyName); string traceDataCollectorDir = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, $@"src\DataCollectors\TraceDataCollector\bin\{IntegrationTestEnvironment.BuildConfiguration}\netstandard2.0"); @@ -91,62 +209,80 @@ private string CreateArguments(RunnerInfo runnerInfo, string targetPlatform, boo this.FrameworkArgValue, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $" /ResultsDirectory:{resultsDirectory}", $" /Diag:{diagFileName}", $" /TestAdapterPath:{traceDataCollectorDir}"); - arguments = string.Concat(arguments, $" /Platform:{targetPlatform}"); + arguments = string.Concat(arguments, $" /Platform:{testParameters.TargetPlatform}"); trxFilePath = Path.Combine(this.resultsDirectory, Guid.NewGuid() + ".trx"); arguments = string.Concat(arguments, " /logger:trx;logfilename=" + trxFilePath); - if (withRunsettings) - { - arguments = string.Concat(arguments, $" /settings:{runSettings}"); - } - else + var defaultRunSettingsPath = Path.Combine( + IntegrationTestEnvironment.TestPlatformRootDirectory, + @"scripts\vstest-codecoverage.runsettings"); + + var runSettings = string.Empty; + switch (testParameters.RunSettingsType) { - // With /collect:"Code Coverage" option. - arguments = string.Concat(arguments, $" /collect:\"Code Coverage\""); + case TestParameters.SettingsType.None: + runSettings = $" /collect:\"Code Coverage\""; + break; + case TestParameters.SettingsType.Default: + runSettings = $" /settings:{defaultRunSettingsPath}"; + break; + case TestParameters.SettingsType.Custom: + runSettings = $" /settings:{testParameters.RunSettingsPath}"; + break; } + arguments = string.Concat(arguments, runSettings); + return arguments; } -#if NET451 - private void ValidateCoverageData(string coverageFile) + private void AssertSkippedMethod(XmlDocument document) { - using (var converageInfo = CoverageInfo.CreateFromFile(coverageFile)) - { - CoverageDS coverageDs = converageInfo.BuildDataSet(); - AssertModuleCoverageCollected(coverageDs); - AssertSourceFileName(coverageDs); - } + var module = this.GetModuleNode(document.DocumentElement, "codecoveragetest.dll"); + Assert.IsNotNull(module); + + var coverage = double.Parse(module.Attributes["block_coverage"].Value); + Assert.IsTrue(coverage > CodeCoverageAcceptanceTestBase.ExpectedMinimalModuleCoverage); + + var testSignFunction = this.GetNode(module, "skipped_function", "TestSign()"); + Assert.IsNotNull(testSignFunction); + Assert.AreEqual("name_excluded", testSignFunction.Attributes["reason"].Value); + + var skippedTestMethod = this.GetNode(module, "skipped_function", "__CxxPureMSILEntry_Test()"); + Assert.IsNotNull(skippedTestMethod); + Assert.AreEqual("name_excluded", skippedTestMethod.Attributes["reason"].Value); + + var testAbsFunction = this.GetNode(module, "function", "TestAbs()"); + Assert.IsNotNull(testAbsFunction); } - private static void AssertSourceFileName(CoverageDS coverageDS) + private void ValidateCoverageData(XmlDocument document, string moduleName) { - var sourceFileNames = from sourceFilePath in coverageDS.GetSourceFiles() - select Path.GetFileName(sourceFilePath); - var expectedFileName = "UnitTest1.cs"; - CollectionAssert.Contains( - sourceFileNames.ToArray(), - expectedFileName, - $"Code Coverage not collected for file: {expectedFileName}"); + var module = this.GetModuleNode(document.DocumentElement, moduleName.ToLower()); + Assert.IsNotNull(module); + + this.AssertCoverage(module, CodeCoverageAcceptanceTestBase.ExpectedMinimalModuleCoverage); + this.AssertSourceFileName(module); } - private void AssertModuleCoverageCollected(CoverageDS coverageDS) + private void AssertSourceFileName(XmlNode module) { - var moduleFound = false; - for (int i = 0; i < coverageDS.Module.Count; i++) + const string ExpectedFileName = "UnitTest1.cs"; + + var found = false; + var sourcesNode = module.SelectSingleNode("./source_files"); + foreach (XmlNode node in sourcesNode.ChildNodes) { - var module = coverageDS.Module[i]; - if (module.ModuleName.Equals(assemblyName, StringComparison.OrdinalIgnoreCase)) + if (node.Attributes["path"].Value.Contains(ExpectedFileName)) { - moduleFound = true; + found = true; break; } } - Assert.IsTrue(moduleFound, $"Code coverage not collected for module: {assemblyName}"); + Assert.IsTrue(found); } -#endif private static string GetCoverageFileNameFromTrx(string trxFilePath, string resultsDirectory) { @@ -179,27 +315,5 @@ private static string GetCoverageFileNameFromTrx(string trxFilePath, string resu return Path.Combine(resultsDirectory, deploymentDir, "In", fileName); } } - - private bool SkipIfRuningInCI(string message) - { - // Setting Console.ForegroundColor to newColor which will be used to determine whether - // test command output is redirecting to file or writing to console. - // If command output is redirecting to file, then Console.ForegroundColor can't be modified. - // So that tests which assert Console.ForegroundColor should not run. - var previousColor = Console.ForegroundColor; - var newColor = previousColor == ConsoleColor.Gray - ? ConsoleColor.Black - : ConsoleColor.Blue; - Console.ForegroundColor = newColor; - if (Console.ForegroundColor != newColor) - { - Console.ForegroundColor = previousColor; - Assert.Inconclusive(message); - } - - Console.ForegroundColor = previousColor; - - return false; - } } } \ No newline at end of file diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj index 928f62332f..dbc75999c4 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj @@ -35,5 +35,8 @@ + + + diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs index 81c1fe51ae..1185e8bd40 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs @@ -1,14 +1,8 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests { - using Castle.Core.Internal; - using Microsoft.TestPlatform.TestUtilities; - using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; - using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; - using Microsoft.VisualStudio.TestPlatform.ObjectModel; - using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.Collections.Generic; using System.Diagnostics; @@ -16,20 +10,17 @@ namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests using System.Linq; using System.Threading; using System.Threading.Tasks; - using System.Xml; - /// - /// The Test run attachments processing tests using VsTestConsoleWrapper API's - /// - [TestClass] - public class CodeCoverageTests : AcceptanceTestBase - { - /* - * Below value is just safe coverage result for which all tests are passing. - * Inspecting this value gives us confidence that there is no big drop in overall coverage. - */ - private const double ExpectedMinimalModuleCoverage = 30.0; + using Microsoft.TestPlatform.TestUtilities; + using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; + using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using Castle.Core.Internal; + + public class CodeCoverageTests : CodeCoverageAcceptanceTestBase + { private IVsTestConsoleWrapper vstestConsoleWrapper; private RunEventHandler runEventHandler; private TestRunAttachmentsProcessingEventHandler testRunAttachmentsProcessingEventHandler; @@ -99,7 +90,7 @@ public async Task TestRunWithCodeCoverageAndAttachmentsProcessing(RunnerInfo run this.vstestConsoleWrapper.RunTests(this.GetTestAssemblies().Take(1), this.GetCodeCoverageRunSettings(1), this.runEventHandler); this.vstestConsoleWrapper.RunTests(this.GetTestAssemblies().Skip(1), this.GetCodeCoverageRunSettings(1), this.runEventHandler); - + Assert.AreEqual(6, this.runEventHandler.TestResults.Count); Assert.AreEqual(2, this.runEventHandler.Attachments.Count); @@ -125,7 +116,7 @@ public async Task TestRunWithCodeCoverageAndAttachmentsProcessing(RunnerInfo run if (testRunAttachmentsProcessingEventHandler.ProgressArgs.Count == 2) { Assert.AreEqual(i == 0 ? 50 : 100, progressArgs.CurrentAttachmentProcessorProgress); - } + } } Assert.AreEqual("Completed", testRunAttachmentsProcessingEventHandler.CompleteArgs.Metrics[TelemetryDataConstants.AttachmentsProcessingState]); @@ -174,7 +165,7 @@ public async Task TestRunWithCodeCoverageAndAttachmentsProcessingNoMetrics(Runne if (testRunAttachmentsProcessingEventHandler.ProgressArgs.Count == 2) { Assert.AreEqual(i == 0 ? 50 : 100, progressArgs.CurrentAttachmentProcessorProgress); - } + } } Assert.IsTrue(testRunAttachmentsProcessingEventHandler.CompleteArgs.Metrics.IsNullOrEmpty()); @@ -221,7 +212,7 @@ public async Task TestRunWithCodeCoverageAndAttachmentsProcessingModuleDuplicate if (testRunAttachmentsProcessingEventHandler.ProgressArgs.Count == 3) { Assert.AreEqual(i == 0 ? 33 : i == 1 ? 66 : 100, progressArgs.CurrentAttachmentProcessorProgress); - } + } } Assert.AreEqual("Completed", testRunAttachmentsProcessingEventHandler.CompleteArgs.Metrics[TelemetryDataConstants.AttachmentsProcessingState]); @@ -251,9 +242,9 @@ public async Task TestRunWithCodeCoverageAndAttachmentsProcessingCancelled(Runne List attachments = Enumerable.Range(0, 1000).Select(i => this.runEventHandler.Attachments.First()).ToList(); CancellationTokenSource cts = new CancellationTokenSource(); - + Task attachmentsProcessing = this.vstestConsoleWrapper.ProcessTestRunAttachmentsAsync(attachments, null, true, true, testRunAttachmentsProcessingEventHandler, cts.Token); - + while (true) { try @@ -291,7 +282,7 @@ public async Task TestRunWithCodeCoverageAndAttachmentsProcessingCancelled(Runne if (i == 0) { Assert.AreEqual(0, progressArgs.CurrentAttachmentProcessorProgress); - } + } } Assert.AreEqual("Canceled", testRunAttachmentsProcessingEventHandler.CompleteArgs.Metrics[TelemetryDataConstants.AttachmentsProcessingState]); @@ -343,13 +334,13 @@ private IList GetProjects() /// Default RunSettings /// /// - public string GetCodeCoverageRunSettings(int cpuCount) + private string GetCodeCoverageRunSettings(int cpuCount) { string runSettingsXml = $@" {FrameworkArgValue} - {GetCodeCoveragePath()} + {this.GetCodeCoveragePath()} {cpuCount} @@ -381,63 +372,14 @@ private void AssertCoverageResults(IList attachments) { if (attachments.Count == 1) { - var xmlCoverage = GetXmlCoverage(attachments.First()); + var xmlCoverage = this.GetXmlCoverage(attachments.First().Attachments.First().Uri.LocalPath); - foreach (var project in GetProjects()) + foreach (var project in this.GetProjects()) { - var moduleNode = GetModuleNode(xmlCoverage.DocumentElement, project.ToLower()); - AssertCoverage(moduleNode, ExpectedMinimalModuleCoverage); + var moduleNode = this.GetModuleNode(xmlCoverage.DocumentElement, project.ToLower()); + this.AssertCoverage(moduleNode, CodeCoverageAcceptanceTestBase.ExpectedMinimalModuleCoverage); } } } - - private XmlDocument GetXmlCoverage(AttachmentSet attachment) - { - string output = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString() + ".xml"); - - var analyze = Process.Start(new ProcessStartInfo - { - FileName = GetCodeCoverageExePath(), - Arguments = $"analyze /include_skipped_functions /include_skipped_modules /output:\"{output}\" \"{attachment.Attachments.First().Uri.LocalPath}\"", - RedirectStandardOutput = true, - UseShellExecute = false - }); - - string analysisOutput = analyze.StandardOutput.ReadToEnd(); - - analyze.WaitForExit(); - Assert.IsTrue(0 == analyze.ExitCode, $"Code Coverage analyze failed: {analysisOutput}"); - - XmlDocument coverage = new XmlDocument(); - coverage.Load(output); - return coverage; - } - - private string GetCodeCoveragePath() - { - return Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "Microsoft.CodeCoverage"); - } - - private string GetCodeCoverageExePath() - { - return Path.Combine(GetCodeCoveragePath(), "CodeCoverage", "CodeCoverage.exe"); - } - - private XmlNode GetModuleNode(XmlNode node, string name) - { - return GetNode(node, "module", name); - } - - private XmlNode GetNode(XmlNode node, string type, string name) - { - return node.SelectSingleNode($"//{type}[@name='{name}']"); - } - - private void AssertCoverage(XmlNode node, double expectedCoverage) - { - var coverage = double.Parse(node.Attributes["block_coverage"].Value); - Console.WriteLine($"Checking coverage for {node.Name} {node.Attributes["name"].Value}. Expected at least: {expectedCoverage}. Result: {coverage}"); - Assert.IsTrue(coverage > expectedCoverage, $"Coverage check failed for {node.Name} {node.Attributes["name"].Value}. Expected at least: {expectedCoverage}. Found: {coverage}"); - } } -} \ No newline at end of file +} diff --git a/test/TestAssets/CodeCoverageTest/CodeCoverageTest.csproj b/test/TestAssets/CodeCoverageTest/CodeCoverageTest.csproj new file mode 100644 index 0000000000..26a8b2276f --- /dev/null +++ b/test/TestAssets/CodeCoverageTest/CodeCoverageTest.csproj @@ -0,0 +1,32 @@ + + + + + + + + + CodeCoverageTest + netcoreapp2.1;net451 + + + + + $(MSTestFrameworkVersion) + + + $(MSTestAdapterVersion) + + + $(NETTestSdkVersion) + + + + + + + + + + + diff --git a/test/TestAssets/CodeCoverageTest/Logic.cs b/test/TestAssets/CodeCoverageTest/Logic.cs new file mode 100644 index 0000000000..a6481b04af --- /dev/null +++ b/test/TestAssets/CodeCoverageTest/Logic.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace CodeCoverageTest +{ + public class Logic + { + public Logic() + { + } + + public int Abs(int x) + { + return (x < 0) ? (-x) : (x); + } + + public int Sign(int x) + { + if (x < 0) return -1; + if (x > 0) return 1; + + return 0; + } + } +} diff --git a/test/TestAssets/CodeCoverageTest/UnitTest1.cs b/test/TestAssets/CodeCoverageTest/UnitTest1.cs new file mode 100644 index 0000000000..071d4dfe93 --- /dev/null +++ b/test/TestAssets/CodeCoverageTest/UnitTest1.cs @@ -0,0 +1,38 @@ +using System; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace CodeCoverageTest +{ + [TestClass] + public class UnitTest1 + { + private Logic logic; + + public UnitTest1() + { + this.logic = new Logic(); + } + + [TestMethod] + public void TestAbs() + { + Assert.AreEqual(logic.Abs(0), 0); + Assert.AreEqual(logic.Abs(-5), 5); + Assert.AreEqual(logic.Abs(7), 7); + } + + [TestMethod] + public void TestSign() + { + Assert.AreEqual(logic.Sign(0), 0); + Assert.AreEqual(logic.Sign(-5), -1); + Assert.AreEqual(logic.Sign(7), 1); + } + + [TestMethod] + public void __CxxPureMSILEntry_Test() + { + Assert.AreEqual(logic.Abs(0), 0); + } + } +} diff --git a/test/TestAssets/TestAssets.sln/TestAssets.sln b/test/TestAssets/TestAssets.sln/TestAssets.sln index e8e695f290..325d36db55 100644 --- a/test/TestAssets/TestAssets.sln/TestAssets.sln +++ b/test/TestAssets/TestAssets.sln/TestAssets.sln @@ -69,8 +69,11 @@ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SimpleClassLibrary", "..\SimpleClassLibrary\SimpleClassLibrary.csproj", "{F37144D7-2C6D-42AF-9E85-EF10E5244A7B}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SelfContainedAppTestProject", "..\SelfContainedAppTestProject\SelfContainedAppTestProject.csproj", "{7F85E9D0-7BCE-431D-B468-4F6104E52DFA}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CoverletCoverageTestProject", "..\CoverletCoverageTestProject\CoverletCoverageTestProject.csproj", "{0D4D59D7-C52F-4858-A220-EAC7484E3827}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodeCoverageTest", "..\CodeCoverageTest\CodeCoverageTest.csproj", "{23790570-66D2-4CD5-9CD0-F8787C5B61BF}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -465,6 +468,18 @@ Global {0D4D59D7-C52F-4858-A220-EAC7484E3827}.Release|x64.Build.0 = Release|Any CPU {0D4D59D7-C52F-4858-A220-EAC7484E3827}.Release|x86.ActiveCfg = Release|Any CPU {0D4D59D7-C52F-4858-A220-EAC7484E3827}.Release|x86.Build.0 = Release|Any CPU + {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Debug|x64.ActiveCfg = Debug|Any CPU + {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Debug|x64.Build.0 = Debug|Any CPU + {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Debug|x86.ActiveCfg = Debug|Any CPU + {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Debug|x86.Build.0 = Debug|Any CPU + {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Release|Any CPU.Build.0 = Release|Any CPU + {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Release|x64.ActiveCfg = Release|Any CPU + {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Release|x64.Build.0 = Release|Any CPU + {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Release|x86.ActiveCfg = Release|Any CPU + {23790570-66D2-4CD5-9CD0-F8787C5B61BF}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE