Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Introduced acceptance tests for default exclusion merging #2454

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions scripts/vstest-codecoverage2.runsettings
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
<RunSettings>
<DataCollectionRunSettings>
<DataCollectors>
<DataCollector friendlyName="Code Coverage" uri="datacollector://Microsoft/CodeCoverage/2.0">
<Configuration>
<CodeCoverage>
<!-- Match assembly file paths: -->
<ModulePaths>
<Exclude>
<ModulePath>.*CodeCoverage.exe$</ModulePath>
</Exclude>
</ModulePaths>

<!-- We recommend you do not change the following values: -->
<UseVerifiableInstrumentation>True</UseVerifiableInstrumentation>
<AllowLowIntegrityProcesses>True</AllowLowIntegrityProcesses>
<CollectFromChildProcesses>True</CollectFromChildProcesses>
<CollectAspDotNet>False</CollectAspDotNet>

<Functions>
<Exclude>
<Function>.*TestSign.*</Function>
</Exclude>
</Functions>

</CodeCoverage>
</Configuration>
</DataCollector>
</DataCollectors>
</DataCollectionRunSettings>
</RunSettings>
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace Microsoft.TestPlatform.AcceptanceTests
{
using System;
using System.Diagnostics;
using System.IO;
using System.Xml;

using Microsoft.TestPlatform.TestUtilities;
using Microsoft.VisualStudio.TestTools.UnitTesting;

public class CodeCoverageAcceptanceTestBase : AcceptanceTestBase
{
/*
* Below value is just safe coverage result for which all tests are passing.
* Inspecting this value gives us confidence that there is no big drop in overall coverage.
*/
protected const double ExpectedMinimalModuleCoverage = 30.0;

protected string GetCodeCoveragePath()
{
return Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "Microsoft.CodeCoverage");
}

protected string GetCodeCoverageExePath()
{
return Path.Combine(this.GetCodeCoveragePath(), "CodeCoverage", "CodeCoverage.exe");
}

protected XmlNode GetModuleNode(XmlNode node, string name)
{
return this.GetNode(node, "module", name);
}

protected XmlNode GetNode(XmlNode node, string type, string name)
{
return node.SelectSingleNode($"//{type}[@name='{name}']");
}

protected XmlDocument GetXmlCoverage(string coverageResult)
{
var codeCoverageExe = this.GetCodeCoverageExePath();
var output = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString() + ".xml");

var watch = new Stopwatch();

Console.WriteLine($"Starting {codeCoverageExe}");
watch.Start();
var analyze = Process.Start(new ProcessStartInfo
{
FileName = codeCoverageExe,
Arguments = $"analyze /include_skipped_functions /include_skipped_modules /output:\"{output}\" \"{coverageResult}\"",
RedirectStandardOutput = true,
UseShellExecute = false
});

string analysisOutput = analyze.StandardOutput.ReadToEnd();

analyze.WaitForExit();
watch.Stop();
Console.WriteLine($"Total execution time: {watch.Elapsed.Duration()}");

Assert.IsTrue(0 == analyze.ExitCode, $"Code Coverage analyze failed: {analysisOutput}");

XmlDocument coverage = new XmlDocument();
coverage.Load(output);
return coverage;
}

protected void AssertCoverage(XmlNode node, double expectedCoverage)
{
var coverage = double.Parse(node.Attributes["block_coverage"].Value);
Console.WriteLine($"Checking coverage for {node.Name} {node.Attributes["name"].Value}. Expected at least: {expectedCoverage}. Result: {coverage}");
Assert.IsTrue(coverage > expectedCoverage, $"Coverage check failed for {node.Name} {node.Attributes["name"].Value}. Expected at least: {expectedCoverage}. Found: {coverage}");
}
}
}
Loading