From dfb2e811a510f8a83221f07b64effeeaee1a470d Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 11:47:14 +0100 Subject: [PATCH 01/64] Fix acceptance tests. --- scripts/build.ps1 | 18 +++ .../AppDomainTests.cs | 13 +- .../ArgumentProcessorTests.cs | 4 +- .../BlameDataCollectorTests.cs | 56 ++++--- .../CUITTest.cs | 11 +- .../CodeCoverageAcceptanceTestBase.cs | 4 +- .../CodeCoverageTests.cs | 17 +-- .../DataCollectionTests.cs | 39 +++-- .../DebugAssertTests.cs | 3 +- .../DeprecateExtensionsPathWarningTests.cs | 15 +- .../DifferentTestFrameworkSimpleTests.cs | 144 +++++++++--------- .../DisableAppdomainTests.cs | 17 +-- .../DiscoveryTests.cs | 8 +- .../DotnetArchitectureSwitchTests.Windows.cs | 8 +- .../DotnetArchitectureSwitchTests.cs | 4 +- .../EventLogCollectorTests.cs | 20 ++- .../ExecutionTests.cs | 47 +++--- .../ExecutionThreadApartmentStateTests.cs | 12 +- .../Extension/RunnnerInfo.cs | 31 ++++ .../FilePatternParserTests.cs | 24 +-- .../FrameworkTests.cs | 12 +- .../LoggerTests.cs | 41 +++-- ...rosoft.TestPlatform.AcceptanceTests.csproj | 6 + .../MultitargetingTestHostTests.cs | 3 +- .../PlatformTests.cs | 7 +- .../PostProcessingTests.cs | 9 +- .../ProcessesInteractionTests.cs | 3 - .../ResultsDirectoryTests.cs | 7 +- .../RunsettingsTests.cs | 105 +++++++------ .../SelfContainedAppTests.cs | 3 +- .../TelemetryTests.cs | 30 ++-- .../TestCaseFilterTests.cs | 34 ++--- .../TestPlatformNugetPackageTests.cs | 2 +- .../CodeCoverageTests.cs | 6 +- .../TranslationLayerTests/RunTests.cs | 3 + ...RunTestsWithDifferentConfigurationTests.cs | 3 +- .../DotnetHostArchitectureVerifierTests.cs | 2 +- .../IntegrationTestBase.cs | 58 ++++--- ...icrosoft.TestPlatform.TestUtilities.csproj | 2 +- .../RandomId.cs | 46 ++++++ .../TempDirectory.cs | 34 +++-- 41 files changed, 487 insertions(+), 424 deletions(-) create mode 100644 test/Microsoft.TestPlatform.TestUtilities/RandomId.cs diff --git a/scripts/build.ps1 b/scripts/build.ps1 index 341d640b26..8e1e80d6ae 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -123,6 +123,24 @@ if ($env:PATH -notlike "*$attachVsPath") { $env:PATH = "$attachVsPath;$env:PATH" } +# VsixUtil gets regularly eaten by antivirus or something. Remove the package dir if it gets broken +# so nuget restores it correctly. +$vsSdkBuildToolsVersion = ([xml](Get-Content $env:TP_ROOT_DIR\scripts\build\TestPlatform.Dependencies.props)).Project.PropertyGroup.VSSdkBuildToolsVersion +$vsixUtilDir = "$env:TP_ROOT_DIR\packages\microsoft.vssdk.buildtools" +if ((Test-Path $vsixUtilDir) -and -not (Test-Path "$vsixUtilDir\$vsSdkBuildToolsVersion\tools\vssdk\bin\VsixUtil.exe")) +{ + Remove-Item -Recurse -Force $vsixUtilDir +} + +# Procdump gets regularly eaten by antivirus or something. Remove the package dir if it gets broken +# so nuget restores it correctly. +$procdumpDir = "$env:TP_ROOT_DIR\packages\procdump" +if ((Test-Path $procdumpDir) -and 2 -ne @(Get-Item "$procdumpDir\0.0.1\bin").Length) +{ + Remove-Item -Recurse -Force $procdumpDir +} + + function Invoke-Build { $timer = Start-Timer diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs index 442b24b0d4..776ad1956e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs @@ -29,22 +29,21 @@ public void RunTestExecutionWithDisableAppDomain(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var testAppDomainDetailFileName = Path.Combine(tempDir.Path, "appdomain_test.txt"); - var dataCollectorAppDomainDetailFileName = Path.Combine(tempDir.Path, "appdomain_datacollector.txt"); + var testAppDomainDetailFileName = Path.Combine(TempDirectory.Path, "appdomain_test.txt"); + var dataCollectorAppDomainDetailFileName = Path.Combine(TempDirectory.Path, "appdomain_datacollector.txt"); // Delete test output files if already exist File.Delete(testAppDomainDetailFileName); File.Delete(dataCollectorAppDomainDetailFileName); - var runsettingsFilePath = GetInProcDataCollectionRunsettingsFile(true, tempDir); + var runsettingsFilePath = GetInProcDataCollectionRunsettingsFile(true, TempDirectory); var arguments = PrepareArguments( GetSampleTestAssembly(), GetTestAdapterPath(), runsettingsFilePath, FrameworkArgValue, runnerInfo.InIsolationValue, - tempDir.Path); + TempDirectory.Path); // Sets the environment variables used by the test project and test data collector. var env = new Dictionary @@ -74,9 +73,9 @@ private static bool IsFilesContentEqual(string filePath1, string filePath2) return string.Equals(content1, content2, StringComparison.Ordinal); } - private string GetInProcDataCollectionRunsettingsFile(bool disableAppDomain, TempDirectory tempDirectory) + private string GetInProcDataCollectionRunsettingsFile(bool disableAppDomain, TempDirectory TempDirectory) { - var runSettings = Path.Combine(tempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); + var runSettings = Path.Combine(TempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); var inprocasm = _testEnvironment.GetTestAsset("SimpleDataCollector.dll"); #if !NET451 var assemblyName = AssemblyLoadContext.GetAssemblyName(inprocasm); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs index 37c8f7b88a..44a4a39dbd 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using Microsoft.TestPlatform.TestUtilities; using Microsoft.VisualStudio.TestTools.UnitTesting; #nullable disable @@ -41,8 +40,7 @@ public void PassingInvalidArgumentsToVsTestConsoleShouldNotPrintHelpMessage(Runn { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /badArgument"); InvokeVsTest(arguments); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs index 66813e4d15..fc00b22d7a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs @@ -6,6 +6,7 @@ using System.IO; using System.Linq; using System.Text; +using System.Text.RegularExpressions; using System.Xml; using Microsoft.TestPlatform.TestUtilities; @@ -23,6 +24,19 @@ public class BlameDataCollectorTests : AcceptanceTestBase { public const string NETCOREANDFX = "net452;net472;netcoreapp3.1"; public const string NET50 = "net5.0"; + private readonly string _procDumpPath; + + public BlameDataCollectorTests() + { + _procDumpPath = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"); + var procDumpExePath = Path.Combine(_procDumpPath, "procdump.exe"); + if (!File.Exists(procDumpExePath)) + { + throw new InvalidOperationException($"Procdump path {procDumpExePath} does not exist. " + + $"It is possible that antivirus deleted it from your nuget cache. " + + $"Delete the whole procdump folder in your nuget cache, and run build, or restore"); + } + } [TestMethod] // netcoreapp2.1 dump is not supported on Linux @@ -31,15 +45,14 @@ public class BlameDataCollectorTests : AcceptanceTestBase [NetCoreTargetFrameworkDataSource] public void BlameDataCollectorShouldGiveCorrectTestCaseName(RunnerInfo runnerInfo) { - using var tempDir = new TempDirectory(); SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = GetAssetFullPath("BlameUnitTestProject.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $" /Blame"); - arguments = string.Concat(arguments, $" /ResultsDirectory:{tempDir.Path}"); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); InvokeVsTest(arguments); - VaildateOutput(tempDir, "BlameUnitTestProject.UnitTest1.TestMethod2"); + VaildateOutput(TempDirectory, "BlameUnitTestProject.UnitTest1.TestMethod2"); } [TestMethod] @@ -49,23 +62,22 @@ public void BlameDataCollectorShouldGiveCorrectTestCaseName(RunnerInfo runnerInf [NetCoreTargetFrameworkDataSource] public void BlameDataCollectorShouldOutputDumpFile(RunnerInfo runnerInfo) { - using var tempDir = new TempDirectory(); SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $" /Blame:CollectDump"); - arguments = string.Concat(arguments, $" /ResultsDirectory:{tempDir.Path}"); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, " /testcasefilter:ExitWithStackoverFlow"); var env = new Dictionary { - ["PROCDUMP_PATH"] = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"), + ["PROCDUMP_PATH"] = _procDumpPath, }; InvokeVsTest(arguments, env); - VaildateOutput(tempDir, "SampleUnitTestProject3.UnitTest1.ExitWithStackoverFlow", validateDumpFile: true); + VaildateOutput(TempDirectory, "SampleUnitTestProject3.UnitTest1.ExitWithStackoverFlow", validateDumpFile: true); } [TestMethod] @@ -75,18 +87,17 @@ public void BlameDataCollectorShouldOutputDumpFile(RunnerInfo runnerInfo) [NetCoreTargetFrameworkDataSource] public void BlameDataCollectorShouldNotOutputDumpFileWhenNoCrashOccurs(RunnerInfo runnerInfo) { - using var tempDir = new TempDirectory(); SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"'); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $" /Blame:CollectDump"); - arguments = string.Concat(arguments, $" /ResultsDirectory:{tempDir.Path}"); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, " /testcasefilter:PassingTest"); var env = new Dictionary { - ["PROCDUMP_PATH"] = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"), + ["PROCDUMP_PATH"] = _procDumpPath }; InvokeVsTest(arguments, env); @@ -101,23 +112,22 @@ public void BlameDataCollectorShouldNotOutputDumpFileWhenNoCrashOccurs(RunnerInf [NetCoreTargetFrameworkDataSource] public void BlameDataCollectorShouldOutputDumpFileWhenNoCrashOccursButCollectAlwaysIsEnabled(RunnerInfo runnerInfo) { - using var tempDir = new TempDirectory(); SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"'); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $" /Blame:CollectDump;CollectAlways=True"); - arguments = string.Concat(arguments, $" /ResultsDirectory:{tempDir.Path}"); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, " /testcasefilter:PassingTest"); var env = new Dictionary { - ["PROCDUMP_PATH"] = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"), + ["PROCDUMP_PATH"] = _procDumpPath }; InvokeVsTest(arguments, env); - Assert.IsTrue(StdOut.Contains(".dmp"), "it should collect dump, even if nothing crashed"); + StringAssert.Matches(StdOut, new Regex("\\.dmp"), "it should collect dump, even if nothing crashed"); } [TestMethod] @@ -133,7 +143,7 @@ public void HangDumpOnTimeout(RunnerInfo runnerInfo) var env = new Dictionary { - ["PROCDUMP_PATH"] = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"), + ["PROCDUMP_PATH"] = _procDumpPath }; InvokeVsTest(arguments, env); @@ -156,7 +166,7 @@ public void CrashDumpWhenThereIsNoTimeout(RunnerInfo runnerInfo) var env = new Dictionary { - ["PROCDUMP_PATH"] = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"), + ["PROCDUMP_PATH"] = _procDumpPath }; InvokeVsTest(arguments, env); @@ -179,7 +189,7 @@ public void CrashDumpOnExit(RunnerInfo runnerInfo) var env = new Dictionary { - ["PROCDUMP_PATH"] = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"), + ["PROCDUMP_PATH"] = _procDumpPath }; InvokeVsTest(arguments, env); @@ -200,7 +210,7 @@ public void CrashDumpOnStackOverflow(RunnerInfo runnerInfo) var env = new Dictionary { - ["PROCDUMP_PATH"] = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"), + ["PROCDUMP_PATH"] = _procDumpPath }; InvokeVsTest(arguments, env); @@ -313,14 +323,14 @@ private void ValidateDump(int expectedDumpCount = 1) } } - private void VaildateOutput(TempDirectory tempDir, string testName, bool validateDumpFile = false) + private void VaildateOutput(TempDirectory TempDirectory, string testName, bool validateDumpFile = false) { bool isSequenceAttachmentReceived = false; bool isDumpAttachmentReceived = false; bool isValid = false; StdErrorContains(testName); StdOutputContains("Sequence_"); - var resultFiles = Directory.GetFiles(tempDir.Path, "*", SearchOption.AllDirectories); + var resultFiles = Directory.GetFiles(TempDirectory.Path, "*", SearchOption.AllDirectories); foreach (var file in resultFiles) { @@ -335,9 +345,9 @@ private void VaildateOutput(TempDirectory tempDir, string testName, bool validat } } - Assert.IsTrue(isSequenceAttachmentReceived); - Assert.IsTrue(!validateDumpFile || isDumpAttachmentReceived); - Assert.IsTrue(isValid); + Assert.IsTrue(isSequenceAttachmentReceived, "Sequence attachment was not received."); + Assert.IsTrue(!validateDumpFile || isDumpAttachmentReceived, "Dump attachment was not received."); + Assert.IsTrue(isValid, "Sequence attachment is not valid."); } private bool IsValidXml(string xmlFilePath) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CUITTest.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CUITTest.cs index 66559416ba..efc524ffde 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CUITTest.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CUITTest.cs @@ -18,20 +18,19 @@ public class CuitTest : AcceptanceTestBase public void CuitRunAllTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - CuitRunAll(runnerInfo.RunnerFramework); + CuitRunAll(runnerInfo); } - private void CuitRunAll(string runnerFramework) + private void CuitRunAll(RunnerInfo runnerInfo) { - if (runnerFramework.StartsWith("netcoreapp")) + if (runnerInfo.IsNetRunner) { - Assert.Inconclusive("CUIT tests are not supported with .Netcore runner."); + Assert.Inconclusive("CUIT tests are not supported with .NET Core runner."); return; } var assemblyAbsolutePath = _testEnvironment.GetTestAsset("CUITTestProject.dll", "net451"); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(assemblyAbsolutePath, string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyAbsolutePath, string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs index 6aaf72b2b7..c7d3aaa1d7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs @@ -59,7 +59,7 @@ protected XmlNode GetNode(XmlNode node, string type, string name) return node.SelectSingleNode($"//{type}[@name='{name}']"); } - protected XmlDocument GetXmlCoverage(string coverageResult, TempDirectory tempDirectory) + protected XmlDocument GetXmlCoverage(string coverageResult, TempDirectory TempDirectory) { var coverage = new XmlDocument(); @@ -70,7 +70,7 @@ protected XmlDocument GetXmlCoverage(string coverageResult, TempDirectory tempDi } var codeCoverageExe = GetCodeCoverageExePath(); - var output = Path.Combine(tempDirectory.Path, Guid.NewGuid().ToString() + ".xml"); + var output = Path.Combine(TempDirectory.Path, Guid.NewGuid().ToString() + ".xml"); var watch = new Stopwatch(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs index fafc3f4708..1dc4c3f618 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs @@ -208,10 +208,9 @@ public void CollectCodeCoverageSpecifyOutputFormatCoberturaOverrideRunSettingsCo private void CollectCodeCoverage(RunnerInfo runnerInfo, TestParameters testParameters) { - using var tempDir = new TempDirectory(); SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = CreateArguments(tempDir, runnerInfo, testParameters, out var trxFilePath); + var arguments = CreateArguments(TempDirectory, runnerInfo, testParameters, out var trxFilePath); InvokeVsTest(arguments); @@ -220,8 +219,8 @@ private void CollectCodeCoverage(RunnerInfo runnerInfo, TestParameters testParam testParameters.ExpectedSkippedTests, testParameters.ExpectedFailedTests); - var actualCoverageFile = GetCoverageFileNameFromTrx(trxFilePath, tempDir.Path); - Console.WriteLine($@"Coverage file: {actualCoverageFile} Results directory: {tempDir.Path} trxfile: {trxFilePath}"); + var actualCoverageFile = GetCoverageFileNameFromTrx(trxFilePath, TempDirectory.Path); + Console.WriteLine($@"Coverage file: {actualCoverageFile} Results directory: {TempDirectory.Path} trxfile: {trxFilePath}"); Assert.IsTrue(File.Exists(actualCoverageFile), "Coverage file not found: {0}", actualCoverageFile); if (testParameters.RunSettingsType == TestParameters.SettingsType.XmlOutput) @@ -237,7 +236,7 @@ private void CollectCodeCoverage(RunnerInfo runnerInfo, TestParameters testParam Assert.IsTrue(actualCoverageFile.EndsWith(".coverage", StringComparison.InvariantCultureIgnoreCase)); } - var coverageDocument = GetXmlCoverage(actualCoverageFile, tempDir); + var coverageDocument = GetXmlCoverage(actualCoverageFile, TempDirectory); if (testParameters.CheckSkipped) { AssertSkippedMethod(coverageDocument); @@ -247,7 +246,7 @@ private void CollectCodeCoverage(RunnerInfo runnerInfo, TestParameters testParam } private string CreateArguments( - TempDirectory tempDir, + TempDirectory TempDirectory, RunnerInfo runnerInfo, TestParameters testParameters, out string trxFilePath) @@ -257,14 +256,14 @@ private string CreateArguments( string traceDataCollectorDir = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "Microsoft.CodeCoverage"); - string diagFileName = Path.Combine(tempDir.Path, "diaglog.txt"); + string diagFileName = Path.Combine(TempDirectory.Path, "diaglog.txt"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, - FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /Diag:{diagFileName}", $" /TestAdapterPath:{traceDataCollectorDir}"); arguments = string.Concat(arguments, $" /Platform:{testParameters.TargetPlatform}"); - trxFilePath = Path.Combine(tempDir.Path, Guid.NewGuid() + ".trx"); + trxFilePath = Path.Combine(TempDirectory.Path, Guid.NewGuid() + ".trx"); arguments = string.Concat(arguments, " /logger:trx;logfilename=" + trxFilePath); var defaultRunSettingsPath = Path.Combine( diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs index f400864e0f..b7907cbdd6 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs @@ -27,29 +27,28 @@ public class DataCollectionTests : AcceptanceTestBase public void ExecuteTestsWithDataCollection(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - string runSettings = GetRunsettingsFilePath(tempDir.Path); - string diagFileName = Path.Combine(tempDir.Path, "diaglog.txt"); + string runSettings = GetRunsettingsFilePath(TempDirectory.Path); + string diagFileName = Path.Combine(TempDirectory.Path, "diaglog.txt"); var extensionsPath = Path.Combine( _testEnvironment.TestAssetsPath, Path.GetFileNameWithoutExtension("OutOfProcDataCollector"), "bin", IntegrationTestEnvironment.BuildConfiguration, _testEnvironment.RunnerFramework); - var arguments = PrepareArguments(assemblyPaths, null, runSettings, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, null, runSettings, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, $" /Diag:{diagFileName}", $" /TestAdapterPath:{extensionsPath}"); var env = new Dictionary { - ["TEST_ASSET_SAMPLE_COLLECTOR_PATH"] = tempDir.Path, + ["TEST_ASSET_SAMPLE_COLLECTOR_PATH"] = TempDirectory.Path, }; InvokeVsTest(arguments, env); ValidateSummaryStatus(1, 1, 1); - VaildateDataCollectorOutput(tempDir.Path); + VaildateDataCollectorOutput(TempDirectory.Path); } [TestMethod] @@ -59,9 +58,8 @@ public void ExecuteTestsWithDataCollectionUsingCollectArgument(RunnerInfo runner { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - string diagFileName = Path.Combine(tempDir.Path, "diaglog.txt"); + string diagFileName = Path.Combine(TempDirectory.Path, "diaglog.txt"); var extensionsPath = Path.Combine( _testEnvironment.TestAssetsPath, Path.GetFileNameWithoutExtension("OutOfProcDataCollector"), @@ -69,18 +67,18 @@ public void ExecuteTestsWithDataCollectionUsingCollectArgument(RunnerInfo runner IntegrationTestEnvironment.BuildConfiguration, _testEnvironment.RunnerFramework); - var arguments = PrepareArguments(assemblyPaths, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /Diag:{diagFileName}", $" /Collect:SampleDataCollector", $" /TestAdapterPath:{extensionsPath}"); var env = new Dictionary { - ["TEST_ASSET_SAMPLE_COLLECTOR_PATH"] = tempDir.Path, + ["TEST_ASSET_SAMPLE_COLLECTOR_PATH"] = TempDirectory.Path, }; InvokeVsTest(arguments, env); ValidateSummaryStatus(1, 1, 1); - VaildateDataCollectorOutput(tempDir.Path); + VaildateDataCollectorOutput(TempDirectory.Path); } [TestMethod] @@ -89,8 +87,7 @@ public void DataCollectorAssemblyLoadingShouldNotThrowErrorForNetCore(RunnerInfo { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetAssetFullPath("AppDomainGetAssembliesTestProject.dll", "netcoreapp2.1"), string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(GetAssetFullPath("AppDomainGetAssembliesTestProject.dll", "netcoreapp2.1"), string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); @@ -103,8 +100,7 @@ public void DataCollectorAssemblyLoadingShouldNotThrowErrorForFullFramework(Runn { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetAssetFullPath("AppDomainGetAssembliesTestProject.dll"), string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(GetAssetFullPath("AppDomainGetAssembliesTestProject.dll"), string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); @@ -117,18 +113,17 @@ public void DataCollectorAttachmentProcessor(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPath = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"'); var secondAssemblyPath = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - string runSettings = GetRunsettingsFilePath(tempDir.Path); - string diagFileName = Path.Combine(tempDir.Path, "diaglog.txt"); + string runSettings = GetRunsettingsFilePath(TempDirectory.Path); + string diagFileName = Path.Combine(TempDirectory.Path, "diaglog.txt"); var extensionsPath = Path.Combine( _testEnvironment.TestAssetsPath, Path.GetFileNameWithoutExtension("AttachmentProcessorDataCollector"), "bin", IntegrationTestEnvironment.BuildConfiguration, "netstandard2.0"); - var arguments = PrepareArguments(new string[] { assemblyPath, secondAssemblyPath }, null, runSettings, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(new string[] { assemblyPath, secondAssemblyPath }, null, runSettings, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, $" /Diag:{diagFileName}", $" /TestAdapterPath:{extensionsPath}"); XElement runSettingsXml = XElement.Load(runSettings); @@ -148,14 +143,14 @@ public void DataCollectorAttachmentProcessor(RunnerInfo runnerInfo) var env = new Dictionary { - ["SampleDataCollectorTempPath"] = tempDir.Path, + ["SampleDataCollectorTempPath"] = TempDirectory.Path, }; InvokeVsTest(arguments, env); ValidateSummaryStatus(2, 2, 2); - string mergedFile = Directory.GetFiles(tempDir.Path, "MergedFile.txt", SearchOption.AllDirectories).Single(); + string mergedFile = Directory.GetFiles(TempDirectory.Path, "MergedFile.txt", SearchOption.AllDirectories).Single(); var fileContent = new List(); using (var streamReader = new StreamReader(mergedFile)) { @@ -169,7 +164,7 @@ public void DataCollectorAttachmentProcessor(RunnerInfo runnerInfo) Assert.AreEqual(2, fileContent.Distinct().Count()); - var dataCollectorsLogs = Directory.GetFiles(tempDir.Path, "*.datacollector.*", SearchOption.TopDirectoryOnly); + var dataCollectorsLogs = Directory.GetFiles(TempDirectory.Path, "*.datacollector.*", SearchOption.TopDirectoryOnly); Assert.AreEqual(2, dataCollectorsLogs.Distinct().Count()); foreach (var dataCollectorLogFile in dataCollectorsLogs) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs index ed293c7e61..fba250c5a2 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs @@ -20,9 +20,8 @@ public void RunningTestWithAFailingDebugAssertDoesNotCrashTheHostingProcess(Runn // is to not crash the process when we are running in debug, and debugger is attached SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPath = BuildMultipleAssemblyPath("CrashingOnDebugAssertTestProject.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPath, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPath, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); // this will have failed tests when our trace listener works and crash the testhost process when it does not diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs index acb36c8e7a..4c10549775 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs @@ -20,18 +20,6 @@ public class DeprecateExtensionsPathWarningTests : AcceptanceTestBase private IList _adapterDependencies; private IList _copiedFiles; - private string BuildConfiguration - { - get - { -#if DEBUG - return "Debug"; -#else - return "Release"; -#endif - } - } - [TestCleanup] public void Cleanup() { @@ -73,8 +61,7 @@ public void CopyAdapterToExtensions() [TestMethod] public void VerifyDeprecatedWarningIsThrownWhenAdaptersPickedFromExtensionDirectory() { - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestAssembly(), null, null, FrameworkArgValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), null, null, FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); StdOutputContains("Adapter lookup is being changed, please follow"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs index 76b2cabe06..fc3209099a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs @@ -20,9 +20,8 @@ public class DifferentTestFrameworkSimpleTests : AcceptanceTestBase public void ChutzpahRunAllTestExecution(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var testJSFileAbsolutePath = Path.Combine(_testEnvironment.TestAssetsPath, "test.js"); - var arguments = PrepareArguments(testJSFileAbsolutePath, GetTestAdapterPath(UnitTestFramework.Chutzpah), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(testJSFileAbsolutePath, GetTestAdapterPath(UnitTestFramework.Chutzpah), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 0); @@ -35,7 +34,7 @@ public void ChutzpahRunAllTestExecution(RunnerInfo runnerInfo) public void CPPRunAllTestExecutionNetFramework(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - CppRunAllTests(runnerInfo.RunnerFramework, "x86"); + CppRunAllTests("x86"); } @@ -47,7 +46,7 @@ public void CPPRunAllTestExecutionNetFramework(RunnerInfo runnerInfo) public void CPPRunAllTestExecutionPlatformx64NetFramework(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - CppRunAllTests(runnerInfo.RunnerFramework, "x64"); + CppRunAllTests("x64"); } [TestMethod] @@ -58,7 +57,7 @@ public void CPPRunAllTestExecutionPlatformx64NetFramework(RunnerInfo runnerInfo) public void CPPRunAllTestExecutionPlatformx64Net(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - CppRunAllTests(runnerInfo.RunnerFramework, "x64"); + CppRunAllTests("x64"); } [TestMethod] @@ -66,9 +65,18 @@ public void CPPRunAllTestExecutionPlatformx64Net(RunnerInfo runnerInfo) [NetFullTargetFrameworkDataSource] public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo) { - using var tempDir = new TempDirectory(); + // This test works on server but not locally, and it fails with + // Exception: Method not found: 'Int64 Microsoft.VisualStudio.TestTools.WebTesting.WebTestRequestStatistics.get_MillisecondsSincePageComplete()'. + // The dll is + + //if (!IsCI) + //{ + // Assert.Inconclusive("Web load tests need special workloads and setup locally"); + //} + + SetTestEnvironment(_testEnvironment, runnerInfo); - var runSettingsFilePath = Path.Combine(tempDir.Path, Guid.NewGuid() + ".runsettings"); + var runSettingsFilePath = Path.Combine(TempDirectory.Path, Guid.NewGuid() + ".runsettings"); //test the iterationCount setting for WebTestRunConfiguration in run settings var runSettingsXml = $@" @@ -81,9 +89,37 @@ public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo) CreateRunSettingsFile(runSettingsFilePath, runSettingsXml); - //minWebTestResultFileSizeInKB is set to 150 here as the web test has a iteration count set to 5 //therefore, the test will run for 5 iterations resulting in web test result file size of at least 150 KB - WebTestRunAllTests(runnerInfo.RunnerFramework, runSettingsFilePath, 150); + var minWebTestResultFileSizeInKB = 150; + if (runnerInfo.IsNetRunner) + { + Assert.Inconclusive("WebTests tests not supported with .Netcore runner."); + return; + } + + string assemblyRelativePath = + @"microsoft.testplatform.qtools.assets\2.0.0\contentFiles\any\any\WebTestAssets\WebTest1.webtest"; + + var assemblyAbsolutePath = Path.Combine(_testEnvironment.PackageDirectory, assemblyRelativePath); + using var resultsDirectory = TempDirectory; + var arguments = PrepareArguments( + assemblyAbsolutePath, + string.Empty, + runSettingsFilePath, FrameworkArgValue, string.Empty, resultsDirectory.Path); + + InvokeVsTest(arguments); + ValidateSummaryStatus(1, 0, 0); + + if (minWebTestResultFileSizeInKB > 0) + { + var dirInfo = new DirectoryInfo(resultsDirectory.Path); + var webtestResultFile = "WebTest1.webtestResult"; + var files = dirInfo.GetFiles(webtestResultFile, SearchOption.AllDirectories); + Assert.IsTrue(files.Length > 0, $"File {webtestResultFile} not found under results directory {resultsDirectory}"); + + var fileSizeInKB = files[0].Length / 1024; + Assert.IsTrue(fileSizeInKB > minWebTestResultFileSizeInKB, $"Size of the file {webtestResultFile} is {fileSizeInKB} KB. It is not greater than {minWebTestResultFileSizeInKB} KB indicating iterationCount in run settings not honored."); + } } [TestMethod] @@ -92,7 +128,21 @@ public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo) public void CodedWebTestRunAllTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - CodedWebTestRunAllTests(runnerInfo.RunnerFramework); + if (runnerInfo.IsNetRunner) + { + Assert.Inconclusive("WebTests tests not supported with .Netcore runner."); + return; + } + + string assemblyRelativePath = @"microsoft.testplatform.qtools.assets\2.0.0\contentFiles\any\any\WebTestAssets\BingWebTest.dll"; + var assemblyAbsolutePath = Path.Combine(_testEnvironment.PackageDirectory, assemblyRelativePath); + var arguments = PrepareArguments( + assemblyAbsolutePath, + string.Empty, + string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); + + InvokeVsTest(arguments); + ValidateSummaryStatus(1, 0, 0); } [TestMethod] @@ -102,12 +152,11 @@ public void NUnitRunAllTestExecution(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( - GetAssetFullPath("NUTestProject.dll"), - GetTestAdapterPath(UnitTestFramework.NUnit), - string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, tempDir.Path); +GetAssetFullPath("NUTestProject.dll"), +GetTestAdapterPath(UnitTestFramework.NUnit), +string.Empty, FrameworkArgValue, +runnerInfo.InIsolationValue, TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 0); } @@ -118,7 +167,6 @@ public void NUnitRunAllTestExecution(RunnerInfo runnerInfo) public void XUnitRunAllTestExecution(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); string testAssemblyPath; // Xunit >= 2.2 won't support net451, Minimum target framework it supports is net452. @@ -135,77 +183,21 @@ public void XUnitRunAllTestExecution(RunnerInfo runnerInfo) testAssemblyPath, GetTestAdapterPath(UnitTestFramework.XUnit), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, tempDir.Path); + runnerInfo.InIsolationValue, TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 0); } - private void CppRunAllTests(string runnerFramework, string platform) + private void CppRunAllTests(string platform) { - using var tempDir = new TempDirectory(); - string assemblyRelativePathFormat = - @"microsoft.testplatform.testasset.nativecpp\2.0.0\contentFiles\any\any\{0}\Microsoft.TestPlatform.TestAsset.NativeCPP.dll"; + string assemblyRelativePathFormat = @"microsoft.testplatform.testasset.nativecpp\2.0.0\contentFiles\any\any\{0}\Microsoft.TestPlatform.TestAsset.NativeCPP.dll"; var assemblyRelativePath = platform.Equals("x64", StringComparison.OrdinalIgnoreCase) ? string.Format(assemblyRelativePathFormat, platform) : string.Format(assemblyRelativePathFormat, ""); var assemblyAbsolutePath = Path.Combine(_testEnvironment.PackageDirectory, assemblyRelativePath); - var arguments = PrepareArguments(assemblyAbsolutePath, string.Empty, string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyAbsolutePath, string.Empty, string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 0); } - - private void WebTestRunAllTests(string runnerFramework, string runSettingsFilePath = null, int minWebTestResultFileSizeInKB = 0) - { - if (runnerFramework.StartsWith("netcoreapp")) - { - Assert.Inconclusive("WebTests tests not supported with .Netcore runner."); - return; - } - - string assemblyRelativePath = - @"microsoft.testplatform.qtools.assets\2.0.0\contentFiles\any\any\WebTestAssets\WebTest1.webtest"; - - var assemblyAbsolutePath = Path.Combine(_testEnvironment.PackageDirectory, assemblyRelativePath); - using var resultsDirectory = new TempDirectory(); - var arguments = PrepareArguments( - assemblyAbsolutePath, - string.Empty, - runSettingsFilePath, FrameworkArgValue, string.Empty, resultsDirectory.Path); - - InvokeVsTest(arguments); - ValidateSummaryStatus(1, 0, 0); - - if (minWebTestResultFileSizeInKB > 0) - { - var dirInfo = new DirectoryInfo(resultsDirectory.Path); - var webtestResultFile = "WebTest1.webtestResult"; - var files = dirInfo.GetFiles(webtestResultFile, SearchOption.AllDirectories); - Assert.IsTrue(files.Length > 0, $"File {webtestResultFile} not found under results directory {resultsDirectory}"); - - var fileSizeInKB = files[0].Length / 1024; - Assert.IsTrue(fileSizeInKB > minWebTestResultFileSizeInKB, $"Size of the file {webtestResultFile} is {fileSizeInKB} KB. It is not greater than {minWebTestResultFileSizeInKB} KB indicating iterationCount in run settings not honored."); - } - } - - private void CodedWebTestRunAllTests(string runnerFramework) - { - if (runnerFramework.StartsWith("netcoreapp")) - { - Assert.Inconclusive("WebTests tests not supported with .Netcore runner."); - return; - } - - using var tempDir = new TempDirectory(); - string assemblyRelativePath = - @"microsoft.testplatform.qtools.assets\2.0.0\contentFiles\any\any\WebTestAssets\BingWebTest.dll"; - var assemblyAbsolutePath = Path.Combine(_testEnvironment.PackageDirectory, assemblyRelativePath); - var arguments = PrepareArguments( - assemblyAbsolutePath, - string.Empty, - string.Empty, FrameworkArgValue, resultsDirectory: tempDir.Path); - - InvokeVsTest(arguments); - ValidateSummaryStatus(1, 0, 0); - } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DisableAppdomainTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DisableAppdomainTests.cs index d78c6fe7e8..78add6fe64 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DisableAppdomainTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DisableAppdomainTests.cs @@ -26,7 +26,7 @@ public void DisableAppdomainTest(RunnerInfo runnerInfo) var diableAppdomainTest1 = _testEnvironment.GetTestAsset("DisableAppdomainTest1.dll", "net451"); var diableAppdomainTest2 = _testEnvironment.GetTestAsset("DisableAppdomainTest2.dll", "net451"); - RunTests(runnerInfo.RunnerFramework, string.Format("{0}\" \"{1}", diableAppdomainTest1, diableAppdomainTest2), 2); + RunTests(runnerInfo, string.Format("{0}\" \"{1}", diableAppdomainTest1, diableAppdomainTest2), 2); } [TestMethod] @@ -38,12 +38,12 @@ public void NewtonSoftDependencyWithDisableAppdomainTest(RunnerInfo runnerInfo) var newtonSoftDependnecyTest = _testEnvironment.GetTestAsset("NewtonSoftDependency.dll", "net451"); - RunTests(runnerInfo.RunnerFramework, newtonSoftDependnecyTest, 1); + RunTests(runnerInfo, newtonSoftDependnecyTest, 1); } - private void RunTests(string runnerFramework, string testAssembly, int passedTestCount) + private void RunTests(RunnerInfo runnerInfo, string testAssembly, int passedTestCount) { - if (runnerFramework.StartsWith("netcoreapp")) + if (runnerInfo.IsNetRunner) { Assert.Inconclusive("This test is not meant for .netcore."); return; @@ -54,20 +54,19 @@ private void RunTests(string runnerFramework, string testAssembly, int passedTes { "DisableAppDomain", "true" } }; - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( testAssembly, string.Empty, - GetRunsettingsFilePath(tempDir, runConfigurationDictionary), - FrameworkArgValue, resultsDirectory: tempDir.Path); + GetRunsettingsFilePath(TempDirectory, runConfigurationDictionary), + FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(passedTestCount, 0, 0); } - private string GetRunsettingsFilePath(TempDirectory tempDir, Dictionary runConfigurationDictionary) + private string GetRunsettingsFilePath(TempDirectory TempDirectory, Dictionary runConfigurationDictionary) { - var runsettingsPath = Path.Combine(tempDir.Path, "test_" + Guid.NewGuid() + ".runsettings"); + var runsettingsPath = Path.Combine(TempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); CreateRunSettingsFile(runsettingsPath, runConfigurationDictionary); return runsettingsPath; } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs index 8bc5843ed2..2f6087aa17 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs @@ -59,14 +59,13 @@ public void MultipleSourcesDiscoverAllTests(RunnerInfo runnerInfo) [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] public void DiscoverFullyQualifiedTests(RunnerInfo runnerInfo) { - using var tempDir = new TempDirectory(); - var dummyFilePath = Path.Combine(tempDir.Path, $"{Guid.NewGuid()}.txt"); + var dummyFilePath = Path.Combine(TempDirectory.Path, $"{Guid.NewGuid()}.txt"); SetTestEnvironment(_testEnvironment, runnerInfo); var listOfTests = new[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", "SampleUnitTestProject.UnitTest1.SkippingTest" }; - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /ListFullyQualifiedTests", " /ListTestsTargetPath:\"" + dummyFilePath + "\""); InvokeVsTest(arguments); @@ -80,10 +79,9 @@ public void DiscoverFullyQualifiedTests(RunnerInfo runnerInfo) public void DiscoverTestsShouldShowProperWarningIfNoTestsOnTestCaseFilter(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assetFullPath = GetAssetFullPath("SimpleTestProject2.dll"); - var arguments = PrepareArguments(assetFullPath, GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assetFullPath, GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /listtests"); arguments = string.Concat(arguments, " /testcasefilter:NonExistTestCaseName"); arguments = string.Concat(arguments, " /logger:\"console;prefix=true\""); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs index 0102a4e692..cbb764e423 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs @@ -1,17 +1,15 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +#if !NET451 using System.Collections.Generic; using System.IO; using System.Linq; -using Microsoft.TestPlatform.TestUtilities; using Microsoft.VisualStudio.TestTools.UnitTesting; using Newtonsoft.Json.Linq; -#if !NET451 - #nullable disable namespace Microsoft.TestPlatform.AcceptanceTests; @@ -25,12 +23,12 @@ public class DotnetArchitectureSwitchTestsWindowsOnly : AcceptanceTestBase [DataRow("X86", "X64")] public void Use_EnvironmentVariables(string architectureFrom, string architectureTo) { - using var workSpace = new TempDirectory(); + using var workSpace = TempDirectory; string dotnetPath = GetDownloadedDotnetMuxerFromTools(architectureFrom); string dotnetPathTo = GetDownloadedDotnetMuxerFromTools(architectureTo); var vstestConsolePath = GetDotnetRunnerPath(); var dotnetRunnerPath = workSpace.CreateDirectory("dotnetrunner"); - workSpace.CopyAll(new DirectoryInfo(Path.GetDirectoryName(vstestConsolePath)), dotnetRunnerPath); + workSpace.CopyDirectory(new DirectoryInfo(Path.GetDirectoryName(vstestConsolePath)), dotnetRunnerPath); // Patch the runner string sdkVersion = GetLatestSdkVersion(dotnetPath); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs index 2e3e384ff6..e2d11506cf 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +#if !NET451 + using System; using System.Collections.Generic; using System.IO; @@ -10,8 +12,6 @@ using Microsoft.TestPlatform.TestUtilities; using Microsoft.VisualStudio.TestTools.UnitTesting; -#if !NET451 - #nullable disable namespace Microsoft.TestPlatform.AcceptanceTests; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/EventLogCollectorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/EventLogCollectorTests.cs index 4b140fd35e..b7e80ebf5e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/EventLogCollectorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/EventLogCollectorTests.cs @@ -26,16 +26,15 @@ public class EventLogCollectorTests : AcceptanceTestBase public void EventLogDataCollectorShoudCreateLogFileHavingEvents(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = _testEnvironment.GetTestAsset("EventLogUnitTestProject.dll"); - string runSettings = GetRunsettingsFilePath(tempDir); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), runSettings, FrameworkArgValue, resultsDirectory: tempDir.Path); + string runSettings = GetRunsettingsFilePath(TempDirectory); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), runSettings, FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(3, 0, 0); - VaildateDataCollectorOutput(tempDir); + VaildateDataCollectorOutput(TempDirectory); StdOutputDoesNotContains("An exception occurred while collecting final entries from the event log"); StdErrorDoesNotContains("event log has encountered an exception, some events might get lost"); StdOutputDoesNotContains("event log may have been cleared during collection; some events may not have been collected"); @@ -49,10 +48,9 @@ public void EventLogDataCollectorShoudCreateLogFileWithoutEventsIfEventsAreNotLo { SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = _testEnvironment.GetTestAsset("SimpleTestProject.dll"); - using var tempDir = new TempDirectory(); - string runSettings = GetRunsettingsFilePath(tempDir); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), runSettings, FrameworkArgValue, resultsDirectory: tempDir.Path); + string runSettings = GetRunsettingsFilePath(TempDirectory); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), runSettings, FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); @@ -63,9 +61,9 @@ public void EventLogDataCollectorShoudCreateLogFileWithoutEventsIfEventsAreNotLo StdErrorDoesNotContains("Unable to read event log"); } - private string GetRunsettingsFilePath(TempDirectory tempDirectory) + private string GetRunsettingsFilePath(TempDirectory TempDirectory) { - var runsettingsPath = Path.Combine(tempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); + var runsettingsPath = Path.Combine(TempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); string runSettingsXml = @" @@ -87,10 +85,10 @@ private string GetRunsettingsFilePath(TempDirectory tempDirectory) return runsettingsPath; } - private void VaildateDataCollectorOutput(TempDirectory tempDir) + private void VaildateDataCollectorOutput(TempDirectory TempDirectory) { // Verify attachments - var di = new DirectoryInfo(tempDir.Path); + var di = new DirectoryInfo(TempDirectory.Path); var resultFiles = di.EnumerateFiles("Event Log.xml", SearchOption.AllDirectories) .OrderBy(d => d.CreationTime) .Select(d => d.FullName) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index accc13eada..b88bf2e02d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -59,13 +59,12 @@ public void RunMultipleTestAssembliesWithoutTestAdapterPath(RunnerInfo runnerInf public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /Parallel"); arguments = string.Concat(arguments, " /Platform:x86"); - arguments += GetDiagArg(tempDir.Path); + arguments += GetDiagArg(TempDirectory.Path); // for the desktop we will run testhost.x86 in two copies, but for core // we will run a combination of testhost.x86 and dotnet, where the dotnet will be @@ -78,7 +77,7 @@ public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) InvokeVsTest(arguments); - AssertExpectedNumberOfHostProcesses(expectedNumOfProcessCreated, tempDir.Path, testHostProcessNames); + AssertExpectedNumberOfHostProcesses(expectedNumOfProcessCreated, TempDirectory.Path, testHostProcessNames); ValidateSummaryStatus(2, 2, 2); ExitCodeEquals(1); // failing tests } @@ -89,11 +88,10 @@ public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) public void TestSessionTimeOutTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /TestCaseFilter:TestSessionTimeoutTest"); // set TestSessionTimeOut = 7 sec @@ -111,10 +109,9 @@ public void TestSessionTimeOutTests(RunnerInfo runnerInfo) public void TestPlatformShouldBeCompatibleWithOldTestHost(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SampleProjectWithOldTestHost.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); @@ -128,10 +125,9 @@ public void TestPlatformShouldBeCompatibleWithOldTestHost(RunnerInfo runnerInfo) public void WorkingDirectoryIsSourceDirectory(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /tests:WorkingDirectoryTest"); InvokeVsTest(arguments); @@ -146,7 +142,6 @@ public void WorkingDirectoryIsSourceDirectory(RunnerInfo runnerInfo) public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); if (IntegrationTestEnvironment.BuildConfiguration.Equals("release", StringComparison.OrdinalIgnoreCase)) { @@ -155,11 +150,11 @@ public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerIn return; } - var diagLogFilePath = Path.Combine(tempDir.Path, $"std_error_log_{Guid.NewGuid()}.txt"); + var diagLogFilePath = Path.Combine(TempDirectory.Path, $"std_error_log_{Guid.NewGuid()}.txt"); File.Delete(diagLogFilePath); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:ExitWithStackoverFlow"); arguments = string.Concat(arguments, $" /diag:{diagLogFilePath}"); @@ -183,14 +178,13 @@ public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerIn public void UnhandleExceptionExceptionShouldBeLoggedToDiagLogFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var diagLogFilePath = Path.Combine(tempDir.Path, $"std_error_log_{Guid.NewGuid()}.txt"); + var diagLogFilePath = Path.Combine(TempDirectory.Path, $"std_error_log_{Guid.NewGuid()}.txt"); File.Delete(diagLogFilePath); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:ExitwithUnhandleException"); arguments = string.Concat(arguments, $" /diag:{diagLogFilePath}"); @@ -207,12 +201,11 @@ public void UnhandleExceptionExceptionShouldBeLoggedToDiagLogFile(RunnerInfo run public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProject3.dll is built for Framework .NETFramework,Version=v4.5.1 and Platform X64"; var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll", "SimpleTestProjectx86.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:PassingTestx86"); InvokeVsTest(arguments); @@ -230,12 +223,11 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo r public void NoIncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProjectx86 is built for Framework .NETFramework,Version=v4.5.1 and Platform X86"; var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProjectx86.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); @@ -251,12 +243,11 @@ public void NoIncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsoleOnlyWhenRunningIn32BitOS(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProject2.dll is built for Framework .NETFramework,Version=v4.5.1 and Platform X64"; var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); @@ -281,11 +272,10 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsoleOnlyWhenRunni public void ExitCodeShouldReturnOneWhenTreatNoTestsAsErrorParameterSetToTrueAndNoTestMatchesFilter(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); // Setting /TestCaseFilter to the test name, which does not exists in the assembly, so we will have 0 tests executed arguments = string.Concat(arguments, " /TestCaseFilter:TestNameThatMatchesNoTestInTheAssembly"); @@ -302,10 +292,9 @@ public void ExitCodeShouldReturnOneWhenTreatNoTestsAsErrorParameterSetToTrueAndN public void ExitCodeShouldReturnZeroWhenTreatNoTestsAsErrorParameterSetToFalseAndNoTestMatchesFilter(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); // Setting /TestCaseFilter to the test name, which does not exists in the assembly, so we will have 0 tests executed arguments = string.Concat(arguments, " /TestCaseFilter:TestNameThatMatchesNoTestInTheAssembly"); @@ -322,11 +311,10 @@ public void ExitCodeShouldReturnZeroWhenTreatNoTestsAsErrorParameterSetToFalseAn public void ExitCodeShouldNotDependOnTreatNoTestsAsErrorTrueValueWhenThereAreAnyTestsToRun(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " -- RunConfiguration.TreatNoTestsAsError=true"); InvokeVsTest(arguments); @@ -341,10 +329,9 @@ public void ExitCodeShouldNotDependOnTreatNoTestsAsErrorTrueValueWhenThereAreAny public void ExitCodeShouldNotDependOnFailTreatNoTestsAsErrorFalseValueWhenThereAreAnyTestsToRun(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " -- RunConfiguration.TreatNoTestsAsError=false"); InvokeVsTest(arguments); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs index d1611403f2..763e0d5016 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs @@ -17,10 +17,9 @@ public class ExecutionThreadApartmentStateTests : AcceptanceTestBase public void UITestShouldPassIfApartmentStateIsSTA(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:UITestMethod"); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); @@ -31,11 +30,10 @@ public void UITestShouldPassIfApartmentStateIsSTA(RunnerInfo runnerInfo) public void WarningShouldBeShownWhenValueIsSTAForNetCore(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:PassingTest2 -- RunConfiguration.ExecutionThreadApartmentState=STA"); InvokeVsTest(arguments); StdOutputContains("ExecutionThreadApartmentState option not supported for framework:"); @@ -47,11 +45,10 @@ public void WarningShouldBeShownWhenValueIsSTAForNetCore(RunnerInfo runnerInfo) public void UITestShouldFailWhenDefaultApartmentStateIsMTA(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:UITestMethod -- RunConfiguration.ExecutionThreadApartmentState=MTA"); InvokeVsTest(arguments); ValidateSummaryStatus(0, 1, 0); @@ -63,11 +60,10 @@ public void UITestShouldFailWhenDefaultApartmentStateIsMTA(RunnerInfo runnerInfo public void CancelTestExectionShouldWorkWhenApartmentStateIsSTA(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /tests:UITestWithSleep1,UITestMethod -- RunConfiguration.ExecutionThreadApartmentState=STA RunConfiguration.TestSessionTimeout=2000"); InvokeVsTest(arguments); StdOutputContains("Canceling test run: test run timeout of"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs index e948649a28..bf7c89fb1d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs @@ -16,6 +16,13 @@ public RunnerInfo(string runnerType, string targetFramework, string inIsolation) RunnerFramework = runnerType; TargetFramework = targetFramework; InIsolationValue = inIsolation; + // The value is netcoreapp2.1. + IsNetRunner = RunnerFramework.StartsWith("netcoreapp"); + // The value is net451. + IsNetFrameworkRunner = !IsNetRunner; + IsNetTarget = TargetFramework.StartsWith("netcoreapp"); + IsNetFrameworkTarget = !IsNetTarget; + } /// /// Gets the target framework. @@ -48,4 +55,28 @@ public override string ToString() { return string.Join(",", new[] { "RunnerFramework = " + RunnerFramework, " TargetFramework = " + TargetFramework, string.IsNullOrEmpty(InIsolationValue) ? " InProcess" : " InIsolation" }); } + + /// + /// Is running via .NET "Core" vstest.console? + /// + /// + public bool IsNetRunner { get; } + + /// + /// Is running via .NET Framework vstest.console? + /// + /// + public bool IsNetFrameworkRunner { get; } + + /// + /// Is running via .NET "Core" testhost? + /// + /// + public bool IsNetTarget { get; } + + /// + /// Is running via .NET Framework testhost? + /// + /// + public bool IsNetFrameworkTarget { get; } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs index 0fc7a719d9..4ab4715e72 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs @@ -19,7 +19,6 @@ public class FilePatternParserTests : AcceptanceTestBase public void WildCardPatternShouldCorrectlyWorkOnFiles(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var testAssembly = GetSampleTestAssembly(); testAssembly = testAssembly.Replace("SimpleTestProject.dll", "*TestProj*.dll"); @@ -28,7 +27,7 @@ public void WildCardPatternShouldCorrectlyWorkOnFiles(RunnerInfo runnerInfo) testAssembly, GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 1); @@ -40,18 +39,21 @@ public void WildCardPatternShouldCorrectlyWorkOnFiles(RunnerInfo runnerInfo) public void WildCardPatternShouldCorrectlyWorkOnArbitraryDepthDirectories(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var testAssembly = GetSampleTestAssembly(); - var oldAssemblyPath = Path.Combine("Debug", _testEnvironment.TargetFramework, "SimpleTestProject.dll"); - var newAssemblyPath = Path.Combine("**", _testEnvironment.TargetFramework, "*TestProj*.dll"); - testAssembly = testAssembly.Replace(oldAssemblyPath, newAssemblyPath); + + // Add one more directory to the temp path, so we can substitute it with ** + // and copy then whole directory there. + TempDirectory.CopyDirectory(Path.GetDirectoryName(testAssembly), Path.Combine(TempDirectory.Path, "dir1")); + + // The path will end up looking like \**\"*TestProj*.dll". + var wildcardedPath = Path.Combine(TempDirectory.Path, "**", "*TestProj*.dll"); var arguments = PrepareArguments( - testAssembly, + wildcardedPath, GetTestAdapterPath(), string.Empty, string.Empty, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 1); @@ -63,7 +65,6 @@ public void WildCardPatternShouldCorrectlyWorkOnArbitraryDepthDirectories(Runner public void WildCardPatternShouldCorrectlyWorkForRelativeAssemblyPath(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var testAssembly = GetSampleTestAssembly(); testAssembly = testAssembly.Replace("SimpleTestProject.dll", "*TestProj*.dll"); @@ -78,7 +79,7 @@ public void WildCardPatternShouldCorrectlyWorkForRelativeAssemblyPath(RunnerInfo testAssembly, GetTestAdapterPath(), string.Empty, string.Empty, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 1); @@ -90,7 +91,6 @@ public void WildCardPatternShouldCorrectlyWorkForRelativeAssemblyPath(RunnerInfo public void WildCardPatternShouldCorrectlyWorkOnMultipleFiles(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var testAssembly = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); testAssembly = testAssembly.Replace("SimpleTestProject.dll", "*TestProj*.dll"); @@ -100,7 +100,7 @@ public void WildCardPatternShouldCorrectlyWorkOnMultipleFiles(RunnerInfo runnerI testAssembly, GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(2, 2, 2); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs index 61cf91196b..394db87a80 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs @@ -18,9 +18,8 @@ public class FrameworkTests : AcceptanceTestBase public void FrameworkArgumentShouldWork(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " ", $"/Framework:{FrameworkArgValue}"); InvokeVsTest(arguments); @@ -33,9 +32,8 @@ public void FrameworkArgumentShouldWork(RunnerInfo runnerInfo) public void FrameworkShortNameArgumentShouldWork(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " ", $"/Framework:{_testEnvironment.TargetFramework}"); InvokeVsTest(arguments); @@ -50,9 +48,8 @@ public void FrameworkShortNameArgumentShouldWork(RunnerInfo runnerInfo) public void OnWrongFrameworkPassedTestRunShouldNotRun(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); if (runnerInfo.TargetFramework.Contains("netcore")) { arguments = string.Concat(arguments, " ", "/Framework:Framework45"); @@ -82,9 +79,8 @@ public void OnWrongFrameworkPassedTestRunShouldNotRun(RunnerInfo runnerInfo) public void RunSpecificTestsShouldWorkWithFrameworkInCompatibleWarning(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " ", "/tests:PassingTest"); arguments = string.Concat(arguments, " ", "/Framework:Framework40"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs index 0f277b407b..784ad8ca3d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs @@ -22,19 +22,18 @@ public class LoggerTests : AcceptanceTestBase public void TrxLoggerWithFriendlyNameShouldProperlyOverwriteFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var trxFileName = "TestResults.trx"; arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); - var trxFilePath = Path.Combine(tempDir.Path, trxFileName); + var trxFilePath = Path.Combine(TempDirectory.Path, trxFileName); Assert.IsTrue(IsValidXml(trxFilePath), "Invalid content in Trx log file"); } @@ -44,19 +43,18 @@ public void TrxLoggerWithFriendlyNameShouldProperlyOverwriteFile(RunnerInfo runn public void HtmlLoggerWithFriendlyNameShouldProperlyOverwriteFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var htmlFileName = "TestResults.html"; arguments = string.Concat(arguments, $" /logger:\"html;LogFileName={htmlFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"html;LogFileName={htmlFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); - var htmlLogFilePath = Path.Combine(tempDir.Path, htmlFileName); + var htmlLogFilePath = Path.Combine(TempDirectory.Path, htmlFileName); IsFileAndContentEqual(htmlLogFilePath); } @@ -65,19 +63,18 @@ public void HtmlLoggerWithFriendlyNameShouldProperlyOverwriteFile(RunnerInfo run public void TrxLoggerWithExecutorUriShouldProperlyOverwriteFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var trxFileName = "TestResults.trx"; arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFileName={trxFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFileName={trxFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); - var trxLogFilePath = Path.Combine(tempDir.Path, trxFileName); + var trxLogFilePath = Path.Combine(TempDirectory.Path, trxFileName); Assert.IsTrue(IsValidXml(trxLogFilePath), "Invalid content in Trx log file"); } @@ -87,19 +84,18 @@ public void TrxLoggerWithExecutorUriShouldProperlyOverwriteFile(RunnerInfo runne public void TrxLoggerWithLogFilePrefixShouldGenerateMultipleTrx(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var trxFileNamePattern = "TestResults"; - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFilePrefix={trxFileNamePattern}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFilePrefix={trxFileNamePattern}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); - var trxFilePaths = Directory.EnumerateFiles(tempDir.Path, trxFileNamePattern + "_net*.trx"); + var trxFilePaths = Directory.EnumerateFiles(TempDirectory.Path, trxFileNamePattern + "_net*.trx"); Assert.IsTrue(trxFilePaths.Count() > 1); } @@ -108,19 +104,18 @@ public void TrxLoggerWithLogFilePrefixShouldGenerateMultipleTrx(RunnerInfo runne public void HtmlLoggerWithExecutorUriShouldProperlyOverwriteFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var htmlFileName = "TestResults.html"; arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/htmlLogger/v1;LogFileName{htmlFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path); + arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/htmlLogger/v1;LogFileName={htmlFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); - var htmlLogFilePath = Path.Combine(tempDir.Path, htmlFileName); + var htmlLogFilePath = Path.Combine(TempDirectory.Path, htmlFileName); IsFileAndContentEqual(htmlLogFilePath); } @@ -130,10 +125,9 @@ public void HtmlLoggerWithExecutorUriShouldProperlyOverwriteFile(RunnerInfo runn public void TrxLoggerResultSummaryOutcomeValueShouldBeFailedIfNoTestsExecutedAndTreatNoTestsAsErrorIsTrue(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); - var trxFilePath = Path.Combine(tempDir.Path, "TrxLogger.trx"); + var trxFilePath = Path.Combine(TempDirectory.Path, "TrxLogger.trx"); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFilePath}\""); @@ -154,10 +148,9 @@ public void TrxLoggerResultSummaryOutcomeValueShouldBeFailedIfNoTestsExecutedAnd public void TrxLoggerResultSummaryOutcomeValueShouldNotChangeIfNoTestsExecutedAndTreatNoTestsAsErrorIsFalse(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); - var trxFilePath = Path.Combine(tempDir.Path, "TrxLogger.trx"); + var trxFilePath = Path.Combine(TempDirectory.Path, "TrxLogger.trx"); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFilePath}\""); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj index 31d535775e..890ed63c39 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj @@ -12,6 +12,12 @@ netcoreapp3.1 Microsoft.TestPlatform.AcceptanceTests + + False + + + False + diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs index f420f861fe..7b1dff7eb9 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs @@ -22,10 +22,9 @@ public class MultitargetingTestHostTests : AcceptanceTestBase public void TestRunInATesthostThatTargetsTheirChosenNETFramework(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPath = BuildMultipleAssemblyPath("MultitargetedNetFrameworkProject.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPath, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPath, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); // Tell the test project which target framework we are expecting it to run as. // It has this value conditionally compiled, so it can compare it. diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs index 4dc7b4552b..b149fa2994 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs @@ -43,18 +43,17 @@ public void RunTestExecutionWithPlatformx86(RunnerInfo runnerInfo) private void RunTestExecutionWithPlatform(string platformArg, string testhostProcessName, int expectedNumOfProcessCreated) { - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, - _testEnvironment.InIsolationValue, resultsDirectory: tempDir.Path); + _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); - arguments = string.Concat(arguments, platformArg, GetDiagArg(tempDir.Path)); + arguments = string.Concat(arguments, platformArg, GetDiagArg(TempDirectory.Path)); InvokeVsTest(arguments); - AssertExpectedNumberOfHostProcesses(expectedNumOfProcessCreated, tempDir.Path, new[] { testhostProcessName }, arguments, GetConsoleRunnerPath()); + AssertExpectedNumberOfHostProcesses(expectedNumOfProcessCreated, TempDirectory.Path, new[] { testhostProcessName }, arguments, GetConsoleRunnerPath()); ValidateSummaryStatus(1, 1, 1); } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/PostProcessingTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/PostProcessingTests.cs index 2204859563..b059a0bb0e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/PostProcessingTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/PostProcessingTests.cs @@ -22,7 +22,6 @@ public class PostProcessingTests : AcceptanceTestBase [TestMethod] public void DotnetSDKSimulation_PostProcessing() { - using var tempDir = new TempDirectory(); var extensionsPath = Path.Combine( _testEnvironment.TestAssetsPath, @@ -32,7 +31,7 @@ public void DotnetSDKSimulation_PostProcessing() "netstandard2.0"); _testEnvironment.RunnerFramework = CoreRunnerFramework; - string runSettings = GetRunsettingsFilePath(tempDir.Path); + string runSettings = GetRunsettingsFilePath(TempDirectory.Path); string correlationSessionId = Guid.NewGuid().ToString(); // Set datacollector parameters @@ -46,7 +45,7 @@ public void DotnetSDKSimulation_PostProcessing() // Build and run tests like msbuild Parallel.For(0, 5, i => { - string projectFolder = Path.Combine(tempDir.Path, i.ToString()); + string projectFolder = Path.Combine(TempDirectory.Path, i.ToString()); ExecuteApplication(GetConsoleRunnerPath(), $"new mstest -o {projectFolder}", out string stdOut, out string stdError, out int exitCode); Assert.AreEqual(exitCode, 0); ExecuteApplication(GetConsoleRunnerPath(), $"build {projectFolder} -c release", out stdOut, out stdError, out exitCode); @@ -54,13 +53,13 @@ public void DotnetSDKSimulation_PostProcessing() string testContainer = Directory.GetFiles(Path.Combine(projectFolder, "bin"), $"{i}.dll", SearchOption.AllDirectories).Single(); - ExecuteVsTestConsole($"{testContainer} --Collect:\"SampleDataCollector\" --TestAdapterPath:\"{extensionsPath}\" --ResultsDirectory:\"{Path.GetDirectoryName(testContainer)}\" --Settings:\"{runSettings}\" --ArtifactsProcessingMode-Collect --TestSessionCorrelationId:\"{correlationSessionId}\" --Diag:\"{tempDir.Path + '/'}\"", out stdOut, out stdError, out exitCode, + ExecuteVsTestConsole($"{testContainer} --Collect:\"SampleDataCollector\" --TestAdapterPath:\"{extensionsPath}\" --ResultsDirectory:\"{Path.GetDirectoryName(testContainer)}\" --Settings:\"{runSettings}\" --ArtifactsProcessingMode-Collect --TestSessionCorrelationId:\"{correlationSessionId}\" --Diag:\"{TempDirectory.Path + '/'}\"", out stdOut, out stdError, out exitCode, new Dictionary() { { "VSTEST_FEATURE_ARTIFACTS_POSTPROCESSING", "1" } }); Assert.AreEqual(exitCode, 0); }); // Post process artifacts - ExecuteVsTestConsole($"--ArtifactsProcessingMode-PostProcess --TestSessionCorrelationId:\"{correlationSessionId}\" --Diag:\"{tempDir.Path + '/'}\"", out string stdOut, out string stdError, out int exitCode, + ExecuteVsTestConsole($"--ArtifactsProcessingMode-PostProcess --TestSessionCorrelationId:\"{correlationSessionId}\" --Diag:\"{TempDirectory.Path + '/'}\"", out string stdOut, out string stdError, out int exitCode, new Dictionary() { { "VSTEST_FEATURE_ARTIFACTS_POSTPROCESSING", "1" } }); Assert.AreEqual(exitCode, 0); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs index 35b7170d8c..b4fe07e22c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs @@ -3,7 +3,6 @@ using System.IO; -using Microsoft.TestPlatform.TestUtilities; using Microsoft.VisualStudio.TestTools.UnitTesting; namespace Microsoft.TestPlatform.AcceptanceTests; @@ -25,10 +24,8 @@ public void WhenTestHostProcessExitsBecauseTheTargetedRuntimeIsNoFoundThenTheMes const string testAssetProjectName = "SimpleTestProjectMessedUpTargetFramework"; var assemblyPath = GetAssetFullPath(testAssetProjectName + ".dll", Core21TargetFramework); UpdateRuntimeConfigJsonWithInvalidFramework(assemblyPath, testAssetProjectName); - using var tempDir = new TempDirectory(); // Act - var arguments = PrepareArguments(assemblyPath, GetTestAdapterPath(), "", FrameworkArgValue, tempDir.Path); InvokeVsTest(assemblyPath); // Assert diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs index b45a06862d..a7b02265a7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs @@ -22,13 +22,12 @@ public void TrxFileShouldBeCreatedInResultsDirectory(RunnerInfo runnerInfo) var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); var trxFileName = "TestResults.trx"; - using var tempDir = new TempDirectory(); - var trxFilePath = Path.Combine(tempDir.Path, trxFileName); + var trxFilePath = Path.Combine(TempDirectory.Path, trxFileName); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); - arguments = string.Concat(arguments, $" /ResultsDirectory:{tempDir.Path}"); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); // Delete if already exists - TempDirectory.TryRemoveDirectory(tempDir.Path); + TempDirectory.TryRemoveDirectory(TempDirectory.Path); InvokeVsTest(arguments); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs index 01ab24c4ab..8ea5a48b5c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs @@ -30,7 +30,10 @@ public void CommandLineRunSettingsShouldWinAmongAllOptions(RunnerInfo runnerInfo var targetPlatform = "x86"; var testhostProcessName = new[] { "testhost.x86" }; - var expectedNumOfProcessCreated = 1; + + // We pass 2 dlls in RunTestWithRunSettings, for .NET Framework they run in + // 1 hosts because that host is Shared. + var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; // passing parallel var runConfigurationDictionary = new Dictionary @@ -67,7 +70,10 @@ public void CLIRunsettingsShouldWinBetweenCLISwitchesAndCLIRunsettings(RunnerInf var targetPlatform = "x86"; var testhostProcessName = new[] { "testhost.x86" }; - var expectedNumOfProcessCreated = 1; + + // We pass 2 dlls in RunTestWithRunSettings, for .NET Framework they run in + // 1 hosts because that host is Shared. + var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; // Pass parallel var additionalArgs = "/Parallel"; @@ -100,7 +106,10 @@ public void CommandLineSwitchesShouldWinBetweenSettingsFileAndCommandLineSwitche SetTestEnvironment(_testEnvironment, runnerInfo); var testhostProcessName = new[] { "testhost.x86" }; - var expectedNumOfProcessCreated = 1; + + // We pass 2 dlls in RunTestWithRunSettings, for .NET Framework they run in + // 1 hosts because that host is Shared. + var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; // passing different platform var runConfigurationDictionary = new Dictionary @@ -126,7 +135,10 @@ public void RunSettingsWithoutParallelAndPlatformX86(RunnerInfo runnerInfo) var targetPlatform = "x86"; var testhostProcessNames = new[] { "testhost.x86" }; - var expectedNumOfProcessCreated = 1; + + // We pass 2 dlls in RunTestWithRunSettings, for .NET Framework they run in + // 1 hosts because that host is Shared. + var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; var runConfigurationDictionary = new Dictionary { @@ -147,7 +159,10 @@ public void RunSettingsParamsAsArguments(RunnerInfo runnerInfo) var targetPlatform = "x86"; var testhostProcessName = new[] { "testhost.x86" }; - var expectedNumOfProcessCreated = 1; + + // We pass 2 dlls in RunTestWithRunSettings, for .NET Framework they run in + // 1 hosts because that host is Shared. + var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; var runSettingsArgs = string.Join( " ", @@ -171,7 +186,11 @@ public void RunSettingsAndRunSettingsParamsAsArguments(RunnerInfo runnerInfo) var targetPlatform = "x86"; var testhostProcessName = new[] { "testhost.x86" }; - var expectedNumOfProcessCreated = 1; + + // We pass 2 dlls in RunTestWithRunSettings, for .NET Framework they run in + // 1 hosts because that host is Shared. + var expectedNumOfProcessCreated = runnerInfo.IsNetFrameworkTarget ? 1 : 2; + var runConfigurationDictionary = new Dictionary { { "MaxCpuCount", "2" }, @@ -220,18 +239,17 @@ public void RunSettingsWithParallelAndPlatformX64(RunnerInfo runnerInfo) public void RunSettingsWithInvalidValueShouldLogError(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var runConfigurationDictionary = new Dictionary - { - { "TargetPlatform", "123" } - }; - var runsettingsFilePath = GetRunsettingsFilePath(runConfigurationDictionary, tempDir); + { + { "TargetPlatform", "123" } + }; + var runsettingsFilePath = GetRunsettingsFilePath(runConfigurationDictionary, TempDirectory); var arguments = PrepareArguments( GetSampleTestAssembly(), string.Empty, runsettingsFilePath, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); StdErrorContains(@"Settings file provided does not conform to required format. An error occurred while loading the settings. Error: Invalid setting 'RunConfiguration'. Invalid value '123' specified for 'TargetPlatform'."); } @@ -242,18 +260,17 @@ public void RunSettingsWithInvalidValueShouldLogError(RunnerInfo runnerInfo) public void TestAdapterPathFromRunSettings(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var runConfigurationDictionary = new Dictionary - { - { "TestAdaptersPaths", GetTestAdapterPath() } - }; - var runsettingsFilePath = GetRunsettingsFilePath(runConfigurationDictionary, tempDir); + { + { "TestAdaptersPaths", GetTestAdapterPath() } + }; + var runsettingsFilePath = GetRunsettingsFilePath(runConfigurationDictionary, TempDirectory); var arguments = PrepareArguments( GetSampleTestAssembly(), string.Empty, runsettingsFilePath, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 1); } @@ -266,7 +283,6 @@ public void TestAdapterPathFromRunSettings(RunnerInfo runnerInfo) public void LegacySettingsWithPlatform(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); _ = Path.GetDirectoryName(testAssemblyPath); @@ -281,13 +297,13 @@ public void LegacySettingsWithPlatform(RunnerInfo runnerInfo) "; - var runsettingsFilePath = GetRunsettingsFilePath(null, tempDir); + var runsettingsFilePath = GetRunsettingsFilePath(null, TempDirectory); File.WriteAllText(runsettingsFilePath, runsettingsXml); var arguments = PrepareArguments( testAssemblyPath, string.Empty, - runsettingsFilePath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runsettingsFilePath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(0, 0, 0); } @@ -298,18 +314,17 @@ public void LegacySettingsWithPlatform(RunnerInfo runnerInfo) public void LegacySettingsWithScripts(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); // Create the script files var guid = Guid.NewGuid(); var setupScriptName = "setupScript_" + guid + ".bat"; - var setupScriptPath = Path.Combine(tempDir.Path, setupScriptName); + var setupScriptPath = Path.Combine(TempDirectory.Path, setupScriptName); File.WriteAllText(setupScriptPath, @"echo > %temp%\ScriptTestingFile.txt"); var cleanupScriptName = "cleanupScript_" + guid + ".bat"; - var cleanupScriptPath = Path.Combine(tempDir.Path, cleanupScriptName); + var cleanupScriptPath = Path.Combine(TempDirectory.Path, cleanupScriptName); File.WriteAllText(cleanupScriptPath, @"del %temp%\ScriptTestingFile.txt"); var runsettingsFormat = @" @@ -323,19 +338,19 @@ public void LegacySettingsWithScripts(RunnerInfo runnerInfo) // Scripts have relative paths to temp directory where the runsettings is created. var runsettingsXml = string.Format(runsettingsFormat, setupScriptName, cleanupScriptName); - var runsettingsPath = GetRunsettingsFilePath(null, tempDir); + var runsettingsPath = GetRunsettingsFilePath(null, TempDirectory); File.WriteAllText(runsettingsPath, runsettingsXml); var arguments = PrepareArguments( testAssemblyPath, string.Empty, - runsettingsPath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runsettingsPath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:Name=ScriptsTest"); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); // Validate cleanup script ran - var scriptPath = Path.Combine(tempDir.Path, "ScriptTestingFile.txt"); + var scriptPath = Path.Combine(TempDirectory.Path, "ScriptTestingFile.txt"); Assert.IsFalse(File.Exists(scriptPath)); } @@ -345,7 +360,6 @@ public void LegacySettingsWithScripts(RunnerInfo runnerInfo) public void LegacySettingsWithDeploymentItem(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); var testAssemblyDirectory = Path.GetDirectoryName(testAssemblyPath); @@ -364,13 +378,13 @@ public void LegacySettingsWithDeploymentItem(RunnerInfo runnerInfo) "; var runsettingsXml = string.Format(runsettingsFormat, deploymentItem); - var runsettingsPath = GetRunsettingsFilePath(null, tempDir); + var runsettingsPath = GetRunsettingsFilePath(null, TempDirectory); File.WriteAllText(runsettingsPath, runsettingsXml); var arguments = PrepareArguments( testAssemblyPath, string.Empty, - runsettingsPath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runsettingsPath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:Name=DeploymentItemTest"); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); @@ -382,7 +396,6 @@ public void LegacySettingsWithDeploymentItem(RunnerInfo runnerInfo) public void LegacySettingsTestTimeout(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); var runsettingsXml = @" @@ -394,9 +407,9 @@ public void LegacySettingsTestTimeout(RunnerInfo runnerInfo) "; - var runsettingsPath = GetRunsettingsFilePath(null, tempDir); + var runsettingsPath = GetRunsettingsFilePath(null, TempDirectory); File.WriteAllText(runsettingsPath, runsettingsXml); - var arguments = PrepareArguments(testAssemblyPath, string.Empty, runsettingsPath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(testAssemblyPath, string.Empty, runsettingsPath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:Name~TimeTest"); InvokeVsTest(arguments); @@ -410,7 +423,6 @@ public void LegacySettingsTestTimeout(RunnerInfo runnerInfo) public void LegacySettingsAssemblyResolution(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); var runsettingsFormat = @" @@ -433,9 +445,9 @@ public void LegacySettingsAssemblyResolution(RunnerInfo runnerInfo) var testAssemblyDirectory = Path.Combine(_testEnvironment.TestAssetsPath, "LegacySettingsUnitTestProject", "DependencyAssembly"); var runsettingsXml = string.Format(runsettingsFormat, testAssemblyDirectory); - var runsettingsPath = GetRunsettingsFilePath(null, tempDir); + var runsettingsPath = GetRunsettingsFilePath(null, TempDirectory); File.WriteAllText(runsettingsPath, runsettingsXml); - var arguments = PrepareArguments(testAssemblyPath, string.Empty, runsettingsPath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(testAssemblyPath, string.Empty, runsettingsPath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:Name=DependencyTest"); InvokeVsTest(arguments); @@ -453,7 +465,6 @@ public void LegacySettingsAssemblyResolution(RunnerInfo runnerInfo) public void EnvironmentVariablesSettingsShouldSetEnvironmentVariables(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var testAssemblyPath = GetAssetFullPath("EnvironmentVariablesTestProject.dll"); @@ -465,13 +476,13 @@ public void EnvironmentVariablesSettingsShouldSetEnvironmentVariables(RunnerInfo "; - var runsettingsPath = GetRunsettingsFilePath(null, tempDir); + var runsettingsPath = GetRunsettingsFilePath(null, TempDirectory); File.WriteAllText(runsettingsPath, runsettingsXml); var arguments = PrepareArguments( testAssemblyPath, string.Empty, - runsettingsPath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runsettingsPath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); } @@ -511,9 +522,9 @@ public void RunSettingsAreLoadedFromProject(RunnerInfo runnerInfo) #endregion - private string GetRunsettingsFilePath(Dictionary runConfigurationDictionary, TempDirectory tempDirectory) + private string GetRunsettingsFilePath(Dictionary runConfigurationDictionary, TempDirectory TempDirectory) { - var runsettingsPath = Path.Combine(tempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); + var runsettingsPath = Path.Combine(TempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); if (runConfigurationDictionary != null) { CreateRunSettingsFile(runsettingsPath, runConfigurationDictionary); @@ -525,20 +536,18 @@ private string GetRunsettingsFilePath(Dictionary runConfiguratio private void RunTestWithRunSettings(Dictionary runConfigurationDictionary, string runSettingsArgs, string additionalArgs, IEnumerable testhostProcessNames, int expectedNumOfProcessCreated) { - using var tempDir = new TempDirectory(); - var assemblyPaths = - BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); var runsettingsPath = string.Empty; if (runConfigurationDictionary != null) { - runsettingsPath = GetRunsettingsFilePath(runConfigurationDictionary, tempDir); + runsettingsPath = GetRunsettingsFilePath(runConfigurationDictionary, TempDirectory); } - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), runsettingsPath, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: tempDir.Path); - arguments += GetDiagArg(tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), runsettingsPath, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); + arguments += GetDiagArg(TempDirectory.Path); if (!string.IsNullOrWhiteSpace(additionalArgs)) { @@ -553,7 +562,7 @@ private void RunTestWithRunSettings(Dictionary runConfigurationD InvokeVsTest(arguments); // assert - AssertExpectedNumberOfHostProcesses(expectedNumOfProcessCreated, tempDir.Path, testhostProcessNames, arguments, GetConsoleRunnerPath()); + AssertExpectedNumberOfHostProcesses(expectedNumOfProcessCreated, TempDirectory.Path, testhostProcessNames, arguments, GetConsoleRunnerPath()); ValidateSummaryStatus(2, 2, 2); //cleanup diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs index 39b94fcaac..c7dc6b7804 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs @@ -25,11 +25,10 @@ public void RunningApplicationThatIsBuiltAsSelfContainedWillNotFailToFindHostpol // that will fail if we run the testhost.exe from the .nuget location, but will work when we run it from the output folder // see https://github.com/dotnet/runtime/issues/3569#issuecomment-595820524 and below for description of how it works SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); // the app is published to win10-x64 because of the runtime identifier in the project var assemblyPath = BuildMultipleAssemblyPath($@"win10-x64{Path.DirectorySeparatorChar}SelfContainedAppTestProject.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPath, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPath, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(passedTestsCount: 1, 0, 0); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs index d5649d4a59..2083b84bde 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs @@ -28,7 +28,7 @@ public void RunTestsShouldPublishMetrics(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - RunTests(runnerInfo.RunnerFramework); + RunTests(runnerInfo); } [TestMethod] @@ -38,12 +38,12 @@ public void DiscoverTestsShouldPublishMetrics(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - DiscoverTests(runnerInfo.RunnerFramework); + DiscoverTests(runnerInfo); } - private void RunTests(string runnerFramework) + private void RunTests(RunnerInfo runnerInfo) { - if (runnerFramework.StartsWith("netcoreapp")) + if (runnerInfo.IsNetRunner) { Assert.Inconclusive("Telemetry API is not supported for .NetCore runner"); return; @@ -51,21 +51,20 @@ private void RunTests(string runnerFramework) var assemblyPaths = GetAssetFullPath("SimpleTestProject2.dll"); - using var tempDir = new TempDirectory(); var env = new Dictionary { - [LOG_TELEMETRY_PATH] = tempDir.Path, + [LOG_TELEMETRY_PATH] = TempDirectory.Path, [TELEMETRY_OPTEDIN] = "1", [LOG_TELEMETRY] = "1", }; InvokeVsTestForExecution(assemblyPaths, GetTestAdapterPath(), FrameworkArgValue, string.Empty, env); - ValidateOutput("Execution", tempDir); + ValidateOutput("Execution", TempDirectory); } - private void DiscoverTests(string runnerFramework) + private void DiscoverTests(RunnerInfo runnerInfo) { - if (runnerFramework.StartsWith("netcoreapp")) + if (runnerInfo.IsNetRunner) { Assert.Inconclusive("Telemetry API is not supported for .NetCore runner"); return; @@ -73,27 +72,26 @@ private void DiscoverTests(string runnerFramework) var assemblyPaths = GetAssetFullPath("SimpleTestProject2.dll"); - using var tempDir = new TempDirectory(); var env = new Dictionary { - [LOG_TELEMETRY_PATH] = tempDir.Path, + [LOG_TELEMETRY_PATH] = TempDirectory.Path, [TELEMETRY_OPTEDIN] = "1", [LOG_TELEMETRY] = "1", }; InvokeVsTestForDiscovery(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, env); - ValidateOutput("Discovery", tempDir); + ValidateOutput("Discovery", TempDirectory); } - private void ValidateOutput(string command, TempDirectory tempDir) + private void ValidateOutput(string command, TempDirectory TempDirectory) { - if (!Directory.Exists(tempDir.Path)) + if (!Directory.Exists(TempDirectory.Path)) { - Assert.Fail("Could not find the telemetry logs folder at {0}", tempDir.Path); + Assert.Fail("Could not find the telemetry logs folder at {0}", TempDirectory.Path); } bool isValid = false; - var directory = new DirectoryInfo(tempDir.Path); + var directory = new DirectoryInfo(TempDirectory.Path); var file = directory.GetFiles().OrderByDescending(f => f.CreationTime).First(); string[] lines = File.ReadAllLines(file.FullName); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs index 08c7972125..9f90e13949 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs @@ -19,13 +19,12 @@ public class TestCaseFilterTests : AcceptanceTestBase public void RunSelectedTestsWithAndOperatorTrait(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /TestCaseFilter:\"(TestCategory=CategoryA&Priority=3)\""); InvokeVsTest(arguments); ValidateSummaryStatus(0, 1, 0); @@ -37,13 +36,12 @@ public void RunSelectedTestsWithAndOperatorTrait(RunnerInfo runnerInfo) public void RunSelectedTestsWithCategoryTraitInMixCase(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /TestCaseFilter:\"TestCategory=Categorya\""); InvokeVsTest(arguments); ValidateSummaryStatus(0, 1, 0); @@ -55,13 +53,12 @@ public void RunSelectedTestsWithCategoryTraitInMixCase(RunnerInfo runnerInfo) public void RunSelectedTestsWithClassNameTrait(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /TestCaseFilter:\"ClassName=SampleUnitTestProject.UnitTest1\""); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 1); @@ -73,13 +70,12 @@ public void RunSelectedTestsWithClassNameTrait(RunnerInfo runnerInfo) public void RunSelectedTestsWithFullyQualifiedNameTrait(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat( arguments, " /TestCaseFilter:\"FullyQualifiedName=SampleUnitTestProject.UnitTest1.FailingTest\""); @@ -93,13 +89,12 @@ public void RunSelectedTestsWithFullyQualifiedNameTrait(RunnerInfo runnerInfo) public void RunSelectedTestsWithNameTrait(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /TestCaseFilter:\"Name=PassingTest\""); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); @@ -111,13 +106,12 @@ public void RunSelectedTestsWithNameTrait(RunnerInfo runnerInfo) public void RunSelectedTestsWithOrOperatorTrait(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /TestCaseFilter:\"(TestCategory=CategoryA|Priority=2)\""); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 0); @@ -129,13 +123,12 @@ public void RunSelectedTestsWithOrOperatorTrait(RunnerInfo runnerInfo) public void RunSelectedTestsWithPriorityTrait(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /TestCaseFilter:\"Priority=2\""); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); @@ -151,13 +144,12 @@ public void RunSelectedTestsWithPriorityTrait(RunnerInfo runnerInfo) public void TestCaseFilterShouldWorkIfOnlyPropertyValueGivenInExpression(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( _testEnvironment.GetTestAsset("SimpleTestProject2.dll"), GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /TestCaseFilter:UnitTest1"); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 1); @@ -171,20 +163,19 @@ public void TestCaseFilterShouldWorkIfOnlyPropertyValueGivenInExpression(RunnerI [NetFullTargetFrameworkDataSource] public void DiscoverMstestV1TestsWithAndOperatorTrait(RunnerInfo runnerInfo) { - if (runnerInfo.RunnerFramework.StartsWith("netcoreapp")) + if (runnerInfo.IsNetRunner) { Assert.Inconclusive("Mstest v1 tests not supported with .Netcore runner."); return; } SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var arguments = PrepareArguments( _testEnvironment.GetTestAsset("MstestV1UnitTestProject.dll"), GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /listtests /TestCaseFilter:\"(TestCategory!=CategoryA&Priority!=3)\""); InvokeVsTest(arguments); @@ -204,21 +195,20 @@ public void DiscoverMstestV1TestsWithAndOperatorTrait(RunnerInfo runnerInfo) [NetFullTargetFrameworkDataSource] public void DiscoverTmiTestsWithOnlyPropertyValue(RunnerInfo runnerInfo) { - if (runnerInfo.RunnerFramework.StartsWith("netcoreapp")) + if (runnerInfo.IsNetRunner) { Assert.Inconclusive("Tmi tests not supported with .Netcore runner."); return; } SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); string testAssemblyPath = _testEnvironment.GetTestAsset("MstestV1UnitTestProject.dll"); var arguments = PrepareArguments( testAssemblyPath, GetTestAdapterPath(), string.Empty, FrameworkArgValue, - runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); string testSettingsPath = Path.Combine(Path.GetDirectoryName(testAssemblyPath), "MstestV1UnitTestProjectTestSettings.testsettings"); arguments = string.Concat(arguments, " /listtests /TestCaseFilter:PassingTest /settings:", testSettingsPath); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs index caa2488f5d..6125803ad0 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs @@ -47,7 +47,7 @@ public static void ClassCleanup() [TestInitialize] public void SetUp() { - _resultsDirectory = new TempDirectory(); + _resultsDirectory = TempDirectory; } [TestCleanup] diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs index 92a25714d9..b14c602758 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs @@ -28,13 +28,13 @@ namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests; public class CodeCoverageTests : CodeCoverageAcceptanceTestBase { private IVsTestConsoleWrapper _vstestConsoleWrapper; - private TempDirectory _tempDirectory; + private TempDirectory _TempDirectory; private RunEventHandler _runEventHandler; private TestRunAttachmentsProcessingEventHandler _testRunAttachmentsProcessingEventHandler; private void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _tempDirectory); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _TempDirectory); _runEventHandler = new RunEventHandler(); _testRunAttachmentsProcessingEventHandler = new TestRunAttachmentsProcessingEventHandler(); } @@ -492,7 +492,7 @@ private void AssertCoverageResults(IList attachments) { if (attachments.Count == 1) { - var xmlCoverage = GetXmlCoverage(attachments.First().Attachments.First().Uri.LocalPath, _tempDirectory); + var xmlCoverage = GetXmlCoverage(attachments.First().Attachments.First().Uri.LocalPath, _TempDirectory); foreach (var project in GetProjects()) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs index c06123c493..f8b15f77c9 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs @@ -71,6 +71,9 @@ public void EndSessionShouldEnsureVstestConsoleProcessDies(RunnerInfo runnerInfo _vstestConsoleWrapper?.EndSession(); // Assert + // TODO: This still works reliably, but it is accidental. Correctly we should look at our "tree" of processes + // but there is no such thing on Windows. We can still replicate it quite well. There is code for it in blame + // hang collector. Assert.AreEqual(numOfProcesses, Process.GetProcessesByName("vstest.console").Length); _vstestConsoleWrapper = null; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs index 00311527e8..0da7554f11 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs @@ -108,8 +108,7 @@ public void RunTestsWithTestSettings(RunnerInfo runnerInfo) ExecuteNotSupportedRunnerFrameworkTests(runnerInfo.RunnerFramework, Netcoreapp, Message); Setup(); - using var tempDir = new TempDirectory(); - var testsettingsFile = Path.Combine(tempDir.Path, "tempsettings.testsettings"); + var testsettingsFile = Path.Combine(TempDirectory.Path, "tempsettings.testsettings"); string testSettingsXml = @""; File.WriteAllText(testsettingsFile, testSettingsXml, Encoding.UTF8); diff --git a/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs b/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs index e64459980a..b701b1aa0d 100644 --- a/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs +++ b/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs @@ -28,7 +28,7 @@ public void VerifyHostArchitecture(string architecture) string dotnetPath = GetDownloadedDotnetMuxerFromTools(architecture); var vstestConsolePath = GetDotnetRunnerPath(); var dotnetRunnerPath = workSpace.CreateDirectory("dotnetrunner"); - workSpace.CopyAll(new DirectoryInfo(Path.GetDirectoryName(vstestConsolePath)), dotnetRunnerPath); + workSpace.CopyDirectory(new DirectoryInfo(Path.GetDirectoryName(vstestConsolePath)), dotnetRunnerPath); // Patch the runner string sdkVersion = GetLatestSdkVersion(dotnetPath); diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 9d714e1033..9cb9a189e0 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -42,6 +42,8 @@ public class IntegrationTestBase protected readonly IntegrationTestEnvironment _testEnvironment; + public string BuildConfiguration { get; } + private readonly string _testAdapterRelativePath = @"mstest.testadapter\{0}\build\_common".Replace('\\', Path.DirectorySeparatorChar); private readonly string _nUnitTestAdapterRelativePath = @"nunit3testadapter\{0}\build".Replace('\\', Path.DirectorySeparatorChar); private readonly string _xUnitTestAdapterRelativePath = @"xunit.runner.visualstudio\{0}\build\_common".Replace('\\', Path.DirectorySeparatorChar); @@ -57,6 +59,8 @@ public enum UnitTestFramework public IntegrationTestBase() { _testEnvironment = new IntegrationTestEnvironment(); + BuildConfiguration = IntegrationTestEnvironment.BuildConfiguration; + TempDirectory = new TempDirectory(); } public string StdOut => _standardTestOutput; @@ -65,6 +69,20 @@ public IntegrationTestBase() public string StdErr => _standardTestError; public string StdErrWithWhiteSpace { get; private set; } = string.Empty; + public TempDirectory TempDirectory { get; } + + public TestContext TestContext { get; set; } + + [TestCleanup] + public void TempDirectoryCleanup() + { + // Delete the directory only when we passed, so we can look at results and logs of failed tests. + if (TestContext.CurrentTestOutcome == UnitTestOutcome.Passed) + { + TempDirectory.Dispose(); + } + } + /// /// Prepare arguments for vstest.console.exe. /// @@ -752,25 +770,7 @@ protected static string GetDiagArg(string rootDir) /// Counts the number of logs following the '*.host.*' pattern in the given folder. /// protected static int CountTestHostLogs(string diagLogsDir, IEnumerable testHostProcessNames) - => Directory.GetFiles(diagLogsDir, "*.host.*") - .Count(filePath => - { - var firstLine = File.ReadLines(filePath).FirstOrDefault(); - return testHostProcessNames.Any(processName => - { - var parts = processName.Split('.'); - if (parts.Length > 2) - { - throw new InvalidOperationException(""); - } - - var hostName = parts[0]; - var platformName = parts.Length > 1 ? @$"\.{parts[1]}" : string.Empty; - - var isMatch = Regex.IsMatch(firstLine, @$",\s{hostName}(?:\.net\d+)?{platformName}\.(?:exe|dll),"); - return isMatch; - }); - }); + => Directory.GetFiles(diagLogsDir, "*.host.*").Count(); protected static void AssertExpectedNumberOfHostProcesses(int expectedNumOfProcessCreated, string diagLogsDir, IEnumerable testHostProcessNames, string arguments = null, string runnerPath = null) { @@ -800,4 +800,24 @@ protected static string GetDownloadedDotnetMuxerFromTools(string architecture) } protected static string GetDotnetRunnerPath() => Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "netcoreapp2.1", "vstest.console.dll"); + + public static void CopyDirectory(string sourceDirectory, string destinationDirectory) + { + if (!Directory.Exists(destinationDirectory)) + Directory.CreateDirectory(destinationDirectory); + string[] files = Directory.GetFiles(sourceDirectory); + foreach (string file in files) + { + string name = Path.GetFileName(file); + string dest = Path.Combine(destinationDirectory, name); + File.Copy(file, dest); + } + string[] folders = Directory.GetDirectories(sourceDirectory); + foreach (string folder in folders) + { + string name = Path.GetFileName(folder); + string dest = Path.Combine(destinationDirectory, name); + CopyDirectory(folder, dest); + } + } } diff --git a/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj b/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj index a91d6a82ca..adb97b9b5a 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj +++ b/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj @@ -33,4 +33,4 @@ - + \ No newline at end of file diff --git a/test/Microsoft.TestPlatform.TestUtilities/RandomId.cs b/test/Microsoft.TestPlatform.TestUtilities/RandomId.cs new file mode 100644 index 0000000000..cad0f84330 --- /dev/null +++ b/test/Microsoft.TestPlatform.TestUtilities/RandomId.cs @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Security.Cryptography; + +namespace Microsoft.TestPlatform.TestUtilities; + +/// +/// Slightly random id that is just good enough for creating disctinct directories for each test. +/// +public static class RandomId +{ + private static readonly RandomNumberGenerator Rng = RandomNumberGenerator.Create(); + private const string Pool = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + + /// + /// 5 character long id from 0-9A-Za-z0, for example fUfko, A6uvM, sOMXa, RY1ei, KvdJZ. + /// + /// + public static string Next() + { + return Next(5); + } + + private static string Next(int length) + { + var poolLength = Pool.Length; + var id = new char[length]; + lock (Pool) + { + for (var idIndex = 0; idIndex < length; idIndex++) + { + var poolIndex = poolLength + 1; + while (poolIndex >= poolLength) + { + var bytes = new byte[1]; + Rng.GetNonZeroBytes(bytes); + poolIndex = bytes[0]; + } + id[idIndex] = Pool[poolIndex]; + } + } + + return new string(id); + } +} diff --git a/test/Microsoft.TestPlatform.TestUtilities/TempDirectory.cs b/test/Microsoft.TestPlatform.TestUtilities/TempDirectory.cs index f218d96f7a..3b45ecc3dc 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/TempDirectory.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/TempDirectory.cs @@ -3,6 +3,8 @@ using System; using System.IO; +using IO = System.IO; + #nullable disable @@ -26,26 +28,31 @@ public void Dispose() } public DirectoryInfo CreateDirectory(string dir) - => Directory.CreateDirectory(System.IO.Path.Combine(Path, dir)); + => Directory.CreateDirectory(IO.Path.Combine(Path, dir)); + + public void CopyDirectory(string sourceDirectory, string targetDirectory) + { + CopyDirectory(new DirectoryInfo(sourceDirectory), new DirectoryInfo(targetDirectory)); + } - public void CopyAll(DirectoryInfo source, DirectoryInfo target) + public void CopyDirectory(DirectoryInfo source, DirectoryInfo target) { Directory.CreateDirectory(target.FullName); // Copy each file into the new directory. foreach (FileInfo fi in source.GetFiles()) { - fi.CopyTo(System.IO.Path.Combine(target.FullName, fi.Name), true); + fi.CopyTo(IO.Path.Combine(target.FullName, fi.Name), true); } // Copy each subdirectory using recursion. foreach (DirectoryInfo diSourceSubDir in source.GetDirectories()) { - DirectoryInfo nextTargetSubDir = - target.CreateSubdirectory(diSourceSubDir.Name); - CopyAll(diSourceSubDir, nextTargetSubDir); + DirectoryInfo nextTargetSubDir = target.CreateSubdirectory(diSourceSubDir.Name); + CopyDirectory(diSourceSubDir, nextTargetSubDir); } } + /// /// Creates an unique temporary directory. /// @@ -54,19 +61,20 @@ public void CopyAll(DirectoryInfo source, DirectoryInfo target) /// internal static string CreateUniqueDirectory() { - // AGENT_TEMPDIRECTORY is AzureDevops variable, which is set to path - // that is cleaned up after every job. This is preferable to use over - // just the normal temp. var temp = GetTempPath(); - var directoryPath = System.IO.Path.Combine(temp, Guid.NewGuid().ToString("n")); + var directoryPath = IO.Path.Combine(temp, "vstest", RandomId.Next()); Directory.CreateDirectory(directoryPath); return directoryPath; } private static string GetTempPath() - => Environment.GetEnvironmentVariable("AGENT_TEMPDIRECTORY") - ?? System.IO.Path.GetTempPath(); + { + // AGENT_TEMPDIRECTORY is AzureDevops variable, which is set to path + // that is cleaned up after every job. This is preferable to use over + // just the normal TEMP, because that is not cleaned up for every run. + return Environment.GetEnvironmentVariable("AGENT_TEMPDIRECTORY") ?? IO.Path.GetTempPath(); + } public static void TryRemoveDirectory(string directory) { @@ -74,7 +82,7 @@ public static void TryRemoveDirectory(string directory) { try { - Directory.Delete(directory, true); + Directory.Delete(directory, recursive: true); } catch { } } From aff2f4a12872d624ae5562352ff732c5e5b09ae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 4 Mar 2022 12:14:56 +0100 Subject: [PATCH 02/64] Apply suggestions from code review --- .../AppDomainTests.cs | 2 +- .../DifferentTestFrameworkSimpleTests.cs | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs index 776ad1956e..235aa4c59f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs @@ -29,7 +29,7 @@ public void RunTestExecutionWithDisableAppDomain(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAppDomainDetailFileName = Path.Combine(TempDirectory.Path, "appdomain_test.txt"); + var testAppDomainDetailFileName = Path.Combine(TempDirectory.Path, "appdomain_test.txt"); var dataCollectorAppDomainDetailFileName = Path.Combine(TempDirectory.Path, "appdomain_datacollector.txt"); // Delete test output files if already exist diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs index fc3209099a..f360caeb84 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs @@ -65,16 +65,6 @@ public void CPPRunAllTestExecutionPlatformx64Net(RunnerInfo runnerInfo) [NetFullTargetFrameworkDataSource] public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo) { - // This test works on server but not locally, and it fails with - // Exception: Method not found: 'Int64 Microsoft.VisualStudio.TestTools.WebTesting.WebTestRequestStatistics.get_MillisecondsSincePageComplete()'. - // The dll is - - //if (!IsCI) - //{ - // Assert.Inconclusive("Web load tests need special workloads and setup locally"); - //} - - SetTestEnvironment(_testEnvironment, runnerInfo); var runSettingsFilePath = Path.Combine(TempDirectory.Path, Guid.NewGuid() + ".runsettings"); From 83a9fc6bae48520da76fd9dc35291b9c322dc6e7 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 12:15:29 +0100 Subject: [PATCH 03/64] Remove more workspace --- .../DotnetArchitectureSwitchTests.Windows.cs | 3 +-- .../DotnetHostArchitectureVerifierTests.cs | 3 +-- .../IntegrationTestBase.cs | 8 ++------ 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs index cbb764e423..bedcb5d62f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs @@ -23,12 +23,11 @@ public class DotnetArchitectureSwitchTestsWindowsOnly : AcceptanceTestBase [DataRow("X86", "X64")] public void Use_EnvironmentVariables(string architectureFrom, string architectureTo) { - using var workSpace = TempDirectory; string dotnetPath = GetDownloadedDotnetMuxerFromTools(architectureFrom); string dotnetPathTo = GetDownloadedDotnetMuxerFromTools(architectureTo); var vstestConsolePath = GetDotnetRunnerPath(); var dotnetRunnerPath = workSpace.CreateDirectory("dotnetrunner"); - workSpace.CopyDirectory(new DirectoryInfo(Path.GetDirectoryName(vstestConsolePath)), dotnetRunnerPath); + TempDirectory.CopyDirectory(new DirectoryInfo(Path.GetDirectoryName(vstestConsolePath)), dotnetRunnerPath); // Patch the runner string sdkVersion = GetLatestSdkVersion(dotnetPath); diff --git a/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs b/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs index b701b1aa0d..8dfe7f2258 100644 --- a/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs +++ b/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs @@ -24,11 +24,10 @@ public class DotnetHostArchitectureVerifierTests : IntegrationTestBase [DataRow("X86")] public void VerifyHostArchitecture(string architecture) { - using var workSpace = new TempDirectory(); string dotnetPath = GetDownloadedDotnetMuxerFromTools(architecture); var vstestConsolePath = GetDotnetRunnerPath(); var dotnetRunnerPath = workSpace.CreateDirectory("dotnetrunner"); - workSpace.CopyDirectory(new DirectoryInfo(Path.GetDirectoryName(vstestConsolePath)), dotnetRunnerPath); + TempDirectory.CopyDirectory(new DirectoryInfo(Path.GetDirectoryName(vstestConsolePath)), dotnetRunnerPath); // Patch the runner string sdkVersion = GetLatestSdkVersion(dotnetPath); diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 9cb9a189e0..2198ba74c2 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -201,9 +201,7 @@ public void InvokeVsTestForExecution(string testAssembly, string runSettings = "", Dictionary environmentVariables = null) { - using var workspace = new TempDirectory(); - - var arguments = PrepareArguments(testAssembly, testAdapterPath, runSettings, framework, _testEnvironment.InIsolationValue, resultsDirectory: workspace.Path); + var arguments = PrepareArguments(testAssembly, testAdapterPath, runSettings, framework, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments, environmentVariables); } @@ -215,9 +213,7 @@ public void InvokeVsTestForExecution(string testAssembly, /// Run settings for execution. public void InvokeVsTestForDiscovery(string testAssembly, string testAdapterPath, string runSettings = "", string targetFramework = "", Dictionary environmentVariables = null) { - using var workspace = new TempDirectory(); - - var arguments = PrepareArguments(testAssembly, testAdapterPath, runSettings, targetFramework, _testEnvironment.InIsolationValue, resultsDirectory: workspace.Path); + var arguments = PrepareArguments(testAssembly, testAdapterPath, runSettings, targetFramework, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /listtests"); InvokeVsTest(arguments, environmentVariables); } From 7536588369978d08c3be0dd5bec607bbf47789d1 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 12:20:18 +0100 Subject: [PATCH 04/64] Revert non-optmize on Release --- .../Microsoft.TestPlatform.AcceptanceTests.csproj | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj index 890ed63c39..31d535775e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj @@ -12,12 +12,6 @@ netcoreapp3.1 Microsoft.TestPlatform.AcceptanceTests - - False - - - False - From a94b1bd7c6b1b554d69022ac4c107999ebf862a4 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 12:30:51 +0100 Subject: [PATCH 05/64] More fixes --- .../DotnetArchitectureSwitchTests.Windows.cs | 12 +++++------ .../IntegrationTestBase.cs | 20 ------------------- 2 files changed, 6 insertions(+), 26 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs index bedcb5d62f..3067a8bcbd 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs @@ -26,7 +26,7 @@ public void Use_EnvironmentVariables(string architectureFrom, string architectur string dotnetPath = GetDownloadedDotnetMuxerFromTools(architectureFrom); string dotnetPathTo = GetDownloadedDotnetMuxerFromTools(architectureTo); var vstestConsolePath = GetDotnetRunnerPath(); - var dotnetRunnerPath = workSpace.CreateDirectory("dotnetrunner"); + var dotnetRunnerPath = TempDirectory.CreateDirectory("dotnetrunner"); TempDirectory.CopyDirectory(new DirectoryInfo(Path.GetDirectoryName(vstestConsolePath)), dotnetRunnerPath); // Patch the runner @@ -42,10 +42,10 @@ public void Use_EnvironmentVariables(string architectureFrom, string architectur [$"DOTNET_ROOT_{architectureTo}"] = Path.GetDirectoryName(dotnetPathTo), ["ExpectedArchitecture"] = architectureTo }; - ExecuteApplication(dotnetPath, "new mstest", out string stdOut, out string stdError, out int exitCode, environmentVariables, workSpace.Path); + ExecuteApplication(dotnetPath, "new mstest", out string stdOut, out string stdError, out int exitCode, environmentVariables, TempDirectory.Path); // Patch test file - File.WriteAllText(Path.Combine(workSpace.Path, "UnitTest1.cs"), + File.WriteAllText(Path.Combine(TempDirectory.Path, "UnitTest1.cs"), @" using Microsoft.VisualStudio.TestTools.UnitTesting; using System; @@ -64,7 +64,7 @@ public void TestMethod1() } }"); - ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out stdOut, out stdError, out exitCode, environmentVariables, workSpace.Path); + ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out stdOut, out stdError, out exitCode, environmentVariables, TempDirectory.Path); Assert.AreEqual(0, exitCode, stdOut); environmentVariables = new Dictionary @@ -73,7 +73,7 @@ public void TestMethod1() ["DOTNET_ROOT"] = Path.GetDirectoryName(dotnetPathTo), ["ExpectedArchitecture"] = architectureTo }; - ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out stdOut, out stdError, out exitCode, environmentVariables, workSpace.Path); + ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out stdOut, out stdError, out exitCode, environmentVariables, TempDirectory.Path); Assert.AreEqual(0, exitCode, stdOut); environmentVariables = new Dictionary @@ -83,7 +83,7 @@ public void TestMethod1() ["DOTNET_ROOT"] = "WE SHOULD PICK THE ABOVE ONE BEFORE FALLBACK TO DOTNET_ROOT", ["ExpectedArchitecture"] = architectureTo }; - ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out stdOut, out stdError, out exitCode, environmentVariables, workSpace.Path); + ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out stdOut, out stdError, out exitCode, environmentVariables, TempDirectory.Path); Assert.AreEqual(0, exitCode, stdOut); } diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 2198ba74c2..91cc6c019a 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -796,24 +796,4 @@ protected static string GetDownloadedDotnetMuxerFromTools(string architecture) } protected static string GetDotnetRunnerPath() => Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "netcoreapp2.1", "vstest.console.dll"); - - public static void CopyDirectory(string sourceDirectory, string destinationDirectory) - { - if (!Directory.Exists(destinationDirectory)) - Directory.CreateDirectory(destinationDirectory); - string[] files = Directory.GetFiles(sourceDirectory); - foreach (string file in files) - { - string name = Path.GetFileName(file); - string dest = Path.Combine(destinationDirectory, name); - File.Copy(file, dest); - } - string[] folders = Directory.GetDirectories(sourceDirectory); - foreach (string folder in folders) - { - string name = Path.GetFileName(folder); - string dest = Path.Combine(destinationDirectory, name); - CopyDirectory(folder, dest); - } - } } From ef668f074b87165bac84d6d2289944bae06987e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 4 Mar 2022 12:31:18 +0100 Subject: [PATCH 06/64] Update test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs --- .../Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 91cc6c019a..8dd9c262f7 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -76,7 +76,7 @@ public IntegrationTestBase() [TestCleanup] public void TempDirectoryCleanup() { - // Delete the directory only when we passed, so we can look at results and logs of failed tests. + // Delete the directory only when the test succeeded, so we can look at results and logs of failed tests. if (TestContext.CurrentTestOutcome == UnitTestOutcome.Passed) { TempDirectory.Dispose(); From 8b04531092c1729b4ff52c8f60d52be706c59223 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 12:56:06 +0100 Subject: [PATCH 07/64] Fix unit test --- .../EventLogDataCollectorTests.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/EventLogDataCollectorTests.cs b/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/EventLogDataCollectorTests.cs index 3096009d27..473c6b8af6 100644 --- a/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/EventLogDataCollectorTests.cs +++ b/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/EventLogDataCollectorTests.cs @@ -217,11 +217,10 @@ public void InitializeShouldSubscribeToDataCollectionEvents() { var testableDataCollectionEvents = new TestableDataCollectionEvents(); _eventLogDataCollector.Initialize(null, testableDataCollectionEvents, _mockDataCollectionSink, _mockDataCollectionLogger.Object, _dataCollectionEnvironmentContext); - Assert.AreEqual(1, testableDataCollectionEvents.GetTestHostLaunchedInvocationList().Length); - Assert.AreEqual(1, testableDataCollectionEvents.GetTestCaseStartInvocationList().Length); - Assert.AreEqual(1, testableDataCollectionEvents.GetTestCaseEndInvocationList().Length); - Assert.AreEqual(1, testableDataCollectionEvents.GetTestSessionEndInvocationList().Length); - Assert.AreEqual(1, testableDataCollectionEvents.GetTestSessionStartInvocationList().Length); + Assert.AreEqual(1, testableDataCollectionEvents.GetTestCaseStartInvocationList()?.Length, "GetTestCaseStartInvocationList"); + Assert.AreEqual(1, testableDataCollectionEvents.GetTestCaseEndInvocationList()?.Length, "GetTestCaseEndInvocationList"); + Assert.AreEqual(1, testableDataCollectionEvents.GetTestSessionEndInvocationList()?.Length, "GetTestSessionEndInvocationList"); + Assert.AreEqual(1, testableDataCollectionEvents.GetTestSessionStartInvocationList().Length, "GetTestSessionStartInvocationList"); } [TestMethod] From e57fe1d281ded602c16723156f287e55db39288c Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 12:57:00 +0100 Subject: [PATCH 08/64] Missing null operator --- .../EventLogDataCollectorTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/EventLogDataCollectorTests.cs b/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/EventLogDataCollectorTests.cs index 473c6b8af6..aca802aa53 100644 --- a/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/EventLogDataCollectorTests.cs +++ b/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/EventLogDataCollectorTests.cs @@ -220,7 +220,7 @@ public void InitializeShouldSubscribeToDataCollectionEvents() Assert.AreEqual(1, testableDataCollectionEvents.GetTestCaseStartInvocationList()?.Length, "GetTestCaseStartInvocationList"); Assert.AreEqual(1, testableDataCollectionEvents.GetTestCaseEndInvocationList()?.Length, "GetTestCaseEndInvocationList"); Assert.AreEqual(1, testableDataCollectionEvents.GetTestSessionEndInvocationList()?.Length, "GetTestSessionEndInvocationList"); - Assert.AreEqual(1, testableDataCollectionEvents.GetTestSessionStartInvocationList().Length, "GetTestSessionStartInvocationList"); + Assert.AreEqual(1, testableDataCollectionEvents.GetTestSessionStartInvocationList()?.Length, "GetTestSessionStartInvocationList"); } [TestMethod] From 98fa6b91517e3497291d022d18f132d806d33863 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 13:03:59 +0100 Subject: [PATCH 09/64] Revert incorrect rename --- scripts/build.ps1 | 1 - .../AppDomainTests.cs | 4 ++-- .../BlameDataCollectorTests.cs | 4 ++-- .../CodeCoverageAcceptanceTestBase.cs | 4 ++-- .../CodeCoverageTests.cs | 8 ++++---- .../DisableAppdomainTests.cs | 4 ++-- .../EventLogCollectorTests.cs | 8 ++++---- .../RunsettingsTests.cs | 4 ++-- .../TelemetryTests.cs | 8 ++++---- 9 files changed, 22 insertions(+), 23 deletions(-) diff --git a/scripts/build.ps1 b/scripts/build.ps1 index c108e8eced..4fe054862f 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -140,7 +140,6 @@ if ((Test-Path $procdumpDir) -and 2 -ne @(Get-Item "$procdumpDir\0.0.1\bin").Len Remove-Item -Recurse -Force $procdumpDir } - function Invoke-Build { $timer = Start-Timer diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs index 235aa4c59f..82676c5ae2 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs @@ -73,9 +73,9 @@ private static bool IsFilesContentEqual(string filePath1, string filePath2) return string.Equals(content1, content2, StringComparison.Ordinal); } - private string GetInProcDataCollectionRunsettingsFile(bool disableAppDomain, TempDirectory TempDirectory) + private string GetInProcDataCollectionRunsettingsFile(bool disableAppDomain, TempDirectory tempDirectory) { - var runSettings = Path.Combine(TempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); + var runSettings = Path.Combine(tempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); var inprocasm = _testEnvironment.GetTestAsset("SimpleDataCollector.dll"); #if !NET451 var assemblyName = AssemblyLoadContext.GetAssemblyName(inprocasm); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs index fc00b22d7a..93a88e9894 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs @@ -323,14 +323,14 @@ private void ValidateDump(int expectedDumpCount = 1) } } - private void VaildateOutput(TempDirectory TempDirectory, string testName, bool validateDumpFile = false) + private void VaildateOutput(TempDirectory tempDirectory, string testName, bool validateDumpFile = false) { bool isSequenceAttachmentReceived = false; bool isDumpAttachmentReceived = false; bool isValid = false; StdErrorContains(testName); StdOutputContains("Sequence_"); - var resultFiles = Directory.GetFiles(TempDirectory.Path, "*", SearchOption.AllDirectories); + var resultFiles = Directory.GetFiles(tempDirectory.Path, "*", SearchOption.AllDirectories); foreach (var file in resultFiles) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs index b8a44734f8..2081de653b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageAcceptanceTestBase.cs @@ -59,7 +59,7 @@ protected XmlNode GetNode(XmlNode node, string type, string name) return node.SelectSingleNode($"//{type}[@name='{name}']") ?? node.SelectSingleNode($"//{type}[@name='{name.ToLower()}']"); } - protected XmlDocument GetXmlCoverage(string coverageResult, TempDirectory TempDirectory) + protected XmlDocument GetXmlCoverage(string coverageResult, TempDirectory tempDirectory) { var coverage = new XmlDocument(); @@ -70,7 +70,7 @@ protected XmlDocument GetXmlCoverage(string coverageResult, TempDirectory TempDi } var codeCoverageExe = GetCodeCoverageExePath(); - var output = Path.Combine(TempDirectory.Path, Guid.NewGuid().ToString() + ".xml"); + var output = Path.Combine(tempDirectory.Path, Guid.NewGuid().ToString() + ".xml"); var watch = new Stopwatch(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs index 1dc4c3f618..b426fc7fd1 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs @@ -246,7 +246,7 @@ private void CollectCodeCoverage(RunnerInfo runnerInfo, TestParameters testParam } private string CreateArguments( - TempDirectory TempDirectory, + TempDirectory tempDirectory, RunnerInfo runnerInfo, TestParameters testParameters, out string trxFilePath) @@ -256,14 +256,14 @@ private string CreateArguments( string traceDataCollectorDir = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "Microsoft.CodeCoverage"); - string diagFileName = Path.Combine(TempDirectory.Path, "diaglog.txt"); + string diagFileName = Path.Combine(tempDirectory.Path, "diaglog.txt"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, - FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + FrameworkArgValue, runnerInfo.InIsolationValue, tempDirectory.Path); arguments = string.Concat(arguments, $" /Diag:{diagFileName}", $" /TestAdapterPath:{traceDataCollectorDir}"); arguments = string.Concat(arguments, $" /Platform:{testParameters.TargetPlatform}"); - trxFilePath = Path.Combine(TempDirectory.Path, Guid.NewGuid() + ".trx"); + trxFilePath = Path.Combine(tempDirectory.Path, Guid.NewGuid() + ".trx"); arguments = string.Concat(arguments, " /logger:trx;logfilename=" + trxFilePath); var defaultRunSettingsPath = Path.Combine( diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DisableAppdomainTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DisableAppdomainTests.cs index 78add6fe64..4136000193 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DisableAppdomainTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DisableAppdomainTests.cs @@ -64,9 +64,9 @@ private void RunTests(RunnerInfo runnerInfo, string testAssembly, int passedTest ValidateSummaryStatus(passedTestCount, 0, 0); } - private string GetRunsettingsFilePath(TempDirectory TempDirectory, Dictionary runConfigurationDictionary) + private string GetRunsettingsFilePath(TempDirectory tempDirectory, Dictionary runConfigurationDictionary) { - var runsettingsPath = Path.Combine(TempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); + var runsettingsPath = Path.Combine(tempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); CreateRunSettingsFile(runsettingsPath, runConfigurationDictionary); return runsettingsPath; } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/EventLogCollectorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/EventLogCollectorTests.cs index b7e80ebf5e..12cee6580a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/EventLogCollectorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/EventLogCollectorTests.cs @@ -61,9 +61,9 @@ public void EventLogDataCollectorShoudCreateLogFileWithoutEventsIfEventsAreNotLo StdErrorDoesNotContains("Unable to read event log"); } - private string GetRunsettingsFilePath(TempDirectory TempDirectory) + private string GetRunsettingsFilePath(TempDirectory tempDirectory) { - var runsettingsPath = Path.Combine(TempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); + var runsettingsPath = Path.Combine(tempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); string runSettingsXml = @" @@ -85,10 +85,10 @@ private string GetRunsettingsFilePath(TempDirectory TempDirectory) return runsettingsPath; } - private void VaildateDataCollectorOutput(TempDirectory TempDirectory) + private void VaildateDataCollectorOutput(TempDirectory tempDirectory) { // Verify attachments - var di = new DirectoryInfo(TempDirectory.Path); + var di = new DirectoryInfo(tempDirectory.Path); var resultFiles = di.EnumerateFiles("Event Log.xml", SearchOption.AllDirectories) .OrderBy(d => d.CreationTime) .Select(d => d.FullName) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs index 8ea5a48b5c..edfa13bcad 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs @@ -522,9 +522,9 @@ public void RunSettingsAreLoadedFromProject(RunnerInfo runnerInfo) #endregion - private string GetRunsettingsFilePath(Dictionary runConfigurationDictionary, TempDirectory TempDirectory) + private string GetRunsettingsFilePath(Dictionary runConfigurationDictionary, TempDirectory tempDirectory) { - var runsettingsPath = Path.Combine(TempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); + var runsettingsPath = Path.Combine(tempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); if (runConfigurationDictionary != null) { CreateRunSettingsFile(runsettingsPath, runConfigurationDictionary); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs index 2083b84bde..fdf49097ce 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs @@ -83,15 +83,15 @@ private void DiscoverTests(RunnerInfo runnerInfo) ValidateOutput("Discovery", TempDirectory); } - private void ValidateOutput(string command, TempDirectory TempDirectory) + private void ValidateOutput(string command, TempDirectory tempDirectory) { - if (!Directory.Exists(TempDirectory.Path)) + if (!Directory.Exists(tempDirectory.Path)) { - Assert.Fail("Could not find the telemetry logs folder at {0}", TempDirectory.Path); + Assert.Fail("Could not find the telemetry logs folder at {0}", tempDirectory.Path); } bool isValid = false; - var directory = new DirectoryInfo(TempDirectory.Path); + var directory = new DirectoryInfo(tempDirectory.Path); var file = directory.GetFiles().OrderByDescending(f => f.CreationTime).First(); string[] lines = File.ReadAllLines(file.FullName); From 6342ee43ad25f54698b8fbe33248fc38015b2be4 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 13:20:59 +0100 Subject: [PATCH 10/64] Fix review remarks --- .../BlameDataCollectorTests.cs | 4 ++-- .../DifferentTestFrameworkSimpleTests.cs | 12 ++++++------ .../Extension/RunnnerInfo.cs | 6 ++++-- .../RunsettingsTests.cs | 4 ++-- .../TestCaseFilterTests.cs | 4 ++-- .../IntegrationTestBase.cs | 10 +++++----- 6 files changed, 21 insertions(+), 19 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs index 93a88e9894..671a91618d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs @@ -33,8 +33,8 @@ public BlameDataCollectorTests() if (!File.Exists(procDumpExePath)) { throw new InvalidOperationException($"Procdump path {procDumpExePath} does not exist. " - + $"It is possible that antivirus deleted it from your nuget cache. " - + $"Delete the whole procdump folder in your nuget cache, and run build, or restore"); + + "It is possible that antivirus deleted it from your nuget cache. " + + "Delete the whole procdump folder in your nuget cache, and run build, or restore"); } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs index f360caeb84..26dfa7ab26 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs @@ -83,7 +83,7 @@ public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo) var minWebTestResultFileSizeInKB = 150; if (runnerInfo.IsNetRunner) { - Assert.Inconclusive("WebTests tests not supported with .Netcore runner."); + Assert.Inconclusive("WebTests tests not supported with .NET Core runner."); return; } @@ -120,7 +120,7 @@ public void CodedWebTestRunAllTests(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); if (runnerInfo.IsNetRunner) { - Assert.Inconclusive("WebTests tests not supported with .Netcore runner."); + Assert.Inconclusive("WebTests tests not supported with .NET Core runner."); return; } @@ -143,10 +143,10 @@ public void NUnitRunAllTestExecution(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( -GetAssetFullPath("NUTestProject.dll"), -GetTestAdapterPath(UnitTestFramework.NUnit), -string.Empty, FrameworkArgValue, -runnerInfo.InIsolationValue, TempDirectory.Path); + GetAssetFullPath("NUTestProject.dll"), + GetTestAdapterPath(UnitTestFramework.NUnit), + string.Empty, FrameworkArgValue, + runnerInfo.InIsolationValue, TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 1, 0); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs index bf7c89fb1d..5357734779 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs @@ -3,6 +3,8 @@ #nullable disable +using System; + namespace Microsoft.TestPlatform.AcceptanceTests; public class RunnerInfo @@ -17,10 +19,10 @@ public RunnerInfo(string runnerType, string targetFramework, string inIsolation) TargetFramework = targetFramework; InIsolationValue = inIsolation; // The value is netcoreapp2.1. - IsNetRunner = RunnerFramework.StartsWith("netcoreapp"); + IsNetRunner = RunnerFramework.StartsWith("netcoreapp", StringComparison.InvariantCultureIgnoreCase); // The value is net451. IsNetFrameworkRunner = !IsNetRunner; - IsNetTarget = TargetFramework.StartsWith("netcoreapp"); + IsNetTarget = TargetFramework.StartsWith("netcoreapp", StringComparison.InvariantCultureIgnoreCase); IsNetFrameworkTarget = !IsNetTarget; } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs index edfa13bcad..2a0809cbaa 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs @@ -242,7 +242,7 @@ public void RunSettingsWithInvalidValueShouldLogError(RunnerInfo runnerInfo) var runConfigurationDictionary = new Dictionary { - { "TargetPlatform", "123" } + { "TargetPlatform", "123" } }; var runsettingsFilePath = GetRunsettingsFilePath(runConfigurationDictionary, TempDirectory); var arguments = PrepareArguments( @@ -263,7 +263,7 @@ public void TestAdapterPathFromRunSettings(RunnerInfo runnerInfo) var runConfigurationDictionary = new Dictionary { - { "TestAdaptersPaths", GetTestAdapterPath() } + { "TestAdaptersPaths", GetTestAdapterPath() } }; var runsettingsFilePath = GetRunsettingsFilePath(runConfigurationDictionary, TempDirectory); var arguments = PrepareArguments( diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs index 9f90e13949..c4363cf825 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs @@ -165,7 +165,7 @@ public void DiscoverMstestV1TestsWithAndOperatorTrait(RunnerInfo runnerInfo) { if (runnerInfo.IsNetRunner) { - Assert.Inconclusive("Mstest v1 tests not supported with .Netcore runner."); + Assert.Inconclusive("Mstest v1 tests not supported with .NET Core runner."); return; } @@ -197,7 +197,7 @@ public void DiscoverTmiTestsWithOnlyPropertyValue(RunnerInfo runnerInfo) { if (runnerInfo.IsNetRunner) { - Assert.Inconclusive("Tmi tests not supported with .Netcore runner."); + Assert.Inconclusive("Tmi tests not supported with .NET Core runner."); return; } diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 8dd9c262f7..9f6b567252 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -42,8 +42,6 @@ public class IntegrationTestBase protected readonly IntegrationTestEnvironment _testEnvironment; - public string BuildConfiguration { get; } - private readonly string _testAdapterRelativePath = @"mstest.testadapter\{0}\build\_common".Replace('\\', Path.DirectorySeparatorChar); private readonly string _nUnitTestAdapterRelativePath = @"nunit3testadapter\{0}\build".Replace('\\', Path.DirectorySeparatorChar); private readonly string _xUnitTestAdapterRelativePath = @"xunit.runner.visualstudio\{0}\build\_common".Replace('\\', Path.DirectorySeparatorChar); @@ -73,6 +71,8 @@ public IntegrationTestBase() public TestContext TestContext { get; set; } + public string BuildConfiguration { get; } + [TestCleanup] public void TempDirectoryCleanup() { @@ -765,12 +765,12 @@ protected static string GetDiagArg(string rootDir) /// /// Counts the number of logs following the '*.host.*' pattern in the given folder. /// - protected static int CountTestHostLogs(string diagLogsDir, IEnumerable testHostProcessNames) - => Directory.GetFiles(diagLogsDir, "*.host.*").Count(); + protected static int CountTestHostLogs(string diagLogsDir) + => Directory.GetFiles(diagLogsDir, "*.host.*").Length; protected static void AssertExpectedNumberOfHostProcesses(int expectedNumOfProcessCreated, string diagLogsDir, IEnumerable testHostProcessNames, string arguments = null, string runnerPath = null) { - var processCreatedCount = CountTestHostLogs(diagLogsDir, testHostProcessNames); + var processCreatedCount = CountTestHostLogs(diagLogsDir); Assert.AreEqual( expectedNumOfProcessCreated, processCreatedCount, From 4f95d819eb975c5212fab64d91c05a8ecc9201a5 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 13:44:59 +0100 Subject: [PATCH 11/64] Build failures --- .../DotnetArchitectureSwitchTests.Windows.cs | 8 ++++---- .../Extension/RunnnerInfo.cs | 2 +- .../TranslationLayerTests/CodeCoverageTests.cs | 4 ++-- .../DotnetHostArchitectureVerifierTests.cs | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs index daa6c32b05..e94f1b83b3 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs @@ -43,7 +43,7 @@ public void Use_EnvironmentVariables(string architectureFrom, string architectur [$"DOTNET_ROOT_{architectureTo}"] = Path.GetDirectoryName(dotnetPathTo), ["ExpectedArchitecture"] = architectureTo }; - ExecuteApplication(dotnetPath, "new mstest", out string stdOut, out string stdError, out int exitCode, environmentVariables, TempDirectory.Path); + ExecuteApplication(dotnetPath, "new mstest", out _, out string _, out _, environmentVariables, TempDirectory.Path); // Patch test file File.WriteAllText(Path.Combine(TempDirectory.Path, "UnitTest1.cs"), @@ -65,7 +65,7 @@ public void TestMethod1() } }"); - ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out stdOut, out stdError, out exitCode, environmentVariables, TempDirectory.Path); + ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out string stdOut, out _, out int exitCode, environmentVariables, TempDirectory.Path); Assert.AreEqual(0, exitCode, stdOut); environmentVariables = new Dictionary @@ -74,7 +74,7 @@ public void TestMethod1() ["DOTNET_ROOT"] = Path.GetDirectoryName(dotnetPathTo), ["ExpectedArchitecture"] = architectureTo }; - ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out stdOut, out stdError, out exitCode, environmentVariables, TempDirectory.Path); + ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out stdOut, out _, out exitCode, environmentVariables, TempDirectory.Path); Assert.AreEqual(0, exitCode, stdOut); environmentVariables = new Dictionary @@ -84,7 +84,7 @@ public void TestMethod1() ["DOTNET_ROOT"] = "WE SHOULD PICK THE ABOVE ONE BEFORE FALLBACK TO DOTNET_ROOT", ["ExpectedArchitecture"] = architectureTo }; - ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out stdOut, out stdError, out exitCode, environmentVariables, TempDirectory.Path); + ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\" --arch {architectureTo.ToLower()} --diag:log.txt", out stdOut, out _, out exitCode, environmentVariables, TempDirectory.Path); Assert.AreEqual(0, exitCode, stdOut); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs index 5357734779..d9f9a14a18 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs @@ -24,8 +24,8 @@ public RunnerInfo(string runnerType, string targetFramework, string inIsolation) IsNetFrameworkRunner = !IsNetRunner; IsNetTarget = TargetFramework.StartsWith("netcoreapp", StringComparison.InvariantCultureIgnoreCase); IsNetFrameworkTarget = !IsNetTarget; - } + /// /// Gets the target framework. /// diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs index dded0392f4..e11224712c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs @@ -29,7 +29,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests; public class CodeCoverageTests : CodeCoverageAcceptanceTestBase { private IVsTestConsoleWrapper _vstestConsoleWrapper; - private TempDirectory _TempDirectory; + private TempDirectory _tempDirectory; private RunEventHandler _runEventHandler; private TestRunAttachmentsProcessingEventHandler _testRunAttachmentsProcessingEventHandler; @@ -42,7 +42,7 @@ static CodeCoverageTests() private void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _TempDirectory); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _tempDirectory); _runEventHandler = new RunEventHandler(); _testRunAttachmentsProcessingEventHandler = new TestRunAttachmentsProcessingEventHandler(); } diff --git a/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs b/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs index 8dfe7f2258..676d55c50c 100644 --- a/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs +++ b/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs @@ -26,7 +26,7 @@ public void VerifyHostArchitecture(string architecture) { string dotnetPath = GetDownloadedDotnetMuxerFromTools(architecture); var vstestConsolePath = GetDotnetRunnerPath(); - var dotnetRunnerPath = workSpace.CreateDirectory("dotnetrunner"); + var dotnetRunnerPath = TempDirectory.CreateDirectory("dotnetrunner"); TempDirectory.CopyDirectory(new DirectoryInfo(Path.GetDirectoryName(vstestConsolePath)), dotnetRunnerPath); // Patch the runner @@ -42,10 +42,10 @@ public void VerifyHostArchitecture(string architecture) ["ExpectedArchitecture"] = architecture }; - ExecuteApplication(dotnetPath, "new mstest", out string stdOut, out string stdError, out int exitCode, environmentVariables, workSpace.Path); + ExecuteApplication(dotnetPath, "new mstest", out _, out _, out _, environmentVariables, TempDirectory.Path); // Patch test file - File.WriteAllText(Path.Combine(workSpace.Path, "UnitTest1.cs"), + File.WriteAllText(Path.Combine(TempDirectory.Path, "UnitTest1.cs"), @" using Microsoft.VisualStudio.TestTools.UnitTesting; using System; @@ -62,7 +62,7 @@ public void TestMethod1() } }"); - ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\"", out stdOut, out stdError, out exitCode, environmentVariables, workSpace.Path); + ExecuteApplication(dotnetPath, $"test -p:VsTestConsolePath=\"{Path.Combine(dotnetRunnerPath.FullName, Path.GetFileName(vstestConsolePath))}\"", out string stdOut, out _, out int exitCode, environmentVariables, TempDirectory.Path); Assert.AreEqual(0, exitCode, stdOut); } From b62a26de2064a874437f452aecf35f35ce92e5c4 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 14:57:54 +0100 Subject: [PATCH 12/64] Upgrade --- ...rm.Extensions.EventLogCollector.UnitTests.csproj | 13 ++++++------- .../AppDomainTests.cs | 4 ++-- .../DotnetArchitectureSwitchTests.Windows.cs | 2 +- .../DotnetArchitectureSwitchTests.cs | 2 +- .../Microsoft.TestPlatform.AcceptanceTests.csproj | 6 +++--- .../PostProcessingTests.cs | 2 ++ ...t.TestPlatform.AdapterUtilities.UnitTests.csproj | 2 +- .../Microsoft.TestPlatform.Build.UnitTests.csproj | 2 +- .../Microsoft.TestPlatform.Client.UnitTests.csproj | 4 ++-- ...crosoft.TestPlatform.Common.PlatformTests.csproj | 4 ++-- .../Microsoft.TestPlatform.Common.UnitTests.csproj | 4 ++-- ...form.CommunicationUtilities.PlatformTests.csproj | 4 ++-- ...Platform.CommunicationUtilities.UnitTests.csproj | 4 ++-- ...soft.TestPlatform.CoreUtilities.UnitTests.csproj | 6 +++--- ...ft.TestPlatform.CrossPlatEngine.UnitTests.csproj | 4 ++-- ...m.Extensions.BlameDataCollector.UnitTests.csproj | 2 +- ...tPlatform.Extensions.HtmlLogger.UnitTests.csproj | 4 ++-- ...stPlatform.Extensions.TrxLogger.UnitTests.csproj | 4 ++-- ...ft.TestPlatform.ObjectModel.PlatformTests.csproj | 5 +++-- .../FrameworkTests.cs | 2 +- ...rosoft.TestPlatform.ObjectModel.UnitTests.csproj | 4 ++-- .../Microsoft.TestPlatform.PerformanceTests.csproj | 6 +++--- .../Microsoft.TestPlatform.SmokeTests.csproj | 6 +++--- ...t.TestPlatform.TestHostProvider.UnitTests.csproj | 2 +- .../AssemblyUtility.cs | 4 ++-- .../Microsoft.TestPlatform.TestUtilities.csproj | 4 ++-- ...icrosoft.TestPlatform.Utilities.UnitTests.csproj | 4 ++-- .../SettingsMigrator.UnitTests.csproj | 3 ++- .../TranslationLayer.UnitTests.csproj | 6 +++--- test/coverlet.collector/coverlet.collector.csproj | 2 +- .../datacollector.PlatformTests.csproj | 2 +- .../datacollector.UnitTests.csproj | 2 +- test/testhost.UnitTests/testhost.UnitTests.csproj | 6 +++--- .../vstest.ProgrammerTests.csproj | 2 +- .../vstest.console.PlatformTests.csproj | 4 ++-- .../vstest.console.UnitTests.csproj | 4 ++-- 36 files changed, 72 insertions(+), 69 deletions(-) diff --git a/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests.csproj b/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests.csproj index 6015fe0778..51460fe09c 100644 --- a/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests.csproj +++ b/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests.csproj @@ -10,19 +10,18 @@ Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests - net451 - - - net451 + + net48 Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests - - + - + + + diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs index 442b24b0d4..5e79b4b26f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs @@ -4,7 +4,7 @@ using System; using System.Collections.Generic; using System.IO; -#if !NET451 +#if !NETFRAMEWORK using System.Runtime.Loader; #else @@ -78,7 +78,7 @@ private string GetInProcDataCollectionRunsettingsFile(bool disableAppDomain, Tem { var runSettings = Path.Combine(tempDirectory.Path, "test_" + Guid.NewGuid() + ".runsettings"); var inprocasm = _testEnvironment.GetTestAsset("SimpleDataCollector.dll"); -#if !NET451 +#if !NETFRAMEWORK var assemblyName = AssemblyLoadContext.GetAssemblyName(inprocasm); #else var assemblyName = AssemblyName.GetAssemblyName(inprocasm); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs index 2e7a2a0807..17fb42a8e8 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -#if !NET451 +#if !NETFRAMEWORK using System.Collections.Generic; using System.IO; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs index e2d11506cf..e60f46ed94 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -#if !NET451 +#if !NETFRAMEWORK using System; using System.Collections.Generic; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj index 31d535775e..be86e43d04 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj @@ -8,7 +8,7 @@ Exe - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Microsoft.TestPlatform.AcceptanceTests @@ -36,11 +36,11 @@ - + - + diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/PostProcessingTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/PostProcessingTests.cs index 2204859563..12f873b1f6 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/PostProcessingTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/PostProcessingTests.cs @@ -14,6 +14,8 @@ using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers; using Microsoft.VisualStudio.TestTools.UnitTesting; +# nullable disable + namespace Microsoft.TestPlatform.AcceptanceTests; [TestClass] diff --git a/test/Microsoft.TestPlatform.AdapterUtilities.UnitTests/Microsoft.TestPlatform.AdapterUtilities.UnitTests.csproj b/test/Microsoft.TestPlatform.AdapterUtilities.UnitTests/Microsoft.TestPlatform.AdapterUtilities.UnitTests.csproj index a6973865bd..1f257d556d 100644 --- a/test/Microsoft.TestPlatform.AdapterUtilities.UnitTests/Microsoft.TestPlatform.AdapterUtilities.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.AdapterUtilities.UnitTests/Microsoft.TestPlatform.AdapterUtilities.UnitTests.csproj @@ -6,7 +6,7 @@ - netcoreapp2.1 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.AdapterUtilities.UnitTests diff --git a/test/Microsoft.TestPlatform.Build.UnitTests/Microsoft.TestPlatform.Build.UnitTests.csproj b/test/Microsoft.TestPlatform.Build.UnitTests/Microsoft.TestPlatform.Build.UnitTests.csproj index 3524adc1a0..5987088a8b 100644 --- a/test/Microsoft.TestPlatform.Build.UnitTests/Microsoft.TestPlatform.Build.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Build.UnitTests/Microsoft.TestPlatform.Build.UnitTests.csproj @@ -7,7 +7,7 @@ - netcoreapp2.1 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.Build.UnitTests diff --git a/test/Microsoft.TestPlatform.Client.UnitTests/Microsoft.TestPlatform.Client.UnitTests.csproj b/test/Microsoft.TestPlatform.Client.UnitTests/Microsoft.TestPlatform.Client.UnitTests.csproj index cd47aeb6bf..516aafb5e1 100644 --- a/test/Microsoft.TestPlatform.Client.UnitTests/Microsoft.TestPlatform.Client.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Client.UnitTests/Microsoft.TestPlatform.Client.UnitTests.csproj @@ -7,7 +7,7 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.Client.UnitTests @@ -15,7 +15,7 @@ - + diff --git a/test/Microsoft.TestPlatform.Common.PlatformTests/Microsoft.TestPlatform.Common.PlatformTests.csproj b/test/Microsoft.TestPlatform.Common.PlatformTests/Microsoft.TestPlatform.Common.PlatformTests.csproj index 9e8b4eec12..0603c1a3c9 100644 --- a/test/Microsoft.TestPlatform.Common.PlatformTests/Microsoft.TestPlatform.Common.PlatformTests.csproj +++ b/test/Microsoft.TestPlatform.Common.PlatformTests/Microsoft.TestPlatform.Common.PlatformTests.csproj @@ -7,7 +7,7 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.Common.PlatformTests @@ -18,7 +18,7 @@ 2.0.0 - + diff --git a/test/Microsoft.TestPlatform.Common.UnitTests/Microsoft.TestPlatform.Common.UnitTests.csproj b/test/Microsoft.TestPlatform.Common.UnitTests/Microsoft.TestPlatform.Common.UnitTests.csproj index f72b78c876..299201bc61 100644 --- a/test/Microsoft.TestPlatform.Common.UnitTests/Microsoft.TestPlatform.Common.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Common.UnitTests/Microsoft.TestPlatform.Common.UnitTests.csproj @@ -7,12 +7,12 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.Common.UnitTests - + diff --git a/test/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests.csproj b/test/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests.csproj index 7c5eae08da..a390f3afd0 100644 --- a/test/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests.csproj +++ b/test/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests.csproj @@ -7,12 +7,12 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.CommunicationUtilities.PlatformTests - + diff --git a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Microsoft.TestPlatform.CommunicationUtilities.UnitTests.csproj b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Microsoft.TestPlatform.CommunicationUtilities.UnitTests.csproj index 33121ffb44..a08483e0fa 100644 --- a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Microsoft.TestPlatform.CommunicationUtilities.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Microsoft.TestPlatform.CommunicationUtilities.UnitTests.csproj @@ -7,12 +7,12 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.CommunicationUtilities.UnitTests - + diff --git a/test/Microsoft.TestPlatform.CoreUtilities.UnitTests/Microsoft.TestPlatform.CoreUtilities.UnitTests.csproj b/test/Microsoft.TestPlatform.CoreUtilities.UnitTests/Microsoft.TestPlatform.CoreUtilities.UnitTests.csproj index afe4bf2809..a3193683f2 100644 --- a/test/Microsoft.TestPlatform.CoreUtilities.UnitTests/Microsoft.TestPlatform.CoreUtilities.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.CoreUtilities.UnitTests/Microsoft.TestPlatform.CoreUtilities.UnitTests.csproj @@ -7,12 +7,12 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.CoreUtilities.UnitTests - + 4.3.0 @@ -26,7 +26,7 @@ 4.3.0 - + diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Microsoft.TestPlatform.CrossPlatEngine.UnitTests.csproj b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Microsoft.TestPlatform.CrossPlatEngine.UnitTests.csproj index f9e9226d18..85e5b5fd38 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Microsoft.TestPlatform.CrossPlatEngine.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Microsoft.TestPlatform.CrossPlatEngine.UnitTests.csproj @@ -8,7 +8,7 @@ Microsoft.TestPlatform.CrossPlatEngine.UnitTests - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe @@ -20,7 +20,7 @@ - + diff --git a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj index 0024aa19b4..6f1d2c843a 100644 --- a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj @@ -19,7 +19,7 @@ Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests - netcoreapp2.1;net472 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests diff --git a/test/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests.csproj b/test/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests.csproj index 25d350e476..409b518469 100644 --- a/test/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests.csproj @@ -7,14 +7,14 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe - + diff --git a/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests.csproj b/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests.csproj index 9b53f2f32d..7a764e60db 100644 --- a/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests.csproj @@ -7,7 +7,7 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests @@ -15,7 +15,7 @@ - + diff --git a/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/Microsoft.TestPlatform.ObjectModel.PlatformTests.csproj b/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/Microsoft.TestPlatform.ObjectModel.PlatformTests.csproj index 3e84049474..c271a4d4b3 100644 --- a/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/Microsoft.TestPlatform.ObjectModel.PlatformTests.csproj +++ b/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/Microsoft.TestPlatform.ObjectModel.PlatformTests.csproj @@ -6,7 +6,7 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.ObjectModel.PlatformTests @@ -14,7 +14,7 @@ - + @@ -22,6 +22,7 @@ + diff --git a/test/Microsoft.TestPlatform.ObjectModel.UnitTests/FrameworkTests.cs b/test/Microsoft.TestPlatform.ObjectModel.UnitTests/FrameworkTests.cs index 0c792d18fb..84a71688ea 100644 --- a/test/Microsoft.TestPlatform.ObjectModel.UnitTests/FrameworkTests.cs +++ b/test/Microsoft.TestPlatform.ObjectModel.UnitTests/FrameworkTests.cs @@ -81,7 +81,7 @@ public void DefaultFrameworkShouldBeNet40OnDesktop() [TestMethod] public void DefaultFrameworkShouldBeNetCoreApp10OnNonDesktop() { -#if !NET451 +#if !NETFRAMEWORK Assert.AreEqual(".NETCoreApp,Version=v1.0", Framework.DefaultFramework.Name); #endif } diff --git a/test/Microsoft.TestPlatform.ObjectModel.UnitTests/Microsoft.TestPlatform.ObjectModel.UnitTests.csproj b/test/Microsoft.TestPlatform.ObjectModel.UnitTests/Microsoft.TestPlatform.ObjectModel.UnitTests.csproj index eca84c2c95..decbf5ef22 100644 --- a/test/Microsoft.TestPlatform.ObjectModel.UnitTests/Microsoft.TestPlatform.ObjectModel.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.ObjectModel.UnitTests/Microsoft.TestPlatform.ObjectModel.UnitTests.csproj @@ -7,7 +7,7 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.ObjectModel.UnitTests @@ -17,7 +17,7 @@ $(JsonNetVersion) - + diff --git a/test/Microsoft.TestPlatform.PerformanceTests/Microsoft.TestPlatform.PerformanceTests.csproj b/test/Microsoft.TestPlatform.PerformanceTests/Microsoft.TestPlatform.PerformanceTests.csproj index f5c20febb5..02254dbfb6 100644 --- a/test/Microsoft.TestPlatform.PerformanceTests/Microsoft.TestPlatform.PerformanceTests.csproj +++ b/test/Microsoft.TestPlatform.PerformanceTests/Microsoft.TestPlatform.PerformanceTests.csproj @@ -7,8 +7,8 @@ - Exe - netcoreapp2.1;net451 + Exe + net6.0;net48 Microsoft.TestPlatform.PerformanceTests @@ -19,7 +19,7 @@ - + diff --git a/test/Microsoft.TestPlatform.SmokeTests/Microsoft.TestPlatform.SmokeTests.csproj b/test/Microsoft.TestPlatform.SmokeTests/Microsoft.TestPlatform.SmokeTests.csproj index 9948d46e82..336ccaee62 100644 --- a/test/Microsoft.TestPlatform.SmokeTests/Microsoft.TestPlatform.SmokeTests.csproj +++ b/test/Microsoft.TestPlatform.SmokeTests/Microsoft.TestPlatform.SmokeTests.csproj @@ -8,19 +8,19 @@ Microsoft.TestPlatform.SmokeTests - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe - + - + 4.3.0 diff --git a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj index e098a6461c..7fa225328d 100644 --- a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj @@ -8,7 +8,7 @@ Microsoft.TestPlatform.TestHostProvider.UnitTests - netcoreapp2.1;net472 + net6.0;net48 netcoreapp3.1 Exe diff --git a/test/Microsoft.TestPlatform.TestUtilities/AssemblyUtility.cs b/test/Microsoft.TestPlatform.TestUtilities/AssemblyUtility.cs index e0702a44f7..a3c67178a7 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/AssemblyUtility.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/AssemblyUtility.cs @@ -2,7 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Reflection; -#if !NET451 +#if !NETFRAMEWORK using System.Runtime.Loader; #endif @@ -22,7 +22,7 @@ public class AssemblyUtility /// public static AssemblyName GetAssemblyName(string assemblyPath) { -#if !NET451 +#if !NETFRAMEWORK return AssemblyLoadContext.GetAssemblyName(assemblyPath); #else return AssemblyName.GetAssemblyName(assemblyPath); diff --git a/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj b/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj index a91d6a82ca..ccb1f81b3f 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj +++ b/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj @@ -6,7 +6,7 @@ Microsoft.TestPlatform.TestUtilities - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 false @@ -22,7 +22,7 @@ - + diff --git a/test/Microsoft.TestPlatform.Utilities.UnitTests/Microsoft.TestPlatform.Utilities.UnitTests.csproj b/test/Microsoft.TestPlatform.Utilities.UnitTests/Microsoft.TestPlatform.Utilities.UnitTests.csproj index b1ada1e9c5..aa15f291aa 100644 --- a/test/Microsoft.TestPlatform.Utilities.UnitTests/Microsoft.TestPlatform.Utilities.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Utilities.UnitTests/Microsoft.TestPlatform.Utilities.UnitTests.csproj @@ -8,7 +8,7 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe Microsoft.TestPlatform.Utilities.UnitTests @@ -22,7 +22,7 @@ - + diff --git a/test/SettingsMigrator.UnitTests/SettingsMigrator.UnitTests.csproj b/test/SettingsMigrator.UnitTests/SettingsMigrator.UnitTests.csproj index 5450d2cc23..c40955205f 100644 --- a/test/SettingsMigrator.UnitTests/SettingsMigrator.UnitTests.csproj +++ b/test/SettingsMigrator.UnitTests/SettingsMigrator.UnitTests.csproj @@ -7,7 +7,8 @@ - net451 + + net48 SettingsMigrator.UnitTests diff --git a/test/TranslationLayer.UnitTests/TranslationLayer.UnitTests.csproj b/test/TranslationLayer.UnitTests/TranslationLayer.UnitTests.csproj index 1b33ca81a5..4efa60842d 100644 --- a/test/TranslationLayer.UnitTests/TranslationLayer.UnitTests.csproj +++ b/test/TranslationLayer.UnitTests/TranslationLayer.UnitTests.csproj @@ -7,15 +7,15 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 - Exe + Exe TranslationLayer.UnitTests - + diff --git a/test/coverlet.collector/coverlet.collector.csproj b/test/coverlet.collector/coverlet.collector.csproj index 7966d51861..700ccdceb0 100644 --- a/test/coverlet.collector/coverlet.collector.csproj +++ b/test/coverlet.collector/coverlet.collector.csproj @@ -1,6 +1,6 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 false false diff --git a/test/datacollector.PlatformTests/datacollector.PlatformTests.csproj b/test/datacollector.PlatformTests/datacollector.PlatformTests.csproj index 0b8bfdcc3b..ade87ab0ee 100644 --- a/test/datacollector.PlatformTests/datacollector.PlatformTests.csproj +++ b/test/datacollector.PlatformTests/datacollector.PlatformTests.csproj @@ -9,7 +9,7 @@ Microsoft.VisualStudio.TestPlatform.DataCollector.PlatformTests - netcoreapp2.1;net472 + net6.0;net48 netcoreapp3.1 Exe datacollector.PlatformTests diff --git a/test/datacollector.UnitTests/datacollector.UnitTests.csproj b/test/datacollector.UnitTests/datacollector.UnitTests.csproj index 5e89f02902..2ce91cd9c1 100644 --- a/test/datacollector.UnitTests/datacollector.UnitTests.csproj +++ b/test/datacollector.UnitTests/datacollector.UnitTests.csproj @@ -11,7 +11,7 @@ Exe - netcoreapp2.1;net472 + net6.0;net48 netcoreapp3.1 datacollector.UnitTests diff --git a/test/testhost.UnitTests/testhost.UnitTests.csproj b/test/testhost.UnitTests/testhost.UnitTests.csproj index b434796078..4487c864f9 100644 --- a/test/testhost.UnitTests/testhost.UnitTests.csproj +++ b/test/testhost.UnitTests/testhost.UnitTests.csproj @@ -7,13 +7,13 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe testhost.UnitTests - x64 + x64 - + diff --git a/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj b/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj index 21a59d0ab8..4b5bd1a84c 100644 --- a/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj +++ b/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj @@ -29,7 +29,7 @@ - + diff --git a/test/vstest.console.PlatformTests/vstest.console.PlatformTests.csproj b/test/vstest.console.PlatformTests/vstest.console.PlatformTests.csproj index f1c029346a..88fec59e33 100644 --- a/test/vstest.console.PlatformTests/vstest.console.PlatformTests.csproj +++ b/test/vstest.console.PlatformTests/vstest.console.PlatformTests.csproj @@ -7,7 +7,7 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe vstest.console.PlatformTests @@ -16,7 +16,7 @@ - + diff --git a/test/vstest.console.UnitTests/vstest.console.UnitTests.csproj b/test/vstest.console.UnitTests/vstest.console.UnitTests.csproj index d41ebf1187..137205af5d 100644 --- a/test/vstest.console.UnitTests/vstest.console.UnitTests.csproj +++ b/test/vstest.console.UnitTests/vstest.console.UnitTests.csproj @@ -7,7 +7,7 @@ - netcoreapp2.1;net451 + net6.0;net48 netcoreapp3.1 Exe vstest.console.UnitTests @@ -22,7 +22,7 @@ true - + From f15ade9c47e1d7be54c95f7e4d1b916404833221 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 16:21:49 +0100 Subject: [PATCH 13/64] Upgrade tests --- scripts/test.ps1 | 20 ++++++++++--------- ...ensions.EventLogCollector.UnitTests.csproj | 1 - ...rosoft.TestPlatform.AcceptanceTests.csproj | 2 +- .../ProcessesInteractionTests.cs | 2 +- ...Platform.AdapterUtilities.UnitTests.csproj | 2 +- ...rosoft.TestPlatform.Build.UnitTests.csproj | 2 +- ...osoft.TestPlatform.Client.UnitTests.csproj | 2 +- ...t.TestPlatform.Common.PlatformTests.csproj | 2 +- ...osoft.TestPlatform.Common.UnitTests.csproj | 2 +- ...ommunicationUtilities.PlatformTests.csproj | 2 +- ...rm.CommunicationUtilities.UnitTests.csproj | 2 +- ...estPlatform.CoreUtilities.UnitTests.csproj | 2 +- ...tPlatform.CrossPlatEngine.UnitTests.csproj | 2 +- ...nsions.BlameDataCollector.UnitTests.csproj | 2 +- .../XmlReaderWriterTests.cs | 9 ++++++--- ...orm.Extensions.HtmlLogger.UnitTests.csproj | 2 +- ...form.Extensions.TrxLogger.UnitTests.csproj | 2 +- ...tPlatform.ObjectModel.PlatformTests.csproj | 2 +- ....TestPlatform.ObjectModel.UnitTests.csproj | 2 +- .../Microsoft.TestPlatform.SmokeTests.csproj | 2 +- .../Hosting/DotnetTestHostManagerTests.cs | 9 +++++++++ ...Platform.TestHostProvider.UnitTests.csproj | 4 ++-- ...ft.TestPlatform.Utilities.UnitTests.csproj | 2 +- .../datacollector.PlatformTests.csproj | 2 +- .../datacollector.UnitTests.csproj | 2 +- .../testhost.UnitTests.csproj | 2 +- .../vstest.ProgrammerTests.csproj | 2 +- .../vstest.console.PlatformTests.csproj | 2 +- .../vstest.console.UnitTests.csproj | 2 +- 29 files changed, 52 insertions(+), 39 deletions(-) diff --git a/scripts/test.ps1 b/scripts/test.ps1 index d0022db77b..c6e30a6113 100644 --- a/scripts/test.ps1 +++ b/scripts/test.ps1 @@ -14,7 +14,7 @@ Param( [System.String] $TargetRuntime = "win7-x64", [Parameter(Mandatory=$false)] - [ValidateSet("net451", "netcoreapp2.1")] + [ValidateSet("net48", "net6.0")] [Alias("f")] [System.String] $TargetFramework, @@ -94,12 +94,14 @@ $env:NUGET_PACKAGES = $env:TP_PACKAGES_DIR # # Test configuration # -$TPT_TargetFrameworkFullCLR = "net451" -$TPT_TargetFrameworkCore20 = "netcoreapp2.1" +$TPT_TargetFrameworkNet451 = "net451" +$TPT_TargetFrameworkNet48 = "net48" +$TPT_TargetFrameworkCore21 = "netcoreapp2.1" +$TPT_TargetFrameworkNet60 = "net6.0" Write-Verbose "Setup build configuration." $Script:TPT_Configuration = $Configuration $Script:TPT_SourceFolders = @("test") -$Script:TPT_TargetFrameworks =@($TPT_TargetFrameworkFullCLR, $TPT_TargetFrameworkCore20) +$Script:TPT_TargetFrameworks =@($TPT_TargetFrameworkNet48, $TPT_TargetFrameworkNet60) $Script:TPT_TargetFramework = $TargetFramework $Script:TPT_TargetRuntime = $TargetRuntime $Script:TPT_SkipProjects = @("_none_"); @@ -225,17 +227,17 @@ function Invoke-Test $testFilter = "/testCaseFilter:`"$TPT_TestFilter`"" } - if($fx -eq $TPT_TargetFrameworkCore20) + if($fx -eq $TPT_TargetFrameworkNet60) { $vstestConsoleFileName = "vstest.console.dll" $targetRunTime = "" - $vstestConsolePath = Join-Path (Get-PackageDirectory $TPT_TargetFrameworkCore20 $targetRuntime) $vstestConsoleFileName + $vstestConsolePath = Join-Path (Get-PackageDirectory $TPT_TargetFrameworkCore21 $targetRuntime) $vstestConsoleFileName } else { $vstestConsoleFileName = "vstest.console.exe" $targetRunTime = $Script:TPT_TargetRuntime - $vstestConsolePath = Join-Path (Get-PackageDirectory $TPT_TargetFrameworkFullCLR $targetRuntime) $vstestConsoleFileName + $vstestConsolePath = Join-Path (Get-PackageDirectory $TPT_TargetFrameworkNet451 $targetRuntime) $vstestConsoleFileName } if (!(Test-Path $vstestConsolePath)) @@ -264,7 +266,7 @@ function Invoke-Test } Set-TestEnvironment - if($fx -eq $TPT_TargetFrameworkFullCLR) + if($fx -eq $TPT_TargetFrameworkNet48) { Write-Verbose "$vstestConsolePath $testContainerSet /parallel /logger:`"trx;LogFileName=$trxLogFileName`" $testFilter $ConsoleLogger" @@ -305,7 +307,7 @@ function Invoke-Test Set-TestEnvironment - if($fx -eq $TPT_TargetFrameworkFullCLR) + if($fx -eq $TPT_TargetFrameworkNet48) { Write-Verbose "$vstestConsolePath $testContainer /logger:`"trx;LogFileName=$trxLogFileName`" $ConsoleLogger $testFilter" diff --git a/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests.csproj b/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests.csproj index 51460fe09c..51e4d723fe 100644 --- a/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests.csproj +++ b/test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests.csproj @@ -20,7 +20,6 @@ - diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj index be86e43d04..9a87dad715 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj @@ -7,7 +7,7 @@ - Exe + Exe net6.0;net48 netcoreapp3.1 Microsoft.TestPlatform.AcceptanceTests diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs index b4fe07e22c..e82ddc9520 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs @@ -37,7 +37,7 @@ static void UpdateRuntimeConfigJsonWithInvalidFramework(string assemblyPath, str // On the contrary to other tests, we need to modify the test asset we are using to replace // the target framework with an invalid framework. This is why we have a specific test asset // that's only meant to be used by this project. - var runtimeConfigJson = Path.Combine(Path.GetDirectoryName(assemblyPath), testAssetProjectName + ".runtimeconfig.json"); + var runtimeConfigJson = Path.Combine(Path.GetDirectoryName(assemblyPath)!, testAssetProjectName + ".runtimeconfig.json"); var fileContent = File.ReadAllText(runtimeConfigJson); var updatedContent = fileContent.Replace("\"version\": \"2.1.0\"", "\"version\": \"0.0.0\""); File.WriteAllText(runtimeConfigJson, updatedContent); diff --git a/test/Microsoft.TestPlatform.AdapterUtilities.UnitTests/Microsoft.TestPlatform.AdapterUtilities.UnitTests.csproj b/test/Microsoft.TestPlatform.AdapterUtilities.UnitTests/Microsoft.TestPlatform.AdapterUtilities.UnitTests.csproj index 1f257d556d..d69a372fc5 100644 --- a/test/Microsoft.TestPlatform.AdapterUtilities.UnitTests/Microsoft.TestPlatform.AdapterUtilities.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.AdapterUtilities.UnitTests/Microsoft.TestPlatform.AdapterUtilities.UnitTests.csproj @@ -8,7 +8,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.AdapterUtilities.UnitTests true $(NoWarn);RS1024 diff --git a/test/Microsoft.TestPlatform.Build.UnitTests/Microsoft.TestPlatform.Build.UnitTests.csproj b/test/Microsoft.TestPlatform.Build.UnitTests/Microsoft.TestPlatform.Build.UnitTests.csproj index 5987088a8b..022d772011 100644 --- a/test/Microsoft.TestPlatform.Build.UnitTests/Microsoft.TestPlatform.Build.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Build.UnitTests/Microsoft.TestPlatform.Build.UnitTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.Build.UnitTests diff --git a/test/Microsoft.TestPlatform.Client.UnitTests/Microsoft.TestPlatform.Client.UnitTests.csproj b/test/Microsoft.TestPlatform.Client.UnitTests/Microsoft.TestPlatform.Client.UnitTests.csproj index 516aafb5e1..3638bd290d 100644 --- a/test/Microsoft.TestPlatform.Client.UnitTests/Microsoft.TestPlatform.Client.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Client.UnitTests/Microsoft.TestPlatform.Client.UnitTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.Client.UnitTests diff --git a/test/Microsoft.TestPlatform.Common.PlatformTests/Microsoft.TestPlatform.Common.PlatformTests.csproj b/test/Microsoft.TestPlatform.Common.PlatformTests/Microsoft.TestPlatform.Common.PlatformTests.csproj index 0603c1a3c9..d0cdd8d84f 100644 --- a/test/Microsoft.TestPlatform.Common.PlatformTests/Microsoft.TestPlatform.Common.PlatformTests.csproj +++ b/test/Microsoft.TestPlatform.Common.PlatformTests/Microsoft.TestPlatform.Common.PlatformTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.Common.PlatformTests diff --git a/test/Microsoft.TestPlatform.Common.UnitTests/Microsoft.TestPlatform.Common.UnitTests.csproj b/test/Microsoft.TestPlatform.Common.UnitTests/Microsoft.TestPlatform.Common.UnitTests.csproj index 299201bc61..26cf91d771 100644 --- a/test/Microsoft.TestPlatform.Common.UnitTests/Microsoft.TestPlatform.Common.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Common.UnitTests/Microsoft.TestPlatform.Common.UnitTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.Common.UnitTests diff --git a/test/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests.csproj b/test/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests.csproj index a390f3afd0..b0098fff88 100644 --- a/test/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests.csproj +++ b/test/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests/Microsoft.TestPlatform.CommunicationUtilities.PlatformTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.CommunicationUtilities.PlatformTests diff --git a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Microsoft.TestPlatform.CommunicationUtilities.UnitTests.csproj b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Microsoft.TestPlatform.CommunicationUtilities.UnitTests.csproj index a08483e0fa..7e35f37f32 100644 --- a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Microsoft.TestPlatform.CommunicationUtilities.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Microsoft.TestPlatform.CommunicationUtilities.UnitTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.CommunicationUtilities.UnitTests diff --git a/test/Microsoft.TestPlatform.CoreUtilities.UnitTests/Microsoft.TestPlatform.CoreUtilities.UnitTests.csproj b/test/Microsoft.TestPlatform.CoreUtilities.UnitTests/Microsoft.TestPlatform.CoreUtilities.UnitTests.csproj index a3193683f2..12d5ab3331 100644 --- a/test/Microsoft.TestPlatform.CoreUtilities.UnitTests/Microsoft.TestPlatform.CoreUtilities.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.CoreUtilities.UnitTests/Microsoft.TestPlatform.CoreUtilities.UnitTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.CoreUtilities.UnitTests diff --git a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Microsoft.TestPlatform.CrossPlatEngine.UnitTests.csproj b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Microsoft.TestPlatform.CrossPlatEngine.UnitTests.csproj index 85e5b5fd38..4379157066 100644 --- a/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Microsoft.TestPlatform.CrossPlatEngine.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Microsoft.TestPlatform.CrossPlatEngine.UnitTests.csproj @@ -10,7 +10,7 @@ Microsoft.TestPlatform.CrossPlatEngine.UnitTests net6.0;net48 netcoreapp3.1 - Exe + Exe diff --git a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj index 6f1d2c843a..8331cbedd0 100644 --- a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj @@ -21,7 +21,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests diff --git a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/XmlReaderWriterTests.cs b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/XmlReaderWriterTests.cs index 7bc177cfe1..4dd782d5e0 100644 --- a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/XmlReaderWriterTests.cs +++ b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/XmlReaderWriterTests.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; @@ -120,7 +121,8 @@ public void WriteTestSequenceShouldWriteFileStream() { // Setup _mockFileHelper.Setup(m => m.Exists(It.IsAny())).Returns(true); - _mockFileHelper.Setup(m => m.GetStream("path.xml", FileMode.Create, FileAccess.ReadWrite)).Returns(_mockStream.Object); + using var stream = new MemoryStream(); + _mockFileHelper.Setup(m => m.GetStream("path.xml", FileMode.Create, FileAccess.ReadWrite)).Returns(stream); _mockStream.Setup(x => x.CanWrite).Returns(true); _mockStream.Setup(x => x.Write(It.IsAny(), It.IsAny(), It.IsAny())); @@ -129,8 +131,9 @@ public void WriteTestSequenceShouldWriteFileStream() // Verify Call to fileHelper _mockFileHelper.Verify(x => x.GetStream("path.xml", FileMode.Create, FileAccess.ReadWrite)); - // Verify Call to stream write - _mockStream.Verify(x => x.Write(It.IsAny(), It.IsAny(), It.IsAny())); + // Assert it has some data + var data = Encoding.UTF8.GetString(stream.ToArray()); + Assert.IsTrue(data.Length > 0, "Stream should have some data."); } /// diff --git a/test/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests.csproj b/test/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests.csproj index 409b518469..5ab7c3446c 100644 --- a/test/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests/Microsoft.TestPlatform.Extensions.HtmlLogger.UnitTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe diff --git a/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests.csproj b/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests.csproj index 7a764e60db..349996bbd3 100644 --- a/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests diff --git a/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/Microsoft.TestPlatform.ObjectModel.PlatformTests.csproj b/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/Microsoft.TestPlatform.ObjectModel.PlatformTests.csproj index c271a4d4b3..45a2a1778a 100644 --- a/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/Microsoft.TestPlatform.ObjectModel.PlatformTests.csproj +++ b/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/Microsoft.TestPlatform.ObjectModel.PlatformTests.csproj @@ -8,7 +8,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.ObjectModel.PlatformTests diff --git a/test/Microsoft.TestPlatform.ObjectModel.UnitTests/Microsoft.TestPlatform.ObjectModel.UnitTests.csproj b/test/Microsoft.TestPlatform.ObjectModel.UnitTests/Microsoft.TestPlatform.ObjectModel.UnitTests.csproj index decbf5ef22..5b34507354 100644 --- a/test/Microsoft.TestPlatform.ObjectModel.UnitTests/Microsoft.TestPlatform.ObjectModel.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.ObjectModel.UnitTests/Microsoft.TestPlatform.ObjectModel.UnitTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.ObjectModel.UnitTests diff --git a/test/Microsoft.TestPlatform.SmokeTests/Microsoft.TestPlatform.SmokeTests.csproj b/test/Microsoft.TestPlatform.SmokeTests/Microsoft.TestPlatform.SmokeTests.csproj index 336ccaee62..f5a1ab7451 100644 --- a/test/Microsoft.TestPlatform.SmokeTests/Microsoft.TestPlatform.SmokeTests.csproj +++ b/test/Microsoft.TestPlatform.SmokeTests/Microsoft.TestPlatform.SmokeTests.csproj @@ -10,7 +10,7 @@ Microsoft.TestPlatform.SmokeTests net6.0;net48 netcoreapp3.1 - Exe + Exe diff --git a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs index 1b130db836..7d4ce97a00 100644 --- a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs +++ b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs @@ -616,6 +616,8 @@ public void GetTestHostProcessStartInfoShouldIncludeTestHostPathFromSourceDirect StringAssert.Contains(startInfo.Arguments, expectedTestHostPath); } + // TODO: Not sure why but this worked before on .NET451, but now I can't get it to work anywhere. Assembly.GetEntryAssembly().Location is null because of running in app domain. +#if NET [TestMethod] public void GetTestHostProcessStartInfoShouldIncludeTestHostPathNextToTestRunnerIfTesthostDllIsNoFoundAndDepsFileNotFound() { @@ -639,6 +641,11 @@ public void GetTestHostProcessStartInfoShouldIncludeTestHostPathNextToTestRunner StringAssert.Contains(startInfo.Arguments, $"--runtimeconfig \"{expectedRuntimeConfigPath}\""); } +#endif + + // TODO: Not sure why but this worked before on .NET451, but now I can't get it to work anywhere. Assembly.GetEntryAssembly().Location is null because of running in app domain. +#if NET + [TestMethod] // we can't put in a "default" value, and we don't have other way to determine if this provided value is the @@ -671,6 +678,8 @@ public void GetTestHostProcessStartInfoShouldIncludeTestHostPathNextToTestRunner StringAssert.Contains(startInfo.Arguments, $"--runtimeconfig \"{expectedRuntimeConfigPath}\""); } +#endif + [TestMethod] public void GetTestHostProcessStartInfoShouldIncludeTestHostPathFromSourceDirectoryIfRunConfigDevFileNotFound() { diff --git a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj index 7fa225328d..40aeb778f2 100644 --- a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj @@ -8,9 +8,9 @@ Microsoft.TestPlatform.TestHostProvider.UnitTests - net6.0;net48 + net6.0;net48;net451 netcoreapp3.1 - Exe + Exe diff --git a/test/Microsoft.TestPlatform.Utilities.UnitTests/Microsoft.TestPlatform.Utilities.UnitTests.csproj b/test/Microsoft.TestPlatform.Utilities.UnitTests/Microsoft.TestPlatform.Utilities.UnitTests.csproj index aa15f291aa..ac2e969a99 100644 --- a/test/Microsoft.TestPlatform.Utilities.UnitTests/Microsoft.TestPlatform.Utilities.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Utilities.UnitTests/Microsoft.TestPlatform.Utilities.UnitTests.csproj @@ -10,7 +10,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe Microsoft.TestPlatform.Utilities.UnitTests diff --git a/test/datacollector.PlatformTests/datacollector.PlatformTests.csproj b/test/datacollector.PlatformTests/datacollector.PlatformTests.csproj index ade87ab0ee..935e9ba419 100644 --- a/test/datacollector.PlatformTests/datacollector.PlatformTests.csproj +++ b/test/datacollector.PlatformTests/datacollector.PlatformTests.csproj @@ -11,7 +11,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe datacollector.PlatformTests diff --git a/test/datacollector.UnitTests/datacollector.UnitTests.csproj b/test/datacollector.UnitTests/datacollector.UnitTests.csproj index 2ce91cd9c1..9810f40bff 100644 --- a/test/datacollector.UnitTests/datacollector.UnitTests.csproj +++ b/test/datacollector.UnitTests/datacollector.UnitTests.csproj @@ -10,7 +10,7 @@ Microsoft.VisualStudio.TestPlatform.DataCollector.UnitTests - Exe + Exe net6.0;net48 netcoreapp3.1 datacollector.UnitTests diff --git a/test/testhost.UnitTests/testhost.UnitTests.csproj b/test/testhost.UnitTests/testhost.UnitTests.csproj index 4487c864f9..843ee2e344 100644 --- a/test/testhost.UnitTests/testhost.UnitTests.csproj +++ b/test/testhost.UnitTests/testhost.UnitTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe testhost.UnitTests x64 diff --git a/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj b/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj index 4b5bd1a84c..793d988e7d 100644 --- a/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj +++ b/test/vstest.ProgrammerTests/vstest.ProgrammerTests.csproj @@ -12,7 +12,7 @@ preview net6.0 netcoreapp3.1 - Exe + Exe Exe diff --git a/test/vstest.console.PlatformTests/vstest.console.PlatformTests.csproj b/test/vstest.console.PlatformTests/vstest.console.PlatformTests.csproj index 88fec59e33..9e48311a14 100644 --- a/test/vstest.console.PlatformTests/vstest.console.PlatformTests.csproj +++ b/test/vstest.console.PlatformTests/vstest.console.PlatformTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe vstest.console.PlatformTests diff --git a/test/vstest.console.UnitTests/vstest.console.UnitTests.csproj b/test/vstest.console.UnitTests/vstest.console.UnitTests.csproj index 137205af5d..d0b644826b 100644 --- a/test/vstest.console.UnitTests/vstest.console.UnitTests.csproj +++ b/test/vstest.console.UnitTests/vstest.console.UnitTests.csproj @@ -9,7 +9,7 @@ net6.0;net48 netcoreapp3.1 - Exe + Exe vstest.console.UnitTests From 100b4ddd4835024ff45c4bdf537d676db5077e11 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 16:43:59 +0100 Subject: [PATCH 14/64] Skip web tests locally --- .../DifferentTestFrameworkSimpleTests.cs | 11 +++++++++++ .../IntegrationTestBase.cs | 3 +++ .../IntegrationTestEnvironment.cs | 3 +++ 3 files changed, 17 insertions(+) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs index 26dfa7ab26..846fd07816 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs @@ -15,6 +15,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests; [TestCategory("Windows-Review")] public class DifferentTestFrameworkSimpleTests : AcceptanceTestBase { + [TestMethod] [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] public void ChutzpahRunAllTestExecution(RunnerInfo runnerInfo) @@ -65,6 +66,11 @@ public void CPPRunAllTestExecutionPlatformx64Net(RunnerInfo runnerInfo) [NetFullTargetFrameworkDataSource] public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo) { + if (IsCI) + { + Assert.Inconclusive("This works on server but not locally, because locall it grabs dll from my GAC and that is old, but has 10.0.0 version as the one in our package."); + } + SetTestEnvironment(_testEnvironment, runnerInfo); var runSettingsFilePath = Path.Combine(TempDirectory.Path, Guid.NewGuid() + ".runsettings"); @@ -117,6 +123,11 @@ public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo) [NetFullTargetFrameworkDataSource] public void CodedWebTestRunAllTests(RunnerInfo runnerInfo) { + if (IsCI) + { + Assert.Inconclusive("This works on server but not locally, because locall it grabs dll from my GAC and that is old, but has 10.0.0 version as the one in our package."); + } + SetTestEnvironment(_testEnvironment, runnerInfo); if (runnerInfo.IsNetRunner) { diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 9f6b567252..abbbbf0967 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -59,6 +59,7 @@ public IntegrationTestBase() _testEnvironment = new IntegrationTestEnvironment(); BuildConfiguration = IntegrationTestEnvironment.BuildConfiguration; TempDirectory = new TempDirectory(); + IsCI = IntegrationTestEnvironment.IsCI; } public string StdOut => _standardTestOutput; @@ -73,6 +74,8 @@ public IntegrationTestBase() public string BuildConfiguration { get; } + public bool IsCI { get; } + [TestCleanup] public void TempDirectoryCleanup() { diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs index 8fe9e840c2..fdded084bf 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs @@ -195,6 +195,9 @@ public string RunnerFramework set; } + // A known AzureDevOps env variable meaning we are running in CI. + public static bool IsCI { get; } = Environment.GetEnvironmentVariable("TF_BUILD") == "True"; + /// /// Gets the full path to a test asset. /// From cdaf3d053f9ecff66b24dc07570c6316557c4760 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 4 Mar 2022 16:51:50 +0100 Subject: [PATCH 15/64] Apply suggestions from code review --- .../Hosting/DotnetTestHostManagerTests.cs | 6 ++++-- ...Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs index 7d4ce97a00..8e917e1046 100644 --- a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs +++ b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs @@ -616,7 +616,8 @@ public void GetTestHostProcessStartInfoShouldIncludeTestHostPathFromSourceDirect StringAssert.Contains(startInfo.Arguments, expectedTestHostPath); } - // TODO: Not sure why but this worked before on .NET451, but now I can't get it to work anywhere. Assembly.GetEntryAssembly().Location is null because of running in app domain. + // TODO: This assembly was previously compiled as net472 and so it was skipped and only ran as netcoreapp2.1. This fails in test, but works in code that is not isolated in appdomain. Might be worth fixing because we get one null here, and another in DotnetTestHostManager. + // Assembly.GetEntryAssembly().Location is null because of running in app domain. #if NET [TestMethod] public void GetTestHostProcessStartInfoShouldIncludeTestHostPathNextToTestRunnerIfTesthostDllIsNoFoundAndDepsFileNotFound() @@ -643,7 +644,8 @@ public void GetTestHostProcessStartInfoShouldIncludeTestHostPathNextToTestRunner #endif - // TODO: Not sure why but this worked before on .NET451, but now I can't get it to work anywhere. Assembly.GetEntryAssembly().Location is null because of running in app domain. + // TODO: This assembly was previously compiled as net472 and so it was skipped and only ran as netcoreapp2.1. This fails in test, but works in code that is not isolated in appdomain. Might be worth fixing because we get one null here, and another in DotnetTestHostManager. + // Assembly.GetEntryAssembly().Location is null because of running in app domain. #if NET [TestMethod] diff --git a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj index 40aeb778f2..c2ee1ef0a6 100644 --- a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj @@ -8,7 +8,7 @@ Microsoft.TestPlatform.TestHostProvider.UnitTests - net6.0;net48;net451 + net6.0;net48 netcoreapp3.1 Exe From 390d80b52fc43f2afd9695d7aec985d754cfe178 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 16:53:44 +0100 Subject: [PATCH 16/64] Skip in non-ci --- .../DifferentTestFrameworkSimpleTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs index 846fd07816..221d806917 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs @@ -66,7 +66,7 @@ public void CPPRunAllTestExecutionPlatformx64Net(RunnerInfo runnerInfo) [NetFullTargetFrameworkDataSource] public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo) { - if (IsCI) + if (!IsCI) { Assert.Inconclusive("This works on server but not locally, because locall it grabs dll from my GAC and that is old, but has 10.0.0 version as the one in our package."); } @@ -123,7 +123,7 @@ public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo) [NetFullTargetFrameworkDataSource] public void CodedWebTestRunAllTests(RunnerInfo runnerInfo) { - if (IsCI) + if (!IsCI) { Assert.Inconclusive("This works on server but not locally, because locall it grabs dll from my GAC and that is old, but has 10.0.0 version as the one in our package."); } From 543c21c47406d74ed48eba70028f759279c43bf9 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Fri, 4 Mar 2022 17:22:12 +0100 Subject: [PATCH 17/64] Change order of tests to run from fastest Use net48 for acceptance --- azure-pipelines.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 62576ab4c0..998b4ffca9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -97,26 +97,26 @@ jobs: reg ADD "HKLM\Software\Wow6432Node\Microsoft\StrongName\Verification\*,*" /f - task: BatchScript@1 - displayName: 'Run Acceptance Tests' + displayName: 'Run Platform Tests' inputs: filename: test.cmd - arguments: '-verbose -configuration $(buildConfiguration) -p AcceptanceTests -f net451' + arguments: '-verbose -configuration $(buildConfiguration) -p platformtests' modifyEnvironment: false failOnStandardError: true - task: BatchScript@1 - displayName: 'Run Platform Tests' + displayName: 'Run Smoke Tests' inputs: filename: test.cmd - arguments: '-verbose -configuration $(buildConfiguration) -p platformtests' + arguments: '-verbose -configuration $(buildConfiguration) -p smoke' modifyEnvironment: false failOnStandardError: true - task: BatchScript@1 - displayName: 'Run Smoke Tests' + displayName: 'Run Acceptance Tests' inputs: filename: test.cmd - arguments: '-verbose -configuration $(buildConfiguration) -p smoke' + arguments: '-verbose -configuration $(buildConfiguration) -p AcceptanceTests -f net48' modifyEnvironment: false failOnStandardError: true From 76021d2c54ac48b91c30b260311e3ffbe0905d3b Mon Sep 17 00:00:00 2001 From: nohwnd Date: Mon, 7 Mar 2022 07:37:01 +0100 Subject: [PATCH 18/64] Newer approach to reference assemblies on MacOS and Linux --- playground/MSTest1/MSTest1.csproj | 1 - .../TestPlatform.Playground/TestPlatform.Playground.csproj | 1 - scripts/build.sh | 2 -- scripts/build/TestPlatform.Settings.targets | 5 +++++ ...crosoft.TestPlatform.Extensions.BlameDataCollector.csproj | 1 - src/package/external/external.csproj | 5 ----- src/testhost.arm64/testhost.arm64.csproj | 1 - src/testhost.x86/testhost.x86.csproj | 1 - src/testhost/testhost.csproj | 1 - ...stPlatform.Extensions.BlameDataCollector.UnitTests.csproj | 1 - .../Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj | 1 - 11 files changed, 5 insertions(+), 15 deletions(-) diff --git a/playground/MSTest1/MSTest1.csproj b/playground/MSTest1/MSTest1.csproj index e952c0fa11..c1a72605a4 100644 --- a/playground/MSTest1/MSTest1.csproj +++ b/playground/MSTest1/MSTest1.csproj @@ -19,7 +19,6 @@ - diff --git a/playground/TestPlatform.Playground/TestPlatform.Playground.csproj b/playground/TestPlatform.Playground/TestPlatform.Playground.csproj index d73746f16e..77ccfd0b7e 100644 --- a/playground/TestPlatform.Playground/TestPlatform.Playground.csproj +++ b/playground/TestPlatform.Playground/TestPlatform.Playground.csproj @@ -30,7 +30,6 @@ - diff --git a/scripts/build.sh b/scripts/build.sh index d6e2e6bf73..9268940821 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -267,8 +267,6 @@ function invoke_build() local start=$SECONDS log ".. .. Build: Source: $TPB_Solution" - # Workaround for https://github.com/dotnet/sdk/issues/335 - export FrameworkPathOverride=$TP_PACKAGES_DIR/microsoft.targetingpack.netframework.v4.7.2/1.0.0/lib/net472/ if [ -z "$PROJECT_NAME_PATTERNS" ] then if [[ $TP_USE_REPO_API = 0 ]]; then diff --git a/scripts/build/TestPlatform.Settings.targets b/scripts/build/TestPlatform.Settings.targets index 36a9550e8a..e883bf8085 100644 --- a/scripts/build/TestPlatform.Settings.targets +++ b/scripts/build/TestPlatform.Settings.targets @@ -49,6 +49,11 @@ + + + + + - diff --git a/src/testhost.arm64/testhost.arm64.csproj b/src/testhost.arm64/testhost.arm64.csproj index 87affaafaf..d3e4c7da5c 100644 --- a/src/testhost.arm64/testhost.arm64.csproj +++ b/src/testhost.arm64/testhost.arm64.csproj @@ -50,7 +50,6 @@ - diff --git a/src/testhost.x86/testhost.x86.csproj b/src/testhost.x86/testhost.x86.csproj index 06e2ad30a7..a9e8ea85ba 100644 --- a/src/testhost.x86/testhost.x86.csproj +++ b/src/testhost.x86/testhost.x86.csproj @@ -49,7 +49,6 @@ - diff --git a/src/testhost/testhost.csproj b/src/testhost/testhost.csproj index cd8477bd59..ac2c7a02cf 100644 --- a/src/testhost/testhost.csproj +++ b/src/testhost/testhost.csproj @@ -50,7 +50,6 @@ - diff --git a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj index 0024aa19b4..1edb0499f2 100644 --- a/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests/Microsoft.TestPlatform.Extensions.BlameDataCollector.UnitTests.csproj @@ -37,7 +37,6 @@ - diff --git a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj index e098a6461c..a6d24cf837 100644 --- a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj +++ b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Microsoft.TestPlatform.TestHostProvider.UnitTests.csproj @@ -20,7 +20,6 @@ - From de7aefccfb09b96ffc80600af97216c97d51291c Mon Sep 17 00:00:00 2001 From: nohwnd Date: Mon, 7 Mar 2022 09:22:54 +0100 Subject: [PATCH 19/64] Revert "Change order of tests to run from fastest" This reverts commit 543c21c47406d74ed48eba70028f759279c43bf9. --- azure-pipelines.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 998b4ffca9..62576ab4c0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -97,26 +97,26 @@ jobs: reg ADD "HKLM\Software\Wow6432Node\Microsoft\StrongName\Verification\*,*" /f - task: BatchScript@1 - displayName: 'Run Platform Tests' + displayName: 'Run Acceptance Tests' inputs: filename: test.cmd - arguments: '-verbose -configuration $(buildConfiguration) -p platformtests' + arguments: '-verbose -configuration $(buildConfiguration) -p AcceptanceTests -f net451' modifyEnvironment: false failOnStandardError: true - task: BatchScript@1 - displayName: 'Run Smoke Tests' + displayName: 'Run Platform Tests' inputs: filename: test.cmd - arguments: '-verbose -configuration $(buildConfiguration) -p smoke' + arguments: '-verbose -configuration $(buildConfiguration) -p platformtests' modifyEnvironment: false failOnStandardError: true - task: BatchScript@1 - displayName: 'Run Acceptance Tests' + displayName: 'Run Smoke Tests' inputs: filename: test.cmd - arguments: '-verbose -configuration $(buildConfiguration) -p AcceptanceTests -f net48' + arguments: '-verbose -configuration $(buildConfiguration) -p smoke' modifyEnvironment: false failOnStandardError: true From 1bddbc87589ef060d891cdc392abb6ca0cd10e23 Mon Sep 17 00:00:00 2001 From: nohwnd Date: Mon, 7 Mar 2022 09:23:28 +0100 Subject: [PATCH 20/64] Acceptance for net48 --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 62576ab4c0..8df66a8969 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -100,7 +100,7 @@ jobs: displayName: 'Run Acceptance Tests' inputs: filename: test.cmd - arguments: '-verbose -configuration $(buildConfiguration) -p AcceptanceTests -f net451' + arguments: '-verbose -configuration $(buildConfiguration) -p AcceptanceTests -f net48' modifyEnvironment: false failOnStandardError: true From c4e7ff1bd86f1e53208a1d11dc448da5b9f171fe Mon Sep 17 00:00:00 2001 From: nohwnd Date: Mon, 7 Mar 2022 09:26:15 +0100 Subject: [PATCH 21/64] Fix filter --- scripts/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test.sh b/scripts/test.sh index a28a96f3ce..9d988cfd52 100644 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -57,7 +57,7 @@ done # # Variables # -PROJECT_NAME_PATTERNS=**$PROJECT_NAME_PATTERNS*bin*$CONFIGURATION*netcoreapp2.1*$PROJECT_NAME_PATTERNS*Tests*dll +PROJECT_NAME_PATTERNS=**$PROJECT_NAME_PATTERNS*bin*$CONFIGURATION*net6.0*$PROJECT_NAME_PATTERNS*Tests*dll TP_ROOT_DIR=$(cd "$(dirname "$0")"; pwd -P) TP_TOOLS_DIR="$TP_ROOT_DIR/tools" TP_PACKAGES_DIR="$TP_ROOT_DIR/packages" From bbd4473c8f5f7fa179c24dbb437a716a29608f0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 7 Mar 2022 09:36:02 +0100 Subject: [PATCH 22/64] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Amaury Levé --- .../DifferentTestFrameworkSimpleTests.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs index 221d806917..faaeb1fd7e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs @@ -15,7 +15,6 @@ namespace Microsoft.TestPlatform.AcceptanceTests; [TestCategory("Windows-Review")] public class DifferentTestFrameworkSimpleTests : AcceptanceTestBase { - [TestMethod] [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] public void ChutzpahRunAllTestExecution(RunnerInfo runnerInfo) @@ -68,7 +67,7 @@ public void WebTestRunAllTestsWithRunSettings(RunnerInfo runnerInfo) { if (!IsCI) { - Assert.Inconclusive("This works on server but not locally, because locall it grabs dll from my GAC and that is old, but has 10.0.0 version as the one in our package."); + Assert.Inconclusive("This works on server but not locally, because locally it grabs old dll from GAC, but has version 10.0.0 as the one in our package."); } SetTestEnvironment(_testEnvironment, runnerInfo); @@ -125,7 +124,7 @@ public void CodedWebTestRunAllTests(RunnerInfo runnerInfo) { if (!IsCI) { - Assert.Inconclusive("This works on server but not locally, because locall it grabs dll from my GAC and that is old, but has 10.0.0 version as the one in our package."); + Assert.Inconclusive("This works on server but not locally, because locally it grabs old dll from GAC, but has version 10.0.0 as the one in our package."); } SetTestEnvironment(_testEnvironment, runnerInfo); From 7c6d9462d86c76af032b0708c2f24270e3c7e079 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 7 Mar 2022 10:24:48 +0100 Subject: [PATCH 23/64] Update scripts/build/TestPlatform.Settings.targets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Amaury Levé --- scripts/build/TestPlatform.Settings.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build/TestPlatform.Settings.targets b/scripts/build/TestPlatform.Settings.targets index e883bf8085..b297c84ab9 100644 --- a/scripts/build/TestPlatform.Settings.targets +++ b/scripts/build/TestPlatform.Settings.targets @@ -51,7 +51,7 @@ - + 17.2.0-dev - 2.1.0 - 2.1.0 + 2.2.8 + 2.2.8 1.0.3-preview + + [2.2.8] + [2.2.9-preview-20220210-07] + [2.1.0] + [2.1.0] + [1.4.0] + 2.3.1 - 2.3.1 - 2.3.1 + $(XUnitFrameworkVersion) + $(XUnitFrameworkVersion) 3.10.1 - 3.10.0 + $(NUnit3FrameworkVersion) 3.8.0 4.4.12 diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs index 4829c3edcb..55b41da95f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs @@ -45,12 +45,18 @@ public class AcceptanceTestBase : IntegrationTestBase public const string DesktopRunnerTargetRuntime = "win7-x64"; public const string CoreRunnerTargetRuntime = ""; public const string InIsolation = "/InIsolation"; - + public const string NETFX452_48 = "net452;net461;net472;net48"; public const string NETFX451_48 = "net452;net461;net472;net48"; public const string NETCORE21_50 = "netcoreapp2.1;netcoreapp3.1;net5.0"; public const string NETFX452_NET50 = "net452;net461;net472;net48;netcoreapp2.1;netcoreapp3.1;net5.0"; public const string NETFX452_NET31 = "net452;net461;net472;net48;netcoreapp2.1;netcoreapp3.1"; + /// + /// Our current defaults for .NET and .NET Framework. + /// + public const string DEFAULT_NETFX_AND_NET = "net451;netcoreapp2.1"; + public const string LATEST_LEGACY = "Latest;LatestStable;LatestPreview;MostDownloaded;PreviousStable;LegacyStable"; + public const string LATESTSTABLE_LEGACY = "LatestStable;LatestPreview;MostDownloaded;PreviousStable;LegacyStable"; public static string And(string left, string right) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index b88bf2e02d..8a2dbcfa97 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -31,6 +31,20 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) ExitCodeEquals(1); // failing tests } + [TestMethod] + [MSTestCompatibilityDataSource] + public void RunMultipleTestAssemblies2(RunnerInfo runnerInfo, MSTestInfo msTestInfo) + { + SetTestEnvironment(_testEnvironment, runnerInfo); + + var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo, "SimpleTestProject.dll").Trim('\"'); + + InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); + + ValidateSummaryStatus(2, 2, 2); + ExitCodeEquals(1); // failing tests + } + [TestMethod] [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] [NetCoreTargetFrameworkDataSource] @@ -59,12 +73,13 @@ public void RunMultipleTestAssembliesWithoutTestAdapterPath(RunnerInfo runnerInf public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /Parallel"); arguments = string.Concat(arguments, " /Platform:x86"); - arguments += GetDiagArg(TempDirectory.Path); + arguments += GetDiagArg(tempDir.Path); // for the desktop we will run testhost.x86 in two copies, but for core // we will run a combination of testhost.x86 and dotnet, where the dotnet will be @@ -77,7 +92,7 @@ public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) InvokeVsTest(arguments); - AssertExpectedNumberOfHostProcesses(expectedNumOfProcessCreated, TempDirectory.Path, testHostProcessNames); + AssertExpectedNumberOfHostProcesses(expectedNumOfProcessCreated, tempDir.Path, testHostProcessNames); ValidateSummaryStatus(2, 2, 2); ExitCodeEquals(1); // failing tests } @@ -88,10 +103,11 @@ public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) public void TestSessionTimeOutTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /TestCaseFilter:TestSessionTimeoutTest"); // set TestSessionTimeOut = 7 sec @@ -109,9 +125,10 @@ public void TestSessionTimeOutTests(RunnerInfo runnerInfo) public void TestPlatformShouldBeCompatibleWithOldTestHost(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SampleProjectWithOldTestHost.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); InvokeVsTest(arguments); @@ -125,9 +142,10 @@ public void TestPlatformShouldBeCompatibleWithOldTestHost(RunnerInfo runnerInfo) public void WorkingDirectoryIsSourceDirectory(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /tests:WorkingDirectoryTest"); InvokeVsTest(arguments); @@ -142,6 +160,7 @@ public void WorkingDirectoryIsSourceDirectory(RunnerInfo runnerInfo) public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); if (IntegrationTestEnvironment.BuildConfiguration.Equals("release", StringComparison.OrdinalIgnoreCase)) { @@ -150,11 +169,11 @@ public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerIn return; } - var diagLogFilePath = Path.Combine(TempDirectory.Path, $"std_error_log_{Guid.NewGuid()}.txt"); + var diagLogFilePath = Path.Combine(tempDir.Path, $"std_error_log_{Guid.NewGuid()}.txt"); File.Delete(diagLogFilePath); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /testcasefilter:ExitWithStackoverFlow"); arguments = string.Concat(arguments, $" /diag:{diagLogFilePath}"); @@ -178,13 +197,14 @@ public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerIn public void UnhandleExceptionExceptionShouldBeLoggedToDiagLogFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); - var diagLogFilePath = Path.Combine(TempDirectory.Path, $"std_error_log_{Guid.NewGuid()}.txt"); + var diagLogFilePath = Path.Combine(tempDir.Path, $"std_error_log_{Guid.NewGuid()}.txt"); File.Delete(diagLogFilePath); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /testcasefilter:ExitwithUnhandleException"); arguments = string.Concat(arguments, $" /diag:{diagLogFilePath}"); @@ -201,11 +221,12 @@ public void UnhandleExceptionExceptionShouldBeLoggedToDiagLogFile(RunnerInfo run public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProject3.dll is built for Framework .NETFramework,Version=v4.5.1 and Platform X64"; var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll", "SimpleTestProjectx86.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /testcasefilter:PassingTestx86"); InvokeVsTest(arguments); @@ -223,11 +244,12 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo r public void NoIncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProjectx86 is built for Framework .NETFramework,Version=v4.5.1 and Platform X86"; var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProjectx86.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); InvokeVsTest(arguments); @@ -243,11 +265,12 @@ public void NoIncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsoleOnlyWhenRunningIn32BitOS(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProject2.dll is built for Framework .NETFramework,Version=v4.5.1 and Platform X64"; var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); InvokeVsTest(arguments); @@ -272,10 +295,11 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsoleOnlyWhenRunni public void ExitCodeShouldReturnOneWhenTreatNoTestsAsErrorParameterSetToTrueAndNoTestMatchesFilter(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); // Setting /TestCaseFilter to the test name, which does not exists in the assembly, so we will have 0 tests executed arguments = string.Concat(arguments, " /TestCaseFilter:TestNameThatMatchesNoTestInTheAssembly"); @@ -292,9 +316,10 @@ public void ExitCodeShouldReturnOneWhenTreatNoTestsAsErrorParameterSetToTrueAndN public void ExitCodeShouldReturnZeroWhenTreatNoTestsAsErrorParameterSetToFalseAndNoTestMatchesFilter(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); // Setting /TestCaseFilter to the test name, which does not exists in the assembly, so we will have 0 tests executed arguments = string.Concat(arguments, " /TestCaseFilter:TestNameThatMatchesNoTestInTheAssembly"); @@ -311,10 +336,11 @@ public void ExitCodeShouldReturnZeroWhenTreatNoTestsAsErrorParameterSetToFalseAn public void ExitCodeShouldNotDependOnTreatNoTestsAsErrorTrueValueWhenThereAreAnyTestsToRun(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " -- RunConfiguration.TreatNoTestsAsError=true"); InvokeVsTest(arguments); @@ -329,9 +355,10 @@ public void ExitCodeShouldNotDependOnTreatNoTestsAsErrorTrueValueWhenThereAreAny public void ExitCodeShouldNotDependOnFailTreatNoTestsAsErrorFalseValueWhenThereAreAnyTestsToRun(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); + using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " -- RunConfiguration.TreatNoTestsAsError=false"); InvokeVsTest(arguments); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs index 4e8ae4ae9d..b9ee94a80e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs @@ -4,7 +4,12 @@ using System; using System.Collections.Generic; using System.Globalization; +using System.IO; +using System.Linq; using System.Reflection; +using System.Text; +using System.Text.RegularExpressions; +using System.Xml; using Microsoft.TestPlatform.TestUtilities; @@ -84,3 +89,77 @@ public string GetDisplayName(MethodInfo methodInfo, object[] data) return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); } } + +public class MSTestCompatibilityDataSource : Attribute, ITestDataSource +{ + /// + /// Initializes a new instance of the class. + /// + /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. + public MSTestCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT_NETFX_AND_NET, string targetFrameworks = AcceptanceTestBase.DEFAULT_NETFX_AND_NET, string msTestVersions = AcceptanceTestBase.LATESTSTABLE_LEGACY) + { + var runnersFrameworks = runners.Split(';'); + var testhostFrameworks = targetFrameworks.Split(';'); + var msTestVersionsToRun = msTestVersions.Split(';'); + + var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); + + // Only run .NET Framework tests on Windows. + Func filter = tfm => isWindows || !tfm.StartsWith("net4"); + + foreach (var runner in runnersFrameworks.Where(filter)) + { + foreach (var fmw in testhostFrameworks.Where(filter)) + { + foreach (var msTestVersion in msTestVersionsToRun) + { + _dataRows.Add(new object[] { new RunnerInfo(runner, fmw), GetMSTestInfo(msTestVersion) }); + } + } + } + } + + private readonly List _dataRows = new(); + private static XmlDocument _depsXml; + + public IEnumerable GetData(MethodInfo methodInfo) + { + return _dataRows; + } + + public string GetDisplayName(MethodInfo methodInfo, object[] data) + { + return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); + } + + private MSTestInfo GetMSTestInfo(string msTestVersion) + { + // TODO: replacing in the result string is lame, but I am not going to fight 20 GetAssetFullPath method overloads right now + // TODO: this could also be cached of course. + + var depsXml = GetDependenciesXml(); + + XmlNode node = depsXml.DocumentElement.SelectSingleNode($"PropertyGroup/MSTestFramework{msTestVersion}Version"); + var version = node?.InnerText.Replace("[", "").Replace("]", ""); + var slash = Path.DirectorySeparatorChar; + var versionSpecificBinPath = $"{slash}bin{slash}MSTest{msTestVersion}-{version}{slash}"; + + return new MSTestInfo(msTestVersion, version, versionSpecificBinPath); + } + + private static XmlDocument GetDependenciesXml() + { + if (_depsXml != null) + return _depsXml; + + var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); + var fileStream = File.OpenRead(depsXmlPath); + var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false }; + var depsXml = new XmlDocument(); + depsXml.Load(xmlTextReader); + + _depsXml = depsXml; + return depsXml; + } +} + diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index abbbbf0967..0277197da9 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -753,13 +753,29 @@ public static void CreateRunSettingsFile(string destinationRunsettingsPath, stri protected string BuildMultipleAssemblyPath(params string[] assetNames) { - var assertFullPaths = new string[assetNames.Length]; + var assetFullPath = new string[assetNames.Length]; for (var i = 0; i < assetNames.Length; i++) { - assertFullPaths[i] = GetAssetFullPath(assetNames[i]).AddDoubleQuote(); + assetFullPath[i] = GetAssetFullPath(assetNames[i]).AddDoubleQuote(); } - return string.Join(" ", assertFullPaths); + return string.Join(" ", assetFullPath); + } + + protected string BuildMultipleAssemblyPath(MSTestInfo msTestInfo, params string[] assetNames) + { + var assetFullPaths = new string[assetNames.Length]; + var slash = Path.DirectorySeparatorChar; + for (var i = 0; i < assetNames.Length; i++) + { + var path = GetAssetFullPath(assetNames[i]); + var updatedPath = msTestInfo.UpdatePath(path); + Assert.IsTrue(File.Exists(updatedPath), "GetTestAsset: Path not found: {0}. Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", updatedPath); + + assetFullPaths[i] = updatedPath.AddDoubleQuote(); + } + + return string.Join(" ", assetFullPaths); } protected static string GetDiagArg(string rootDir) @@ -800,3 +816,29 @@ protected static string GetDownloadedDotnetMuxerFromTools(string architecture) protected static string GetDotnetRunnerPath() => Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "netcoreapp2.1", "vstest.console.dll"); } + +public class MSTestInfo +{ + public MSTestInfo(string versionType, string version, string path) + { + VersionType = versionType; + Version = version; + Path = path; + } + + public string VersionType { get; } + public string Version { get; } + public string Path { get; } + + public override string ToString() => $" MSTest = {Version} [{VersionType}]"; + + public string UpdatePath(string path) + { + // Version is not directly used, below, but if it is not populated the path will be incorrect. + // We don't want to throw when creating MSTestInfo because that is happening too early, and has worse error reporting. + if (Version == null) + throw new InvalidOperationException($"Version was not correctly populated from TestPlatform.Dependencies.props, review that there is entry for MSTestFramework{VersionType}Version."); + + return path.Replace($"{System.IO.Path.DirectorySeparatorChar}bin{System.IO.Path.DirectorySeparatorChar}", Path); + } +} diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs index 2de0b4dd03..51008831c2 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs @@ -235,7 +235,7 @@ public string GetTestAsset(string assetName, string targetFramework) targetFramework, assetName); - Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: {0}.", assetPath); + Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: {0}. Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", assetPath); return assetPath; } From ecdb18eddf4099708f9f8c24bf4a44cc49d1acc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Thu, 10 Mar 2022 10:51:11 +0100 Subject: [PATCH 26/64] MSTest adapter tests --- scripts/build.ps1 | 28 ++-- scripts/build/TestPlatform.Dependencies.props | 7 +- .../AcceptanceTestBase.cs | 8 +- .../ExecutionTests.cs | 36 ++--- .../MSTestCompatibilityDataSource.cs | 118 +++++++++++++++ .../Extension/NetCoreRunner.cs | 25 ++-- .../NetCoreTargetFrameworkDataSource.cs | 136 +++++------------- .../Extension/NetFrameworkRunner.cs | 27 ++-- .../NetFullTargetFrameworkDataSource.cs | 67 ++++----- .../Extension/RunnnerInfo.cs | 15 +- .../Extension/TestDataSource.cs | 71 +++++++++ .../IntegrationTestBase.cs | 58 ++++---- .../IntegrationTestEnvironment.cs | 4 + .../MSTestInfo.cs | 34 +++++ 14 files changed, 405 insertions(+), 229 deletions(-) create mode 100644 test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs create mode 100644 test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs create mode 100644 test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs diff --git a/scripts/build.ps1 b/scripts/build.ps1 index 625a7838bc..81475960c4 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -174,7 +174,11 @@ function Invoke-TestAssetsBuild $dependenciesPath = "$env:TP_ROOT_DIR\scripts\build\TestPlatform.Dependencies.props" $dependenciesXml = [xml](Get-Content -Raw -Encoding UTF8 $dependenciesPath) - $project = "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject\SimpleTestProject.csproj" + $projects = @( + "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject\SimpleTestProject.csproj" + "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject2\SimpleTestProject2.csproj" + ) + $versionProperties = @( "MSTestFrameworkLatestStableVersion" "MSTestFrameworkLatestPreviewVersion" @@ -182,18 +186,20 @@ function Invoke-TestAssetsBuild "MSTestFrameworkPreviousStableVersion" "MSTestFrameworkLegacyStableVersion" ) + + foreach ($project in $projects) { + foreach ($propertyName in $versionProperties) { + $mstestVersion = $dependenciesXml.Project.PropertyGroup.$propertyName - foreach ($propertyName in $versionProperties) { - $mstestVersion = $dependenciesXml.Project.PropertyGroup.$propertyName - - if (-not $mstestVersion) - { - throw "MSTestVersion for $propertyName is empty." + if (-not $mstestVersion) + { + throw "MSTestVersion for $propertyName is empty." + } + + $dirVersion = $mstestVersion -replace "\[|\]" + $dirPropertyName = $propertyName -replace "Framework" -replace "Version" + Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:MSTestFrameworkVersion=$mstestVersion -p:MSTestAdapterVersion=$mstestVersion -p:BaseOutputPath=""bin\$dirPropertyName-$dirVersion\\""" } - - $dirVersion = $mstestVersion -replace "\[|\]" - $dirPropertyName = $propertyName -replace "Framework" -replace "Version" - Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -bl:""$env:TP_OUT_DIR\log\$Configuration\TestAssets.binlog"" -p:MSTestFrameworkVersion=$mstestVersion -p:MSTestAdapterVersion=$mstestVersion -p:BaseOutputPath=""bin\$dirPropertyName-$dirVersion\\""" } } finally { diff --git a/scripts/build/TestPlatform.Dependencies.props b/scripts/build/TestPlatform.Dependencies.props index 4cc9cca978..6d505c671e 100644 --- a/scripts/build/TestPlatform.Dependencies.props +++ b/scripts/build/TestPlatform.Dependencies.props @@ -13,11 +13,14 @@ from a build parameter would not be available, so I am writing this version from the build.ps1 script to keep it in sync --> 17.2.0-dev + 2.2.8 2.2.8 1.0.3-preview - + [2.2.8] [2.2.9-preview-20220210-07] [2.1.0] @@ -29,7 +32,7 @@ $(XUnitFrameworkVersion) 3.10.1 - $(NUnit3FrameworkVersion) + 3.11.0 3.8.0 4.4.12 diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs index 55b41da95f..3a108e5cf7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs @@ -51,10 +51,12 @@ public class AcceptanceTestBase : IntegrationTestBase public const string NETCORE21_50 = "netcoreapp2.1;netcoreapp3.1;net5.0"; public const string NETFX452_NET50 = "net452;net461;net472;net48;netcoreapp2.1;netcoreapp3.1;net5.0"; public const string NETFX452_NET31 = "net452;net461;net472;net48;netcoreapp2.1;netcoreapp3.1"; + public const string DEFAULT_RUNNER_NETFX = "net451"; /// /// Our current defaults for .NET and .NET Framework. /// - public const string DEFAULT_NETFX_AND_NET = "net451;netcoreapp2.1"; + public const string DEFAULT_RUNNER_NETFX_AND_NET = $"{DEFAULT_RUNNER_NETFX};netcoreapp2.1"; + public const string DEFAULT_HOST_NETFX_AND_NET = "net451;netcoreapp2.1"; public const string LATEST_LEGACY = "Latest;LatestStable;LatestPreview;MostDownloaded;PreviousStable;LegacyStable"; public const string LATESTSTABLE_LEGACY = "LatestStable;LatestPreview;MostDownloaded;PreviousStable;LegacyStable"; @@ -70,6 +72,10 @@ protected static void SetTestEnvironment(IntegrationTestEnvironment testEnvironm testEnvironment.RunnerFramework = runnerInfo.RunnerFramework; testEnvironment.TargetFramework = runnerInfo.TargetFramework; testEnvironment.InIsolationValue = runnerInfo.InIsolationValue; + testEnvironment.DebugVSTestConsole = runnerInfo.DebugVSTestConsole; + testEnvironment.DebugTesthost = runnerInfo.DebugTesthost; + testEnvironment.DebugDataCollector = runnerInfo.DebugDataCollector; + testEnvironment.NoDefaultBreakpoints = runnerInfo.NoDefaultBreakpoints; } protected static string DeriveFrameworkArgValue(IntegrationTestEnvironment testEnvironment) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 8a2dbcfa97..9a7f2b275e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -16,28 +16,15 @@ namespace Microsoft.TestPlatform.AcceptanceTests; [TestClass] public class ExecutionTests : AcceptanceTestBase { - [TestMethod] - [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] - [NetCoreTargetFrameworkDataSource] - public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) - { - SetTestEnvironment(_testEnvironment, runnerInfo); - - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); - - InvokeVsTestForExecution(assemblyPaths, GetTestAdapterPath(), FrameworkArgValue, string.Empty); - - ValidateSummaryStatus(2, 2, 2); - ExitCodeEquals(1); // failing tests - } + // TODO: It looks like the first 3 tests would be useful to multiply by all 3 test frameworks, should we make the test even more generic, or duplicate them? [TestMethod] - [MSTestCompatibilityDataSource] - public void RunMultipleTestAssemblies2(RunnerInfo runnerInfo, MSTestInfo msTestInfo) + [MSTestCompatibilityDataSource(InProcess = true)] + public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, MSTestInfo msTestInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo, "SimpleTestProject.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); @@ -45,6 +32,10 @@ public void RunMultipleTestAssemblies2(RunnerInfo runnerInfo, MSTestInfo msTestI ExitCodeEquals(1); // failing tests } + + // TODO: This one mixes different frameworks, I can make it work, but it is worth it? We are going to test + // the two respective versions together (e.g. latest xunit and latest mstest), but does using two different test + // frameworks have any added value over using 2 mstest dlls? [TestMethod] [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)] [NetCoreTargetFrameworkDataSource] @@ -58,7 +49,7 @@ public void RunMultipleTestAssembliesWithoutTestAdapterPath(RunnerInfo runnerInf _testEnvironment.GetTestAsset("XUTestProject.dll"); assemblyPaths = string.Concat(assemblyPaths, "\" \"", xunitAssemblyPath); - InvokeVsTestForExecution(assemblyPaths, string.Empty, FrameworkArgValue, string.Empty); + InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty); ValidateSummaryStatus(2, 2, 1); ExitCodeEquals(1); // failing tests @@ -68,15 +59,14 @@ public void RunMultipleTestAssembliesWithoutTestAdapterPath(RunnerInfo runnerInf // and after --arch feature implementation we won't find correct muxer on CI. [TestCategory("Windows")] [TestMethod] - [NetFullTargetFrameworkDataSource] - [NetCoreTargetFrameworkDataSource] - public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) + [MSTestCompatibilityDataSource] + public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo, MSTestInfo msTestInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); using var tempDir = new TempDirectory(); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo,"SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var arguments = PrepareArguments(assemblyPaths, testAdapterPath: null, runSettings: null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /Parallel"); arguments = string.Concat(arguments, " /Platform:x86"); arguments += GetDiagArg(tempDir.Path); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs new file mode 100644 index 0000000000..d560945109 --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs @@ -0,0 +1,118 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Xml; + +using Microsoft.TestPlatform.TestUtilities; + + +namespace Microsoft.TestPlatform.AcceptanceTests; + +public sealed class MSTestCompatibilityDataSource : TestDataSource +{ + private static XmlDocument? _depsXml; + private readonly string[] _runnerFrameworks; + private readonly string[] _targetFrameworks; + private readonly string[] _msTestVersions; + + /// + /// Initializes a new instance. + /// + /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. + public MSTestCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT_RUNNER_NETFX_AND_NET, string targetFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, string msTestVersions = AcceptanceTestBase.LATESTSTABLE_LEGACY) + { + _runnerFrameworks = runners.Split(';'); + _targetFrameworks = targetFrameworks.Split(';'); + _msTestVersions = msTestVersions.Split(';'); + + // Do not generate the data rows here, properties (e.g. InProcess) are not populated until after constructor is done. + } + + /// + /// Add also run for in-process using the runner. + /// + // TODO: Can we somehow assert that we actually ran in process? + public bool InProcess { get; set; } + public bool DebugVSTestConsole { get; set; } + public bool DebugTesthost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + public override void CreateData(MethodInfo methodInfo) + { + var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); + // Only run .NET Framework tests on Windows. + Func filter = tfm => isWindows || !tfm.StartsWith("net4"); + + if (InProcess) + { + foreach (var msTestVersion in _msTestVersions) + { + var runnerInfo = new RunnerInfo(AcceptanceTestBase.DEFAULT_RUNNER_NETFX, AcceptanceTestBase.DEFAULT_RUNNER_NETFX, inIsolation: null, + DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); + var msTestInfo = GetMSTestInfo(msTestVersion); + // We run in the .NET Framework runner process, the runner and target framework must agree. + AddData(runnerInfo, msTestInfo); + } + } + + foreach (var runner in _runnerFrameworks.Where(filter)) + { + foreach (var fmw in _targetFrameworks.Where(filter)) + { + foreach (var msTestVersion in _msTestVersions) + { + var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation, + DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); + var msTestInfo = GetMSTestInfo(msTestVersion); + + AddData(runnerInfo, msTestInfo); + } + } + } + } + + public string GetDisplayName(MethodInfo methodInfo, object[] data) + { + return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); + } + + private MSTestInfo GetMSTestInfo(string msTestVersion) + { + // TODO: replacing in the result string is lame, but I am not going to fight 20 GetAssetFullPath method overloads right now + // TODO: this could also be cached of course. + + var depsXml = GetDependenciesXml(); + + // It is okay when node is null, we check that Version has value when we update paths by using MSTestInfo, and throw. + // This way it throws in the body of the test which has better error reporting than throwing in the data source. + XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/MSTestFramework{msTestVersion}Version"); + var version = node?.InnerText.Replace("[", "").Replace("]", ""); + var slash = Path.DirectorySeparatorChar; + var versionSpecificBinPath = $"{slash}bin{slash}MSTest{msTestVersion}-{version}{slash}"; + + return new MSTestInfo(msTestVersion, version, versionSpecificBinPath); + } + + private static XmlDocument GetDependenciesXml() + { + if (_depsXml != null) + return _depsXml; + + var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); + var fileStream = File.OpenRead(depsXmlPath); + var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false }; + var depsXml = new XmlDocument(); + depsXml.Load(xmlTextReader); + + _depsXml = depsXml; + return depsXml; + } +} + diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs index 67cc4fdc68..86f2b05b33 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs @@ -23,27 +23,34 @@ namespace Microsoft.TestPlatform.AcceptanceTests; /// public class NetCoreRunner : Attribute, ITestDataSource { + private readonly string _targetFrameworks; + /// /// Initializes a new instance of the class. /// /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. public NetCoreRunner(string targetFrameworks = AcceptanceTestBase.NETFX452_NET50) { + _targetFrameworks = targetFrameworks; + } + + public bool DebugVSTestConsole { get; set; } + public bool DebugTesthost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + public IEnumerable GetData(MethodInfo methodInfo) + { + var dataRows = new List(); var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); // on non-windows we want to filter down only to netcoreapp runner, and net5.0 and newer. Func filter = tfm => isWindows || !tfm.StartsWith("net4"); - foreach (var fmw in targetFrameworks.Split(';').Where(filter)) + foreach (var fmw in _targetFrameworks.Split(';').Where(filter)) { - _dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, fmw) }); + dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, fmw, inIsolation: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); } - } - - private readonly List _dataRows = new(); - - public IEnumerable GetData(MethodInfo methodInfo) - { - return _dataRows; + return dataRows; } public string GetDisplayName(MethodInfo methodInfo, object[] data) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs index b9ee94a80e..786b477bbc 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs @@ -4,12 +4,7 @@ using System; using System.Collections.Generic; using System.Globalization; -using System.IO; -using System.Linq; using System.Reflection; -using System.Text; -using System.Text.RegularExpressions; -using System.Xml; using Microsoft.TestPlatform.TestUtilities; @@ -27,7 +22,11 @@ namespace Microsoft.TestPlatform.AcceptanceTests; /// public class NetCoreTargetFrameworkDataSource : Attribute, ITestDataSource { - private readonly List _dataRows = new(); + private readonly bool _useDesktopRunner; + private readonly bool _useCoreRunner; + private readonly bool _useNetCore21Target; + private readonly bool _useNetCore31Target; + /// /// Initializes a new instance of the class. /// @@ -43,45 +42,56 @@ public NetCoreTargetFrameworkDataSource( // all tests to avoid changing all acceptance tests right now bool useNetCore31Target = false) { + _useDesktopRunner = useDesktopRunner; + _useCoreRunner = useCoreRunner; + _useNetCore21Target = useNetCore21Target; + _useNetCore31Target = useNetCore31Target; + } + + public bool DebugVSTestConsole { get; set; } + public bool DebugTesthost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + private void AddRunnerDataRow(List dataRows, string runnerFramework, string targetFramework) + { + var runnerInfo = new RunnerInfo(runnerFramework, targetFramework, inIsolation: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); + dataRows.Add(new object[] { runnerInfo }); + } + + public IEnumerable GetData(MethodInfo methodInfo) + { + var dataRows = new List(); var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); - if (useDesktopRunner && isWindows) + if (_useDesktopRunner && isWindows) { var runnerFramework = IntegrationTestBase.DesktopRunnerFramework; - if (useNetCore21Target) + if (_useNetCore21Target) { - AddRunnerDataRow(runnerFramework, AcceptanceTestBase.Core21TargetFramework); + AddRunnerDataRow(dataRows, runnerFramework, AcceptanceTestBase.Core21TargetFramework); } - if (useNetCore31Target) + if (_useNetCore31Target) { - AddRunnerDataRow(runnerFramework, AcceptanceTestBase.Core31TargetFramework); + AddRunnerDataRow(dataRows, runnerFramework, AcceptanceTestBase.Core31TargetFramework); } } - if (useCoreRunner) + if (_useCoreRunner) { var runnerFramework = IntegrationTestBase.CoreRunnerFramework; - if (useNetCore21Target) + if (_useNetCore21Target) { - AddRunnerDataRow(runnerFramework, AcceptanceTestBase.Core21TargetFramework); + AddRunnerDataRow(dataRows, runnerFramework, AcceptanceTestBase.Core21TargetFramework); } - if (useNetCore31Target) + if (_useNetCore31Target) { - AddRunnerDataRow(runnerFramework, AcceptanceTestBase.Core31TargetFramework); + AddRunnerDataRow(dataRows, runnerFramework, AcceptanceTestBase.Core31TargetFramework); } } - } - private void AddRunnerDataRow(string runnerFramework, string targetFramework) - { - var runnerInfo = new RunnerInfo(runnerFramework, targetFramework); - _dataRows.Add(new object[] { runnerInfo }); - } - - public IEnumerable GetData(MethodInfo methodInfo) - { - return _dataRows; + return dataRows; } public string GetDisplayName(MethodInfo methodInfo, object[] data) @@ -89,77 +99,3 @@ public string GetDisplayName(MethodInfo methodInfo, object[] data) return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); } } - -public class MSTestCompatibilityDataSource : Attribute, ITestDataSource -{ - /// - /// Initializes a new instance of the class. - /// - /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. - public MSTestCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT_NETFX_AND_NET, string targetFrameworks = AcceptanceTestBase.DEFAULT_NETFX_AND_NET, string msTestVersions = AcceptanceTestBase.LATESTSTABLE_LEGACY) - { - var runnersFrameworks = runners.Split(';'); - var testhostFrameworks = targetFrameworks.Split(';'); - var msTestVersionsToRun = msTestVersions.Split(';'); - - var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); - - // Only run .NET Framework tests on Windows. - Func filter = tfm => isWindows || !tfm.StartsWith("net4"); - - foreach (var runner in runnersFrameworks.Where(filter)) - { - foreach (var fmw in testhostFrameworks.Where(filter)) - { - foreach (var msTestVersion in msTestVersionsToRun) - { - _dataRows.Add(new object[] { new RunnerInfo(runner, fmw), GetMSTestInfo(msTestVersion) }); - } - } - } - } - - private readonly List _dataRows = new(); - private static XmlDocument _depsXml; - - public IEnumerable GetData(MethodInfo methodInfo) - { - return _dataRows; - } - - public string GetDisplayName(MethodInfo methodInfo, object[] data) - { - return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); - } - - private MSTestInfo GetMSTestInfo(string msTestVersion) - { - // TODO: replacing in the result string is lame, but I am not going to fight 20 GetAssetFullPath method overloads right now - // TODO: this could also be cached of course. - - var depsXml = GetDependenciesXml(); - - XmlNode node = depsXml.DocumentElement.SelectSingleNode($"PropertyGroup/MSTestFramework{msTestVersion}Version"); - var version = node?.InnerText.Replace("[", "").Replace("]", ""); - var slash = Path.DirectorySeparatorChar; - var versionSpecificBinPath = $"{slash}bin{slash}MSTest{msTestVersion}-{version}{slash}"; - - return new MSTestInfo(msTestVersion, version, versionSpecificBinPath); - } - - private static XmlDocument GetDependenciesXml() - { - if (_depsXml != null) - return _depsXml; - - var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); - var fileStream = File.OpenRead(depsXmlPath); - var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false }; - var depsXml = new XmlDocument(); - depsXml.Load(xmlTextReader); - - _depsXml = depsXml; - return depsXml; - } -} - diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs index 49f33f8d02..b8386149f0 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs @@ -28,24 +28,31 @@ public class NetFrameworkRunner : Attribute, ITestDataSource /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. public NetFrameworkRunner(string targetFrameworks = AcceptanceTestBase.NETFX452_NET50) { + _targetFrameworks = targetFrameworks; + } + + public bool DebugVSTestConsole { get; set; } + public bool DebugTesthost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + private readonly string _targetFrameworks; + + public IEnumerable GetData(MethodInfo methodInfo) + { + var dataRows = new List(); var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); if (!isWindows) { - return; + return dataRows; } - foreach (var fmw in targetFrameworks.Split(';')) + foreach (var fmw in _targetFrameworks.Split(';')) { - _dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, fmw, AcceptanceTestBase.InIsolation) }); + dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, fmw, AcceptanceTestBase.InIsolation, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); } - } - - private readonly List _dataRows = new(); - - public IEnumerable GetData(MethodInfo methodInfo) - { - return _dataRows; + return dataRows; } public string GetDisplayName(MethodInfo methodInfo, object[] data) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs index 626d11422f..a18eeb8271 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs @@ -22,6 +22,11 @@ namespace Microsoft.TestPlatform.AcceptanceTests; /// public class NetFullTargetFrameworkDataSource : Attribute, ITestDataSource { + private readonly bool _inIsolation; + private readonly bool _inProcess; + private readonly bool _useDesktopRunner; + private readonly bool _useCoreRunner; + /// /// Initializes a new instance of the class. /// @@ -31,60 +36,40 @@ public class NetFullTargetFrameworkDataSource : Attribute, ITestDataSource /// To run tests with core runner(dotnet vstest.console.dll) public NetFullTargetFrameworkDataSource(bool inIsolation = true, bool inProcess = false, bool useDesktopRunner = true, bool useCoreRunner = true) { - _dataRows = new List(); + _inIsolation = inIsolation; + _inProcess = inProcess; + _useDesktopRunner = useDesktopRunner; + _useCoreRunner = useCoreRunner; + } - var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); - if (useCoreRunner && isWindows) - { - _dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, AcceptanceTestBase.DesktopTargetFramework) }); - } + public bool DebugVSTestConsole { get; set; } + public bool DebugTesthost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; - if (useDesktopRunner && isWindows) + public IEnumerable GetData(MethodInfo methodInfo) + { + var dataRows = new List(); + var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); + if (_useCoreRunner && isWindows) { - if (inIsolation) - { - _dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, AcceptanceTestBase.InIsolation) }); - } - - if (inProcess) - { - _dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework) }); - } + dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, inIsolation: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); } - } - - /// - /// Initializes a new instance of the class. - /// - /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. - public NetFullTargetFrameworkDataSource(string[] targetFrameworks, bool inIsolation = true, bool inProcess = false) - { - if (inIsolation) + if (_useDesktopRunner && isWindows) { - foreach (var fmw in targetFrameworks) + if (_inIsolation) { - _dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, fmw, AcceptanceTestBase.InIsolation) }); + dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, AcceptanceTestBase.InIsolation, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); } - } - if (inProcess) - { - foreach (var fmw in targetFrameworks) + if (_inProcess) { - _dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, fmw) }); + dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, inIsolation: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); } } - } - - /// - /// Gets or sets the data rows. - /// - private readonly List _dataRows = new(); - public IEnumerable GetData(MethodInfo methodInfo) - { - return _dataRows; + return dataRows; } public string GetDisplayName(MethodInfo methodInfo, object[] data) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs index d9f9a14a18..5f73d8a0c5 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs @@ -9,15 +9,16 @@ namespace Microsoft.TestPlatform.AcceptanceTests; public class RunnerInfo { - public RunnerInfo(string runnerType, string targetFramework) : this(runnerType, targetFramework, "") - { - } - - public RunnerInfo(string runnerType, string targetFramework, string inIsolation) + public RunnerInfo(string runnerType, string targetFramework, string inIsolation, + bool debugVSTestConsole, bool debugTesthost, bool debugDataCollector, bool noDefaultBreakpoints) { RunnerFramework = runnerType; TargetFramework = targetFramework; InIsolationValue = inIsolation; + DebugVSTestConsole = debugVSTestConsole; + DebugTesthost = debugTesthost; + DebugDataCollector = debugDataCollector; + NoDefaultBreakpoints = noDefaultBreakpoints; // The value is netcoreapp2.1. IsNetRunner = RunnerFramework.StartsWith("netcoreapp", StringComparison.InvariantCultureIgnoreCase); // The value is net451. @@ -43,6 +44,10 @@ public string InIsolationValue { get; set; } + public bool DebugVSTestConsole { get; } + public bool DebugTesthost { get; } + public bool DebugDataCollector { get; } + public bool NoDefaultBreakpoints { get; } /// /// Gets the application type. diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs new file mode 100644 index 0000000000..104aee8d8a --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Reflection; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Microsoft.TestPlatform.AcceptanceTests; + +[AttributeUsage(AttributeTargets.Method)] +public abstract class TestDataSource : Attribute, ITestDataSource where T1 : notnull +{ + private readonly List _data = new(); + + public abstract void CreateData(MethodInfo methodInfo); + + public void AddData(T1 value1) + { + _data.Add(new object[] { value1 }); + } + + public virtual string GetDisplayName(MethodInfo methodInfo, T1 value1) + { + return $"{methodInfo.Name} ({value1})"; + } + + IEnumerable ITestDataSource.GetData(MethodInfo methodInfo) + { + CreateData(methodInfo); + return _data; + } + + string ITestDataSource.GetDisplayName(MethodInfo methodInfo, object[] data) + { + return GetDisplayName(methodInfo, (T1)data[0]); + } +} + +[AttributeUsage(AttributeTargets.Method)] +public abstract class TestDataSource : Attribute, ITestDataSource + where T1 : notnull + where T2 : notnull +{ + private readonly List _data = new(); + + public abstract void CreateData(MethodInfo methodInfo); + + public void AddData(T1 value1, T2 value2) + { + _data.Add(new object[] { value1, value2 }); + } + + public virtual string GetDisplayName(MethodInfo methodInfo, T1 value1, T2 value2) + { + return $"{methodInfo.Name} ({value1}, {value2})"; + } + + IEnumerable ITestDataSource.GetData(MethodInfo methodInfo) + { + CreateData(methodInfo); + return _data; + } + + string ITestDataSource.GetDisplayName(MethodInfo methodInfo, object[] data) + { + return GetDisplayName(methodInfo, (T1)data[0], (T2)data[1]); + } +} + diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index c7076109ed..a5d8ca3b5a 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -128,6 +128,11 @@ public static string PrepareArguments(string[] testAssemblies, string testAdapte if (!string.IsNullOrWhiteSpace(inIsolation)) { + if (inIsolation != "/InIsolation") + { + // TODO: The whole inIsolation should be just a bool, but it is not, and it's changing in other PR. + throw new InvalidOperationException("InIsolation value must be '/InIsolation'"); + } arguments = string.Concat(arguments, " ", inIsolation); } @@ -205,6 +210,32 @@ public void InvokeVsTestForExecution(string testAssembly, Dictionary environmentVariables = null) { var arguments = PrepareArguments(testAssembly, testAdapterPath, runSettings, framework, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); + + if (_testEnvironment.DebugVSTestConsole || _testEnvironment.DebugTesthost || _testEnvironment.DebugDataCollector) + { + environmentVariables ??= new Dictionary(); + + if (_testEnvironment.DebugVSTestConsole) + { + environmentVariables.Add("VSTEST_RUNNER_DEBUG_ATTACHVS", "1"); + } + + if (_testEnvironment.DebugTesthost) + { + environmentVariables.Add("VSTEST_HOST_DEBUG_ATTACHVS", "1"); + } + + if (_testEnvironment.DebugDataCollector) + { + environmentVariables.Add("VSTEST_DATACOLLECTOR_DEBUG_ATTACHVS", "1"); + } + + if (_testEnvironment.NoDefaultBreakpoints) + { + environmentVariables.Add("VSTEST_DEBUG_NOBP", "1"); + } + } + InvokeVsTest(arguments, environmentVariables); } @@ -780,7 +811,6 @@ protected string BuildMultipleAssemblyPath(params string[] assetNames) protected string BuildMultipleAssemblyPath(MSTestInfo msTestInfo, params string[] assetNames) { var assetFullPaths = new string[assetNames.Length]; - var slash = Path.DirectorySeparatorChar; for (var i = 0; i < assetNames.Length; i++) { var path = GetAssetFullPath(assetNames[i]); @@ -831,29 +861,3 @@ protected static string GetDownloadedDotnetMuxerFromTools(string architecture) protected static string GetDotnetRunnerPath() => Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "netcoreapp2.1", "vstest.console.dll"); } - -public class MSTestInfo -{ - public MSTestInfo(string versionType, string version, string path) - { - VersionType = versionType; - Version = version; - Path = path; - } - - public string VersionType { get; } - public string Version { get; } - public string Path { get; } - - public override string ToString() => $" MSTest = {Version} [{VersionType}]"; - - public string UpdatePath(string path) - { - // Version is not directly used, below, but if it is not populated the path will be incorrect. - // We don't want to throw when creating MSTestInfo because that is happening too early, and has worse error reporting. - if (Version == null) - throw new InvalidOperationException($"Version was not correctly populated from TestPlatform.Dependencies.props, review that there is entry for MSTestFramework{VersionType}Version."); - - return path.Replace($"{System.IO.Path.DirectorySeparatorChar}bin{System.IO.Path.DirectorySeparatorChar}", Path); - } -} diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs index 51008831c2..66195fba24 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs @@ -193,6 +193,10 @@ public string RunnerFramework // A known AzureDevOps env variable meaning we are running in CI. public static bool IsCI { get; } = Environment.GetEnvironmentVariable("TF_BUILD") == "True"; + public bool DebugVSTestConsole { get; set; } + public bool DebugTesthost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } /// /// Gets the full path to a test asset. diff --git a/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs new file mode 100644 index 0000000000..55453ed196 --- /dev/null +++ b/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; + +namespace Microsoft.TestPlatform.TestUtilities; + +public class MSTestInfo +{ + public MSTestInfo(string versionType, string? version, string path) + { + VersionType = versionType; + // Version can be null when we fail to find the respective propertin in TestPlatform.Dependencies.props + // when that happens we throw when we try to update the path. + Version = version; + Path = path; + } + + public string VersionType { get; } + public string? Version { get; } + public string Path { get; } + + public override string ToString() => $" MSTest = {Version} [{VersionType}]"; + + public string UpdatePath(string path) + { + // Version is not directly used, below, but if it is not populated the path will be incorrect. + // We don't want to throw when creating MSTestInfo because that is happening too early, and has worse error reporting. + if (Version == null) + throw new InvalidOperationException($"Version was not correctly populated from TestPlatform.Dependencies.props, review that there is entry for MSTestFramework{VersionType}Version."); + + return path.Replace($"{System.IO.Path.DirectorySeparatorChar}bin{System.IO.Path.DirectorySeparatorChar}", Path); + } +} From a65c00ab7812582b4b3097350fab3a70c7faef6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Thu, 10 Mar 2022 17:21:47 +0100 Subject: [PATCH 27/64] Add vstest console data source, even though it is names translation layer. --- scripts/build.ps1 | 81 +++++++++- scripts/build/TestPlatform.Dependencies.props | 41 +++-- scripts/verify-nupkgs.ps1 | 4 +- .../AcceptanceTestBase.cs | 5 +- .../MSTestCompatibilityDataSource.cs | 6 +- ...TranslationLayerCompatibilityDataSource.cs | 140 ++++++++++++++++++ .../TranslationLayerTests/DiscoverTests.cs | 12 +- .../IntegrationTestBase.cs | 35 +++-- .../MSTestInfo.cs | 1 + .../VSTestConsoleInfo.cs | 23 +++ 10 files changed, 306 insertions(+), 42 deletions(-) create mode 100644 test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs create mode 100644 test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs diff --git a/scripts/build.ps1 b/scripts/build.ps1 index 81475960c4..0b39b742d6 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -169,19 +169,23 @@ function Invoke-TestAssetsBuild try { Write-Log ".. .. Build: Source: $TPB_TestAssets_Solution -- add NuGet source" Invoke-Exe -IgnoreExitCode 1 $nugetExe -Arguments "sources add -Name ""locally-built-testplatform-packages"" -Source $env:TP_TESTARTIFACTS\packages\ -ConfigFile ""$nugetConfig""" - Invoke-Exe $dotnetExe -Arguments "build $TPB_TestAssets_Solution --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -bl:""$env:TP_OUT_DIR\log\$Configuration\TestAssets.binlog""" + #Invoke-Exe $dotnetExe -Arguments "build $TPB_TestAssets_Solution --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -bl:""$env:TP_OUT_DIR\log\$Configuration\TestAssets.binlog""" + # Compatibility matrix build $dependenciesPath = "$env:TP_ROOT_DIR\scripts\build\TestPlatform.Dependencies.props" $dependenciesXml = [xml](Get-Content -Raw -Encoding UTF8 $dependenciesPath) + + # build with multiple versions of MSTest $projects = @( "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject\SimpleTestProject.csproj" "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject2\SimpleTestProject2.csproj" ) $versionProperties = @( - "MSTestFrameworkLatestStableVersion" "MSTestFrameworkLatestPreviewVersion" + "MSTestFrameworkLatestStableVersion" + "MSTestFrameworkRecentStableVersion" "MSTestFrameworkMostDownloadedVersion" "MSTestFrameworkPreviousStableVersion" "MSTestFrameworkLegacyStableVersion" @@ -201,6 +205,68 @@ function Invoke-TestAssetsBuild Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:MSTestFrameworkVersion=$mstestVersion -p:MSTestAdapterVersion=$mstestVersion -p:BaseOutputPath=""bin\$dirPropertyName-$dirVersion\\""" } } + + # restore previous versions of TestPlatform (for vstest.console.exe), and TestPlatform.CLI (for vstest.console.dll) + $versionProperties = @( + "NETTestSdkVersion" + "VSTestConsoleLatestPreviewVersion" + "VSTestConsoleLatestStableVersion" + "VSTestConsoleRecentStableVersion" + "VSTestConsoleMostDownloadedVersion" + "VSTestConsolePreviousStableVersion" + "VSTestConsoleLegacyStableVersion" + ) + + foreach ($propertyName in $versionProperties) { + if ("VSTestConsoleLatestVersion" -eq $propertyName) { + # NETTestSdkVersion has the version of the locally built package. + $vsTestConsoleVersion = $dependenciesXml.Project.PropertyGroup."NETTestSdkVersion" + } + else { + $vsTestConsoleVersion = $dependenciesXml.Project.PropertyGroup.$propertyName + } + + # The command line tool does not like the package ranges. + $vsTestConsoleVersion = $vsTestConsoleVersion -replace "(\[|\])" + if (-not $vsTestConsoleVersion) + { + throw "VSTestConsoleVersion for $propertyName is empty." + } + + $packages = @( + "Microsoft.TestPlatform" + "Microsoft.TestPlatform.CLI", + "Microsoft.TestPlatform.TranslationLayer" + "Microsoft.NET.Test.SDK" + ) + + foreach ($package in $packages) { + $packagePath = "$env:TP_ROOT_DIR\packages\$($package.ToLower())" + $cachePath = "$packagePath\$vstestConsoleVersion" + + if ((Test-Path -Path $cachePath) -and (Get-ChildItem $cachePath)) { + "Package $package $vsTestConsoleVersion is already in nuget cache at $cachePath." + continue + } + + Invoke-Exe $nugetExe -Arguments "install $package -Version $vsTestConsoleVersion -OutputDirectory $packagePath -ConfigFile ""$nugetConfig""" + + # Install puts it in packages/microsoft.testplatform/Microsoft.TestPlatform.17.1.0, + # because we use that as our output folder. And it also caches it in packages/microsoft.testplatform/17.1.0 + # unless the package is from local source, then it does not do that. So we need to rename the folder and remove + # the original one. + if (-not (Test-Path -Path $cachePath) -or -not (Get-ChildItem $cachePath)) { + Rename-Item "$packagePath\$package.$vsTestConsoleVersion" $cachePath + # nuget locks the locally copied package it seems. + Start-Sleep -Milliseconds 300 + } + if (Test-Path "$packagePath\$package.$vsTestConsoleVersion") { + Remove-Item -Recurse -Force "$packagePath\$package.$vsTestConsoleVersion" + } + } + } + + # end } finally { Write-Log ".. .. Build: Source: $TPB_TestAssets_Solution -- remove NuGet source" @@ -835,9 +901,14 @@ function Create-NugetPackages Copy-Item (Join-Path $env:TP_PACKAGE_PROJ_DIR "Icon.png") $stagingDir -Force + # Remove all locally built nuget packages before we start creating them + # we are leaving them in the folder after uzipping them for easier review. + if (Test-Path $packageOutputDir) { + Remove-Item $packageOutputDir -Recurse -Force + } if (-not (Test-Path $packageOutputDir)) { - New-Item $packageOutputDir -type directory -Force + New-Item $packageOutputDir -Type directory -Force } $tpNuspecDir = Join-Path $env:TP_PACKAGE_PROJ_DIR "nuspec" @@ -1258,9 +1329,9 @@ Get-Variable | Where-Object -FilterScript { $_.Name.StartsWith("TPB_") } | Forma # } if ($Force -or $Steps -contains "PrepareAcceptanceTests") { - # Publish-PatchedDotnet + #Publish-PatchedDotnet Invoke-TestAssetsBuild - # Publish-Tests + #Publish-Tests } if ($Script:ScriptFailed) { diff --git a/scripts/build/TestPlatform.Dependencies.props b/scripts/build/TestPlatform.Dependencies.props index 6d505c671e..30969e3253 100644 --- a/scripts/build/TestPlatform.Dependencies.props +++ b/scripts/build/TestPlatform.Dependencies.props @@ -13,23 +13,16 @@ from a build parameter would not be available, so I am writing this version from the build.ps1 script to keep it in sync --> 17.2.0-dev - + 2.2.8 2.2.8 1.0.3-preview - - [2.2.8] - [2.2.9-preview-20220210-07] - [2.1.0] - [2.1.0] - [1.4.0] - + 2.3.1 - $(XUnitFrameworkVersion) - $(XUnitFrameworkVersion) + 2.3.1 + 2.3.1 3.10.1 3.11.0 @@ -37,6 +30,30 @@ 4.4.12 + + + [2.2.9-preview-20220210-07] + [2.2.8] + [2.2.7] + [2.1.0] + [2.1.0] + [1.4.0] + + + + + 17.2.0-preview-20220131-20 + [17.1.0] + [17.0.0] + [16.9.4] + [16.11.0] + [15.9.2] + + 5.11.0 5.0.0 diff --git a/scripts/verify-nupkgs.ps1 b/scripts/verify-nupkgs.ps1 index 86f09edb87..432bfbbcb1 100644 --- a/scripts/verify-nupkgs.ps1 +++ b/scripts/verify-nupkgs.ps1 @@ -56,7 +56,9 @@ function Verify-Nuget-Packages($packageDirectory, $version) Write-Error "Number of files are not equal $unzipNugetPackageDir, expected: $($expectedNumOfFiles[$packageKey]) actual: $actualNumOfFiles" } - Remove-Item -Force -Recurse $unzipNugetPackageDir | Out-Null + # Don't remove the directories after you unpacked them + # they are useful for reviewing what is in the package. + # Remove-Item -Force -Recurse $unzipNugetPackageDir | Out-Null } Write-Log "Completed Verify-Nuget-Packages." diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs index 3a108e5cf7..09e2d9fa4c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs @@ -57,8 +57,9 @@ public class AcceptanceTestBase : IntegrationTestBase /// public const string DEFAULT_RUNNER_NETFX_AND_NET = $"{DEFAULT_RUNNER_NETFX};netcoreapp2.1"; public const string DEFAULT_HOST_NETFX_AND_NET = "net451;netcoreapp2.1"; - public const string LATEST_LEGACY = "Latest;LatestStable;LatestPreview;MostDownloaded;PreviousStable;LegacyStable"; - public const string LATESTSTABLE_LEGACY = "LatestStable;LatestPreview;MostDownloaded;PreviousStable;LegacyStable"; + public const string LATEST_LEGACY = "Latest;LatestPreview;LatestStable;RecentStable;MostDownloaded;PreviousStable;LegacyStable"; + public const string LATESTPREVIEW_LEGACY = "LatestPreview;LatestStable;RecentStable;MostDownloaded;PreviousStable;LegacyStable"; + public const string LATEST = "Latest"; public static string And(string left, string right) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs index d560945109..b491bda22e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs @@ -25,7 +25,7 @@ public sealed class MSTestCompatibilityDataSource : TestDataSource /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. - public MSTestCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT_RUNNER_NETFX_AND_NET, string targetFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, string msTestVersions = AcceptanceTestBase.LATESTSTABLE_LEGACY) + public MSTestCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT_RUNNER_NETFX_AND_NET, string targetFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, string msTestVersions = AcceptanceTestBase.LATESTPREVIEW_LEGACY) { _runnerFrameworks = runners.Split(';'); _targetFrameworks = targetFrameworks.Split(';'); @@ -85,9 +85,6 @@ public string GetDisplayName(MethodInfo methodInfo, object[] data) private MSTestInfo GetMSTestInfo(string msTestVersion) { - // TODO: replacing in the result string is lame, but I am not going to fight 20 GetAssetFullPath method overloads right now - // TODO: this could also be cached of course. - var depsXml = GetDependenciesXml(); // It is okay when node is null, we check that Version has value when we update paths by using MSTestInfo, and throw. @@ -115,4 +112,3 @@ private static XmlDocument GetDependenciesXml() return depsXml; } } - diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs new file mode 100644 index 0000000000..2d1b637399 --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs @@ -0,0 +1,140 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Xml; + +using Microsoft.TestPlatform.TestUtilities; + + +namespace Microsoft.TestPlatform.AcceptanceTests; + +public sealed class TranslationLayerCompatibilityDataSource : TestDataSource +{ + private static XmlDocument? _depsXml; + private readonly string[] _runnerFrameworks; + private readonly string[] _targetFrameworks; + private readonly string[] _translationLayerVersions; + private readonly string[] _vstestConsoleVersions; + + /// + /// Initializes a new instance. + /// + /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. + public TranslationLayerCompatibilityDataSource( + string runners = AcceptanceTestBase.DEFAULT_RUNNER_NETFX_AND_NET, + string targetFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + // string translationLayerVersions = AcceptanceTestBase.LATEST_LEGACY, + string vstestConsoleVersions = AcceptanceTestBase.LATEST_LEGACY) + { + _runnerFrameworks = runners.Split(';'); + _targetFrameworks = targetFrameworks.Split(';'); + // _translationLayerVersions = translationLayerVersions.Split(';'); + _vstestConsoleVersions = vstestConsoleVersions.Split(';'); + + // Do not generate the data rows here, properties (e.g. DebugVSTestConsole) are not populated until after constructor is done. + } + + public bool DebugVSTestConsole { get; set; } + public bool DebugTesthost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + public override void CreateData(MethodInfo methodInfo) + { + var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); + // Only run .NET Framework tests on Windows. + Func filter = tfm => isWindows || !tfm.StartsWith("net4"); + + + foreach (var runner in _runnerFrameworks.Where(filter)) + { + foreach (var fmw in _targetFrameworks.Where(filter)) + { + foreach (var vstestConsoleVersion in _vstestConsoleVersions) + { + var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation, + DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); + var vsTestConsoleInfo = GetVSTestConsoleInfo(vstestConsoleVersion, runnerInfo); + + AddData(runnerInfo, vsTestConsoleInfo); + } + } + } + } + + public string GetDisplayName(MethodInfo methodInfo, object[] data) + { + return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); + } + + private VSTestConsoleInfo GetVSTestConsoleInfo(string vstestConsoleVersion, RunnerInfo runnerInfo) + { + var depsXml = GetDependenciesXml(); + + // When version is Latest, we built it locally, but it gets restored into our nuget cache on build + // same as other versions, we just need to grab the version from a different property. + + var propertyName = vstestConsoleVersion == AcceptanceTestBase.LATEST + ? $"NETTestSdkVersion" + : $"VSTestConsole{vstestConsoleVersion}Version"; + + var packageName = runnerInfo.IsNetFrameworkRunner + ? "microsoft.testplatform" + : "microsoft.testplatform.cli"; + + // It is okay when node is null, we will fail to find the executable later, and throw. + // This way it throws in the body of the test which has better error reporting than throwing in the data source. + // And we can easily find out what is going on because --WRONG-VERSION-- sticks out, and is easy to find in the codebase. + XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/{propertyName}"); + var version = node?.InnerText.Replace("[", "").Replace("]", "") ?? "--WRONG-VERSION--"; + var vstestConsolePath = runnerInfo.IsNetFrameworkRunner + ? Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "packages", packageName, version, + "tools", "net451", "Common7", "IDE", "Extensions", "TestPlatform", "vstest.console.exe") + : Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "packages", packageName, version, + "contentFiles", "any", "netcoreapp2.1", "vstest.console.dll"); + + if (version.StartsWith("15.")) + { + vstestConsolePath = vstestConsolePath.Replace("netcoreapp2.1", "netcoreapp2.0"); + } + + return new VSTestConsoleInfo(vstestConsoleVersion, version, vstestConsolePath); + } + + //private MSTestInfo GetTranslationLayerInfo(string translationLayerVersion) + //{ + // // TODO: replacing in the result string is lame, but I am not going to fight 20 GetAssetFullPath method overloads right now + // // TODO: this could also be cached of course. + + // var depsXml = GetDependenciesXml(); + + // // It is okay when node is null, we check that Version has value when we update paths by using MSTestInfo, and throw. + // // This way it throws in the body of the test which has better error reporting than throwing in the data source. + // XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/TranslationLayer{translationLayerVersion}Version"); + // var version = node?.InnerText.Replace("[", "").Replace("]", ""); + // var slash = Path.DirectorySeparatorChar; + // var dllPath = $"{slash}bin{slash}MSTest{translationLayerVersion}-{version}{slash}"; + + // return new TranslationLayerInfo(dllPath, version, versionSpecificBinPath); + //} + + private static XmlDocument GetDependenciesXml() + { + if (_depsXml != null) + return _depsXml; + + var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); + var fileStream = File.OpenRead(depsXmlPath); + var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false }; + var depsXml = new XmlDocument(); + depsXml.Load(xmlTextReader); + + _depsXml = depsXml; + return depsXml; + } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index c891784174..482af33303 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -40,15 +40,17 @@ public void Cleanup() } [TestMethod] - [NetFullTargetFrameworkDataSource] - [NetCoreTargetFrameworkDataSource] - public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo) + [TranslationLayerCompatibilityDataSource()] + public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - Setup(); + // Setup(); + _discoveryEventHandler = new DiscoveryEventHandler(); + _discoveryEventHandler2 = new DiscoveryEventHandler2(); - _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), _discoveryEventHandler); + var vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), _discoveryEventHandler); // Assert. Assert.AreEqual(6, _discoveryEventHandler.DiscoveredTestCases.Count); diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index a5d8ca3b5a..1399274b77 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -555,7 +555,7 @@ protected virtual string SetVSTestConsoleDLLPathInArgs(string args) /// public IVsTestConsoleWrapper GetVsTestConsoleWrapper() { - return GetVsTestConsoleWrapper(TempDirectory); + return GetVsTestConsoleWrapper(TempDirectory, null); } /// @@ -564,21 +564,24 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper() public IVsTestConsoleWrapper GetVsTestConsoleWrapper(out TempDirectory logFileDir) { logFileDir = new TempDirectory(); - return GetVsTestConsoleWrapper(logFileDir); + return GetVsTestConsoleWrapper(logFileDir, null); } /// /// Returns the VsTestConsole Wrapper. /// - /// - public IVsTestConsoleWrapper GetVsTestConsoleWrapper(TempDirectory logFileDir) + public IVsTestConsoleWrapper GetVsTestConsoleWrapper(TempDirectory tempDirectory) { - if (!Directory.Exists(logFileDir.Path)) - { - Directory.CreateDirectory(logFileDir.Path); - } + return GetVsTestConsoleWrapper(tempDirectory, vsTestConsoleInfo: null); + } - // Directory is already unique so there is no need to have a unique file name. + /// + /// Returns the VsTestConsole Wrapper. + /// + /// + public IVsTestConsoleWrapper GetVsTestConsoleWrapper(TempDirectory logFileDir, VSTestConsoleInfo vsTestConsoleInfo) + { + // Temp directory is already unique so there is no need to have a unique file name. var logFilePath = Path.Combine(logFileDir.Path, "log.txt"); if (!File.Exists(logFilePath)) { @@ -587,9 +590,11 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper(TempDirectory logFileDir) Console.WriteLine($"Logging diagnostics in {logFilePath}"); - var consoleRunnerPath = IsNetCoreRunner() - ? Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll") - : GetConsoleRunnerPath(); + var consoleRunnerPath = vsTestConsoleInfo != null + ? vsTestConsoleInfo.Path + : IsNetCoreRunner() + ? Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll") + : GetConsoleRunnerPath(); var executablePath = IsWindows ? @"dotnet\dotnet.exe" : @"dotnet-linux/dotnet"; var dotnetPath = Path.Combine(_testEnvironment.ToolsDirectory, executablePath); @@ -598,6 +603,12 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper(TempDirectory logFileDir) throw new FileNotFoundException($"File '{dotnetPath}' was not found."); } + if (!File.Exists(consoleRunnerPath)) + { + throw new FileNotFoundException($"File '{consoleRunnerPath}' was not found."); + } + + Console.WriteLine($"Console runner path: {consoleRunnerPath}"); var vstestConsoleWrapper = new VsTestConsoleWrapper(consoleRunnerPath, dotnetPath, new ConsoleParameters() { LogFilePath = logFilePath }); vstestConsoleWrapper.StartSession(); diff --git a/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs index 55453ed196..132d06a8c5 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs @@ -29,6 +29,7 @@ public string UpdatePath(string path) if (Version == null) throw new InvalidOperationException($"Version was not correctly populated from TestPlatform.Dependencies.props, review that there is entry for MSTestFramework{VersionType}Version."); + // TODO: replacing in the result string is lame, but I am not going to fight 20 GetAssetFullPath method overloads right now return path.Replace($"{System.IO.Path.DirectorySeparatorChar}bin{System.IO.Path.DirectorySeparatorChar}", Path); } } diff --git a/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs new file mode 100644 index 0000000000..d88ad20375 --- /dev/null +++ b/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.TestUtilities; + +public class VSTestConsoleInfo +{ + public VSTestConsoleInfo(string versionType, string? version, string path) + { + VersionType = versionType; + // Version can be null when we fail to find the respective propertin in TestPlatform.Dependencies.props + // when that happens we throw when we try to update the path. + Version = version; + Path = path; + } + + public string VersionType { get; } + public string? Version { get; } + public string Path { get; } + + public override string ToString() => $" vstest.console = {Version} [{VersionType}]"; +} + From 0fc39cf263e3147cf8e7fe51282d65b9b297cd1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 11 Mar 2022 05:54:21 +0100 Subject: [PATCH 28/64] Second test --- .../TranslationLayerTests/DiscoverTests.cs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index 482af33303..a8452835a0 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -57,14 +57,17 @@ public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo, VSTe } [TestMethod] - [NetFullTargetFrameworkDataSource] - [NetCoreTargetFrameworkDataSource] - public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedOut(RunnerInfo runnerInfo) + [TranslationLayerCompatibilityDataSource()] + public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedOut(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - Setup(); + // Setup(); - _vstestConsoleWrapper.DiscoverTests( + _discoveryEventHandler = new DiscoveryEventHandler(); + _discoveryEventHandler2 = new DiscoveryEventHandler2(); + + var vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + vstestConsoleWrapper.DiscoverTests( GetTestAssemblies(), GetDefaultRunSettings(), new TestPlatformOptions() { CollectMetrics = false }, From aaecf7fddca17ff494ecf39a1c7610caf1067aa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 11 Mar 2022 07:55:05 +0100 Subject: [PATCH 29/64] Remove unused field --- .../Extension/TranslationLayerCompatibilityDataSource.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs index 2d1b637399..879fbe3830 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs @@ -18,7 +18,7 @@ public sealed class TranslationLayerCompatibilityDataSource : TestDataSource From 82cb607959539f87c618f7a260a1d4cab42e5e47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 11 Mar 2022 14:39:29 +0100 Subject: [PATCH 30/64] Translation layer x vstest.console --- scripts/build.ps1 | 76 ++++----- .../ConsoleParameters.cs | 7 - .../PublicAPI/PublicAPI.Shipped.txt | 4 +- .../PublicAPI/net451/PublicAPI.Shipped.txt | 2 - .../VsTestConsoleProcessManager.cs | 3 +- src/SettingsMigrator/ConsoleParameters.cs | 93 +++++++++++ .../DebugTests.cs | 76 +++++++++ .../{ => Extension}/IsExternalInit.cs | 0 .../MSTestCompatibilityDataSource.cs | 3 +- .../Extension/NetCoreRunner.cs | 2 +- .../NetCoreTargetFrameworkDataSource.cs | 2 +- .../NetFullTargetFrameworkDataSource.cs | 4 +- .../Extension/RunnnerInfo.cs | 9 +- ...TranslationLayerCompatibilityDataSource.cs | 52 +++++- ...rosoft.TestPlatform.AcceptanceTests.csproj | 2 + .../CustomTestHostLauncher.cs | 54 ------- .../CustomTestHostTests.cs | 151 ++++++++++++++++-- .../TranslationLayerTests/DiscoverTests.cs | 2 +- .../EventHandler/RunEventHandler.cs | 2 +- .../TranslationLayerTests/RunTests.cs | 11 +- .../RunTestsWithFilterTests.cs | 23 ++- .../IntegrationTestBase.cs | 38 ++++- 22 files changed, 464 insertions(+), 152 deletions(-) create mode 100644 src/SettingsMigrator/ConsoleParameters.cs create mode 100644 test/Microsoft.TestPlatform.AcceptanceTests/DebugTests.cs rename test/Microsoft.TestPlatform.AcceptanceTests/{ => Extension}/IsExternalInit.cs (100%) delete mode 100644 test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostLauncher.cs diff --git a/scripts/build.ps1 b/scripts/build.ps1 index 0b39b742d6..16160aab93 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -169,14 +169,14 @@ function Invoke-TestAssetsBuild try { Write-Log ".. .. Build: Source: $TPB_TestAssets_Solution -- add NuGet source" Invoke-Exe -IgnoreExitCode 1 $nugetExe -Arguments "sources add -Name ""locally-built-testplatform-packages"" -Source $env:TP_TESTARTIFACTS\packages\ -ConfigFile ""$nugetConfig""" - #Invoke-Exe $dotnetExe -Arguments "build $TPB_TestAssets_Solution --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -bl:""$env:TP_OUT_DIR\log\$Configuration\TestAssets.binlog""" + Invoke-Exe $dotnetExe -Arguments "build $TPB_TestAssets_Solution --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -bl:""$env:TP_OUT_DIR\log\$Configuration\TestAssets.binlog""" - # Compatibility matrix build + # Compatibility matrix build. $dependenciesPath = "$env:TP_ROOT_DIR\scripts\build\TestPlatform.Dependencies.props" $dependenciesXml = [xml](Get-Content -Raw -Encoding UTF8 $dependenciesPath) - # build with multiple versions of MSTest + # Build with multiple versions of MSTest. $projects = @( "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject\SimpleTestProject.csproj" "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject2\SimpleTestProject2.csproj" @@ -206,7 +206,7 @@ function Invoke-TestAssetsBuild } } - # restore previous versions of TestPlatform (for vstest.console.exe), and TestPlatform.CLI (for vstest.console.dll) + # Restore previous versions of TestPlatform (for vstest.console.exe), and TestPlatform.CLI (for vstest.console.dll). $versionProperties = @( "NETTestSdkVersion" "VSTestConsoleLatestPreviewVersion" @@ -257,7 +257,7 @@ function Invoke-TestAssetsBuild # the original one. if (-not (Test-Path -Path $cachePath) -or -not (Get-ChildItem $cachePath)) { Rename-Item "$packagePath\$package.$vsTestConsoleVersion" $cachePath - # nuget locks the locally copied package it seems. + # Nuget locks the locally copied package from time to time. Start-Sleep -Milliseconds 300 } if (Test-Path "$packagePath\$package.$vsTestConsoleVersion") { @@ -1296,42 +1296,42 @@ Get-ChildItem env: | Where-Object -FilterScript { $_.Name.StartsWith("TP_") } | Write-Log "Test platform build variables: " Get-Variable | Where-Object -FilterScript { $_.Name.StartsWith("TPB_") } | Format-Table -# if ($Force -or $Steps -contains "InstallDotnet") { -# Install-DotNetCli -# } - -# if ($Force -or $Steps -contains "Restore") { -# Clear-Package -# Restore-Package -# } - -# if ($Force -or $Steps -contains "UpdateLocalization") { -# Update-LocalizedResources -# } - -# if ($Force -or $Steps -contains "Build") { -# Invoke-Build -# } - -# if ($Force -or $Steps -contains "Publish") { -# Publish-Package -# Create-VsixPackage -# Create-NugetPackages -# } - -# if ($Force -or $Steps -contains "Publish" -or $Steps -contains "Manifest") { -# Generate-Manifest -PackageFolder $TPB_PackageOutDir -# if (Test-Path $TPB_SourceBuildPackageOutDir) -# { -# Generate-Manifest -PackageFolder $TPB_SourceBuildPackageOutDir -# } -# Copy-PackageIntoStaticDirectory -# } +if ($Force -or $Steps -contains "InstallDotnet") { + Install-DotNetCli +} + +if ($Force -or $Steps -contains "Restore") { + Clear-Package + Restore-Package +} + +if ($Force -or $Steps -contains "UpdateLocalization") { + Update-LocalizedResources +} + +if ($Force -or $Steps -contains "Build") { + Invoke-Build +} + +if ($Force -or $Steps -contains "Publish") { + Publish-Package + Create-VsixPackage + Create-NugetPackages +} + +if ($Force -or $Steps -contains "Publish" -or $Steps -contains "Manifest") { + Generate-Manifest -PackageFolder $TPB_PackageOutDir + if (Test-Path $TPB_SourceBuildPackageOutDir) + { + Generate-Manifest -PackageFolder $TPB_SourceBuildPackageOutDir + } + Copy-PackageIntoStaticDirectory +} if ($Force -or $Steps -contains "PrepareAcceptanceTests") { - #Publish-PatchedDotnet + Publish-PatchedDotnet Invoke-TestAssetsBuild - #Publish-Tests + Publish-Tests } if ($Script:ScriptFailed) { diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/ConsoleParameters.cs b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/ConsoleParameters.cs index a1abda01df..7386f07d06 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/ConsoleParameters.cs +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/ConsoleParameters.cs @@ -1,9 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -#if NETFRAMEWORK using System.Collections.Generic; -#endif using System.Diagnostics; using System.IO; @@ -41,16 +39,11 @@ public ConsoleParameters(IFileHelper fileHelper) _fileHelper = fileHelper; } -#if NETFRAMEWORK - /// - /// TODO: Remove the #if when project is targeted to netstandard2.0 /// Environment variables to be set for the process /// public Dictionary EnvironmentVariables { get; set; } -#endif - /// /// Trace level for logs. /// diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt index cb8161d464..e9c4b6ee04 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt @@ -196,4 +196,6 @@ static Microsoft.VisualStudio.TestPlatform.VsTestConsole.TranslationLayer.Resour static Microsoft.VisualStudio.TestPlatform.VsTestConsole.TranslationLayer.Resources.Resources.InvalidFilePath.get -> string static Microsoft.VisualStudio.TestPlatform.VsTestConsole.TranslationLayer.Resources.Resources.ResourceManager.get -> System.Resources.ResourceManager static Microsoft.VisualStudio.TestPlatform.VsTestConsole.TranslationLayer.Resources.Resources.VsTestProcessExitedAbnormally.get -> string -virtual Microsoft.TestPlatform.VsTestConsole.TranslationLayer.TestSession.Dispose(bool disposing) -> void \ No newline at end of file +virtual Microsoft.TestPlatform.VsTestConsole.TranslationLayer.TestSession.Dispose(bool disposing) -> void +Microsoft.TestPlatform.VsTestConsole.TranslationLayer.ConsoleParameters.EnvironmentVariables.get -> System.Collections.Generic.Dictionary +Microsoft.TestPlatform.VsTestConsole.TranslationLayer.ConsoleParameters.EnvironmentVariables.set -> void diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/net451/PublicAPI.Shipped.txt b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/net451/PublicAPI.Shipped.txt index 2b1c581fe8..e69de29bb2 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/net451/PublicAPI.Shipped.txt +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/net451/PublicAPI.Shipped.txt @@ -1,2 +0,0 @@ -Microsoft.TestPlatform.VsTestConsole.TranslationLayer.ConsoleParameters.EnvironmentVariables.get -> System.Collections.Generic.Dictionary -Microsoft.TestPlatform.VsTestConsole.TranslationLayer.ConsoleParameters.EnvironmentVariables.set -> void \ No newline at end of file diff --git a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleProcessManager.cs b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleProcessManager.cs index 48b804836e..323ff2d95a 100644 --- a/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleProcessManager.cs +++ b/src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleProcessManager.cs @@ -121,7 +121,6 @@ public void StartProcess(ConsoleParameters consoleParameters) EqtTrace.Verbose("VsTestCommandLineWrapper: Process Start Info {0} {1}", info.FileName, info.Arguments); -#if NETFRAMEWORK if (consoleParameters.EnvironmentVariables != null) { info.EnvironmentVariables.Clear(); @@ -133,7 +132,7 @@ public void StartProcess(ConsoleParameters consoleParameters) } } } -#endif + try { _process = Process.Start(info); diff --git a/src/SettingsMigrator/ConsoleParameters.cs b/src/SettingsMigrator/ConsoleParameters.cs new file mode 100644 index 0000000000..80470abdff --- /dev/null +++ b/src/SettingsMigrator/ConsoleParameters.cs @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +#if NETFRAMEWORK +using System.Collections.Generic; +#endif +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; + +using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Extensions; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers; +using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; + +#nullable disable + +namespace Microsoft.TestPlatform.VsTestConsole.TranslationLayer; + +/// +/// Class which defines additional specifiable parameters for vstest.console.exe +/// +public class ConsoleParameters +{ + internal static readonly ConsoleParameters Default = new(); + + private string _logFilePath; + private readonly IFileHelper _fileHelper; + + /// + /// Create instance of + /// + public ConsoleParameters() : this(new FileHelper()) + { } + + /// + /// Create instance of + /// + /// Object of type + public ConsoleParameters(IFileHelper fileHelper) + { + _fileHelper = fileHelper; + } + + /// + /// Environment variables to be set for the process + /// + public Dictionary EnvironmentVariables { get; set; } + + /// + /// Trace level for logs. + /// + public TraceLevel TraceLevel { get; set; } = TraceLevel.Verbose; + + /// + /// Full path for the log file + /// + public string LogFilePath + { + get + { + return _logFilePath; + } + + set + { + ValidateArg.NotNullOrEmpty(value, "LogFilePath"); + var directoryPath = Path.GetDirectoryName(value); + if (!string.IsNullOrEmpty(directoryPath) && !_fileHelper.DirectoryExists(directoryPath)) + { + Directory.CreateDirectory(directoryPath); + } + + // Ensure path is double quoted. if path has white space then it can create problem. + _logFilePath = value.AddDoubleQuote(); + } + } + + /// + /// Port Number for communication + /// vstest.console will need this port number to communicate with this component - translation layer + /// Currently Internal as we are not intentionally exposing this to consumers of translation layer + /// + internal int PortNumber { get; set; } + + /// + /// Parent Process ID of the process whose lifetime should dictate the life time of vstest.console.exe + /// vstest.console will need this process ID to know when the process exits. + /// If parent process dies/crashes without invoking EndSession, vstest.console should exit immediately + /// Currently Internal as we are not intentionally exposing this to consumers of translation layer + /// + internal int ParentProcessId { get; set; } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DebugTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DebugTests.cs new file mode 100644 index 0000000000..57898375ff --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DebugTests.cs @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.Linq; + +using Microsoft.TestPlatform.TestUtilities; +using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +#nullable disable + +namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests; + +/// +/// The Run Tests using VsTestConsoleWrapper API's +/// +[TestClass] +public class DebugTests : AcceptanceTestBase +{ + private IVsTestConsoleWrapper _vstestConsoleWrapper; + + [TestCleanup] + public void Cleanup() + { + _vstestConsoleWrapper?.EndSession(); + } + + [TestMethod] + [TranslationLayerCompatibilityDataSource("net451", "net451", "LegacyStable")] + public void AttachDebugger(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) + { + SetTestEnvironment(_testEnvironment, runnerInfo); + // Setup(); + + var runEventHandler = new AttachDebuggerRunEventHandler(attachDebuggerToProcessResponse: true); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + _vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetDefaultRunSettings(), runEventHandler); + + // Assert + Assert.AreEqual(6, runEventHandler.TestResults.Count); + Assert.AreEqual(2, runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed)); + Assert.AreEqual(2, runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed)); + Assert.AreEqual(2, runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped)); + } + + private IList GetTestAssemblies() + { + var testAssemblies = new List + { + GetAssetFullPath("SimpleTestProject.dll"), + GetAssetFullPath("SimpleTestProject2.dll") + }; + + return testAssemblies; + } + + class AttachDebuggerRunEventHandler : RunEventHandler + { + private readonly bool _attachDebuggerToProcessResult; + + public AttachDebuggerRunEventHandler(bool attachDebuggerToProcessResponse) + { + _attachDebuggerToProcessResult = attachDebuggerToProcessResponse; + } + + public int Pid { get; private set; } + + public override bool AttachDebuggerToProcess(int pid) + { + Pid = pid; + return _attachDebuggerToProcessResult; + } + } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/IsExternalInit.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/IsExternalInit.cs similarity index 100% rename from test/Microsoft.TestPlatform.AcceptanceTests/IsExternalInit.cs rename to test/Microsoft.TestPlatform.AcceptanceTests/Extension/IsExternalInit.cs diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs index b491bda22e..a032a26649 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs @@ -2,7 +2,6 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; -using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; @@ -54,7 +53,7 @@ public override void CreateData(MethodInfo methodInfo) { foreach (var msTestVersion in _msTestVersions) { - var runnerInfo = new RunnerInfo(AcceptanceTestBase.DEFAULT_RUNNER_NETFX, AcceptanceTestBase.DEFAULT_RUNNER_NETFX, inIsolation: null, + var runnerInfo = new RunnerInfo(AcceptanceTestBase.DEFAULT_RUNNER_NETFX, AcceptanceTestBase.DEFAULT_RUNNER_NETFX, InIsolationValue: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); var msTestInfo = GetMSTestInfo(msTestVersion); // We run in the .NET Framework runner process, the runner and target framework must agree. diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs index 86f2b05b33..1a3d855e29 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs @@ -47,7 +47,7 @@ public IEnumerable GetData(MethodInfo methodInfo) Func filter = tfm => isWindows || !tfm.StartsWith("net4"); foreach (var fmw in _targetFrameworks.Split(';').Where(filter)) { - dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, fmw, inIsolation: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); + dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, fmw, InIsolationValue: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); } return dataRows; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs index 786b477bbc..6788d56074 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs @@ -55,7 +55,7 @@ public NetCoreTargetFrameworkDataSource( private void AddRunnerDataRow(List dataRows, string runnerFramework, string targetFramework) { - var runnerInfo = new RunnerInfo(runnerFramework, targetFramework, inIsolation: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); + var runnerInfo = new RunnerInfo(runnerFramework, targetFramework, InIsolationValue: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); dataRows.Add(new object[] { runnerInfo }); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs index a18eeb8271..c3b0af0e6c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs @@ -53,7 +53,7 @@ public IEnumerable GetData(MethodInfo methodInfo) var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); if (_useCoreRunner && isWindows) { - dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, inIsolation: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); + dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, InIsolationValue: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); } if (_useDesktopRunner && isWindows) @@ -65,7 +65,7 @@ public IEnumerable GetData(MethodInfo methodInfo) if (_inProcess) { - dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, inIsolation: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); + dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, InIsolationValue: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs index 7bbe7ce046..c6f67359da 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs @@ -11,9 +11,14 @@ namespace Microsoft.TestPlatform.AcceptanceTests; /// /// /// Supported value = /InIsolation. -public record RunnerInfo(string RunnerFramework, string TargetFramework, string InIsolationValue = "", - bool DebugVSTestConsole = false, bool DebugTestHost = false, bool DebugDataCollector = false, bool NoDefaultBreakpoints = true) +public record RunnerInfo(string RunnerFramework, string TargetFramework, string? InIsolationValue = "", + bool DebugVSTestConsole = false, bool DebugTesthost = false, bool DebugDataCollector = false, bool NoDefaultBreakpoints = true) { + /// + /// Is running via .NET "Core" vstest.console? + /// + public bool IsNetRunner => !IsNetFrameworkRunner; + /// /// Is running via .NET Framework vstest.console? /// diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs index 879fbe3830..fc5149e0d4 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; @@ -10,6 +11,7 @@ using Microsoft.TestPlatform.TestUtilities; +using Semver; namespace Microsoft.TestPlatform.AcceptanceTests; @@ -28,12 +30,13 @@ public sealed class TranslationLayerCompatibilityDataSource : TestDataSource filter = tfm => isWindows || !tfm.StartsWith("net4"); - + // TODO: maybe we should throw if we don't end up generating any data + // because none of the versions match, or some other way to identify tests that will never run because they are very outdated. + // We probably don't have that need right now, because legacy version is 15.x.x, which is very old, and we are still keeping + // compatibility. foreach (var runner in _runnerFrameworks.Where(filter)) { foreach (var fmw in _targetFrameworks.Where(filter)) @@ -61,6 +89,12 @@ public override void CreateData(MethodInfo methodInfo) DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); var vsTestConsoleInfo = GetVSTestConsoleInfo(vstestConsoleVersion, runnerInfo); + if (beforeVersion != null && vsTestConsoleInfo.Version > beforeVersion) + continue; + + if (afterVersion != null && vsTestConsoleInfo.Version < afterVersion) + continue; + AddData(runnerInfo, vsTestConsoleInfo); } } @@ -138,3 +172,15 @@ private static XmlDocument GetDependenciesXml() return depsXml; } } + +public readonly record struct Feature(string Version, string Issue); + +public static class Features +{ + public const string ATTACH_DEBUGGER = nameof(ATTACH_DEBUGGER); + + public static Dictionary Table { get; } = new Dictionary + { + [ATTACH_DEBUGGER] = new(Version: "v16.7.0-preview-20200519-01", Issue: "https://github.com/microsoft/vstest/pull/2325") + }; +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj index c5a1f80019..c33244d042 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj @@ -28,6 +28,7 @@ + all runtime; build; native; contentfiles; analyzers; buildtransitive @@ -35,6 +36,7 @@ + diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostLauncher.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostLauncher.cs deleted file mode 100644 index e7fe325916..0000000000 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostLauncher.cs +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System.Diagnostics; -using System.Threading; - -using Microsoft.VisualStudio.TestPlatform.ObjectModel; -using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; - -#nullable disable - -namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests; - -/// -/// The custom test host launcher. -/// -public class CustomTestHostLauncher : ITestHostLauncher2 -{ - public int ProcessId - { - get; - private set; - } - - /// - public bool IsDebug => true; - - public bool AttachDebuggerToProcess(int pid) => AttachDebuggerToProcess(pid, CancellationToken.None); - - public bool AttachDebuggerToProcess(int pid, CancellationToken cancellationToken) => true; - - /// - public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo) - { - return LaunchTestHost(defaultTestHostStartInfo, CancellationToken.None); - } - - /// - public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo, CancellationToken cancellationToken) - { - var processInfo = new ProcessStartInfo( - defaultTestHostStartInfo.FileName, - defaultTestHostStartInfo.Arguments) - { - WorkingDirectory = defaultTestHostStartInfo.WorkingDirectory - }; - processInfo.UseShellExecute = false; - - var process = new Process { StartInfo = processInfo }; - process.Start(); - - return process != null ? process.Id : -1; - } -} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs index 829b2e7eda..646fda4dbf 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs @@ -2,10 +2,16 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Collections.Generic; +using System.Diagnostics; using System.Linq; +using System.Threading; +using FluentAssertions; + +using Microsoft.TestPlatform.TestUtilities; using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; using Microsoft.VisualStudio.TestTools.UnitTesting; #nullable disable @@ -21,36 +27,99 @@ public class CustomTestHostTests : AcceptanceTestBase private IVsTestConsoleWrapper _vstestConsoleWrapper; private RunEventHandler _runEventHandler; - private void Setup() + [TestCleanup] + public void Cleanup() + { + _vstestConsoleWrapper?.EndSession(); + } + + [TestMethod] + [TranslationLayerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] + public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncher(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _); + SetTestEnvironment(_testEnvironment, runnerInfo); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); _runEventHandler = new RunEventHandler(); + + var customTestHostLauncher = new TestHostLauncherV1(); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler, customTestHostLauncher); + + // Assert + customTestHostLauncher.Should().BeAssignableTo(); + customTestHostLauncher.LaunchProcessProcessId.Should().NotBeNull("we should launch some real process and save the pid of it"); + + _runEventHandler.TestResults.Should().HaveCount(6); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed).Should().Be(2); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed).Should().Be(2); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped).Should().Be(2); } - [TestCleanup] - public void Cleanup() + [TestMethod] + [TranslationLayerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] + public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncherWhenITestHostLauncher2IsProvided(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { - _vstestConsoleWrapper?.EndSession(); + SetTestEnvironment(_testEnvironment, runnerInfo); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + _runEventHandler = new RunEventHandler(); + + var customTestHostLauncher = new TestHostLauncherV2(); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler, customTestHostLauncher); + + // Assert + customTestHostLauncher.Should().BeAssignableTo(); + customTestHostLauncher.LaunchProcessProcessId.Should().NotBeNull("we should launch some real process and save the pid of it"); + customTestHostLauncher.AttachDebuggerProcessId.Should().BeNull("we should not be asked to attach to a debugger, that flow is not used when vstest.console does not support it yet, even when it is given ITestHostLauncher2"); + + _runEventHandler.TestResults.Should().HaveCount(6); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed).Should().Be(2); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed).Should().Be(2); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped).Should().Be(2); } + [TestMethod] + [TranslationLayerCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER)] + public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvidedLauncher(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) + { + SetTestEnvironment(_testEnvironment, runnerInfo); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + _runEventHandler = new RunEventHandler(); + + var customTestHostLauncher = new TestHostLauncherV2(); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler, customTestHostLauncher); + + // Assert + customTestHostLauncher.Should().BeAssignableTo(); + customTestHostLauncher.AttachDebuggerProcessId.Should().NotBeNull("we should be asked to attach a debugger to some process and save the pid of the process"); + customTestHostLauncher.LaunchProcessProcessId.Should().BeNull("we should not be asked to launch some real process, that flow is not used when vstest.console supports it and is given ITestHostLauncher2"); + + _runEventHandler.TestResults.Should().HaveCount(6); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed).Should().Be(2); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed).Should().Be(2); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped).Should().Be(2); + } [TestMethod] - [NetFullTargetFrameworkDataSource] - [NetCoreTargetFrameworkDataSource] - public void RunTestsWithCustomTestHostLaunch(RunnerInfo runnerInfo) + [Ignore("This is not working. The compatibility code only checks the protocol version (in handler), which is dictated by testhost. " + + "It sees 6 but does not realize that the provided CustomTesthostLauncher is not supporting the new feature, it ends up calling back to EditoAttachDebugger" + + "in translation layer, and that just silently skips the call.")] + [TranslationLayerCompatibilityDataSource("net451", "net451", "Latest", AfterFeature = Features.ATTACH_DEBUGGER, DebugVSTestConsole = true, DebugTesthost=true)] + public void RunTestsWithCustomTestHostLauncherUsesLaunchWhenGivenAnOutdatedITestHostLauncher(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - Setup(); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + _runEventHandler = new RunEventHandler(); - var customTestHostLauncher = new CustomTestHostLauncher(); + var customTestHostLauncher = new TestHostLauncherV1(); _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler, customTestHostLauncher); // Assert - Assert.AreEqual(6, _runEventHandler.TestResults.Count); - Assert.IsTrue(customTestHostLauncher.ProcessId != -1); - Assert.AreEqual(2, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed)); - Assert.AreEqual(2, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed)); - Assert.AreEqual(2, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped)); + customTestHostLauncher.Should().NotBeAssignableTo(); + customTestHostLauncher.LaunchProcessProcessId.Should().NotBeNull("we should launch some real process and save the pid of it"); + + _runEventHandler.TestResults.Should().HaveCount(6); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed).Should().Be(2); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed).Should().Be(2); + _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped).Should().Be(2); } private IList GetTestAssemblies() @@ -63,4 +132,56 @@ private IList GetTestAssemblies() return testAssemblies; } + + /// + /// The custom test host launcher implementing ITestHostLauncher. + /// + public class TestHostLauncherV1 : ITestHostLauncher + { + public int? LaunchProcessProcessId { get; private set; } + + /// + public bool IsDebug => true; + + /// + public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo) + { + return LaunchTestHost(defaultTestHostStartInfo, CancellationToken.None); + } + + /// + public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo, CancellationToken cancellationToken) + { + var processInfo = new ProcessStartInfo( + defaultTestHostStartInfo.FileName, + defaultTestHostStartInfo.Arguments) + { + WorkingDirectory = defaultTestHostStartInfo.WorkingDirectory + }; + processInfo.UseShellExecute = false; + + var process = new Process { StartInfo = processInfo }; + process.Start(); + + LaunchProcessProcessId = process?.Id; + return LaunchProcessProcessId ?? -1; + } + } + + /// + /// The custom test host launcher implementing ITestHostLauncher2. + /// + public class TestHostLauncherV2 : TestHostLauncherV1, ITestHostLauncher2 + { + + public int? AttachDebuggerProcessId { get; private set; } + + public bool AttachDebuggerToProcess(int pid) => AttachDebuggerToProcess(pid, CancellationToken.None); + + public bool AttachDebuggerToProcess(int pid, CancellationToken cancellationToken) + { + AttachDebuggerProcessId = pid; + return true; + } + } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index a8452835a0..3fd97f543e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -40,7 +40,7 @@ public void Cleanup() } [TestMethod] - [TranslationLayerCompatibilityDataSource()] + [TranslationLayerCompatibilityDataSource] public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/RunEventHandler.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/RunEventHandler.cs index b02ecf0730..848dff8996 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/RunEventHandler.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/EventHandler/RunEventHandler.cs @@ -117,7 +117,7 @@ public int LaunchProcessWithDebuggerAttached(TestProcessStartInfo testProcessSta return -1; } - public bool AttachDebuggerToProcess(int pid) + public virtual bool AttachDebuggerToProcess(int pid) { // No op return true; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs index f8b15f77c9..4c4197078e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs @@ -40,14 +40,15 @@ public void Cleanup() } [TestMethod] - [NetFullTargetFrameworkDataSource] - [NetCoreTargetFrameworkDataSource] - public void RunAllTests(RunnerInfo runnerInfo) + [TranslationLayerCompatibilityDataSource] + public void RunAllTests(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - Setup(); + // Setup(); - _vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler); + var vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + _runEventHandler = new RunEventHandler(); + vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler); // Assert Assert.AreEqual(6, _runEventHandler.TestResults.Count); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs index 694f23409f..010e916d5b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Linq; +using Microsoft.TestPlatform.TestUtilities; using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; @@ -35,19 +36,18 @@ public void Cleanup() } [TestMethod] - [NetFullTargetFrameworkDataSource] - [NetCoreTargetFrameworkDataSource] - public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo) + [TranslationLayerCompatibilityDataSource] + public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - Setup(); + // Setup(); - var sources = new List - { - GetAssetFullPath("SimpleTestProject.dll") - }; + _runEventHandler = new RunEventHandler(); - _vstestConsoleWrapper.RunTests( + var vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + var sources = new List { GetAssetFullPath("SimpleTestProject.dll") }; + + vstestConsoleWrapper.RunTests( sources, GetDefaultRunSettings(), new TestPlatformOptions() { TestCaseFilter = "FullyQualifiedName=SampleUnitTestProject.UnitTest1.PassingTest" }, @@ -66,10 +66,7 @@ public void RunTestsWithFastFilter(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - var sources = new List - { - GetAssetFullPath("SimpleTestProject.dll") - }; + var sources = new List { GetAssetFullPath("SimpleTestProject.dll") }; _vstestConsoleWrapper.RunTests( sources, diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 1399274b77..1b309be65c 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +using System.Collections; using System.Collections.Generic; using System.Diagnostics; using System.IO; @@ -605,11 +606,44 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper(TempDirectory logFileDir, V if (!File.Exists(consoleRunnerPath)) { - throw new FileNotFoundException($"File '{consoleRunnerPath}' was not found."); + throw new FileNotFoundException($"File '{consoleRunnerPath}' was not found.") ; } Console.WriteLine($"Console runner path: {consoleRunnerPath}"); - var vstestConsoleWrapper = new VsTestConsoleWrapper(consoleRunnerPath, dotnetPath, new ConsoleParameters() { LogFilePath = logFilePath }); + + VsTestConsoleWrapper vstestConsoleWrapper; + if (_testEnvironment.DebugVSTestConsole || _testEnvironment.DebugTesthost || _testEnvironment.DebugDataCollector) + { + var environmentVariables = new Dictionary(); + Environment.GetEnvironmentVariables().OfType().ToList().ForEach(e=> environmentVariables.Add(e.Key.ToString(), e.Value.ToString())); + + if (_testEnvironment.DebugVSTestConsole) + { + environmentVariables.Add("VSTEST_RUNNER_DEBUG_ATTACHVS", "1"); + } + + if (_testEnvironment.DebugTesthost) + { + environmentVariables.Add("VSTEST_HOST_DEBUG_ATTACHVS", "1"); + } + + if (_testEnvironment.DebugDataCollector) + { + environmentVariables.Add("VSTEST_DATACOLLECTOR_DEBUG_ATTACHVS", "1"); + } + + if (_testEnvironment.NoDefaultBreakpoints) + { + environmentVariables.Add("VSTEST_DEBUG_NOBP", "1"); + } + + // This clears all variables + vstestConsoleWrapper = new VsTestConsoleWrapper(consoleRunnerPath, dotnetPath, new ConsoleParameters() { LogFilePath = logFilePath, EnvironmentVariables = environmentVariables }); + } + else + { + vstestConsoleWrapper = new VsTestConsoleWrapper(consoleRunnerPath, dotnetPath, new ConsoleParameters() { LogFilePath = logFilePath }); + } vstestConsoleWrapper.StartSession(); return vstestConsoleWrapper; From 07cdf1c87f0a4c247a476b5b5b8f2b36bbad59b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 14 Mar 2022 13:37:19 +0100 Subject: [PATCH 31/64] Fixup console run --- scripts/build.ps1 | 171 ++++++++++-------- scripts/build/TestPlatform.Dependencies.props | 2 +- .../AcceptanceTestBase.cs | 6 +- .../DebugTests.cs | 76 -------- .../ExecutionTests.cs | 14 ++ .../MSTestCompatibilityDataSource.cs | 3 +- .../Extension/TestDataSource.cs | 31 ++++ .../TesthostCompatibilityDataSource.cs | 139 ++++++++++++++ ...TranslationLayerCompatibilityDataSource.cs | 12 +- .../CodeCoverageTests.cs | 6 +- .../CustomTestHostTests.cs | 16 +- .../DifferentTestFrameworkSimpleTests.cs | 2 +- .../TranslationLayerTests/DiscoverTests.cs | 10 +- .../LiveUnitTestingTests.cs | 2 +- .../TranslationLayerTests/RunSelectedTests.cs | 2 +- .../TranslationLayerTests/RunTests.cs | 25 ++- ...RunTestsWithDifferentConfigurationTests.cs | 4 +- .../RunTestsWithFilterTests.cs | 6 +- .../DllInfo.cs | 39 ++++ .../IntegrationTestBase.cs | 62 +++---- .../IntegrationTestEnvironment.cs | 1 + .../MSTestInfo.cs | 27 +-- .../TesthostInfo.cs | 12 ++ 23 files changed, 412 insertions(+), 256 deletions(-) delete mode 100644 test/Microsoft.TestPlatform.AcceptanceTests/DebugTests.cs create mode 100644 test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs create mode 100644 test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs create mode 100644 test/Microsoft.TestPlatform.TestUtilities/TesthostInfo.cs diff --git a/scripts/build.ps1 b/scripts/build.ps1 index 16160aab93..2a27c513e4 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -175,40 +175,10 @@ function Invoke-TestAssetsBuild $dependenciesPath = "$env:TP_ROOT_DIR\scripts\build\TestPlatform.Dependencies.props" $dependenciesXml = [xml](Get-Content -Raw -Encoding UTF8 $dependenciesPath) - - # Build with multiple versions of MSTest. - $projects = @( - "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject\SimpleTestProject.csproj" - "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject2\SimpleTestProject2.csproj" - ) - - $versionProperties = @( - "MSTestFrameworkLatestPreviewVersion" - "MSTestFrameworkLatestStableVersion" - "MSTestFrameworkRecentStableVersion" - "MSTestFrameworkMostDownloadedVersion" - "MSTestFrameworkPreviousStableVersion" - "MSTestFrameworkLegacyStableVersion" - ) - - foreach ($project in $projects) { - foreach ($propertyName in $versionProperties) { - $mstestVersion = $dependenciesXml.Project.PropertyGroup.$propertyName - - if (-not $mstestVersion) - { - throw "MSTestVersion for $propertyName is empty." - } - - $dirVersion = $mstestVersion -replace "\[|\]" - $dirPropertyName = $propertyName -replace "Framework" -replace "Version" - Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:MSTestFrameworkVersion=$mstestVersion -p:MSTestAdapterVersion=$mstestVersion -p:BaseOutputPath=""bin\$dirPropertyName-$dirVersion\\""" - } - } - # Restore previous versions of TestPlatform (for vstest.console.exe), and TestPlatform.CLI (for vstest.console.dll). - $versionProperties = @( - "NETTestSdkVersion" + # These properties are coming from TestPlatform.Dependencies.props. + $vstestConsoleVersionProperties = @( + "VSTestConsoleLatestVersion" "VSTestConsoleLatestPreviewVersion" "VSTestConsoleLatestStableVersion" "VSTestConsoleRecentStableVersion" @@ -217,7 +187,7 @@ function Invoke-TestAssetsBuild "VSTestConsoleLegacyStableVersion" ) - foreach ($propertyName in $versionProperties) { + foreach ($propertyName in $vstestConsoleVersionProperties) { if ("VSTestConsoleLatestVersion" -eq $propertyName) { # NETTestSdkVersion has the version of the locally built package. $vsTestConsoleVersion = $dependenciesXml.Project.PropertyGroup."NETTestSdkVersion" @@ -266,6 +236,59 @@ function Invoke-TestAssetsBuild } } + + # Build with multiple versions of MSTest. + $projects = @( + "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject\SimpleTestProject.csproj" + "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject2\SimpleTestProject2.csproj" + ) + + $msTestVersionProperties = @( + "MSTestFrameworkLatestPreviewVersion" + "MSTestFrameworkLatestStableVersion" + "MSTestFrameworkRecentStableVersion" + "MSTestFrameworkMostDownloadedVersion" + "MSTestFrameworkPreviousStableVersion" + "MSTestFrameworkLegacyStableVersion" + ) + + foreach ($project in $projects) { + foreach ($propertyName in $msTestVersionProperties) { + $mstestVersion = $dependenciesXml.Project.PropertyGroup.$propertyName + + if (-not $mstestVersion) + { + throw "MSTestVersion for $propertyName is empty." + } + + $dirMSTestVersion = $mstestVersion -replace "\[|\]" + $dirMSTestPropertyName = $propertyName -replace "Framework" -replace "Version" + Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:MSTestFrameworkVersion=$mstestVersion -p:MSTestAdapterVersion=$mstestVersion -p:BaseOutputPath=""bin\$dirMSTestPropertyName-$dirMSTestVersion\\""" + } + } + + foreach ($project in $projects) { + # We use the same version properties for NET.Test.Sdk as for VSTestConsole, for now. + foreach ($propertyName in $vstestConsoleVersionProperties) { + if ("VSTestConsoleLatestVersion" -eq $propertyName) { + # NETTestSdkVersion has the version of the locally built package. + $netTestSdkVersion = $dependenciesXml.Project.PropertyGroup."NETTestSdkVersion" + } + else { + $netTestSdkVersion = $dependenciesXml.Project.PropertyGroup.$propertyName + } + + if (-not $netTestSdkVersion) + { + throw "NetTestSdkVersion for $propertyName is empty." + } + + $dirNetTestSdkVersion = $netTestSdkVersion -replace "\[|\]" + $dirNetTestSdkPropertyName = $propertyName -replace "Framework" -replace "Version" -replace "VSTestConsole", "NETTestSdk" + Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:NETTestSdkVersion=$netTestSdkVersion -p:BaseOutputPath=""bin\$dirNetTestSdkPropertyName-$dirNetTestSdkVersion\\""" + } + } + # end } finally { @@ -1289,49 +1312,49 @@ if ($ProjectNamePatterns.Count -ne 0) } # Execute build -$timer = Start-Timer -Write-Log "Build started: args = '$args'" -Write-Log "Test platform environment variables: " -Get-ChildItem env: | Where-Object -FilterScript { $_.Name.StartsWith("TP_") } | Format-Table -Write-Log "Test platform build variables: " -Get-Variable | Where-Object -FilterScript { $_.Name.StartsWith("TPB_") } | Format-Table - -if ($Force -or $Steps -contains "InstallDotnet") { - Install-DotNetCli -} - -if ($Force -or $Steps -contains "Restore") { - Clear-Package - Restore-Package -} - -if ($Force -or $Steps -contains "UpdateLocalization") { - Update-LocalizedResources -} - -if ($Force -or $Steps -contains "Build") { - Invoke-Build -} - -if ($Force -or $Steps -contains "Publish") { - Publish-Package - Create-VsixPackage - Create-NugetPackages -} - -if ($Force -or $Steps -contains "Publish" -or $Steps -contains "Manifest") { - Generate-Manifest -PackageFolder $TPB_PackageOutDir - if (Test-Path $TPB_SourceBuildPackageOutDir) - { - Generate-Manifest -PackageFolder $TPB_SourceBuildPackageOutDir - } - Copy-PackageIntoStaticDirectory -} +# $timer = Start-Timer +# Write-Log "Build started: args = '$args'" +# Write-Log "Test platform environment variables: " +# Get-ChildItem env: | Where-Object -FilterScript { $_.Name.StartsWith("TP_") } | Format-Table +# Write-Log "Test platform build variables: " +# Get-Variable | Where-Object -FilterScript { $_.Name.StartsWith("TPB_") } | Format-Table + +# if ($Force -or $Steps -contains "InstallDotnet") { +# Install-DotNetCli +# } + +# if ($Force -or $Steps -contains "Restore") { +# Clear-Package +# Restore-Package +# } + +# if ($Force -or $Steps -contains "UpdateLocalization") { +# Update-LocalizedResources +# } + +# if ($Force -or $Steps -contains "Build") { +# Invoke-Build +# } + +# if ($Force -or $Steps -contains "Publish") { +# Publish-Package +# Create-VsixPackage +# Create-NugetPackages +# } + +# if ($Force -or $Steps -contains "Publish" -or $Steps -contains "Manifest") { +# Generate-Manifest -PackageFolder $TPB_PackageOutDir +# if (Test-Path $TPB_SourceBuildPackageOutDir) +# { +# Generate-Manifest -PackageFolder $TPB_SourceBuildPackageOutDir +# } +# Copy-PackageIntoStaticDirectory +# } if ($Force -or $Steps -contains "PrepareAcceptanceTests") { - Publish-PatchedDotnet + #Publish-PatchedDotnet Invoke-TestAssetsBuild - Publish-Tests + #Publish-Tests } if ($Script:ScriptFailed) { diff --git a/scripts/build/TestPlatform.Dependencies.props b/scripts/build/TestPlatform.Dependencies.props index 30969e3253..03fd5958bd 100644 --- a/scripts/build/TestPlatform.Dependencies.props +++ b/scripts/build/TestPlatform.Dependencies.props @@ -46,7 +46,7 @@ See Invoke-TestAssetsBuild in scripts/build.ps1. Exact versions are used to avoid Nuget substituting them by closest match, if we make a typo. These versions need to be "statically" readable because we read this file as xml in our build and tests. --> - 17.2.0-preview-20220131-20 + [17.2.0-preview-20220131-20] [17.1.0] [17.0.0] [16.9.4] diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs index 09e2d9fa4c..48e1d98556 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs @@ -68,8 +68,12 @@ public static string And(string left, string right) protected string FrameworkArgValue => DeriveFrameworkArgValue(_testEnvironment); - protected static void SetTestEnvironment(IntegrationTestEnvironment testEnvironment, RunnerInfo runnerInfo) + protected static void SetTestEnvironment(IntegrationTestEnvironment testEnvironment, RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo = null) { + if (vsTestConsoleInfo != null) + { + testEnvironment.VSTestConsolePath = vsTestConsoleInfo.Path; + } testEnvironment.RunnerFramework = runnerInfo.RunnerFramework; testEnvironment.TargetFramework = runnerInfo.TargetFramework; testEnvironment.InIsolationValue = runnerInfo.InIsolationValue; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DebugTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DebugTests.cs deleted file mode 100644 index 57898375ff..0000000000 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DebugTests.cs +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System.Collections.Generic; -using System.Linq; - -using Microsoft.TestPlatform.TestUtilities; -using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; -using Microsoft.VisualStudio.TestPlatform.ObjectModel; -using Microsoft.VisualStudio.TestTools.UnitTesting; - -#nullable disable - -namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests; - -/// -/// The Run Tests using VsTestConsoleWrapper API's -/// -[TestClass] -public class DebugTests : AcceptanceTestBase -{ - private IVsTestConsoleWrapper _vstestConsoleWrapper; - - [TestCleanup] - public void Cleanup() - { - _vstestConsoleWrapper?.EndSession(); - } - - [TestMethod] - [TranslationLayerCompatibilityDataSource("net451", "net451", "LegacyStable")] - public void AttachDebugger(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) - { - SetTestEnvironment(_testEnvironment, runnerInfo); - // Setup(); - - var runEventHandler = new AttachDebuggerRunEventHandler(attachDebuggerToProcessResponse: true); - _vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); - _vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetDefaultRunSettings(), runEventHandler); - - // Assert - Assert.AreEqual(6, runEventHandler.TestResults.Count); - Assert.AreEqual(2, runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed)); - Assert.AreEqual(2, runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed)); - Assert.AreEqual(2, runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped)); - } - - private IList GetTestAssemblies() - { - var testAssemblies = new List - { - GetAssetFullPath("SimpleTestProject.dll"), - GetAssetFullPath("SimpleTestProject2.dll") - }; - - return testAssemblies; - } - - class AttachDebuggerRunEventHandler : RunEventHandler - { - private readonly bool _attachDebuggerToProcessResult; - - public AttachDebuggerRunEventHandler(bool attachDebuggerToProcessResponse) - { - _attachDebuggerToProcessResult = attachDebuggerToProcessResponse; - } - - public int Pid { get; private set; } - - public override bool AttachDebuggerToProcess(int pid) - { - Pid = pid; - return _attachDebuggerToProcessResult; - } - } -} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 9a7f2b275e..89ad03fdaa 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -32,6 +32,20 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, MSTestInfo msTestIn ExitCodeEquals(1); // failing tests } + [TestMethod] + [TesthostCompatibilityDataSource] + public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo, TesthostInfo testhostInfo) + { + SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + + var assemblyPaths = BuildMultipleAssemblyPath(testhostInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + + InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); + + ValidateSummaryStatus(2, 2, 2); + ExitCodeEquals(1); // failing tests + } + // TODO: This one mixes different frameworks, I can make it work, but it is worth it? We are going to test // the two respective versions together (e.g. latest xunit and latest mstest), but does using two different test diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs index a032a26649..0fe704ad78 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs @@ -46,7 +46,7 @@ public MSTestCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT public override void CreateData(MethodInfo methodInfo) { var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); - // Only run .NET Framework tests on Windows. + // Run .NET Framework tests only on Windows. Func filter = tfm => isWindows || !tfm.StartsWith("net4"); if (InProcess) @@ -111,3 +111,4 @@ private static XmlDocument GetDependenciesXml() return depsXml; } } + diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs index 104aee8d8a..2aeb5fa0c1 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs @@ -69,3 +69,34 @@ string ITestDataSource.GetDisplayName(MethodInfo methodInfo, object[] data) } } +[AttributeUsage(AttributeTargets.Method)] +public abstract class TestDataSource : Attribute, ITestDataSource + where T1 : notnull + where T2 : notnull + where T3 : notnull +{ + private readonly List _data = new(); + + public abstract void CreateData(MethodInfo methodInfo); + + public void AddData(T1 value1, T2 value2, T3 value3) + { + _data.Add(new object[] { value1, value2, value3 }); + } + + public virtual string GetDisplayName(MethodInfo methodInfo, T1 value1, T2 value2, T3 value3) + { + return $"{methodInfo.Name} ({value1}, {value2}, {value3})"; + } + + IEnumerable ITestDataSource.GetData(MethodInfo methodInfo) + { + CreateData(methodInfo); + return _data; + } + + string ITestDataSource.GetDisplayName(MethodInfo methodInfo, object[] data) + { + return GetDisplayName(methodInfo, (T1)data[0], (T2)data[1], (T3)data[2]); + } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs new file mode 100644 index 0000000000..fc5d78bf95 --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs @@ -0,0 +1,139 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Xml; + +using Microsoft.TestPlatform.TestUtilities; + + +namespace Microsoft.TestPlatform.AcceptanceTests; + +public sealed class TesthostCompatibilityDataSource : TestDataSource +{ + private static XmlDocument? s_depsXml; + private readonly string[] _runnerFrameworks; + private readonly string[] _targetFrameworks; + private readonly string[] _testhostVersions; + + /// + /// Initializes a new instance. + /// + /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. + public TesthostCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT_RUNNER_NETFX_AND_NET, string targetFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string runnerVersions = AcceptanceTestBase.LATESTPREVIEW_LEGACY, string testhostVersions = AcceptanceTestBase.LATESTPREVIEW_LEGACY) + { + _runnerFrameworks = runners.Split(';'); + _targetFrameworks = targetFrameworks.Split(';'); + _testhostVersions = testhostVersions.Split(';'); + + // Do not generate the data rows here, properties (e.g. InProcess) are not populated until after constructor is done. + } + + public bool DebugVSTestConsole { get; set; } + public bool DebugTesthost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + public override void CreateData(MethodInfo methodInfo) + { + var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); + // Run .NET Framework tests only on Windows. + Func filter = tfm => isWindows || tfm.StartsWith("net4"); + var onlyLatest = new[] { AcceptanceTestBase.LATEST }; + + // Test different versions of vstest console together with latest testhost. + // There is no point in proving that old testhost does not work with old vstest.console + // when we can patch neither. + foreach (var runner in _runnerFrameworks.Where(filter)) + { + foreach (var fmw in _targetFrameworks.Where(filter)) + { + // For .NET Framework generate only latest console with latest testhost, + // we cannot control the version of testhost because it is shipped with the console. + var testhostVersions = runner.StartsWith("net4") ? onlyLatest : _testhostVersions; + foreach (var testhostVersion in testhostVersions) + { + var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation, + DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); + + var vstestConsoleInfo = TranslationLayerCompatibilityDataSource.GetVSTestConsoleInfo(AcceptanceTestBase.LATEST, runnerInfo); + var testhostInfo = GetTesthostInfo(testhostVersion); + + AddData(runnerInfo, vstestConsoleInfo, testhostInfo); + } + } + } + + // Test different versions of vstest console together with latest vstest.console. + // There is no point in proving that old testhost does not work with old vstest.console + // when we can patch neither. + foreach (var runner in _runnerFrameworks.Where(filter)) + { + foreach (var fmw in _targetFrameworks.Where(filter)) + { + // For .NET Framework generate only latest console with latest testhost, + // we cannot control the version of testhost because it is shipped with the console. + var consoleVersions = runner.StartsWith("net4") ? onlyLatest : _testhostVersions; + foreach (var consoleVersion in consoleVersions) + { + var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation, + DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); + + var vstestConsoleInfo = TranslationLayerCompatibilityDataSource.GetVSTestConsoleInfo(consoleVersion, runnerInfo); + // Generate only for latest testhsot, + var testhostInfo = GetTesthostInfo(AcceptanceTestBase.LATEST); + + AddData(runnerInfo, vstestConsoleInfo, testhostInfo); + } + } + } + } + + public string GetDisplayName(MethodInfo methodInfo, object[] data) + { + return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); + } + + private TesthostInfo GetTesthostInfo(string testhostVersionType) + { + var depsXml = GetDependenciesXml(); + + // When version is Latest, we built it locally, but it gets restored into our nuget cache on build + // same as other versions, we just need to grab the version from a different property. + + var propertyName = testhostVersionType == AcceptanceTestBase.LATEST + ? $"NETTestSdkVersion" + : $"VSTestConsole{testhostVersionType}Version"; + + // It is okay when node is null, we check that Version has value when we update paths by using TesthostInfo, and throw. + // This way it throws in the body of the test which has better error reporting than throwing in the data source. + // + // We use the VSTestConsole properties to figure out testhost version, for now. + XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/{propertyName}"); + var version = node?.InnerText.Replace("[", "").Replace("]", ""); + var slash = Path.DirectorySeparatorChar; + var versionSpecificBinPath = $"{slash}bin{slash}NETTestSdk{testhostVersionType}-{version}{slash}"; + + return new TesthostInfo(testhostVersionType, version, versionSpecificBinPath); + } + + private static XmlDocument GetDependenciesXml() + { + if (s_depsXml != null) + return s_depsXml; + + var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); + var fileStream = File.OpenRead(depsXmlPath); + var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false }; + var depsXml = new XmlDocument(); + depsXml.Load(xmlTextReader); + + s_depsXml = depsXml; + return depsXml; + } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs index fc5149e0d4..4b7f02dfbb 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs @@ -17,7 +17,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests; public sealed class TranslationLayerCompatibilityDataSource : TestDataSource { - private static XmlDocument? _depsXml; + private static XmlDocument? s_depsXml; private readonly string[] _runnerFrameworks; private readonly string[] _targetFrameworks; // private readonly string[] _translationLayerVersions; @@ -72,7 +72,7 @@ public override void CreateData(MethodInfo methodInfo) } var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); - // Only run .NET Framework tests on Windows. + // Run .NET Framework tests only on Windows. Func filter = tfm => isWindows || !tfm.StartsWith("net4"); // TODO: maybe we should throw if we don't end up generating any data @@ -106,7 +106,7 @@ public string GetDisplayName(MethodInfo methodInfo, object[] data) return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); } - private VSTestConsoleInfo GetVSTestConsoleInfo(string vstestConsoleVersion, RunnerInfo runnerInfo) + internal static VSTestConsoleInfo GetVSTestConsoleInfo(string vstestConsoleVersion, RunnerInfo runnerInfo) { var depsXml = GetDependenciesXml(); @@ -159,8 +159,8 @@ private VSTestConsoleInfo GetVSTestConsoleInfo(string vstestConsoleVersion, Runn private static XmlDocument GetDependenciesXml() { - if (_depsXml != null) - return _depsXml; + if (s_depsXml != null) + return s_depsXml; var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); var fileStream = File.OpenRead(depsXmlPath); @@ -168,7 +168,7 @@ private static XmlDocument GetDependenciesXml() var depsXml = new XmlDocument(); depsXml.Load(xmlTextReader); - _depsXml = depsXml; + s_depsXml = depsXml; return depsXml; } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs index efa4b69597..c77faab8ab 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs @@ -11,7 +11,6 @@ using Castle.Core.Internal; -using Microsoft.TestPlatform.TestUtilities; using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -28,13 +27,12 @@ namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests; public class CodeCoverageTests : CodeCoverageAcceptanceTestBase { private IVsTestConsoleWrapper _vstestConsoleWrapper; - private TempDirectory _tempDirectory; private RunEventHandler _runEventHandler; private TestRunAttachmentsProcessingEventHandler _testRunAttachmentsProcessingEventHandler; private void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _tempDirectory); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); _testRunAttachmentsProcessingEventHandler = new TestRunAttachmentsProcessingEventHandler(); } @@ -493,7 +491,7 @@ private void AssertCoverageResults(IList attachments) { foreach (var attachment in attachmentSet.Attachments) { - var xmlCoverage = GetXmlCoverage(attachments.First().Attachments.First().Uri.LocalPath, _tempDirectory); + var xmlCoverage = GetXmlCoverage(attachments.First().Attachments.First().Uri.LocalPath, TempDirectory); foreach (var project in GetProjects()) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs index 646fda4dbf..f0524f871e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs @@ -37,8 +37,8 @@ public void Cleanup() [TranslationLayerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncher(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo); - _vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); var customTestHostLauncher = new TestHostLauncherV1(); @@ -58,8 +58,8 @@ public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvided [TranslationLayerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncherWhenITestHostLauncher2IsProvided(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo); - _vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); var customTestHostLauncher = new TestHostLauncherV2(); @@ -80,8 +80,8 @@ public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvided [TranslationLayerCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER)] public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvidedLauncher(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo); - _vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); var customTestHostLauncher = new TestHostLauncherV2(); @@ -105,8 +105,8 @@ public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvided [TranslationLayerCompatibilityDataSource("net451", "net451", "Latest", AfterFeature = Features.ATTACH_DEBUGGER, DebugVSTestConsole = true, DebugTesthost=true)] public void RunTestsWithCustomTestHostLauncherUsesLaunchWhenGivenAnOutdatedITestHostLauncher(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo); - _vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); var customTestHostLauncher = new TestHostLauncherV1(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs index 92150afe0c..b08763da05 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs @@ -26,7 +26,7 @@ public class DifferentTestFrameworkSimpleTests : AcceptanceTestBase private void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index 3fd97f543e..3a59d0de0f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -28,7 +28,7 @@ public class DiscoverTests : AcceptanceTestBase public void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _discoveryEventHandler = new DiscoveryEventHandler(); _discoveryEventHandler2 = new DiscoveryEventHandler2(); } @@ -43,13 +43,13 @@ public void Cleanup() [TranslationLayerCompatibilityDataSource] public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo); + SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); // Setup(); _discoveryEventHandler = new DiscoveryEventHandler(); _discoveryEventHandler2 = new DiscoveryEventHandler2(); - var vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + var vstestConsoleWrapper = GetVsTestConsoleWrapper(); vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), _discoveryEventHandler); // Assert. @@ -60,13 +60,13 @@ public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo, VSTe [TranslationLayerCompatibilityDataSource()] public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedOut(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo); + SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); // Setup(); _discoveryEventHandler = new DiscoveryEventHandler(); _discoveryEventHandler2 = new DiscoveryEventHandler2(); - var vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + var vstestConsoleWrapper = GetVsTestConsoleWrapper(); vstestConsoleWrapper.DiscoverTests( GetTestAssemblies(), GetDefaultRunSettings(), diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs index 5faeaa83d2..0beac2e85c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs @@ -21,7 +21,7 @@ public class LiveUnitTestingTests : AcceptanceTestBase public void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _discoveryEventHandler = new DiscoveryEventHandler(); _runEventHandler = new RunEventHandler(); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs index 2aeb7a7680..73b07981b6 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs @@ -23,7 +23,7 @@ public class RunSelectedTests : AcceptanceTestBase private void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); _discoveryEventHandler = new DiscoveryEventHandler(); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs index 4c4197078e..d966022eeb 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs @@ -29,7 +29,7 @@ public class RunTests : AcceptanceTestBase private void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); } @@ -43,18 +43,17 @@ public void Cleanup() [TranslationLayerCompatibilityDataSource] public void RunAllTests(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo); - // Setup(); - - var vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); - _runEventHandler = new RunEventHandler(); - vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler); + SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + + var vstestConsoleWrapper = GetVsTestConsoleWrapper(); + var runEventHandler = new RunEventHandler(); + vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetDefaultRunSettings(), runEventHandler); // Assert - Assert.AreEqual(6, _runEventHandler.TestResults.Count); - Assert.AreEqual(2, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed)); - Assert.AreEqual(2, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed)); - Assert.AreEqual(2, _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped)); + Assert.AreEqual(6, runEventHandler.TestResults.Count); + Assert.AreEqual(2, runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed)); + Assert.AreEqual(2, runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed)); + Assert.AreEqual(2, runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped)); } [TestMethod] @@ -189,12 +188,10 @@ public void RunTestsShouldShowProperWarningOnNoTestsForTestCaseFilter(RunnerInfo private IList GetTestAssemblies() { - var testAssemblies = new List + return new List { GetAssetFullPath("SimpleTestProject.dll"), GetAssetFullPath("SimpleTestProject2.dll") }; - - return testAssemblies; } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs index 0da7554f11..4f6e641fb2 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs @@ -31,8 +31,8 @@ public class RunTestsWithDifferentConfigurationTests : AcceptanceTestBase private void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out var logsDir); - _logsDir = logsDir; + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); + _logsDir = TempDirectory; _runEventHandler = new RunEventHandler(); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs index 010e916d5b..9edda9281e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs @@ -25,7 +25,7 @@ public class RunTestsWithFilterTests : AcceptanceTestBase private void Setup() { - _vstestConsoleWrapper = GetVsTestConsoleWrapper(out _); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); } @@ -39,12 +39,12 @@ public void Cleanup() [TranslationLayerCompatibilityDataSource] public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo); + SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); // Setup(); _runEventHandler = new RunEventHandler(); - var vstestConsoleWrapper = GetVsTestConsoleWrapper(TempDirectory, vsTestConsoleInfo); + var vstestConsoleWrapper = GetVsTestConsoleWrapper(); var sources = new List { GetAssetFullPath("SimpleTestProject.dll") }; vstestConsoleWrapper.RunTests( diff --git a/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs new file mode 100644 index 0000000000..e417ef9db7 --- /dev/null +++ b/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; + +namespace Microsoft.TestPlatform.TestUtilities; + +public abstract class DllInfo +{ + protected DllInfo(string name, string propertyName, string versionType, string? version, string path) + { + Name = name; + PropertyName = propertyName; + VersionType = versionType; + // Version can be null when we fail to find the respective propertin in TestPlatform.Dependencies.props + // when that happens we throw when we try to update the path. + Version = version; + Path = path; + } + + public string Name { get; } + public string PropertyName { get; } + public string VersionType { get; } + public string? Version { get; } + public string Path { get; } + + public override string ToString() => $" {Name} = {Version} [{VersionType}]"; + + public string UpdatePath(string path) + { + // Version is not directly used, below, but if it is not populated the path will be incorrect. + // We don't want to throw when creating SourcePathInfo because that is happening too early, and has worse error reporting. + if (Version == null) + throw new InvalidOperationException($"Version was not correctly populated from TestPlatform.Dependencies.props, review that there is entry for {PropertyName}{VersionType}Version."); + + // TODO: replacing in the result string is lame, but I am not going to fight 20 GetAssetFullPath method overloads right now + return path.Replace($"{System.IO.Path.DirectorySeparatorChar}bin{System.IO.Path.DirectorySeparatorChar}", Path); + } +} diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 1b309be65c..121d767536 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -467,6 +467,17 @@ protected string GetAssetFullPath(string assetName, string targetFramework) return _testEnvironment.GetTestAsset(assetName, targetFramework); } + protected List GetAssetFullPath(DllInfo dllInfo, params string[] assetNames) + { + var assets = new List(); + foreach (var assetName in assetNames) + { + assets.Add(dllInfo.UpdatePath(GetAssetFullPath(assetName))); + } + + return assets; + } + protected string GetProjectFullPath(string projectName) { return _testEnvironment.GetTestProject(projectName); @@ -524,7 +535,14 @@ public virtual string GetConsoleRunnerPath() if (IsDesktopRunner()) { - consoleRunnerPath = Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.exe"); + if (!string.IsNullOrWhiteSpace(_testEnvironment.VSTestConsolePath)) + { + consoleRunnerPath = _testEnvironment.VSTestConsolePath; + } + else + { + consoleRunnerPath = Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.exe"); + } } else if (IsNetCoreRunner()) { @@ -536,6 +554,8 @@ public virtual string GetConsoleRunnerPath() Assert.Fail("Unknown Runner framework - [{0}]", _testEnvironment.RunnerFramework); } + + Assert.IsTrue(File.Exists(consoleRunnerPath), "GetConsoleRunnerPath: Path not found: {0}", consoleRunnerPath); return consoleRunnerPath; } @@ -551,39 +571,15 @@ protected virtual string SetVSTestConsoleDLLPathInArgs(string args) return args; } - /// - /// Returns the VsTestConsole Wrapper. - /// - public IVsTestConsoleWrapper GetVsTestConsoleWrapper() - { - return GetVsTestConsoleWrapper(TempDirectory, null); - } - - /// - /// Returns the VsTestConsole Wrapper. - /// - public IVsTestConsoleWrapper GetVsTestConsoleWrapper(out TempDirectory logFileDir) - { - logFileDir = new TempDirectory(); - return GetVsTestConsoleWrapper(logFileDir, null); - } - - /// - /// Returns the VsTestConsole Wrapper. - /// - public IVsTestConsoleWrapper GetVsTestConsoleWrapper(TempDirectory tempDirectory) - { - return GetVsTestConsoleWrapper(tempDirectory, vsTestConsoleInfo: null); - } /// /// Returns the VsTestConsole Wrapper. /// /// - public IVsTestConsoleWrapper GetVsTestConsoleWrapper(TempDirectory logFileDir, VSTestConsoleInfo vsTestConsoleInfo) + public IVsTestConsoleWrapper GetVsTestConsoleWrapper() { // Temp directory is already unique so there is no need to have a unique file name. - var logFilePath = Path.Combine(logFileDir.Path, "log.txt"); + var logFilePath = Path.Combine(TempDirectory.Path, "log.txt"); if (!File.Exists(logFilePath)) { File.Create(logFilePath).Close(); @@ -591,10 +587,10 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper(TempDirectory logFileDir, V Console.WriteLine($"Logging diagnostics in {logFilePath}"); - var consoleRunnerPath = vsTestConsoleInfo != null - ? vsTestConsoleInfo.Path - : IsNetCoreRunner() - ? Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll") + var consoleRunnerPath = IsNetCoreRunner() + ? _testEnvironment.VSTestConsolePath != null + ? _testEnvironment.VSTestConsolePath + : Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll") : GetConsoleRunnerPath(); var executablePath = IsWindows ? @"dotnet\dotnet.exe" : @"dotnet-linux/dotnet"; var dotnetPath = Path.Combine(_testEnvironment.ToolsDirectory, executablePath); @@ -853,13 +849,13 @@ protected string BuildMultipleAssemblyPath(params string[] assetNames) return string.Join(" ", assetFullPath); } - protected string BuildMultipleAssemblyPath(MSTestInfo msTestInfo, params string[] assetNames) + protected string BuildMultipleAssemblyPath(DllInfo dllInfo, params string[] assetNames) { var assetFullPaths = new string[assetNames.Length]; for (var i = 0; i < assetNames.Length; i++) { var path = GetAssetFullPath(assetNames[i]); - var updatedPath = msTestInfo.UpdatePath(path); + var updatedPath = dllInfo.UpdatePath(path); Assert.IsTrue(File.Exists(updatedPath), "GetTestAsset: Path not found: {0}. Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", updatedPath); assetFullPaths[i] = updatedPath.AddDoubleQuote(); diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs index 66195fba24..7d4429d58c 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs @@ -197,6 +197,7 @@ public string RunnerFramework public bool DebugTesthost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } + public string VSTestConsolePath { get; set; } /// /// Gets the full path to a test asset. diff --git a/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs index 132d06a8c5..e4c128774e 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs @@ -1,35 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System; - namespace Microsoft.TestPlatform.TestUtilities; -public class MSTestInfo +public class MSTestInfo : DllInfo { public MSTestInfo(string versionType, string? version, string path) + : base(name: "MSTest", propertyName: "MSTest", versionType, version, path) { - VersionType = versionType; - // Version can be null when we fail to find the respective propertin in TestPlatform.Dependencies.props - // when that happens we throw when we try to update the path. - Version = version; - Path = path; - } - - public string VersionType { get; } - public string? Version { get; } - public string Path { get; } - - public override string ToString() => $" MSTest = {Version} [{VersionType}]"; - - public string UpdatePath(string path) - { - // Version is not directly used, below, but if it is not populated the path will be incorrect. - // We don't want to throw when creating MSTestInfo because that is happening too early, and has worse error reporting. - if (Version == null) - throw new InvalidOperationException($"Version was not correctly populated from TestPlatform.Dependencies.props, review that there is entry for MSTestFramework{VersionType}Version."); - - // TODO: replacing in the result string is lame, but I am not going to fight 20 GetAssetFullPath method overloads right now - return path.Replace($"{System.IO.Path.DirectorySeparatorChar}bin{System.IO.Path.DirectorySeparatorChar}", Path); } } diff --git a/test/Microsoft.TestPlatform.TestUtilities/TesthostInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/TesthostInfo.cs new file mode 100644 index 0000000000..f29006fd38 --- /dev/null +++ b/test/Microsoft.TestPlatform.TestUtilities/TesthostInfo.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.TestUtilities; + +public class TesthostInfo : DllInfo +{ + public TesthostInfo(string versionType, string? version, string path) + : base(name: "Testhost", propertyName: "VSTestConsole", versionType, version, path) + { + } +} From 1b0202a79deb8ae735a63bd9c851cc3f71e740a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 14 Mar 2022 17:38:25 +0100 Subject: [PATCH 32/64] Runs tests from the big data source, but test sdk is not used. --- .../AcceptanceTestBase.cs | 5 +- .../ExecutionTests.cs | 17 ++ .../MSTestCompatibilityDataSource.cs | 2 +- .../Extension/RunnnerInfo.cs | 2 +- .../TestPlatformCompatibilityDataSource.cs | 284 ++++++++++++++++++ .../TesthostCompatibilityDataSource.cs | 2 +- ...TranslationLayerCompatibilityDataSource.cs | 21 +- .../PortableNugetPackageTests.cs | 34 --- .../IntegrationTestBase.cs | 85 +++++- 9 files changed, 379 insertions(+), 73 deletions(-) create mode 100644 test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs index 48e1d98556..4f74da75e7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs @@ -57,9 +57,10 @@ public class AcceptanceTestBase : IntegrationTestBase /// public const string DEFAULT_RUNNER_NETFX_AND_NET = $"{DEFAULT_RUNNER_NETFX};netcoreapp2.1"; public const string DEFAULT_HOST_NETFX_AND_NET = "net451;netcoreapp2.1"; - public const string LATEST_LEGACY = "Latest;LatestPreview;LatestStable;RecentStable;MostDownloaded;PreviousStable;LegacyStable"; - public const string LATESTPREVIEW_LEGACY = "LatestPreview;LatestStable;RecentStable;MostDownloaded;PreviousStable;LegacyStable"; + public const string LATEST_TO_LEGACY = "Latest;LatestPreview;LatestStable;RecentStable;MostDownloaded;PreviousStable;LegacyStable"; + public const string LATESTPREVIEW_TO_LEGACY = "LatestPreview;LatestStable;RecentStable;MostDownloaded;PreviousStable;LegacyStable"; public const string LATEST = "Latest"; + internal const string MSTEST="MSTest"; public static string And(string left, string right) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 89ad03fdaa..749b8f6eaf 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -32,6 +32,23 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, MSTestInfo msTestIn ExitCodeEquals(1); // failing tests } + [TestMethod] + //[TestPlatformCompatibilityDataSource("netcoreapp2.1", "LegacyStable", "netcoreapp2.1", "Latest", "LatestPreview")] + [TestPlatformCompatibilityDataSource("netcoreapp2.1", AcceptanceTestBase.LATEST_TO_LEGACY, "netcoreapp2.1", AcceptanceTestBase.LATEST_TO_LEGACY, AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY)] + //[TestPlatformCompatibilityDataSource()] + + public void RunMultipleTestAssemblies223(RunnerInfo runnerInfo, VSTestConsoleInfo consoleInfo, MSTestInfo msTestInfo) + { + SetTestEnvironment(_testEnvironment, runnerInfo, consoleInfo); + + var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + + InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); + + ValidateSummaryStatus(2, 2, 2); + ExitCodeEquals(1); // failing tests + } + [TestMethod] [TesthostCompatibilityDataSource] public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo, TesthostInfo testhostInfo) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs index 0fe704ad78..c9e40513b0 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs @@ -24,7 +24,7 @@ public sealed class MSTestCompatibilityDataSource : TestDataSource /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. - public MSTestCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT_RUNNER_NETFX_AND_NET, string targetFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, string msTestVersions = AcceptanceTestBase.LATESTPREVIEW_LEGACY) + public MSTestCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT_RUNNER_NETFX_AND_NET, string targetFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, string msTestVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY) { _runnerFrameworks = runners.Split(';'); _targetFrameworks = targetFrameworks.Split(';'); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs index c6f67359da..da0d836e89 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs @@ -11,7 +11,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests; /// /// /// Supported value = /InIsolation. -public record RunnerInfo(string RunnerFramework, string TargetFramework, string? InIsolationValue = "", +public record struct RunnerInfo(string RunnerFramework, string TargetFramework, string? InIsolationValue = "", bool DebugVSTestConsole = false, bool DebugTesthost = false, bool DebugDataCollector = false, bool NoDefaultBreakpoints = true) { /// diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs new file mode 100644 index 0000000000..63ae2f14de --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs @@ -0,0 +1,284 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Xml; + +using Microsoft.TestPlatform.TestUtilities; + +using Semver; + +namespace Microsoft.TestPlatform.AcceptanceTests; + +public sealed class TestPlatformCompatibilityDataSource : TestDataSource +{ + private static XmlDocument? s_depsXml; + private readonly string[] _runnerFrameworks; + private readonly string[] _runnerVersions; + private readonly string[] _hostFrameworks; + private readonly string[] _adapterVersions; + private readonly string[] _adapters; + private readonly string[] _hostVersions; + + public TestPlatformCompatibilityDataSource( + string runnerFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string runnerVersions = AcceptanceTestBase.LATEST_TO_LEGACY, + string hostFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string hostVersions = AcceptanceTestBase.LATEST_TO_LEGACY, + string adapterVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY, + string adapters = AcceptanceTestBase.MSTEST) + { + // TODO: We actually don't generate values to use different translation layers, because we don't have a good way to do + // that right now. Translation layer is loaded directly into the acceptance test, and so we don't have easy way to substitute it. + // I am keeping this source separate from vstest console compatibility data source, to be able to easily add this feature later. + _runnerFrameworks = runnerFrameworks.Split(';'); + _runnerVersions = runnerVersions.Split(';'); + _hostFrameworks = hostFrameworks.Split(';'); + _hostVersions = hostVersions.Split(';'); + _adapterVersions = adapterVersions.Split(';'); + _adapters = adapters.Split(';'); + + // Do not generate the data rows here, properties (e.g. DebugVSTestConsole) are not populated until after constructor is done. + } + + public bool DebugVSTestConsole { get; set; } + public bool DebugTesthost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + public string? BeforeFeature { get; set; } + public string? AfterFeature { get; set; } + public string? BeforeAdapterFeature { get; set; } + public string? AfterAdapterFeature { get; set; } + + public override void CreateData(MethodInfo methodInfo) + { + var dataRows = new List>>(); + AddEveryVersionOfRunner(dataRows); + + // with every version of host + AddEveryVersionOfHost(dataRows); + + AddEveryVersionOfAdapter(dataRows); + + AddOlderConfigurations(dataRows); + + var c = dataRows.Count(); + + if (BeforeFeature != null && AfterFeature != null) + { + throw new InvalidOperationException($"You cannot specify {nameof(BeforeFeature)} and {nameof(AfterFeature)} at the same time"); + } + + var minVersion = SemVersion.Parse("0.0.0-alpha.1"); + SemVersion? beforeVersion = null; + SemVersion? afterVersion = null; + if (BeforeFeature != null) + { + var feature = Features.Table[BeforeFeature]; + beforeVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + } + if (AfterFeature != null) + { + var feature = Features.Table[AfterFeature]; + afterVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + } + + var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); + // Run .NET Framework tests only on Windows. + Func filter = tfm => isWindows || !tfm.StartsWith("net4"); + + // TODO: maybe we should throw if we don't end up generating any data + // because none of the versions match, or some other way to identify tests that will never run because they are very outdated. + // We probably don't have that need right now, because legacy version is 15.x.x, which is very old, and we are still keeping + // compatibility. + + foreach (var dataRow in dataRows) + { + AddData(dataRow.Key, dataRow.Value.Key, dataRow.Value.Value); + } + } + + private void AddOlderConfigurations(List>> dataRows) + { + // Older configurations where the runner, host and adapter version are the same. + // We already added the row where all are newest when adding combination with all runners. + foreach (var runnerVersion in _runnerVersions.Skip(1)) + { + foreach (var runnerFramework in _runnerFrameworks) + { + foreach (var hostFramework in _hostFrameworks) + { + var isNetFramework = hostFramework.StartsWith("net4"); + var hostVersion = runnerVersion; + foreach (var adapter in _adapters) + { + var adapterVersion = runnerVersion; + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion); + } + } + } + } + } + + private void AddEveryVersionOfAdapter(List>> dataRows) + { + var runnerVersion = _runnerVersions[0]; + foreach (var runnerFramework in _runnerFrameworks) + { + foreach (var hostFramework in _hostFrameworks) + { + var isNetFramework = hostFramework.StartsWith("net4"); + // .NET Framework testhost ships with the runner, and the version from the + // runner directory is always selected, otherwise select the newest version from _hostFrameworks. + var hostVersion = isNetFramework ? runnerVersion : _hostVersions[0]; + foreach (var adapter in _adapters) + { + // We already used the newest when adding combination with every runner + foreach (var adapterVersion in _adapterVersions.Skip(1)) + { + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion); + } + } + } + } + } + + private void AddEveryVersionOfHost(List>> dataRows) + { + var runnerVersion = _runnerVersions[0]; + + foreach (var runnerFramework in _runnerFrameworks) + { + foreach (var hostFramework in _hostFrameworks) + { + var isNetFramework = hostFramework.StartsWith("net4"); + // .NET Framework testhost ships with the runner, and the version from the + // runner directory is always the same as the runner. There are no variations + // so we just need to add host versions for .NET testhosts. We also skip the + // newest version because we already added it when AddEveryVersionOfRunner + var hostVersions = isNetFramework ? Array.Empty() : _hostVersions.Skip(1).ToArray(); + foreach (var hostVersion in hostVersions) + { + foreach (var adapter in _adapters) + { + // use the newest + var adapterVersion = _adapterVersions[0]; + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion); + } + } + } + } + } + + private void AddEveryVersionOfRunner(List>> dataRows) + { + foreach (var runnerVersion in _runnerVersions) + { + foreach (var runnerFramework in _runnerFrameworks) + { + foreach (var hostFramework in _hostFrameworks) + { + var isNetFramework = hostFramework.StartsWith("net4"); + // .NET Framework testhost ships with the runner, and the version from the + // runner directory is always selected, otherwise select the newest version from _hostFrameworks. + var hostVersion = isNetFramework ? runnerVersion : _hostVersions[0]; + foreach (var adapter in _adapters) + { + // use the newest + var adapterVersion = _adapterVersions[0]; + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion); + } + } + } + } + } + + private void AddRow(List>> dataRows, + string runnerVersion, string runnerFramework, string hostVersion, string hostFramework, string adapter, string adapterVersion) + { + var mstestInfo = GetMSTestInfo(adapterVersion); + RunnerInfo runnerInfo = GetRunnerInfo(runnerFramework, hostFramework, inIsolation: true); + var vstestConsoleInfo = GetVSTestConsoleInfo(runnerVersion, runnerInfo); + dataRows.Add(new KeyValuePair>(runnerInfo, new KeyValuePair(vstestConsoleInfo, mstestInfo))); + } + + private RunnerInfo GetRunnerInfo(string runnerFramework, string hostFramework, bool inIsolation) + { + return new RunnerInfo(runnerFramework, hostFramework, inIsolation ? AcceptanceTestBase.InIsolation : null, + DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); + } + + public string GetDisplayName(MethodInfo methodInfo, object[] data) + { + return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); + } + + private MSTestInfo GetMSTestInfo(string msTestVersion) + { + var depsXml = GetDependenciesXml(); + + // It is okay when node is null, we check that Version has value when we update paths by using MSTestInfo, and throw. + // This way it throws in the body of the test which has better error reporting than throwing in the data source. + XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/MSTestFramework{msTestVersion}Version"); + var version = node?.InnerText.Replace("[", "").Replace("]", ""); + var slash = Path.DirectorySeparatorChar; + var versionSpecificBinPath = $"{slash}bin{slash}MSTest{msTestVersion}-{version}{slash}"; + + return new MSTestInfo(msTestVersion, version, versionSpecificBinPath); + } + + internal static VSTestConsoleInfo GetVSTestConsoleInfo(string vstestConsoleVersion, RunnerInfo runnerInfo) + { + var depsXml = GetDependenciesXml(); + + // When version is Latest, we built it locally, but it gets restored into our nuget cache on build + // same as other versions, we just need to grab the version from a different property. + + var propertyName = vstestConsoleVersion == AcceptanceTestBase.LATEST + ? $"NETTestSdkVersion" + : $"VSTestConsole{vstestConsoleVersion}Version"; + + var packageName = runnerInfo.IsNetFrameworkRunner + ? "microsoft.testplatform" + : "microsoft.testplatform.cli"; + + // It is okay when node is null, we will fail to find the executable later, and throw. + // This way it throws in the body of the test which has better error reporting than throwing in the data source. + // And we can easily find out what is going on because --WRONG-VERSION-- sticks out, and is easy to find in the codebase. + XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/{propertyName}"); + var version = node?.InnerText.Replace("[", "").Replace("]", "") ?? "--WRONG-VERSION--"; + var vstestConsolePath = runnerInfo.IsNetFrameworkRunner + ? Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "packages", packageName, version, + "tools", "net451", "Common7", "IDE", "Extensions", "TestPlatform", "vstest.console.exe") + : Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "packages", packageName, version, + "contentFiles", "any", "netcoreapp2.1", "vstest.console.dll"); + + if (version.StartsWith("15.")) + { + vstestConsolePath = vstestConsolePath.Replace("netcoreapp2.1", "netcoreapp2.0"); + } + + return new VSTestConsoleInfo(vstestConsoleVersion, version, vstestConsolePath); + } + + private static XmlDocument GetDependenciesXml() + { + if (s_depsXml != null) + return s_depsXml; + + var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); + var fileStream = File.OpenRead(depsXmlPath); + var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false }; + var depsXml = new XmlDocument(); + depsXml.Load(xmlTextReader); + + s_depsXml = depsXml; + return depsXml; + } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs index fc5d78bf95..8039ecce7c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs @@ -25,7 +25,7 @@ public sealed class TesthostCompatibilityDataSource : TestDataSource /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. public TesthostCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT_RUNNER_NETFX_AND_NET, string targetFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, - string runnerVersions = AcceptanceTestBase.LATESTPREVIEW_LEGACY, string testhostVersions = AcceptanceTestBase.LATESTPREVIEW_LEGACY) + string runnerVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY, string testhostVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY) { _runnerFrameworks = runners.Split(';'); _targetFrameworks = targetFrameworks.Split(';'); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs index 4b7f02dfbb..d5e8d2ee77 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs @@ -30,7 +30,7 @@ public sealed class TranslationLayerCompatibilityDataSource : TestDataSource environmen /// Arguments provided to vstest.console.exe public void InvokeDotnetTest(string arguments) { - var vstestConsolePath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "netcoreapp2.1", "vstest.console.dll"); + var vstestConsolePath = GetDotnetRunnerPath(); var env = "VSTEST_CONSOLE_PATH"; var originalVstestConsolePath = Environment.GetEnvironmentVariable(env); @@ -275,6 +276,13 @@ public void ExecuteNotSupportedRunnerFrameworkTests(string runnerFramework, stri /// Skipped test count public void ValidateSummaryStatus(int passedTestsCount, int failedTestsCount, int skippedTestsCount) { + // TODO: Switch on the actual version of vstest console when we have that set on test environment. + if (_testEnvironment.VSTestConsolePath.Contains($"{Path.DirectorySeparatorChar}15.")) + { + ValidateSummaryStatusv15(passedTestsCount, failedTestsCount, skippedTestsCount); + return; + } + var totalTestCount = passedTestsCount + failedTestsCount + skippedTestsCount; if (totalTestCount == 0) { @@ -323,6 +331,57 @@ public void ValidateSummaryStatus(int passedTestsCount, int failedTestsCount, in } } + /// + /// Validate if the overall test count and results are matching. + /// + /// Passed test count + /// Failed test count + /// Skipped test count + public void ValidateSummaryStatusv15(int passedTestsCount, int failedTestsCount, int skippedTestsCount) + { + // example: Total tests: 6. Passed: 2. Failed: 2. Skipped: 2. + var totalTestCount = passedTestsCount + failedTestsCount + skippedTestsCount; + if (totalTestCount == 0) + { + // No test should be found/run + StringAssert.DoesNotMatch( + _standardTestOutput, + new Regex("Total tests\\:"), + "Excepted: There should not be test summary{2}Actual: {0}{2}Standard Error: {1}{2}Arguments: {3}{2}", + _standardTestOutput, + _standardTestError, + Environment.NewLine, + _arguments); + } + else + { + var summaryStatus = $"Total tests: {totalTestCount}."; + if (passedTestsCount != 0) + { + summaryStatus += $" Passed: {passedTestsCount}."; + } + + if (failedTestsCount != 0) + { + summaryStatus += $" Failed: {failedTestsCount}."; + } + + if (skippedTestsCount != 0) + { + summaryStatus += $" Skipped: {skippedTestsCount}."; + } + + Assert.IsTrue( + _standardTestOutput.Contains(summaryStatus), + "The Test summary does not match.{3}Expected summary: {1}{3}Test Output: {0}{3}Standard Error: {2}{3}Arguments: {4}{3}", + _standardTestOutput, + summaryStatus, + _standardTestError, + Environment.NewLine, + _arguments); + } + } + public void StdErrorContains(string substring) { Assert.IsTrue(_standardTestError.Contains(substring), "StdErrorOutput - [{0}] did not contain expected string '{1}'", _standardTestError, substring); @@ -417,7 +476,7 @@ public void ValidateDiscoveredTests(params string[] discoveredTestsList) || _standardTestOutput.Contains(GetTestMethodName(test)); Assert.IsTrue(flag, $"Test {test} does not appear in discovered tests list." + $"{Environment.NewLine}Std Output: {_standardTestOutput}" + - $"{Environment.NewLine}Std Error: { _standardTestError}"); + $"{Environment.NewLine}Std Error: {_standardTestError}"); } } @@ -433,7 +492,7 @@ public void ValidateTestsNotDiscovered(params string[] testsList) || _standardTestOutput.Contains(GetTestMethodName(test)); Assert.IsFalse(flag, $"Test {test} should not appear in discovered tests list." + $"{Environment.NewLine}Std Output: {_standardTestOutput}" + - $"{Environment.NewLine}Std Error: { _standardTestError}"); + $"{Environment.NewLine}Std Error: {_standardTestError}"); } } @@ -448,7 +507,7 @@ public void ValidateFullyQualifiedDiscoveredTests(string filePath, params string || fileOutput.Contains(GetTestMethodName(test)); Assert.IsTrue(flag, $"Test {test} does not appear in discovered tests list." + $"{Environment.NewLine}Std Output: {_standardTestOutput}" + - $"{Environment.NewLine}Std Error: { _standardTestError}"); + $"{Environment.NewLine}Std Error: {_standardTestError}"); } } @@ -472,7 +531,7 @@ protected List GetAssetFullPath(DllInfo dllInfo, params string[] assetNa var assets = new List(); foreach (var assetName in assetNames) { - assets.Add(dllInfo.UpdatePath(GetAssetFullPath(assetName))); + assets.Add(dllInfo.UpdatePath(GetAssetFullPath(assetName))); } return assets; @@ -524,7 +583,7 @@ protected bool IsNetCoreRunner() } /// - /// Gets the path to vstest.console.exe. + /// Gets the path to vstest.console.exe or dotnet.exe. /// /// /// Full path to test runner @@ -554,15 +613,13 @@ public virtual string GetConsoleRunnerPath() Assert.Fail("Unknown Runner framework - [{0}]", _testEnvironment.RunnerFramework); } - - Assert.IsTrue(File.Exists(consoleRunnerPath), "GetConsoleRunnerPath: Path not found: {0}", consoleRunnerPath); return consoleRunnerPath; } protected virtual string SetVSTestConsoleDLLPathInArgs(string args) { - var vstestConsoleDll = Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll"); + var vstestConsoleDll = GetDotnetRunnerPath(); vstestConsoleDll = vstestConsoleDll.AddDoubleQuote(); args = string.Concat( vstestConsoleDll, @@ -588,9 +645,7 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper() Console.WriteLine($"Logging diagnostics in {logFilePath}"); var consoleRunnerPath = IsNetCoreRunner() - ? _testEnvironment.VSTestConsolePath != null - ? _testEnvironment.VSTestConsolePath - : Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll") + ? GetDotnetRunnerPath() : GetConsoleRunnerPath(); var executablePath = IsWindows ? @"dotnet\dotnet.exe" : @"dotnet-linux/dotnet"; var dotnetPath = Path.Combine(_testEnvironment.ToolsDirectory, executablePath); @@ -602,7 +657,7 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper() if (!File.Exists(consoleRunnerPath)) { - throw new FileNotFoundException($"File '{consoleRunnerPath}' was not found.") ; + throw new FileNotFoundException($"File '{consoleRunnerPath}' was not found."); } Console.WriteLine($"Console runner path: {consoleRunnerPath}"); @@ -611,7 +666,7 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper() if (_testEnvironment.DebugVSTestConsole || _testEnvironment.DebugTesthost || _testEnvironment.DebugDataCollector) { var environmentVariables = new Dictionary(); - Environment.GetEnvironmentVariables().OfType().ToList().ForEach(e=> environmentVariables.Add(e.Key.ToString(), e.Value.ToString())); + Environment.GetEnvironmentVariables().OfType().ToList().ForEach(e => environmentVariables.Add(e.Key.ToString(), e.Value.ToString())); if (_testEnvironment.DebugVSTestConsole) { @@ -900,5 +955,5 @@ protected static string GetDownloadedDotnetMuxerFromTools(string architecture) return path; } - protected static string GetDotnetRunnerPath() => Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "netcoreapp2.1", "vstest.console.dll"); + protected string GetDotnetRunnerPath() => _testEnvironment.VSTestConsolePath ?? Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll"); } From d5565250a072a1b86fdb9ec2e6a228cbc370a0ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 14 Mar 2022 18:04:42 +0100 Subject: [PATCH 33/64] Compose path to testhost + adapter. --- .../ExecutionTests.cs | 10 +++--- .../Extension/TestDataSource.cs | 32 +++++++++++++++++++ .../TestPlatformCompatibilityDataSource.cs | 21 ++++++------ .../TesthostCompatibilityDataSource.cs | 2 +- .../IntegrationTestBase.cs | 15 +++++++++ 5 files changed, 64 insertions(+), 16 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 749b8f6eaf..57be416dd5 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -20,7 +20,7 @@ public class ExecutionTests : AcceptanceTestBase [TestMethod] [MSTestCompatibilityDataSource(InProcess = true)] - public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, MSTestInfo msTestInfo) + public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, TesthostInfo testhostInfo, MSTestInfo msTestInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -33,15 +33,15 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, MSTestInfo msTestIn } [TestMethod] - //[TestPlatformCompatibilityDataSource("netcoreapp2.1", "LegacyStable", "netcoreapp2.1", "Latest", "LatestPreview")] - [TestPlatformCompatibilityDataSource("netcoreapp2.1", AcceptanceTestBase.LATEST_TO_LEGACY, "netcoreapp2.1", AcceptanceTestBase.LATEST_TO_LEGACY, AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY)] + [TestPlatformCompatibilityDataSource("netcoreapp2.1", "LegacyStable", "netcoreapp2.1", "Latest", "LatestPreview")] + // [TestPlatformCompatibilityDataSource("netcoreapp2.1", LATEST_TO_LEGACY, "netcoreapp2.1", LATEST_TO_LEGACY, LATESTPREVIEW_TO_LEGACY)] //[TestPlatformCompatibilityDataSource()] - public void RunMultipleTestAssemblies223(RunnerInfo runnerInfo, VSTestConsoleInfo consoleInfo, MSTestInfo msTestInfo) + public void RunMultipleTestAssemblies223(RunnerInfo runnerInfo, VSTestConsoleInfo consoleInfo, TesthostInfo testhostInfo, MSTestInfo msTestInfo) { SetTestEnvironment(_testEnvironment, runnerInfo, consoleInfo); - var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath(testhostInfo, msTestInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs index 2aeb5fa0c1..abfead224e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs @@ -100,3 +100,35 @@ string ITestDataSource.GetDisplayName(MethodInfo methodInfo, object[] data) return GetDisplayName(methodInfo, (T1)data[0], (T2)data[1], (T3)data[2]); } } + +[AttributeUsage(AttributeTargets.Method)] +public abstract class TestDataSource : Attribute, ITestDataSource + where T1 : notnull + where T2 : notnull + where T3 : notnull +{ + private readonly List _data = new(); + + public abstract void CreateData(MethodInfo methodInfo); + + public void AddData(T1 value1, T2 value2, T3 value3, T4 value4) + { + _data.Add(new object[] { value1, value2, value3, value4 }); + } + + public virtual string GetDisplayName(MethodInfo methodInfo, T1 value1, T2 value2, T3 value3, T4 value4) + { + return $"{methodInfo.Name} ({value1}, {value2}, {value3}, {value4})"; + } + + IEnumerable ITestDataSource.GetData(MethodInfo methodInfo) + { + CreateData(methodInfo); + return _data; + } + + string ITestDataSource.GetDisplayName(MethodInfo methodInfo, object[] data) + { + return GetDisplayName(methodInfo, (T1)data[0], (T2)data[1], (T3)data[2], (T4)data[3]); + } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs index 63ae2f14de..63d0a7105e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs @@ -15,7 +15,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests; -public sealed class TestPlatformCompatibilityDataSource : TestDataSource +public sealed class TestPlatformCompatibilityDataSource : TestDataSource { private static XmlDocument? s_depsXml; private readonly string[] _runnerFrameworks; @@ -58,7 +58,7 @@ public TestPlatformCompatibilityDataSource( public override void CreateData(MethodInfo methodInfo) { - var dataRows = new List>>(); + var dataRows = new List<(RunnerInfo runnerInfo, VSTestConsoleInfo vstestConsoleInfo, TesthostInfo testhostInfo, MSTestInfo mstestInfo)>(); AddEveryVersionOfRunner(dataRows); // with every version of host @@ -100,11 +100,11 @@ public override void CreateData(MethodInfo methodInfo) foreach (var dataRow in dataRows) { - AddData(dataRow.Key, dataRow.Value.Key, dataRow.Value.Value); + AddData(dataRow.runnerInfo, dataRow.vstestConsoleInfo, dataRow.testhostInfo, dataRow.mstestInfo); } } - private void AddOlderConfigurations(List>> dataRows) + private void AddOlderConfigurations(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows) { // Older configurations where the runner, host and adapter version are the same. // We already added the row where all are newest when adding combination with all runners. @@ -126,7 +126,7 @@ private void AddOlderConfigurations(List>> dataRows) + private void AddEveryVersionOfAdapter(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows) { var runnerVersion = _runnerVersions[0]; foreach (var runnerFramework in _runnerFrameworks) @@ -149,7 +149,7 @@ private void AddEveryVersionOfAdapter(List>> dataRows) + private void AddEveryVersionOfHost(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows) { var runnerVersion = _runnerVersions[0]; @@ -176,7 +176,7 @@ private void AddEveryVersionOfHost(List>> dataRows) + private void AddEveryVersionOfRunner(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows) { foreach (var runnerVersion in _runnerVersions) { @@ -199,13 +199,14 @@ private void AddEveryVersionOfRunner(List>> dataRows, + private void AddRow(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows, string runnerVersion, string runnerFramework, string hostVersion, string hostFramework, string adapter, string adapterVersion) { - var mstestInfo = GetMSTestInfo(adapterVersion); RunnerInfo runnerInfo = GetRunnerInfo(runnerFramework, hostFramework, inIsolation: true); var vstestConsoleInfo = GetVSTestConsoleInfo(runnerVersion, runnerInfo); - dataRows.Add(new KeyValuePair>(runnerInfo, new KeyValuePair(vstestConsoleInfo, mstestInfo))); + var testhostInfo = TesthostCompatibilityDataSource.GetTesthostInfo(hostVersion); + var mstestInfo = GetMSTestInfo(adapterVersion); + dataRows.Add(new(runnerInfo, vstestConsoleInfo, testhostInfo, mstestInfo)); } private RunnerInfo GetRunnerInfo(string runnerFramework, string hostFramework, bool inIsolation) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs index 8039ecce7c..6892b4a210 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs @@ -99,7 +99,7 @@ public string GetDisplayName(MethodInfo methodInfo, object[] data) return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); } - private TesthostInfo GetTesthostInfo(string testhostVersionType) + internal static TesthostInfo GetTesthostInfo(string testhostVersionType) { var depsXml = GetDependenciesXml(); diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 7836d28ed5..1e84f3b094 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -919,6 +919,21 @@ protected string BuildMultipleAssemblyPath(DllInfo dllInfo, params string[] asse return string.Join(" ", assetFullPaths); } + protected string BuildMultipleAssemblyPath(TesthostInfo testhostInfo, DllInfo adapterInfo, params string[] assetNames) + { + var assetFullPaths = new string[assetNames.Length]; + for (var i = 0; i < assetNames.Length; i++) + { + var path = GetAssetFullPath(assetNames[i]); + var updatedPath = testhostInfo.UpdatePath(adapterInfo.UpdatePath(path)); + Assert.IsTrue(File.Exists(updatedPath), "GetTestAsset: Path not found: {0}. Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", updatedPath); + + assetFullPaths[i] = updatedPath.AddDoubleQuote(); + } + + return string.Join(" ", assetFullPaths); + } + protected static string GetDiagArg(string rootDir) => " --diag:" + Path.Combine(rootDir, "log.txt"); From d243b59ca398f4a326b07562e76c7123ce71ec7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Tue, 15 Mar 2022 18:21:58 +0100 Subject: [PATCH 34/64] One big unified data source. Still no version filtering. --- scripts/build.ps1 | 374 ++++++++---------- .../ExecutionTests.cs | 7 +- .../MSTestCompatibilityDataSource.cs | 8 +- .../TestPlatformCompatibilityDataSource.cs | 65 ++- .../IntegrationTestBase.cs | 1 - .../MSTestProject2/MSTestProject2.csproj | Bin 0 -> 2194 bytes test/TestAssets/MSTestProject2/UnitTest1.cs | 53 +++ test/TestAssets/TestAssets.sln | 16 +- 8 files changed, 298 insertions(+), 226 deletions(-) create mode 100644 test/TestAssets/MSTestProject2/MSTestProject2.csproj create mode 100644 test/TestAssets/MSTestProject2/UnitTest1.cs diff --git a/scripts/build.ps1 b/scripts/build.ps1 index 2a27c513e4..78b06fd19e 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -3,47 +3,47 @@ [CmdletBinding()] Param( - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [ValidateSet("Debug", "Release")] [Alias("c")] [System.String] $Configuration = "Debug", - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("r")] [System.String] $TargetRuntime = "win7-x64", # Versioning scheme = Major(15).Minor(RTW, Updates).SubUpdates(preview4, preview5, RC etc) # E.g. VS 2017 Update 1 Preview will have version 15.1.1 - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("v")] [System.String] $Version, # Will set this later by reading TestPlatform.Settings.targets file. - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("vs")] [System.String] $VersionSuffix = "dev", - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("bn")] [System.String] $BuildNumber = "20991231-99", - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("ff")] [System.Boolean] $FailFast = $true, - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("noloc")] [Switch] $DisableLocalizedBuild, - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("ci")] [Switch] $CIBuild, - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("pt")] [Switch] $PublishTestArtifacts, # Build specific projects - [Parameter(Mandatory=$false)] + [Parameter(Mandatory = $false)] [Alias("p")] [System.String[]] $ProjectNamePatterns = @(), @@ -61,11 +61,10 @@ $ErrorView = 'Normal' # Set Version from scripts/build/TestPlatform.Settings.targets, when we are running locally and not providing the version as the parameter # or when the build is done directly in VS -if([string]::IsNullOrWhiteSpace($Version)) -{ +if ([string]::IsNullOrWhiteSpace($Version)) { $Version = ([xml](Get-Content $env:TP_ROOT_DIR\scripts\build\TestPlatform.Settings.targets)).Project.PropertyGroup[0].TPVersionPrefix | - ForEach-Object { $_.Trim() } | - Select-Object -First 1 + ForEach-Object { $_.Trim() } | + Select-Object -First 1 Write-Verbose "Version was not provided using version '$Version' from TestPlatform.Settings.targets" } @@ -128,21 +127,18 @@ if ($env:PATH -notlike "*$attachVsPath") { # so nuget restores it correctly. $vsSdkBuildToolsVersion = ([xml](Get-Content $env:TP_ROOT_DIR\scripts\build\TestPlatform.Dependencies.props)).Project.PropertyGroup.VSSdkBuildToolsVersion $vsixUtilDir = "$env:TP_ROOT_DIR\packages\microsoft.vssdk.buildtools" -if ((Test-Path $vsixUtilDir) -and -not (Test-Path "$vsixUtilDir\$vsSdkBuildToolsVersion\tools\vssdk\bin\VsixUtil.exe")) -{ +if ((Test-Path $vsixUtilDir) -and -not (Test-Path "$vsixUtilDir\$vsSdkBuildToolsVersion\tools\vssdk\bin\VsixUtil.exe")) { Remove-Item -Recurse -Force $vsixUtilDir } # Procdump gets regularly eaten by antivirus or something. Remove the package dir if it gets broken # so nuget restores it correctly. $procdumpDir = "$env:TP_ROOT_DIR\packages\procdump" -if ((Test-Path $procdumpDir) -and 2 -ne @(Get-Item "$procdumpDir\0.0.1\bin").Length) -{ +if ((Test-Path $procdumpDir) -and 2 -ne @(Get-Item "$procdumpDir\0.0.1\bin").Length) { Remove-Item -Recurse -Force $procdumpDir } -function Invoke-Build -{ +function Invoke-Build { $timer = Start-Timer Write-Log "Invoke-Build: Start build." $dotnetExe = Get-DotNetPath @@ -157,8 +153,7 @@ function Invoke-Build Write-Log "Invoke-Build: Complete. {$(Get-ElapsedTime($timer))}" } -function Invoke-TestAssetsBuild -{ +function Invoke-TestAssetsBuild { $timer = Start-Timer Write-Log "Invoke-TestAssetsBuild: Start test assets build." $dotnetExe = Get-DotNetPath @@ -198,8 +193,7 @@ function Invoke-TestAssetsBuild # The command line tool does not like the package ranges. $vsTestConsoleVersion = $vsTestConsoleVersion -replace "(\[|\])" - if (-not $vsTestConsoleVersion) - { + if (-not $vsTestConsoleVersion) { throw "VSTestConsoleVersion for $propertyName is empty." } @@ -240,7 +234,9 @@ function Invoke-TestAssetsBuild # Build with multiple versions of MSTest. $projects = @( "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject\SimpleTestProject.csproj" - "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject2\SimpleTestProject2.csproj" + "$env:TP_ROOT_DIR\test\TestAssets\MSTestProject2\MSTestProject2.csproj" + # Don't use this one, it does not use the variables for mstest and test sdk. + # "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject2\SimpleTestProject2.csproj" ) $msTestVersionProperties = @( @@ -252,40 +248,35 @@ function Invoke-TestAssetsBuild "MSTestFrameworkLegacyStableVersion" ) - foreach ($project in $projects) { - foreach ($propertyName in $msTestVersionProperties) { - $mstestVersion = $dependenciesXml.Project.PropertyGroup.$propertyName - - if (-not $mstestVersion) - { - throw "MSTestVersion for $propertyName is empty." - } - - $dirMSTestVersion = $mstestVersion -replace "\[|\]" - $dirMSTestPropertyName = $propertyName -replace "Framework" -replace "Version" - Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:MSTestFrameworkVersion=$mstestVersion -p:MSTestAdapterVersion=$mstestVersion -p:BaseOutputPath=""bin\$dirMSTestPropertyName-$dirMSTestVersion\\""" - } - } - foreach ($project in $projects) { # We use the same version properties for NET.Test.Sdk as for VSTestConsole, for now. - foreach ($propertyName in $vstestConsoleVersionProperties) { - if ("VSTestConsoleLatestVersion" -eq $propertyName) { + foreach ($sdkPropertyName in $vstestConsoleVersionProperties) { + if ("VSTestConsoleLatestVersion" -eq $sdkPropertyName) { # NETTestSdkVersion has the version of the locally built package. $netTestSdkVersion = $dependenciesXml.Project.PropertyGroup."NETTestSdkVersion" } else { - $netTestSdkVersion = $dependenciesXml.Project.PropertyGroup.$propertyName + $netTestSdkVersion = $dependenciesXml.Project.PropertyGroup.$sdkPropertyName } - - if (-not $netTestSdkVersion) - { - throw "NetTestSdkVersion for $propertyName is empty." + + if (-not $netTestSdkVersion) { + throw "NetTestSdkVersion for $sdkPropertyName is empty." } + + $dirNetTestSdkVersion = $netTestSdkVersion -replace "\[|\]" + $dirNetTestSdkPropertyName = $sdkPropertyName -replace "Framework" -replace "Version" -replace "VSTestConsole", "NETTestSdk" + + foreach ($propertyName in $msTestVersionProperties) { + $mstestVersion = $dependenciesXml.Project.PropertyGroup.$propertyName - $dirNetTestSdkVersion = $netTestSdkVersion -replace "\[|\]" - $dirNetTestSdkPropertyName = $propertyName -replace "Framework" -replace "Version" -replace "VSTestConsole", "NETTestSdk" - Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:NETTestSdkVersion=$netTestSdkVersion -p:BaseOutputPath=""bin\$dirNetTestSdkPropertyName-$dirNetTestSdkVersion\\""" + if (-not $mstestVersion) { + throw "MSTestVersion for $propertyName is empty." + } + + $dirMSTestVersion = $mstestVersion -replace "\[|\]" + $dirMSTestPropertyName = $propertyName -replace "Framework" -replace "Version" + Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:MSTestFrameworkVersion=$mstestVersion -p:MSTestAdapterVersion=$mstestVersion -p:NETTestSdkVersion=$netTestSdkVersion -p:BaseOutputPath=""bin\$dirNetTestSdkPropertyName-$dirNetTestSdkVersion\$dirMSTestPropertyName-$dirMSTestVersion\\"" -bl:""$env:TP_OUT_DIR\log\$Configuration\perm.binlog""" + } } } @@ -328,8 +319,7 @@ function Publish-PatchedDotnet { Copy-Item $buildArtifactsPath $dotnetTestArtifactsSdkPath -Force } -function Publish-Package -{ +function Publish-Package { $timer = Start-Timer Write-Log "Publish-Package: Started." $fullCLRPackage451Dir = Get-FullCLRPackageDirectory @@ -475,73 +465,73 @@ function Publish-Package # into the output folder (for some reason), and we overwrite it with actual uap10.0 version below Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.ObjectModel\bin\$TPB_Configuration") ` - -files @{ - $TPB_TargetFramework45 = $fullCLRPackage45Dir # net45 - $TPB_TargetFramework451 = $fullCLRPackage451Dir # net451 - $TPB_TargetFrameworkCore10 = $coreCLR10PackageDir # netcoreapp1.0 - $TPB_TargetFrameworkCore20 = $coreCLR20PackageDir # netcoreapp2.1 - $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 - $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 - $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 - $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 - } + -files @{ + $TPB_TargetFramework45 = $fullCLRPackage45Dir # net45 + $TPB_TargetFramework451 = $fullCLRPackage451Dir # net451 + $TPB_TargetFrameworkCore10 = $coreCLR10PackageDir # netcoreapp1.0 + $TPB_TargetFrameworkCore20 = $coreCLR20PackageDir # netcoreapp2.1 + $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 + $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 + $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 + $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 + } Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.ObjectModel\bin\$TPB_Configuration") ` - -files @{ - $TPB_TargetFrameworkUap100 = $testhostUapPackageDir # uap10.0 - testhost - } + -files @{ + $TPB_TargetFrameworkUap100 = $testhostUapPackageDir # uap10.0 - testhost + } ################################################################################ # Publish Microsoft.TestPlatform.PlatformAbstractions Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.PlatformAbstractions\bin\$TPB_Configuration") ` - -files @{ - $TPB_TargetFramework45 = $fullCLRPackage45Dir # net45 - $TPB_TargetFramework451 = $fullCLRPackage451Dir # net451 - $TPB_TargetFrameworkCore20 = $coreCLR20PackageDir # netcoreapp2.1 - $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 - $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 - $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 - $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 - } + -files @{ + $TPB_TargetFramework45 = $fullCLRPackage45Dir # net45 + $TPB_TargetFramework451 = $fullCLRPackage451Dir # net451 + $TPB_TargetFrameworkCore20 = $coreCLR20PackageDir # netcoreapp2.1 + $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 + $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 + $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 + $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 + } Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.PlatformAbstractions\bin\$TPB_Configuration") ` - -files @{ - $TPB_TargetFrameworkUap100 = $testhostUapPackageDir # uap10.0 - testhost - } + -files @{ + $TPB_TargetFrameworkUap100 = $testhostUapPackageDir # uap10.0 - testhost + } ################################################################################ # Publish Microsoft.TestPlatform.CoreUtilities Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.CoreUtilities\bin\$TPB_Configuration") ` - -files @{ - $TPB_TargetFramework45 = $fullCLRPackage45Dir # net45 - $TPB_TargetFramework451 = $fullCLRPackage451Dir # net451 - $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 - $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 - $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 - $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 - } + -files @{ + $TPB_TargetFramework45 = $fullCLRPackage45Dir # net45 + $TPB_TargetFramework451 = $fullCLRPackage451Dir # net451 + $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 + $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 + $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 + $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 + } Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.CoreUtilities\bin\$TPB_Configuration") ` - -files @{ - $TPB_TargetFrameworkUap100 = $testhostUapPackageDir # uap10.0 - testhost - } + -files @{ + $TPB_TargetFrameworkUap100 = $testhostUapPackageDir # uap10.0 - testhost + } ################################################################################ # Publish Microsoft.TestPlatform.AdapterUtilities Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.AdapterUtilities\bin\$TPB_Configuration") ` - -files @{ - # "net20" = $net20PackageDir # net20 - "net45/any" = $net45PackageDir # $net4 - $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 - $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 - $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 - } + -files @{ + # "net20" = $net20PackageDir # net20 + "net45/any" = $net45PackageDir # $net4 + $TPB_TargetFrameworkNS10 = $netstandard10PackageDir # netstandard1_0 + $TPB_TargetFrameworkNS20 = $netstandard20PackageDir # netstandard2_0 + $TPB_TargetFrameworkUap100 = $uap100PackageDir # uap10.0 + } ################################################################################ # Publish Microsoft.TestPlatform.CrossPlatEngine Copy-Bulk -root (Join-Path $env:TP_ROOT_DIR "src\Microsoft.TestPlatform.CrossPlatEngine\bin\$TPB_Configuration") ` - -files @{ - $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 - } + -files @{ + $TPB_TargetFrameworkNS13 = $netstandard13PackageDir # netstandard1_3 + } ################################################################################ # Publish msdia @@ -572,7 +562,7 @@ function Publish-Package "Microsoft.VisualStudio.TestPlatform.Extensions.Html.TestLogger.dll", "Microsoft.VisualStudio.TestPlatform.Extensions.Html.TestLogger.pdb" ) - foreach($file in $loggers) { + foreach ($file in $loggers) { Write-Verbose "Move-Item $fullCLRPackage451Dir\$file $fullCLRExtensionsDir -Force" Move-Item $fullCLRPackage451Dir\$file $fullCLRExtensionsDir -Force @@ -581,7 +571,7 @@ function Publish-Package } # Move logger resource dlls - if($TPB_LocalizedBuild) { + if ($TPB_LocalizedBuild) { Move-Loc-Files $fullCLRPackage451Dir $fullCLRExtensionsDir "Microsoft.VisualStudio.TestPlatform.Extensions.Trx.TestLogger.resources.dll" Move-Loc-Files $coreCLR20PackageDir $coreCLRExtensionsDir "Microsoft.VisualStudio.TestPlatform.Extensions.Trx.TestLogger.resources.dll" Move-Loc-Files $fullCLRPackage451Dir $fullCLRExtensionsDir "Microsoft.VisualStudio.TestPlatform.Extensions.Html.TestLogger.resources.dll" @@ -622,7 +612,7 @@ function Publish-Package # Copy-Item $blameDataCollectorNetStandard\procdump $coreCLRExtensionsDir\procdump -Force # Copy blame data collector resource dlls - if($TPB_LocalizedBuild) { + if ($TPB_LocalizedBuild) { Copy-Loc-Files $blameDataCollectorNetFull $fullCLRExtensionsDir "Microsoft.TestPlatform.Extensions.BlameDataCollector.resources.dll" Copy-Loc-Files $blameDataCollectorNetStandard $coreCLRExtensionsDir "Microsoft.TestPlatform.Extensions.BlameDataCollector.resources.dll" } @@ -636,7 +626,7 @@ function Publish-Package Copy-Item $eventLogDataCollectorNetFull\Microsoft.TestPlatform.Extensions.EventLogCollector.pdb $coreCLRExtensionsDir -Force # Copy EventLogCollector resource dlls - if($TPB_LocalizedBuild) { + if ($TPB_LocalizedBuild) { Copy-Loc-Files $eventLogDataCollectorNetFull $fullCLRExtensionsDir "Microsoft.TestPlatform.Extensions.EventLogCollector.resources.dll" Copy-Loc-Files $eventLogDataCollectorNetFull $coreCLRExtensionsDir "Microsoft.TestPlatform.Extensions.EventLogCollector.resources.dll" } @@ -645,13 +635,13 @@ function Publish-Package $codeCoverageExternalsVersion = ([xml](Get-Content $env:TP_ROOT_DIR\eng\Versions.props)).Project.PropertyGroup.MicrosoftInternalCodeCoverageVersion $codeCoverageIOPackagesDir = Join-Path $env:TP_PACKAGES_DIR "microsoft.visualstudio.coverage.io\$codeCoverageExternalsVersion\lib\$TPB_TargetFrameworkStandard" Copy-Item $codeCoverageIOPackagesDir\Microsoft.VisualStudio.Coverage.IO.dll $coreCLR20PackageDir -Force - if($TPB_LocalizedBuild) { + if ($TPB_LocalizedBuild) { Copy-Loc-Files $codeCoverageIOPackagesDir $coreCLR20PackageDir "Microsoft.VisualStudio.Coverage.IO.resources.dll" } # If there are some dependencies for the TestHostRuntimeProvider assemblies, those need to be moved too. $runtimeproviders = @("Microsoft.TestPlatform.TestHostRuntimeProvider.dll", "Microsoft.TestPlatform.TestHostRuntimeProvider.pdb") - foreach($file in $runtimeproviders) { + foreach ($file in $runtimeproviders) { Write-Verbose "Move-Item $fullCLRPackage451Dir\$file $fullCLRExtensionsDir -Force" Move-Item $fullCLRPackage451Dir\$file $fullCLRExtensionsDir -Force @@ -690,8 +680,8 @@ function Publish-Package Copy-Item $testhostCore20PackageDir\testhost.pdb $coreCLR20PackageDir -Force Get-Item "$testhostCore20PackageDir\*" | - Where-Object { $_.Name -notin ("x64", "x86", "win7-x64", "win7-x86", "testhost.deps.json", "testhost.runtimeconfig.json")} | - Copy-Item -Recurse -Destination $fullCLRTestHostDir -Force + Where-Object { $_.Name -notin ("x64", "x86", "win7-x64", "win7-x86", "testhost.deps.json", "testhost.runtimeconfig.json") } | + Copy-Item -Recurse -Destination $fullCLRTestHostDir -Force Copy-Item $standaloneTesthost $fullCLRTestHostDir -Force # For libraries that are externally published, copy the output into artifacts. These will be signed and packaged independently. @@ -737,10 +727,8 @@ function Publish-Package Write-Log "Publish-Package: Complete. {$(Get-ElapsedTime($timer))}" } -function Publish-Tests -{ - if($TPB_PublishTests) - { +function Publish-Tests { + if ($TPB_PublishTests) { Write-Log "Publish-Tests: Started." # Adding only Perf project for now @@ -764,21 +752,18 @@ function Publish-Tests } } -function Publish-PackageInternal($packagename, $framework, $output) -{ +function Publish-PackageInternal($packagename, $framework, $output) { $dotnetExe = Get-DotNetPath Invoke-Exe $dotnetExe -Arguments "publish $packagename --configuration $TPB_Configuration --framework $framework --output $output -v:minimal -p:Version=$TPB_Version -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild" } -function Publish-PackageWithRuntimeInternal($packagename, $framework, $runtime, $selfcontained, $output) -{ +function Publish-PackageWithRuntimeInternal($packagename, $framework, $runtime, $selfcontained, $output) { $dotnetExe = Get-DotNetPath Invoke-Exe $dotnetExe -Arguments "publish $packagename --configuration $TPB_Configuration --framework $framework --runtime $runtime --self-contained $selfcontained --output $output -v:minimal -p:Version=$TPB_Version -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild" } -function Copy-Loc-Files($sourceDir, $destinationDir, $dllName) -{ - foreach($lang in $language) { +function Copy-Loc-Files($sourceDir, $destinationDir, $dllName) { + foreach ($lang in $language) { $dllToCopy = Join-Path $sourceDir\$lang $dllName $destinationFolder = Join-Path $destinationDir $lang if (-not (Test-Path $destinationFolder)) { @@ -788,9 +773,8 @@ function Copy-Loc-Files($sourceDir, $destinationDir, $dllName) } } -function Move-Loc-Files($sourceDir, $destinationDir, $dllName) -{ - foreach($lang in $language) { +function Move-Loc-Files($sourceDir, $destinationDir, $dllName) { + foreach ($lang in $language) { $dllToCopy = Join-Path $sourceDir\$lang $dllName $destinationFolder = Join-Path $destinationDir $lang if (-not (Test-Path $destinationFolder)) { @@ -800,8 +784,7 @@ function Move-Loc-Files($sourceDir, $destinationDir, $dllName) } } -function Create-VsixPackage -{ +function Create-VsixPackage { Write-Log "Create-VsixPackage: Started." $timer = Start-Timer @@ -820,7 +803,7 @@ function Create-VsixPackage # Copy Microsoft.VisualStudio.TraceDataCollector to Extensions $traceDataCollectorPackageDirectory = Join-Path $env:TP_PACKAGES_DIR "Microsoft.VisualStudio.TraceDataCollector\$codeCoverageExternalsVersion\lib\$TPB_TargetFramework472" Copy-Item $traceDataCollectorPackageDirectory\Microsoft.VisualStudio.TraceDataCollector.dll $extensionsPackageDir -Force - if($TPB_LocalizedBuild) { + if ($TPB_LocalizedBuild) { Copy-Loc-Files $traceDataCollectorPackageDirectory $extensionsPackageDir "Microsoft.VisualStudio.TraceDataCollector.resources.dll" } @@ -841,7 +824,7 @@ function Create-VsixPackage # Copy Microsoft.VisualStudio.IO to root $codeCoverageIOPackageDirectory = Join-Path $env:TP_PACKAGES_DIR "Microsoft.VisualStudio.Coverage.IO\$codeCoverageExternalsVersion\lib\$TPB_TargetFramework451" Copy-Item $codeCoverageIOPackageDirectory\Microsoft.VisualStudio.Coverage.IO.dll $packageDir -Force - if($TPB_LocalizedBuild) { + if ($TPB_LocalizedBuild) { Copy-Loc-Files $codeCoverageIOPackageDirectory $packageDir "Microsoft.VisualStudio.Coverage.IO.resources.dll" } @@ -894,8 +877,7 @@ function Create-VsixPackage $msbuildPath = Locate-MSBuildPath # Create vsix only when msbuild is installed. - if(![string]::IsNullOrEmpty($msbuildPath)) - { + if (![string]::IsNullOrEmpty($msbuildPath)) { # Copy the vsix project to artifacts directory to modify manifest New-Item $vsixProjectDir -Type Directory -Force Copy-Item -Recurse $vsixSourceDir\* $vsixProjectDir -Force @@ -906,16 +888,14 @@ function Create-VsixPackage # Build vsix project to get TestPlatform.vsix Invoke-Exe $msbuildPath -Arguments """$vsixProjectDir\TestPlatform.csproj"" -p:Configuration=$Configuration" } - else - { + else { throw ".. Create-VsixPackage: Cannot generate vsix as msbuild.exe not found at '$msbuildPath'." } Write-Log "Create-VsixPackage: Complete. {$(Get-ElapsedTime($timer))}" } -function Create-NugetPackages -{ +function Create-NugetPackages { $timer = Start-Timer Write-Log "Create-NugetPackages: Started." @@ -1049,8 +1029,7 @@ function Create-NugetPackages Write-Log "Create-NugetPackages: Complete. {$(Get-ElapsedTime($timer))}" } -function Copy-CodeCoverage-Package-Artifacts -{ +function Copy-CodeCoverage-Package-Artifacts { # Copy TraceDataCollector to Microsoft.CodeCoverage folder. $codeCoverageExternalsVersion = ([xml](Get-Content $env:TP_ROOT_DIR\eng\Versions.props)).Project.PropertyGroup.MicrosoftInternalCodeCoverageVersion $traceDataCollectorPackagesDir = Join-Path $env:TP_PACKAGES_DIR "microsoft.visualstudio.tracedatacollector\$codeCoverageExternalsVersion\lib\$TPB_TargetFrameworkNS20" @@ -1085,13 +1064,12 @@ function Copy-CodeCoverage-Package-Artifacts Copy-Item $codeCoverageImMacosPackagesDir\x64 $microsoftCodeCoveragePackageDir\InstrumentationEngine\macos\ -Force -Recurse # Copy TraceDataCollector resource dlls - if($TPB_LocalizedBuild) { + if ($TPB_LocalizedBuild) { Copy-Loc-Files $traceDataCollectorPackagesDir $microsoftCodeCoveragePackageDir "Microsoft.VisualStudio.TraceDataCollector.resources.dll" } } -function Copy-PackageItems($packageName) -{ +function Copy-PackageItems($packageName) { # Packages published separately are copied into their own artifacts directory # E.g. src\Microsoft.TestPlatform.ObjectModel\bin\Debug\net451\* is copied # to artifacts\Debug\Microsoft.TestPlatform.ObjectModel\net451 @@ -1106,8 +1084,7 @@ function Copy-PackageItems($packageName) Copy-Item -Path $binariesDirectory -Destination $publishDirectory -Recurse -Force } -function Update-LocalizedResources -{ +function Update-LocalizedResources { $timer = Start-Timer $dotnetExe = Get-DotNetPath @@ -1125,8 +1102,7 @@ function Update-LocalizedResources # # Helper functions # -function Get-DotNetPath -{ +function Get-DotNetPath { $dotnetPath = Join-Path $env:TP_TOOLS_DIR "dotnet\dotnet.exe" if (-not (Test-Path $dotnetPath)) { Write-Error "Dotnet.exe not found at $dotnetPath. Did the dotnet cli installation succeed?" @@ -1135,33 +1111,27 @@ function Get-DotNetPath return $dotnetPath } -function Get-FullCLRPackageDirectory -{ +function Get-FullCLRPackageDirectory { return $(Join-Path $env:TP_OUT_DIR "$TPB_Configuration\$TPB_TargetFramework451\$TPB_TargetRuntime") } -function Get-FullCLRPackageDirectory45 -{ +function Get-FullCLRPackageDirectory45 { return $(Join-Path $env:TP_OUT_DIR "$TPB_Configuration\$TPB_TargetFramework45\$TPB_TargetRuntime") } -function Get-CoreCLR20PackageDirectory -{ +function Get-CoreCLR20PackageDirectory { return $(Join-Path $env:TP_OUT_DIR "$TPB_Configuration\$TPB_TargetFrameworkCore20") } -function Get-CoreCLR10PackageDirectory -{ +function Get-CoreCLR10PackageDirectory { return $(Join-Path $env:TP_OUT_DIR "$TPB_Configuration\$TPB_TargetFrameworkCore10") } -function Get-CoreCLR20TestHostPackageDirectory -{ +function Get-CoreCLR20TestHostPackageDirectory { return $(Join-Path $env:TP_OUT_DIR "$TPB_Configuration\$TPB_TargetFrameworkCore20\TestHost") } -function Locate-MSBuildPath -{ +function Locate-MSBuildPath { $vsInstallPath = Locate-VsInstallPath $msbuildPath = Get-ChildItem (Join-Path -path $vsInstallPath -childPath "MSBuild\*\Bin\MSBuild.exe") @@ -1176,66 +1146,61 @@ function Locate-MSBuildPath return $msBuild.FullName } -function Locate-VsInstallPath -{ - $vswhere = Join-Path -path $env:TP_PACKAGES_DIR -ChildPath "vswhere\$env:VSWHERE_VERSION\tools\vswhere.exe" - if (!(Test-Path -path $vswhere)) { - throw "Unable to locate vswhere in path '$vswhere'." - } - - Write-Verbose "Using '$vswhere' to locate VS installation path." - - $requiredPackageIds = @("Microsoft.Component.MSBuild", "Microsoft.Net.Component.4.6.TargetingPack", "Microsoft.VisualStudio.Component.VSSDK") - Write-Verbose "VSInstallation requirements : $requiredPackageIds" - - Try - { - if ($TPB_CIBuild) { - $vsInstallPath = Invoke-Exe $vswhere -CaptureOutput -Arguments "-version (15.0 -products * -requires $requiredPackageIds -property installationPath" - } - else { - # Allow using pre release versions of VS for dev builds - $vsInstallPath = Invoke-Exe $vswhere -CaptureOutput -Arguments "-version (15.0 -prerelease -products * -requires $requiredPackageIds -property installationPath" - } - } - Catch [System.Management.Automation.MethodInvocationException] - { - throw "Failed to find VS installation with requirements: $requiredPackageIds" - } - - if ($null -eq $vsInstallPath -or 0 -eq @($vsInstallPath).Count) { +function Locate-VsInstallPath { + $vswhere = Join-Path -path $env:TP_PACKAGES_DIR -ChildPath "vswhere\$env:VSWHERE_VERSION\tools\vswhere.exe" + if (!(Test-Path -path $vswhere)) { + throw "Unable to locate vswhere in path '$vswhere'." + } + + Write-Verbose "Using '$vswhere' to locate VS installation path." + + $requiredPackageIds = @("Microsoft.Component.MSBuild", "Microsoft.Net.Component.4.6.TargetingPack", "Microsoft.VisualStudio.Component.VSSDK") + Write-Verbose "VSInstallation requirements : $requiredPackageIds" + + Try { + if ($TPB_CIBuild) { + $vsInstallPath = Invoke-Exe $vswhere -CaptureOutput -Arguments "-version (15.0 -products * -requires $requiredPackageIds -property installationPath" + } + else { + # Allow using pre release versions of VS for dev builds + $vsInstallPath = Invoke-Exe $vswhere -CaptureOutput -Arguments "-version (15.0 -prerelease -products * -requires $requiredPackageIds -property installationPath" + } + } + Catch [System.Management.Automation.MethodInvocationException] { throw "Failed to find VS installation with requirements: $requiredPackageIds" - } - else { + } + + if ($null -eq $vsInstallPath -or 0 -eq @($vsInstallPath).Count) { + throw "Failed to find VS installation with requirements: $requiredPackageIds" + } + else { Write-Verbose "Found VS installation with requirements '$($requiredPackageIds -join "','")' : '$($vsInstallPath -join "','")'." - } + } - $vsPath = $vsInstallPath | Select-Object -First 1 - Write-Verbose "VSInstallPath is : $vsPath" - return $vsPath + $vsPath = $vsInstallPath | Select-Object -First 1 + Write-Verbose "VSInstallPath is : $vsPath" + return $vsPath } -function Update-VsixVersion($vsixProjectDir) -{ +function Update-VsixVersion($vsixProjectDir) { Write-Log "Update-VsixVersion: Started." $vsixVersion = $Version # Build number comes in the form 20170111-01(yyyymmdd-buildNoOfThatDay) # So Version of the vsix will be 15.1.0.2017011101 $vsixVersionSuffix = $BuildNumber.Split("-"); - if($vsixVersionSuffix.Length -ige 2) { + if ($vsixVersionSuffix.Length -ige 2) { $vsixVersion = "$vsixVersion.$($vsixVersionSuffix[0])$($vsixVersionSuffix[1])" } - $manifestContentWithVersion = Get-Content "$vsixProjectDir\source.extension.vsixmanifest" -raw | ForEach-Object {$_.ToString().Replace("`$version`$", "$vsixVersion") } + $manifestContentWithVersion = Get-Content "$vsixProjectDir\source.extension.vsixmanifest" -raw | ForEach-Object { $_.ToString().Replace("`$version`$", "$vsixVersion") } Set-Content -path "$vsixProjectDir\source.extension.vsixmanifest" -value $manifestContentWithVersion Write-Log "Update-VsixVersion: Completed." } -function Generate-Manifest ($PackageFolder) -{ - $packagesFolderName = [System.IO.Path]::GetFileName($PackageFolder) +function Generate-Manifest ($PackageFolder) { + $packagesFolderName = [System.IO.Path]::GetFileName($PackageFolder) Write-Log "Generate-Manifest ($packagesFolderName): Started." $generateManifestPath = Join-Path $env:TP_ROOT_DIR "scripts\build\GenerateManifest.proj" @@ -1246,8 +1211,7 @@ function Generate-Manifest ($PackageFolder) Write-Log "Generate-Manifest ($packagesFolderName): Completed." } -function Build-SpecificProjects -{ +function Build-SpecificProjects { Write-Log "Build-SpecificProjects: Started for pattern: $ProjectNamePatterns" # FrameworksAndOutDirs format ("", ""). $FrameworksAndOutDirs = ( @@ -1263,18 +1227,18 @@ function Build-SpecificProjects # Get projects to build. Get-ChildItem -Recurse -Path $env:TP_ROOT_DIR -Include *.csproj | ForEach-Object { foreach ($ProjectNamePattern in $ProjectNamePatterns) { - if($_.FullName -match $ProjectNamePattern) { - $ProjectsToBuild += ,"$_" + if ($_.FullName -match $ProjectNamePattern) { + $ProjectsToBuild += , "$_" } } } - if( $null -eq $ProjectsToBuild){ + if ( $null -eq $ProjectsToBuild) { Write-Error "No csproj name match for given pattern: $ProjectNamePatterns" } # Build Projects. - foreach($ProjectToBuild in $ProjectsToBuild) { + foreach ($ProjectToBuild in $ProjectsToBuild) { Write-Log "Building Project $ProjectToBuild" # Restore and Build $output = Invoke-Exe $dotnetPath -Arguments "restore $ProjectToBuild" @@ -1289,13 +1253,13 @@ function Build-SpecificProjects # Copy artifacts $ProjectDir = [System.IO.Path]::GetDirectoryName($ProjectToBuild) - foreach($FrameworkAndOutDir in $FrameworksAndOutDirs) { + foreach ($FrameworkAndOutDir in $FrameworksAndOutDirs) { $fromDir = $([System.IO.Path]::Combine($ProjectDir, "bin", $TPB_Configuration, $FrameworkAndOutDir[0])) $toDir = $([System.IO.Path]::Combine($env:TP_OUT_DIR, $TPB_Configuration, $FrameworkAndOutDir[1])) - if ( Test-Path $fromDir){ + if ( Test-Path $fromDir) { Write-Log "Copying artifacts from $fromDir to $toDir" Get-ChildItem $fromDir | ForEach-Object { - if(-not ($_.PSIsContainer)) { + if (-not ($_.PSIsContainer)) { Copy-Item $_.FullName $toDir } } @@ -1304,8 +1268,7 @@ function Build-SpecificProjects } } -if ($ProjectNamePatterns.Count -ne 0) -{ +if ($ProjectNamePatterns.Count -ne 0) { # Build Specific projects. Build-SpecificProjects Exit @@ -1352,7 +1315,7 @@ if ($ProjectNamePatterns.Count -ne 0) # } if ($Force -or $Steps -contains "PrepareAcceptanceTests") { - #Publish-PatchedDotnet + #Publish-PatchedDotnet Invoke-TestAssetsBuild #Publish-Tests } @@ -1360,7 +1323,8 @@ if ($Force -or $Steps -contains "PrepareAcceptanceTests") { if ($Script:ScriptFailed) { Write-Log "Build failed. {$(Get-ElapsedTime($timer))}" -Level "Error" Exit 1 -} else { +} +else { Write-Log "Build succeeded. {$(Get-ElapsedTime($timer))}" Exit 0 } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 57be416dd5..e10173e02a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -17,7 +17,6 @@ namespace Microsoft.TestPlatform.AcceptanceTests; public class ExecutionTests : AcceptanceTestBase { // TODO: It looks like the first 3 tests would be useful to multiply by all 3 test frameworks, should we make the test even more generic, or duplicate them? - [TestMethod] [MSTestCompatibilityDataSource(InProcess = true)] public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, TesthostInfo testhostInfo, MSTestInfo msTestInfo) @@ -33,15 +32,15 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, TesthostInfo testho } [TestMethod] - [TestPlatformCompatibilityDataSource("netcoreapp2.1", "LegacyStable", "netcoreapp2.1", "Latest", "LatestPreview")] + //[TestPlatformCompatibilityDataSource("netcoreapp2.1", "LegacyStable", "netcoreapp2.1", "Latest", "LatestPreview")] // [TestPlatformCompatibilityDataSource("netcoreapp2.1", LATEST_TO_LEGACY, "netcoreapp2.1", LATEST_TO_LEGACY, LATESTPREVIEW_TO_LEGACY)] - //[TestPlatformCompatibilityDataSource()] + [TestPlatformCompatibilityDataSource( WithInProcess = true)] public void RunMultipleTestAssemblies223(RunnerInfo runnerInfo, VSTestConsoleInfo consoleInfo, TesthostInfo testhostInfo, MSTestInfo msTestInfo) { SetTestEnvironment(_testEnvironment, runnerInfo, consoleInfo); - var assemblyPaths = BuildMultipleAssemblyPath(testhostInfo, msTestInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath(testhostInfo, msTestInfo, "SimpleTestProject.dll", "MSTestProject2.dll").Trim('\"'); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs index c9e40513b0..388217cea8 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs @@ -15,7 +15,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests; public sealed class MSTestCompatibilityDataSource : TestDataSource { - private static XmlDocument? _depsXml; + private static XmlDocument? s_depsXml; private readonly string[] _runnerFrameworks; private readonly string[] _targetFrameworks; private readonly string[] _msTestVersions; @@ -98,8 +98,8 @@ private MSTestInfo GetMSTestInfo(string msTestVersion) private static XmlDocument GetDependenciesXml() { - if (_depsXml != null) - return _depsXml; + if (s_depsXml != null) + return s_depsXml; var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); var fileStream = File.OpenRead(depsXmlPath); @@ -107,7 +107,7 @@ private static XmlDocument GetDependenciesXml() var depsXml = new XmlDocument(); depsXml.Load(xmlTextReader); - _depsXml = depsXml; + s_depsXml = depsXml; return depsXml; } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs index 63d0a7105e..262b3f1b85 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs @@ -51,6 +51,20 @@ public TestPlatformCompatibilityDataSource( public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; + /// + /// Add run for in-process using the selected .NET Framework runners, and and all selected adapters. + /// + public bool WithInProcess { get; set; } + + public bool WithEveryVersionOfRunner { get; set; } = true; + + public bool WithEveryVersionOfHost { get; set; } = true; + + public bool WithEveryVersionOfAdapter { get; set; } = true; + + public bool WithOlderConfigurations { get; set; } = true; + + public string? BeforeFeature { get; set; } public string? AfterFeature { get; set; } public string? BeforeAdapterFeature { get; set; } @@ -59,14 +73,21 @@ public TestPlatformCompatibilityDataSource( public override void CreateData(MethodInfo methodInfo) { var dataRows = new List<(RunnerInfo runnerInfo, VSTestConsoleInfo vstestConsoleInfo, TesthostInfo testhostInfo, MSTestInfo mstestInfo)>(); - AddEveryVersionOfRunner(dataRows); - // with every version of host - AddEveryVersionOfHost(dataRows); + if (WithEveryVersionOfRunner) + AddEveryVersionOfRunner(dataRows); - AddEveryVersionOfAdapter(dataRows); + if (WithEveryVersionOfHost) + AddEveryVersionOfHost(dataRows); - AddOlderConfigurations(dataRows); + if (WithEveryVersionOfAdapter) + AddEveryVersionOfAdapter(dataRows); + + if (WithOlderConfigurations) + AddOlderConfigurations(dataRows); + + if (WithInProcess) + AddInProcess(dataRows); var c = dataRows.Count(); @@ -104,6 +125,28 @@ public override void CreateData(MethodInfo methodInfo) } } + private void AddInProcess(List<(RunnerInfo runnerInfo, VSTestConsoleInfo vstestConsoleInfo, TesthostInfo testhostInfo, MSTestInfo mstestInfo)> dataRows) + { + foreach (var runnerFramework in _runnerFrameworks) + { + if (!runnerFramework.StartsWith("net4")) + { + continue; + } + + foreach (var runnerVersion in _runnerVersions) + { + foreach (var adapter in _adapters) + { + foreach (var adapterVersion in _adapterVersions) + { + AddRow(dataRows, runnerVersion, runnerFramework, runnerVersion, runnerFramework, adapter, adapterVersion, inIsolation: false); + } + } + } + } + } + private void AddOlderConfigurations(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows) { // Older configurations where the runner, host and adapter version are the same. @@ -119,7 +162,7 @@ private void AddOlderConfigurations(List<(RunnerInfo, VSTestConsoleInfo, Testhos foreach (var adapter in _adapters) { var adapterVersion = runnerVersion; - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion); + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); } } } @@ -142,7 +185,7 @@ private void AddEveryVersionOfAdapter(List<(RunnerInfo, VSTestConsoleInfo, Testh // We already used the newest when adding combination with every runner foreach (var adapterVersion in _adapterVersions.Skip(1)) { - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion); + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); } } } @@ -169,7 +212,7 @@ private void AddEveryVersionOfHost(List<(RunnerInfo, VSTestConsoleInfo, Testhost { // use the newest var adapterVersion = _adapterVersions[0]; - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion); + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); } } } @@ -192,7 +235,7 @@ private void AddEveryVersionOfRunner(List<(RunnerInfo, VSTestConsoleInfo, Testho { // use the newest var adapterVersion = _adapterVersions[0]; - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion); + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); } } } @@ -200,9 +243,9 @@ private void AddEveryVersionOfRunner(List<(RunnerInfo, VSTestConsoleInfo, Testho } private void AddRow(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows, - string runnerVersion, string runnerFramework, string hostVersion, string hostFramework, string adapter, string adapterVersion) + string runnerVersion, string runnerFramework, string hostVersion, string hostFramework, string adapter, string adapterVersion, bool inIsolation) { - RunnerInfo runnerInfo = GetRunnerInfo(runnerFramework, hostFramework, inIsolation: true); + RunnerInfo runnerInfo = GetRunnerInfo(runnerFramework, hostFramework, inIsolation); var vstestConsoleInfo = GetVSTestConsoleInfo(runnerVersion, runnerInfo); var testhostInfo = TesthostCompatibilityDataSource.GetTesthostInfo(hostVersion); var mstestInfo = GetMSTestInfo(adapterVersion); diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 1e84f3b094..f979ad4906 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -7,7 +7,6 @@ using System.Diagnostics; using System.IO; using System.Linq; -using System.Runtime.CompilerServices; using System.Text; using System.Text.RegularExpressions; using System.Xml; diff --git a/test/TestAssets/MSTestProject2/MSTestProject2.csproj b/test/TestAssets/MSTestProject2/MSTestProject2.csproj new file mode 100644 index 0000000000000000000000000000000000000000..18ce8041acd97ea3ce45df23fd308ba411dedd89 GIT binary patch literal 2194 zcmd6pPjAye5XIjaiSJ-4q|!qTEq^Y!sY)m)hoXv9oN_U))0U>GWhYR+JoGoaacw7Q zoPv;06ld+3dHd$=%s9V)McUVmt~64h?@E=c(1orw)}A)BqfjsS4X})r@cb0(SZB^U z)kH&WCcz_6pQuY0aYS_DtOXuJtf4;XP$&F)uZKwP9J zpl`G4!7w8tC!*lK%le9s@eGJgc{~o#6POxtg5l|_6TsQH~v83*q)y1+LU%DLsB&)8@!A_{m zLl}%&ROwKyF + /// The unit test 1. + /// + [TestClass] + public class UnitTest1 + { + /// + /// The passing test. + /// + [Priority(2)] + [TestMethod] + public void PassingTest() + { + Assert.AreEqual(2, 2); + } + + /// + /// The failing test. + /// + [TestCategory("CategoryA")] + [Priority(3)] + [TestMethod] + public void FailingTest() + { +#if NETFRAMEWORK + // current App domain should be write to file to test DisableAppDomain acceptance test. + var appDomainFilePath = Environment.GetEnvironmentVariable("TEST_ASSET_APPDOMAIN_TEST_PATH") ?? Path.Combine(Path.GetTempPath(), "appdomain_test.txt"); + File.WriteAllText(appDomainFilePath, "AppDomain FriendlyName: " + AppDomain.CurrentDomain.FriendlyName); +#endif + Assert.AreEqual(2, 3); + } + + /// + /// The skipping test. + /// + [Ignore] + [TestMethod] + public void SkippingTest() + { + } + } +} diff --git a/test/TestAssets/TestAssets.sln b/test/TestAssets/TestAssets.sln index fb41d58e19..c5d6cba585 100644 --- a/test/TestAssets/TestAssets.sln +++ b/test/TestAssets/TestAssets.sln @@ -94,7 +94,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AttachmentProcessorDataColl EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ArchitectureSwitch", "ArchitectureSwitch\ArchitectureSwitch.csproj", "{452352E1-71CA-436E-8165-F284EE36C924}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SimpleTestProjectMessedUpTargetFramework", "SimpleTestProjectMessedUpTargetFramework\SimpleTestProjectMessedUpTargetFramework.csproj", "{08C44607-EB80-4EE5-927D-08C34AA277AF}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SimpleTestProjectMessedUpTargetFramework", "SimpleTestProjectMessedUpTargetFramework\SimpleTestProjectMessedUpTargetFramework.csproj", "{08C44607-EB80-4EE5-927D-08C34AA277AF}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MSTestProject2", "MSTestProject2\MSTestProject2.csproj", "{10AA955C-B412-41A8-899F-8609AAE19F61}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -634,6 +636,18 @@ Global {08C44607-EB80-4EE5-927D-08C34AA277AF}.Release|x64.Build.0 = Release|Any CPU {08C44607-EB80-4EE5-927D-08C34AA277AF}.Release|x86.ActiveCfg = Release|Any CPU {08C44607-EB80-4EE5-927D-08C34AA277AF}.Release|x86.Build.0 = Release|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Debug|Any CPU.Build.0 = Debug|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Debug|x64.ActiveCfg = Debug|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Debug|x64.Build.0 = Debug|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Debug|x86.ActiveCfg = Debug|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Debug|x86.Build.0 = Debug|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|Any CPU.ActiveCfg = Release|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|Any CPU.Build.0 = Release|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|x64.ActiveCfg = Release|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|x64.Build.0 = Release|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|x86.ActiveCfg = Release|Any CPU + {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 1d22cd99f8ec2f442a7b0d87c366e802ee5ac58c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 16 Mar 2022 13:04:33 +0100 Subject: [PATCH 35/64] Remove quote path trimming. --- build.cmd | 1 + scripts/build.ps1 | 13 ++-- sln.cmd | 14 ++++ .../BlameDataCollectorTests.cs | 6 +- .../DataCollectionTests.cs | 8 +-- .../DebugAssertTests.cs | 4 +- .../DiscoveryTests.cs | 2 +- .../DotnetTestTests.cs | 4 +- .../ExecutionTests.cs | 40 ++++++----- .../ExecutionThreadApartmentStateTests.cs | 8 +-- .../FilePatternParserTests.cs | 2 +- .../MultitargetingTestHostTests.cs | 4 +- .../PortableNugetPackageTests.cs | 2 +- .../RunsettingsTests.cs | 2 +- .../SelfContainedAppTests.cs | 4 +- .../TestPlatformNugetPackageTests.cs | 4 +- .../Hosting/DefaultTestHostManagerTests.cs | 4 +- .../IntegrationTestBase.cs | 64 ++++++++++-------- .../IntegrationTestEnvironment.cs | 6 +- .../MSTestProject1/MSTestProject1.csproj | 28 ++++++++ test/TestAssets/MSTestProject1/UnitTest1.cs | 28 ++++++++ .../MSTestProject2/MSTestProject2.csproj | Bin 2194 -> 1063 bytes test/TestAssets/MSTestProject2/UnitTest1.cs | 59 +++++----------- test/TestAssets/TestAssets.sln | 22 +++++- 24 files changed, 203 insertions(+), 126 deletions(-) create mode 100644 sln.cmd create mode 100644 test/TestAssets/MSTestProject1/MSTestProject1.csproj create mode 100644 test/TestAssets/MSTestProject1/UnitTest1.cs diff --git a/build.cmd b/build.cmd index f2a9a38318..51620a6421 100644 --- a/build.cmd +++ b/build.cmd @@ -3,4 +3,5 @@ REM Copyright (c) Microsoft. All rights reserved. powershell -ExecutionPolicy Bypass -NoProfile -NoLogo -Command "%~dp0scripts\build.ps1 %*; exit $LastExitCode;" + if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/scripts/build.ps1 b/scripts/build.ps1 index 78b06fd19e..23c5872736 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -231,9 +231,10 @@ function Invoke-TestAssetsBuild { } - # Build with multiple versions of MSTest. + # Build with multiple versions of MSTest. The projects are directly in the root. + # The folder structure in VS is not echoed in the TestAssets directory. $projects = @( - "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject\SimpleTestProject.csproj" + "$env:TP_ROOT_DIR\test\TestAssets\MSTestProject1\MSTestProject1.csproj" "$env:TP_ROOT_DIR\test\TestAssets\MSTestProject2\MSTestProject2.csproj" # Don't use this one, it does not use the variables for mstest and test sdk. # "$env:TP_ROOT_DIR\test\TestAssets\SimpleTestProject2\SimpleTestProject2.csproj" @@ -275,7 +276,7 @@ function Invoke-TestAssetsBuild { $dirMSTestVersion = $mstestVersion -replace "\[|\]" $dirMSTestPropertyName = $propertyName -replace "Framework" -replace "Version" - Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:MSTestFrameworkVersion=$mstestVersion -p:MSTestAdapterVersion=$mstestVersion -p:NETTestSdkVersion=$netTestSdkVersion -p:BaseOutputPath=""bin\$dirNetTestSdkPropertyName-$dirNetTestSdkVersion\$dirMSTestPropertyName-$dirMSTestVersion\\"" -bl:""$env:TP_OUT_DIR\log\$Configuration\perm.binlog""" + Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration --no-restore -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:MSTestFrameworkVersion=$mstestVersion -p:MSTestAdapterVersion=$mstestVersion -p:NETTestSdkVersion=$netTestSdkVersion -p:BaseOutputPath=""bin\$dirNetTestSdkPropertyName-$dirNetTestSdkVersion\$dirMSTestPropertyName-$dirMSTestVersion\\"" -bl:""$env:TP_OUT_DIR\log\$Configuration\perm.binlog""" } } } @@ -1282,9 +1283,9 @@ if ($ProjectNamePatterns.Count -ne 0) { # Write-Log "Test platform build variables: " # Get-Variable | Where-Object -FilterScript { $_.Name.StartsWith("TPB_") } | Format-Table -# if ($Force -or $Steps -contains "InstallDotnet") { -# Install-DotNetCli -# } +if ($Force -or $Steps -contains "InstallDotnet") { + Install-DotNetCli +} # if ($Force -or $Steps -contains "Restore") { # Clear-Package diff --git a/sln.cmd b/sln.cmd new file mode 100644 index 0000000000..8ba8ab0bb9 --- /dev/null +++ b/sln.cmd @@ -0,0 +1,14 @@ +@echo off + +REM Copyright (c) Microsoft. All rights reserved. + +REM set DOTNET_ROOT to point at the locally installed dotnet, +REM to avoid problems with .NET Core 2.1 that we run our tests and tools against, +REM but that is regularly corrupted. + +set DOTNET_ROOT=%~dp0tools\dotnet +set DOTNET_ROOT(x86)=%~dp0tools\dotnet_x86 + +start %~dp0TestPlatform.sln + +if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs index 671a91618d..e9e6af25ee 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs @@ -64,7 +64,7 @@ public void BlameDataCollectorShouldOutputDumpFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $" /Blame:CollectDump"); arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); @@ -89,7 +89,7 @@ public void BlameDataCollectorShouldNotOutputDumpFileWhenNoCrashOccurs(RunnerInf { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $" /Blame:CollectDump"); arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); @@ -114,7 +114,7 @@ public void BlameDataCollectorShouldOutputDumpFileWhenNoCrashOccursButCollectAlw { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $" /Blame:CollectDump;CollectAlways=True"); arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs index b7907cbdd6..2486a605ff 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs @@ -28,7 +28,7 @@ public void ExecuteTestsWithDataCollection(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); string runSettings = GetRunsettingsFilePath(TempDirectory.Path); string diagFileName = Path.Combine(TempDirectory.Path, "diaglog.txt"); var extensionsPath = Path.Combine( @@ -58,7 +58,7 @@ public void ExecuteTestsWithDataCollectionUsingCollectArgument(RunnerInfo runner { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); string diagFileName = Path.Combine(TempDirectory.Path, "diaglog.txt"); var extensionsPath = Path.Combine( _testEnvironment.TestAssetsPath, @@ -113,8 +113,8 @@ public void DataCollectorAttachmentProcessor(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPath = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"'); - var secondAssemblyPath = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + var assemblyPath = BuildMultipleAssemblyPath("SimpleTestProject.dll"); + var secondAssemblyPath = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); string runSettings = GetRunsettingsFilePath(TempDirectory.Path); string diagFileName = Path.Combine(TempDirectory.Path, "diaglog.txt"); var extensionsPath = Path.Combine( diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs index fba250c5a2..a03280f37c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DebugAssertTests.cs @@ -20,13 +20,13 @@ public void RunningTestWithAFailingDebugAssertDoesNotCrashTheHostingProcess(Runn // is to not crash the process when we are running in debug, and debugger is attached SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPath = BuildMultipleAssemblyPath("CrashingOnDebugAssertTestProject.dll").Trim('\"'); + var assemblyPath = BuildMultipleAssemblyPath("CrashingOnDebugAssertTestProject.dll"); var arguments = PrepareArguments(assemblyPath, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); // this will have failed tests when our trace listener works and crash the testhost process when it does not // because crashing processes is what a failed Debug.Assert does by default, unless you have a debugger attached - ValidateSummaryStatus(passedTestsCount: 4, failedTestsCount: 4, 0); + ValidateSummaryStatus(passed: 4, failed: 4, 0); StringAssert.Contains(StdOut, "threw exception: Microsoft.VisualStudio.TestPlatform.TestHost.DebugAssertException:"); } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs index 2f6087aa17..fa9bb8627b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs @@ -38,7 +38,7 @@ public void DiscoverAllTests(RunnerInfo runnerInfo) public void MultipleSourcesDiscoverAllTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); var listOfTests = new[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetTestTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetTestTests.cs index 4fb0d68eca..3e50b7c219 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetTestTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetTestTests.cs @@ -38,7 +38,7 @@ public void RunDotnetTestWithDll(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPath = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"'); + var assemblyPath = BuildMultipleAssemblyPath("SimpleTestProject.dll"); InvokeDotnetTest($@"{assemblyPath} --logger:""Console;Verbosity=normal"""); // ensure our dev version is used @@ -70,7 +70,7 @@ public void PassInlineSettingsToDll(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPath = BuildMultipleAssemblyPath("ParametrizedTestProject.dll").Trim('\"'); + var assemblyPath = BuildMultipleAssemblyPath("ParametrizedTestProject.dll"); InvokeDotnetTest($@"{assemblyPath} --logger:""Console;Verbosity=normal"" -- TestRunParameters.Parameter(name=\""weburl\"", value=\""http://localhost//def\"")"); ValidateSummaryStatus(1, 0, 0); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index e10173e02a..14daacdf20 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -19,11 +19,11 @@ public class ExecutionTests : AcceptanceTestBase // TODO: It looks like the first 3 tests would be useful to multiply by all 3 test frameworks, should we make the test even more generic, or duplicate them? [TestMethod] [MSTestCompatibilityDataSource(InProcess = true)] - public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, TesthostInfo testhostInfo, MSTestInfo msTestInfo) + public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, MSTestInfo msTestInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll"); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); @@ -32,19 +32,17 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, TesthostInfo testho } [TestMethod] - //[TestPlatformCompatibilityDataSource("netcoreapp2.1", "LegacyStable", "netcoreapp2.1", "Latest", "LatestPreview")] - // [TestPlatformCompatibilityDataSource("netcoreapp2.1", LATEST_TO_LEGACY, "netcoreapp2.1", LATEST_TO_LEGACY, LATESTPREVIEW_TO_LEGACY)] - [TestPlatformCompatibilityDataSource( WithInProcess = true)] + [TestPlatformCompatibilityDataSource(WithInProcess = true)] - public void RunMultipleTestAssemblies223(RunnerInfo runnerInfo, VSTestConsoleInfo consoleInfo, TesthostInfo testhostInfo, MSTestInfo msTestInfo) + public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo, VSTestConsoleInfo consoleInfo, TesthostInfo testhostInfo, MSTestInfo msTestInfo) { SetTestEnvironment(_testEnvironment, runnerInfo, consoleInfo); - var assemblyPaths = BuildMultipleAssemblyPath(testhostInfo, msTestInfo, "SimpleTestProject.dll", "MSTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath(testhostInfo, msTestInfo, "MSTestProject1.dll", "MSTestProject2.dll"); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); - ValidateSummaryStatus(2, 2, 2); + ValidateSummaryStatus(passed: 2, failed: 2, skipped: 2); ExitCodeEquals(1); // failing tests } @@ -54,7 +52,7 @@ public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(Ru { SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); - var assemblyPaths = BuildMultipleAssemblyPath(testhostInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath(testhostInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll"); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); @@ -73,7 +71,7 @@ public void RunMultipleTestAssembliesWithoutTestAdapterPath(RunnerInfo runnerInf { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll"); var xunitAssemblyPath = _testEnvironment.TargetFramework.Equals("net451") ? _testEnvironment.GetTestAsset("XUTestProject.dll", "net46") : _testEnvironment.GetTestAsset("XUTestProject.dll"); @@ -95,7 +93,7 @@ public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo, MSTestInf SetTestEnvironment(_testEnvironment, runnerInfo); using var tempDir = new TempDirectory(); - var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo,"SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo,"SimpleTestProject.dll", "SimpleTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, testAdapterPath: null, runSettings: null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /Parallel"); arguments = string.Concat(arguments, " /Platform:x86"); @@ -126,7 +124,7 @@ public void TestSessionTimeOutTests(RunnerInfo runnerInfo) using var tempDir = new TempDirectory(); var assemblyPaths = - BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /TestCaseFilter:TestSessionTimeoutTest"); @@ -147,7 +145,7 @@ public void TestPlatformShouldBeCompatibleWithOldTestHost(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); using var tempDir = new TempDirectory(); - var assemblyPaths = BuildMultipleAssemblyPath("SampleProjectWithOldTestHost.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SampleProjectWithOldTestHost.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); InvokeVsTest(arguments); @@ -164,7 +162,7 @@ public void WorkingDirectoryIsSourceDirectory(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); using var tempDir = new TempDirectory(); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /tests:WorkingDirectoryTest"); @@ -192,7 +190,7 @@ public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerIn var diagLogFilePath = Path.Combine(tempDir.Path, $"std_error_log_{Guid.NewGuid()}.txt"); File.Delete(diagLogFilePath); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /testcasefilter:ExitWithStackoverFlow"); arguments = string.Concat(arguments, $" /diag:{diagLogFilePath}"); @@ -223,7 +221,7 @@ public void UnhandleExceptionExceptionShouldBeLoggedToDiagLogFile(RunnerInfo run File.Delete(diagLogFilePath); var assemblyPaths = - BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /testcasefilter:ExitwithUnhandleException"); arguments = string.Concat(arguments, $" /diag:{diagLogFilePath}"); @@ -245,7 +243,7 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo r var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProject3.dll is built for Framework .NETFramework,Version=v4.5.1 and Platform X64"; var assemblyPaths = - BuildMultipleAssemblyPath("SimpleTestProject3.dll", "SimpleTestProjectx86.dll").Trim('\"'); + BuildMultipleAssemblyPath("SimpleTestProject3.dll", "SimpleTestProjectx86.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /testcasefilter:PassingTestx86"); @@ -317,7 +315,7 @@ public void ExitCodeShouldReturnOneWhenTreatNoTestsAsErrorParameterSetToTrueAndN SetTestEnvironment(_testEnvironment, runnerInfo); using var tempDir = new TempDirectory(); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); @@ -338,7 +336,7 @@ public void ExitCodeShouldReturnZeroWhenTreatNoTestsAsErrorParameterSetToFalseAn SetTestEnvironment(_testEnvironment, runnerInfo); using var tempDir = new TempDirectory(); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); // Setting /TestCaseFilter to the test name, which does not exists in the assembly, so we will have 0 tests executed @@ -358,7 +356,7 @@ public void ExitCodeShouldNotDependOnTreatNoTestsAsErrorTrueValueWhenThereAreAny SetTestEnvironment(_testEnvironment, runnerInfo); using var tempDir = new TempDirectory(); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); @@ -377,7 +375,7 @@ public void ExitCodeShouldNotDependOnFailTreatNoTestsAsErrorFalseValueWhenThereA SetTestEnvironment(_testEnvironment, runnerInfo); using var tempDir = new TempDirectory(); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " -- RunConfiguration.TreatNoTestsAsError=false"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs index 763e0d5016..7548f73d3f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionThreadApartmentStateTests.cs @@ -18,7 +18,7 @@ public void UITestShouldPassIfApartmentStateIsSTA(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:UITestMethod"); InvokeVsTest(arguments); @@ -32,7 +32,7 @@ public void WarningShouldBeShownWhenValueIsSTAForNetCore(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = - BuildMultipleAssemblyPath("SimpleTestProject2.dll").Trim('\"'); + BuildMultipleAssemblyPath("SimpleTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:PassingTest2 -- RunConfiguration.ExecutionThreadApartmentState=STA"); InvokeVsTest(arguments); @@ -47,7 +47,7 @@ public void UITestShouldFailWhenDefaultApartmentStateIsMTA(RunnerInfo runnerInfo SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = - BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:UITestMethod -- RunConfiguration.ExecutionThreadApartmentState=MTA"); InvokeVsTest(arguments); @@ -62,7 +62,7 @@ public void CancelTestExectionShouldWorkWhenApartmentStateIsSTA(RunnerInfo runne SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = - BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"'); + BuildMultipleAssemblyPath("SimpleTestProject3.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /tests:UITestWithSleep1,UITestMethod -- RunConfiguration.ExecutionThreadApartmentState=STA RunConfiguration.TestSessionTimeout=2000"); InvokeVsTest(arguments); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs index 4ab4715e72..3d8f36218b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs @@ -92,7 +92,7 @@ public void WildCardPatternShouldCorrectlyWorkOnMultipleFiles(RunnerInfo runnerI { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssembly = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var testAssembly = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); testAssembly = testAssembly.Replace("SimpleTestProject.dll", "*TestProj*.dll"); testAssembly = testAssembly.Replace("SimpleTestProject2.dll", "*TestProj*.dll"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs index 7b1dff7eb9..70a132de3a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/MultitargetingTestHostTests.cs @@ -23,7 +23,7 @@ public void TestRunInATesthostThatTargetsTheirChosenNETFramework(RunnerInfo runn { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPath = BuildMultipleAssemblyPath("MultitargetedNetFrameworkProject.dll").Trim('\"'); + var assemblyPath = BuildMultipleAssemblyPath("MultitargetedNetFrameworkProject.dll"); var arguments = PrepareArguments(assemblyPath, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); // Tell the test project which target framework we are expecting it to run as. @@ -35,6 +35,6 @@ public void TestRunInATesthostThatTargetsTheirChosenNETFramework(RunnerInfo runn InvokeVsTest(arguments, env); - ValidateSummaryStatus(passedTestsCount: 1, failedTestsCount: 0, 0); + ValidateSummaryStatus(passed: 1, failed: 0, 0); } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs index 5510ef2aed..d5f227fccf 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs @@ -43,7 +43,7 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); InvokeVsTestForExecution(assemblyPaths, GetTestAdapterPath(), FrameworkArgValue, string.Empty); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs index 2a0809cbaa..3d5dbcbea7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs @@ -537,7 +537,7 @@ private void RunTestWithRunSettings(Dictionary runConfigurationD string runSettingsArgs, string additionalArgs, IEnumerable testhostProcessNames, int expectedNumOfProcessCreated) { - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); var runsettingsPath = string.Empty; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs index c7dc6b7804..ed5e10fc8b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/SelfContainedAppTests.cs @@ -27,10 +27,10 @@ public void RunningApplicationThatIsBuiltAsSelfContainedWillNotFailToFindHostpol SetTestEnvironment(_testEnvironment, runnerInfo); // the app is published to win10-x64 because of the runtime identifier in the project - var assemblyPath = BuildMultipleAssemblyPath($@"win10-x64{Path.DirectorySeparatorChar}SelfContainedAppTestProject.dll").Trim('\"'); + var assemblyPath = BuildMultipleAssemblyPath($@"win10-x64{Path.DirectorySeparatorChar}SelfContainedAppTestProject.dll"); var arguments = PrepareArguments(assemblyPath, null, null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); - ValidateSummaryStatus(passedTestsCount: 1, 0, 0); + ValidateSummaryStatus(passed: 1, 0, 0); } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs index 6125803ad0..d3304c795d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TestPlatformNugetPackageTests.cs @@ -64,7 +64,7 @@ public void RunMultipleTestAssembliesWithCodeCoverage(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll").Trim('\"'); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); var arguments = CreateCodeCoverageArguments(runnerInfo, assemblyPaths, out var trxFilePath); InvokeVsTest(arguments); @@ -85,7 +85,7 @@ public override string GetConsoleRunnerPath() consoleRunnerPath = Path.Combine(s_nugetPackageFolder, "tools", "net451", "Common7", "IDE", "Extensions", "TestPlatform", "vstest.console.exe"); } - Assert.IsTrue(File.Exists(consoleRunnerPath), "GetConsoleRunnerPath: Path not found: {0}", consoleRunnerPath); + Assert.IsTrue(File.Exists(consoleRunnerPath), "GetConsoleRunnerPath: Path not found: \"{0}\"", consoleRunnerPath); return consoleRunnerPath; } diff --git a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DefaultTestHostManagerTests.cs b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DefaultTestHostManagerTests.cs index 7fd3915a62..b18464992d 100644 --- a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DefaultTestHostManagerTests.cs +++ b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DefaultTestHostManagerTests.cs @@ -142,7 +142,7 @@ public void GetTestHostProcessStartInfoShouldIncludeTestSourcePathInArgumentsIfN { _testHostManager.Initialize(_mockMessageLogger.Object, $" {Architecture.X86} {Framework.DefaultFramework} {true} "); var connectionInfo = new TestRunnerConnectionInfo { Port = 123, ConnectionInfo = new TestHostConnectionInfo { Endpoint = "127.0.0.0:123", Role = ConnectionRole.Client, Transport = Transport.Sockets }, RunnerProcessId = 101 }; - var source = "C:\temp\a.dll"; + var source = @"C:\temp\a.dll"; var info = _testHostManager.GetTestHostProcessStartInfo( new List() { source }, @@ -158,7 +158,7 @@ public void GetTestHostProcessStartInfoShouldUseMonoAsHostOnNonWindowsIfNotStart _mockProcessHelper.Setup(p => p.GetCurrentProcessFileName()).Returns("/usr/bin/dotnet"); _mockEnvironment.Setup(e => e.OperatingSystem).Returns(PlatformOperatingSystem.Unix); _mockDotnetHostHelper.Setup(d => d.GetMonoPath()).Returns("/usr/bin/mono"); - var source = "C:\temp\a.dll"; + var source = @"C:\temp\a.dll"; var info = _testHostManager.GetTestHostProcessStartInfo( new List() { source }, diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index f979ad4906..6155bfa57a 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -102,7 +102,17 @@ public static string PrepareArguments(string[] testAssemblies, string testAdapte var arguments = ""; foreach (var path in testAssemblies) { - arguments += path.AddDoubleQuote() + " "; + // The incoming testAssembly path is either a single dll path in quotes or without quotes. + // Or multiple assembly paths in a single string each double quoted and joined by space. + // We trim, and add quotes here to get either: + // C:\1.dll -> "C:\1.dll" + // "C:\1.dll" -> "C:\1.dll" + // "C:\1.dll" "C:\2.dll" -> "C:\1.dll" "C:\2.dll" + // + // For unquoted multi path string C:\1.dll C:\2.dll, we will get "C:\1.dll C:\2.dll" + // which is wrong and will fail later, but it's the test's fault for doing it wrong + // rather than providing an array of strings that this overload takes. + arguments += path.Trim('\"').AddDoubleQuote() + " "; } arguments = arguments.Trim(); @@ -270,19 +280,19 @@ public void ExecuteNotSupportedRunnerFrameworkTests(string runnerFramework, stri /// /// Validate if the overall test count and results are matching. /// - /// Passed test count - /// Failed test count - /// Skipped test count - public void ValidateSummaryStatus(int passedTestsCount, int failedTestsCount, int skippedTestsCount) + /// Passed test count + /// Failed test count + /// Skipped test count + public void ValidateSummaryStatus(int passed, int failed, int skipped) { // TODO: Switch on the actual version of vstest console when we have that set on test environment. if (_testEnvironment.VSTestConsolePath.Contains($"{Path.DirectorySeparatorChar}15.")) { - ValidateSummaryStatusv15(passedTestsCount, failedTestsCount, skippedTestsCount); + ValidateSummaryStatusv15(passed, failed, skipped); return; } - var totalTestCount = passedTestsCount + failedTestsCount + skippedTestsCount; + var totalTestCount = passed + failed + skipped; if (totalTestCount == 0) { // No test should be found/run @@ -304,19 +314,19 @@ public void ValidateSummaryStatus(int passedTestsCount, int failedTestsCount, in else { var summaryStatus = string.Format(TotalTestsMessage, totalTestCount); - if (passedTestsCount != 0) + if (passed != 0) { - summaryStatus += string.Format(PassedTestsMessage, passedTestsCount); + summaryStatus += string.Format(PassedTestsMessage, passed); } - if (failedTestsCount != 0) + if (failed != 0) { - summaryStatus += string.Format(FailedTestsMessage, failedTestsCount); + summaryStatus += string.Format(FailedTestsMessage, failed); } - if (skippedTestsCount != 0) + if (skipped != 0) { - summaryStatus += string.Format(SkippedTestsMessage, skippedTestsCount); + summaryStatus += string.Format(SkippedTestsMessage, skipped); } Assert.IsTrue( @@ -333,13 +343,13 @@ public void ValidateSummaryStatus(int passedTestsCount, int failedTestsCount, in /// /// Validate if the overall test count and results are matching. /// - /// Passed test count - /// Failed test count - /// Skipped test count - public void ValidateSummaryStatusv15(int passedTestsCount, int failedTestsCount, int skippedTestsCount) + /// Passed test count + /// Failed test count + /// Skipped test count + public void ValidateSummaryStatusv15(int passed, int failed, int skipped) { // example: Total tests: 6. Passed: 2. Failed: 2. Skipped: 2. - var totalTestCount = passedTestsCount + failedTestsCount + skippedTestsCount; + var totalTestCount = passed + failed + skipped; if (totalTestCount == 0) { // No test should be found/run @@ -355,19 +365,19 @@ public void ValidateSummaryStatusv15(int passedTestsCount, int failedTestsCount, else { var summaryStatus = $"Total tests: {totalTestCount}."; - if (passedTestsCount != 0) + if (passed != 0) { - summaryStatus += $" Passed: {passedTestsCount}."; + summaryStatus += $" Passed: {passed}."; } - if (failedTestsCount != 0) + if (failed != 0) { - summaryStatus += $" Failed: {failedTestsCount}."; + summaryStatus += $" Failed: {failed}."; } - if (skippedTestsCount != 0) + if (skipped != 0) { - summaryStatus += $" Skipped: {skippedTestsCount}."; + summaryStatus += $" Skipped: {skipped}."; } Assert.IsTrue( @@ -612,7 +622,7 @@ public virtual string GetConsoleRunnerPath() Assert.Fail("Unknown Runner framework - [{0}]", _testEnvironment.RunnerFramework); } - Assert.IsTrue(File.Exists(consoleRunnerPath), "GetConsoleRunnerPath: Path not found: {0}", consoleRunnerPath); + Assert.IsTrue(File.Exists(consoleRunnerPath), "GetConsoleRunnerPath: Path not found: \"{0}\"", consoleRunnerPath); return consoleRunnerPath; } @@ -910,7 +920,7 @@ protected string BuildMultipleAssemblyPath(DllInfo dllInfo, params string[] asse { var path = GetAssetFullPath(assetNames[i]); var updatedPath = dllInfo.UpdatePath(path); - Assert.IsTrue(File.Exists(updatedPath), "GetTestAsset: Path not found: {0}. Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", updatedPath); + Assert.IsTrue(File.Exists(updatedPath), "GetTestAsset: Path not found: \"{0}\". Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", updatedPath); assetFullPaths[i] = updatedPath.AddDoubleQuote(); } @@ -925,7 +935,7 @@ protected string BuildMultipleAssemblyPath(TesthostInfo testhostInfo, DllInfo ad { var path = GetAssetFullPath(assetNames[i]); var updatedPath = testhostInfo.UpdatePath(adapterInfo.UpdatePath(path)); - Assert.IsTrue(File.Exists(updatedPath), "GetTestAsset: Path not found: {0}. Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", updatedPath); + Assert.IsTrue(File.Exists(updatedPath), "GetTestAsset: Path not found: \"{0}\". Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", updatedPath); assetFullPaths[i] = updatedPath.AddDoubleQuote(); } diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs index 7d4429d58c..1762200b2f 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs @@ -240,8 +240,10 @@ public string GetTestAsset(string assetName, string targetFramework) targetFramework, assetName); - Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: {0}. Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", assetPath); + Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: \"{0}\". Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", assetPath); + // If you are thinking about wrapping the path in double quotes here, + // then don't. File.Exist cannot handle quoted paths, and we use it in a lot of places. return assetPath; } @@ -315,7 +317,7 @@ public string GetTestProject(string assetName) simpleAssetName, assetName); - Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: {0}.", assetPath); + Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: \"{0}\".", assetPath); return assetPath; } diff --git a/test/TestAssets/MSTestProject1/MSTestProject1.csproj b/test/TestAssets/MSTestProject1/MSTestProject1.csproj new file mode 100644 index 0000000000..6e15698454 --- /dev/null +++ b/test/TestAssets/MSTestProject1/MSTestProject1.csproj @@ -0,0 +1,28 @@ + + + + + + netcoreapp2.1;net451 + netcoreapp3.1 + false + Preview + + + + + $(MSTestFrameworkVersion) + + + $(MSTestAdapterVersion) + + + $(NETTestSdkVersion) + + + + + + + + diff --git a/test/TestAssets/MSTestProject1/UnitTest1.cs b/test/TestAssets/MSTestProject1/UnitTest1.cs new file mode 100644 index 0000000000..17c1760f18 --- /dev/null +++ b/test/TestAssets/MSTestProject1/UnitTest1.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace MSTestProject1; + +[TestClass] +public class UnitTest1 +{ + [TestMethod] + public void PassingTest() + { + Assert.AreEqual(2, 2); + } + + [TestMethod] + public void FailingTest() + { + Assert.AreEqual(2, 3); + } + + [Ignore] + [TestMethod] + public void SkippedTest() + { + } +} diff --git a/test/TestAssets/MSTestProject2/MSTestProject2.csproj b/test/TestAssets/MSTestProject2/MSTestProject2.csproj index 18ce8041acd97ea3ce45df23fd308ba411dedd89..6e15698454aeb3c77999dfba81759f0cd428ef05 100644 GIT binary patch literal 1063 zcmb_b!A|2a5WV*+EDNb{Sck&GWs@Mb3#igAs+wv~y%=X&LX+4UPg?l2 zDTh~B-TPBF2t;PtisHYojp}qp3bIV>DF2xD-3F5ISb;RKkp72 zKze&XT^avJSDQ*)0NtZ3V6Aa!;}S?|1Zm8L82oTw->(K?@E`2(X*Selno+$g>z|zd z()0-Q(~otJx=3_|3eNg0iddhpfimWRR-%Y*e!Z>f{r}CHXdwmDr@Vv%d?pZ@ccN*u!AY zig#nWiRGWhYR+JoGoaacw7Q zoPv;06ld+3dHd$=%s9V)McUVmt~64h?@E=c(1orw)}A)BqfjsS4X})r@cb0(SZB^U z)kH&WCcz_6pQuY0aYS_DtOXuJtf4;XP$&F)uZKwP9J zpl`G4!7w8tC!*lK%le9s@eGJgc{~o#6POxtg5l|_6TsQH~v83*q)y1+LU%DLsB&)8@!A_{m zLl}%&ROwKyF - /// The unit test 1. - /// - [TestClass] - public class UnitTest1 + [TestMethod] + public void PassingTest() { - /// - /// The passing test. - /// - [Priority(2)] - [TestMethod] - public void PassingTest() - { - Assert.AreEqual(2, 2); - } + Assert.AreEqual(2, 2); + } - /// - /// The failing test. - /// - [TestCategory("CategoryA")] - [Priority(3)] - [TestMethod] - public void FailingTest() - { -#if NETFRAMEWORK - // current App domain should be write to file to test DisableAppDomain acceptance test. - var appDomainFilePath = Environment.GetEnvironmentVariable("TEST_ASSET_APPDOMAIN_TEST_PATH") ?? Path.Combine(Path.GetTempPath(), "appdomain_test.txt"); - File.WriteAllText(appDomainFilePath, "AppDomain FriendlyName: " + AppDomain.CurrentDomain.FriendlyName); -#endif - Assert.AreEqual(2, 3); - } + [TestMethod] + public void FailingTest() + { + Assert.AreEqual(2, 3); + } - /// - /// The skipping test. - /// - [Ignore] - [TestMethod] - public void SkippingTest() - { - } + [Ignore] + [TestMethod] + public void SkippedTest() + { } } diff --git a/test/TestAssets/TestAssets.sln b/test/TestAssets/TestAssets.sln index c5d6cba585..1a05bdef96 100644 --- a/test/TestAssets/TestAssets.sln +++ b/test/TestAssets/TestAssets.sln @@ -96,7 +96,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ArchitectureSwitch", "Archi EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SimpleTestProjectMessedUpTargetFramework", "SimpleTestProjectMessedUpTargetFramework\SimpleTestProjectMessedUpTargetFramework.csproj", "{08C44607-EB80-4EE5-927D-08C34AA277AF}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MSTestProject2", "MSTestProject2\MSTestProject2.csproj", "{10AA955C-B412-41A8-899F-8609AAE19F61}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MSTestProject2", "MSTestProject2\MSTestProject2.csproj", "{10AA955C-B412-41A8-899F-8609AAE19F61}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MSTestProject1", "MSTestProject1\MSTestProject1.csproj", "{E166D337-4033-4209-863F-8F77675EAEE8}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "basic", "basic", "{2633D125-64A7-456C-AD37-F8A6B56C2403}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -648,10 +652,26 @@ Global {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|x64.Build.0 = Release|Any CPU {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|x86.ActiveCfg = Release|Any CPU {10AA955C-B412-41A8-899F-8609AAE19F61}.Release|x86.Build.0 = Release|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Debug|x64.ActiveCfg = Debug|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Debug|x64.Build.0 = Debug|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Debug|x86.ActiveCfg = Debug|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Debug|x86.Build.0 = Debug|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Release|Any CPU.Build.0 = Release|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Release|x64.ActiveCfg = Release|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Release|x64.Build.0 = Release|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Release|x86.ActiveCfg = Release|Any CPU + {E166D337-4033-4209-863F-8F77675EAEE8}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {10AA955C-B412-41A8-899F-8609AAE19F61} = {2633D125-64A7-456C-AD37-F8A6B56C2403} + {E166D337-4033-4209-863F-8F77675EAEE8} = {2633D125-64A7-456C-AD37-F8A6B56C2403} + EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {D2334DAA-F7B2-450E-ABA4-FBC185152500} EndGlobalSection From 82ad32023f5e4374a5e9ca69017e2196e5a483a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 16 Mar 2022 14:27:38 +0100 Subject: [PATCH 36/64] Some tests run. --- .../AcceptanceTestBase.cs | 14 ++-- .../ExecutionTests.cs | 20 +++--- .../MSTestCompatibilityDataSource.cs | 33 +++++++--- .../Extension/NetCoreRunner.cs | 10 ++- .../NetCoreTargetFrameworkDataSource.cs | 9 ++- .../Extension/NetFrameworkRunner.cs | 11 +++- .../NetFullTargetFrameworkDataSource.cs | 30 ++++++++- .../Extension/RunnnerInfo.cs | 28 +++++++- .../TestPlatformCompatibilityDataSource.cs | 50 +++++++++----- .../TesthostCompatibilityDataSource.cs | 45 ++++++++----- ...TranslationLayerCompatibilityDataSource.cs | 25 +++++-- .../CustomTestHostTests.cs | 16 ++--- .../TranslationLayerTests/DiscoverTests.cs | 8 +-- .../TranslationLayerTests/RunTests.cs | 4 +- .../RunTestsWithFilterTests.cs | 4 +- .../DebugInfo.cs | 17 +++++ .../DllInfo.cs | 1 + .../IntegrationTestBase.cs | 65 +++++++------------ .../IntegrationTestEnvironment.cs | 9 ++- .../MSTestInfo.cs | 3 + .../TesthostInfo.cs | 7 +- .../VSTestConsoleInfo.cs | 3 + 22 files changed, 271 insertions(+), 141 deletions(-) create mode 100644 test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs index 4f74da75e7..c44ec9775e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs @@ -69,19 +69,15 @@ public static string And(string left, string right) protected string FrameworkArgValue => DeriveFrameworkArgValue(_testEnvironment); - protected static void SetTestEnvironment(IntegrationTestEnvironment testEnvironment, RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo = null) + protected static void SetTestEnvironment(IntegrationTestEnvironment testEnvironment, RunnerInfo runnerInfo) { - if (vsTestConsoleInfo != null) - { - testEnvironment.VSTestConsolePath = vsTestConsoleInfo.Path; - } + testEnvironment.VSTestConsoleInfo = runnerInfo.VSTestConsoleInfo; + testEnvironment.DllInfos = runnerInfo.DllInfos; + testEnvironment.DebugInfo = runnerInfo.DebugInfo; + testEnvironment.RunnerFramework = runnerInfo.RunnerFramework; testEnvironment.TargetFramework = runnerInfo.TargetFramework; testEnvironment.InIsolationValue = runnerInfo.InIsolationValue; - testEnvironment.DebugVSTestConsole = runnerInfo.DebugVSTestConsole; - testEnvironment.DebugTesthost = runnerInfo.DebugTesthost; - testEnvironment.DebugDataCollector = runnerInfo.DebugDataCollector; - testEnvironment.NoDefaultBreakpoints = runnerInfo.NoDefaultBreakpoints; } protected static string DeriveFrameworkArgValue(IntegrationTestEnvironment testEnvironment) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 14daacdf20..3faa9178ab 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -19,11 +19,11 @@ public class ExecutionTests : AcceptanceTestBase // TODO: It looks like the first 3 tests would be useful to multiply by all 3 test frameworks, should we make the test even more generic, or duplicate them? [TestMethod] [MSTestCompatibilityDataSource(InProcess = true)] - public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, MSTestInfo msTestInfo) + public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll"); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); @@ -32,13 +32,13 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo, MSTestInfo msTestIn } [TestMethod] - [TestPlatformCompatibilityDataSource(WithInProcess = true)] + [TestPlatformCompatibilityDataSource(WithInProcess = true, WithEveryVersionOfAdapter = false, WithEveryVersionOfHost = false, WithEveryVersionOfRunner = false, WithOlderConfigurations = false)] - public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo, VSTestConsoleInfo consoleInfo, TesthostInfo testhostInfo, MSTestInfo msTestInfo) + public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo, consoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath(testhostInfo, msTestInfo, "MSTestProject1.dll", "MSTestProject2.dll"); + var assemblyPaths = BuildMultipleAssemblyPath("MSTestProject1.dll", "MSTestProject2.dll"); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); @@ -48,11 +48,11 @@ public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo, VSTestCo [TestMethod] [TesthostCompatibilityDataSource] - public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo, TesthostInfo testhostInfo) + public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(RunnerInfo runnerInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath(testhostInfo, "SimpleTestProject.dll", "SimpleTestProject2.dll"); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); @@ -93,7 +93,7 @@ public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo, MSTestInf SetTestEnvironment(_testEnvironment, runnerInfo); using var tempDir = new TempDirectory(); - var assemblyPaths = BuildMultipleAssemblyPath(msTestInfo,"SimpleTestProject.dll", "SimpleTestProject2.dll"); + var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, testAdapterPath: null, runSettings: null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /Parallel"); arguments = string.Concat(arguments, " /Platform:x86"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs index 388217cea8..af3101c971 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs @@ -13,7 +13,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests; -public sealed class MSTestCompatibilityDataSource : TestDataSource +public sealed class MSTestCompatibilityDataSource : TestDataSource { private static XmlDocument? s_depsXml; private readonly string[] _runnerFrameworks; @@ -53,11 +53,18 @@ public override void CreateData(MethodInfo methodInfo) { foreach (var msTestVersion in _msTestVersions) { - var runnerInfo = new RunnerInfo(AcceptanceTestBase.DEFAULT_RUNNER_NETFX, AcceptanceTestBase.DEFAULT_RUNNER_NETFX, InIsolationValue: null, - DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); - var msTestInfo = GetMSTestInfo(msTestVersion); // We run in the .NET Framework runner process, the runner and target framework must agree. - AddData(runnerInfo, msTestInfo); + var runnerInfo = new RunnerInfo(AcceptanceTestBase.DEFAULT_RUNNER_NETFX, AcceptanceTestBase.DEFAULT_RUNNER_NETFX, inIsolationValue: null); + runnerInfo.DebugInfo = new DebugInfo + { + DebugVSTestConsole = DebugVSTestConsole, + DebugTesthost = DebugTesthost, + DebugDataCollector = DebugDataCollector, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; + runnerInfo.DllInfos.Add(GetMSTestInfo(msTestVersion)); + + AddData(runnerInfo); } } @@ -67,11 +74,17 @@ public override void CreateData(MethodInfo methodInfo) { foreach (var msTestVersion in _msTestVersions) { - var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation, - DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); - var msTestInfo = GetMSTestInfo(msTestVersion); - - AddData(runnerInfo, msTestInfo); + var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation); + runnerInfo.DebugInfo = new DebugInfo + { + DebugVSTestConsole = DebugVSTestConsole, + DebugTesthost = DebugTesthost, + DebugDataCollector = DebugDataCollector, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; + runnerInfo.DllInfos.Add(GetMSTestInfo(msTestVersion)); + + AddData(runnerInfo); } } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs index 1a3d855e29..e12d7f6011 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs @@ -47,7 +47,15 @@ public IEnumerable GetData(MethodInfo methodInfo) Func filter = tfm => isWindows || !tfm.StartsWith("net4"); foreach (var fmw in _targetFrameworks.Split(';').Where(filter)) { - dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, fmw, InIsolationValue: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); + var runnerInfo = new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, fmw, inIsolationValue: null); + runnerInfo.DebugInfo = new DebugInfo + { + DebugVSTestConsole = DebugVSTestConsole, + DebugTesthost = DebugTesthost, + DebugDataCollector = DebugDataCollector, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; + dataRows.Add(new object[] { runnerInfo }); } return dataRows; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs index 6788d56074..d5bfd3b4ce 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs @@ -55,7 +55,14 @@ public NetCoreTargetFrameworkDataSource( private void AddRunnerDataRow(List dataRows, string runnerFramework, string targetFramework) { - var runnerInfo = new RunnerInfo(runnerFramework, targetFramework, InIsolationValue: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); + var runnerInfo = new RunnerInfo(runnerFramework, targetFramework, inIsolationValue: null); + runnerInfo.DebugInfo = new DebugInfo + { + DebugDataCollector = DebugDataCollector, + DebugTesthost = DebugTesthost, + DebugVSTestConsole = DebugVSTestConsole, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; dataRows.Add(new object[] { runnerInfo }); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs index b8386149f0..07f30ee823 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs @@ -49,7 +49,16 @@ public IEnumerable GetData(MethodInfo methodInfo) foreach (var fmw in _targetFrameworks.Split(';')) { - dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, fmw, AcceptanceTestBase.InIsolation, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); + var runnerInfo = new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, fmw, AcceptanceTestBase.InIsolation); + runnerInfo.DebugInfo = new DebugInfo + { + DebugVSTestConsole = DebugVSTestConsole, + DebugTesthost = DebugTesthost, + DebugDataCollector = DebugDataCollector, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; + + dataRows.Add(new object[] { runnerInfo }); } return dataRows; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs index c3b0af0e6c..3b45a19e74 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs @@ -53,19 +53,43 @@ public IEnumerable GetData(MethodInfo methodInfo) var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); if (_useCoreRunner && isWindows) { - dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, InIsolationValue: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); + var runnerInfo = new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, inIsolationValue: null); + runnerInfo.DebugInfo = new DebugInfo + { + DebugVSTestConsole = DebugVSTestConsole, + DebugTesthost = DebugTesthost, + DebugDataCollector = DebugDataCollector, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; + dataRows.Add(new object[] { runnerInfo }); } if (_useDesktopRunner && isWindows) { if (_inIsolation) { - dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, AcceptanceTestBase.InIsolation, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); + var runnerInfo = new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, AcceptanceTestBase.InIsolation); + runnerInfo.DebugInfo = new DebugInfo + { + DebugVSTestConsole = DebugVSTestConsole, + DebugTesthost = DebugTesthost, + DebugDataCollector = DebugDataCollector, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; + dataRows.Add(new object[] { runnerInfo }); } if (_inProcess) { - dataRows.Add(new object[] { new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, InIsolationValue: null, DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints) }); + var runnerInfo = new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, inIsolationValue: null); + runnerInfo.DebugInfo = new DebugInfo + { + DebugVSTestConsole = DebugVSTestConsole, + DebugTesthost = DebugTesthost, + DebugDataCollector = DebugDataCollector, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; + dataRows.Add(new object[] { runnerInfo }); } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs index da0d836e89..fcf9cca46e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs @@ -2,6 +2,9 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +using System.Collections.Generic; + +using Microsoft.TestPlatform.TestUtilities; namespace Microsoft.TestPlatform.AcceptanceTests; @@ -11,9 +14,28 @@ namespace Microsoft.TestPlatform.AcceptanceTests; /// /// /// Supported value = /InIsolation. -public record struct RunnerInfo(string RunnerFramework, string TargetFramework, string? InIsolationValue = "", - bool DebugVSTestConsole = false, bool DebugTesthost = false, bool DebugDataCollector = false, bool NoDefaultBreakpoints = true) +[Serializable] +public class RunnerInfo { + public RunnerInfo(string runnerFramework, string targetFramework, string? inIsolationValue = "") + { + RunnerFramework = runnerFramework; + TargetFramework = targetFramework; + InIsolationValue = inIsolationValue; + } + + public string RunnerFramework { get; set; } + + public VSTestConsoleInfo VSTestConsoleInfo { get; set; } + + + public string TargetFramework { get; set; } + public string? InIsolationValue { get; set; } + + public DebugInfo? DebugInfo { get; set; } + + public List DllInfos { get; set; } = new(); + /// /// Is running via .NET "Core" vstest.console? /// @@ -34,5 +56,5 @@ public record struct RunnerInfo(string RunnerFramework, string TargetFramework, /// public bool IsNetFrameworkTarget => TargetFramework.StartsWith("net4", StringComparison.InvariantCultureIgnoreCase); - public override string ToString() => $"RunnerFramework = {RunnerFramework}, TargetFramework = {TargetFramework}, {(string.IsNullOrEmpty(InIsolationValue) ? "InProcess" : "InIsolation")}"; + public override string ToString() => $"Runner = {RunnerFramework}, TargetFramework = {TargetFramework}, {(string.IsNullOrEmpty(InIsolationValue) ? "InProcess" : "InIsolation")}, {VSTestConsoleInfo}, {string.Join(",", DllInfos)}"; } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs index 262b3f1b85..eca9ac55e6 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs @@ -15,7 +15,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests; -public sealed class TestPlatformCompatibilityDataSource : TestDataSource +public sealed class TestPlatformCompatibilityDataSource : TestDataSource { private static XmlDocument? s_depsXml; private readonly string[] _runnerFrameworks; @@ -54,7 +54,7 @@ public TestPlatformCompatibilityDataSource( /// /// Add run for in-process using the selected .NET Framework runners, and and all selected adapters. /// - public bool WithInProcess { get; set; } + public bool WithInProcess { get; set; } = true; public bool WithEveryVersionOfRunner { get; set; } = true; @@ -72,7 +72,7 @@ public TestPlatformCompatibilityDataSource( public override void CreateData(MethodInfo methodInfo) { - var dataRows = new List<(RunnerInfo runnerInfo, VSTestConsoleInfo vstestConsoleInfo, TesthostInfo testhostInfo, MSTestInfo mstestInfo)>(); + var dataRows = new List(); if (WithEveryVersionOfRunner) AddEveryVersionOfRunner(dataRows); @@ -121,11 +121,11 @@ public override void CreateData(MethodInfo methodInfo) foreach (var dataRow in dataRows) { - AddData(dataRow.runnerInfo, dataRow.vstestConsoleInfo, dataRow.testhostInfo, dataRow.mstestInfo); + AddData(dataRow); } } - private void AddInProcess(List<(RunnerInfo runnerInfo, VSTestConsoleInfo vstestConsoleInfo, TesthostInfo testhostInfo, MSTestInfo mstestInfo)> dataRows) + private void AddInProcess(List dataRows) { foreach (var runnerFramework in _runnerFrameworks) { @@ -147,7 +147,7 @@ private void AddInProcess(List<(RunnerInfo runnerInfo, VSTestConsoleInfo vstestC } } - private void AddOlderConfigurations(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows) + private void AddOlderConfigurations(List dataRows) { // Older configurations where the runner, host and adapter version are the same. // We already added the row where all are newest when adding combination with all runners. @@ -169,7 +169,7 @@ private void AddOlderConfigurations(List<(RunnerInfo, VSTestConsoleInfo, Testhos } } - private void AddEveryVersionOfAdapter(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows) + private void AddEveryVersionOfAdapter(List dataRows) { var runnerVersion = _runnerVersions[0]; foreach (var runnerFramework in _runnerFrameworks) @@ -192,7 +192,7 @@ private void AddEveryVersionOfAdapter(List<(RunnerInfo, VSTestConsoleInfo, Testh } } - private void AddEveryVersionOfHost(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows) + private void AddEveryVersionOfHost(List dataRows) { var runnerVersion = _runnerVersions[0]; @@ -219,7 +219,7 @@ private void AddEveryVersionOfHost(List<(RunnerInfo, VSTestConsoleInfo, Testhost } } - private void AddEveryVersionOfRunner(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows) + private void AddEveryVersionOfRunner(List dataRows) { foreach (var runnerVersion in _runnerVersions) { @@ -242,20 +242,38 @@ private void AddEveryVersionOfRunner(List<(RunnerInfo, VSTestConsoleInfo, Testho } } - private void AddRow(List<(RunnerInfo, VSTestConsoleInfo, TesthostInfo, MSTestInfo)> dataRows, + private void AddRow(List dataRows, string runnerVersion, string runnerFramework, string hostVersion, string hostFramework, string adapter, string adapterVersion, bool inIsolation) { RunnerInfo runnerInfo = GetRunnerInfo(runnerFramework, hostFramework, inIsolation); - var vstestConsoleInfo = GetVSTestConsoleInfo(runnerVersion, runnerInfo); - var testhostInfo = TesthostCompatibilityDataSource.GetTesthostInfo(hostVersion); - var mstestInfo = GetMSTestInfo(adapterVersion); - dataRows.Add(new(runnerInfo, vstestConsoleInfo, testhostInfo, mstestInfo)); + runnerInfo.DebugInfo = GetDebugInfo(); + runnerInfo.VSTestConsoleInfo = GetVSTestConsoleInfo(runnerVersion, runnerInfo); + + // The order in which we add them matters. We end up both modifying the same path + // and adding to it. So the first one added will be later in the path. E.g.: + // Adding testSdk first: + // C:\p\vstest\test\TestAssets\MSTestProject1\bin\MSTestLatestPreview-2.2.9-preview-20220210-07\NETTestSdkLatest-17.2.0-dev\Debug\net451\MSTestProject1.dll + // versus adding testSdk second: + // C:\p\vstest\test\TestAssets\MSTestProject1\bin\NETTestSdkLatest-17.2.0-dev\MSTestLatestPreview-2.2.9-preview-20220210-07\Debug\net451\MSTestProject1.dll + runnerInfo.DllInfos.Add(GetMSTestInfo(adapterVersion)); + runnerInfo.DllInfos.Add(TesthostCompatibilityDataSource.GetNetTestSdkInfo(hostVersion)); + dataRows.Add(runnerInfo); + } + + private DebugInfo GetDebugInfo() + { + return new DebugInfo + { + DebugDataCollector = DebugDataCollector, + DebugTesthost = DebugTesthost, + DebugVSTestConsole = DebugVSTestConsole, + NoDefaultBreakpoints = NoDefaultBreakpoints + }; } private RunnerInfo GetRunnerInfo(string runnerFramework, string hostFramework, bool inIsolation) { - return new RunnerInfo(runnerFramework, hostFramework, inIsolation ? AcceptanceTestBase.InIsolation : null, - DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); + return new RunnerInfo(runnerFramework, hostFramework, inIsolation ? AcceptanceTestBase.InIsolation : null); } public string GetDisplayName(MethodInfo methodInfo, object[] data) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs index 6892b4a210..a74c45adc9 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs @@ -13,7 +13,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests; -public sealed class TesthostCompatibilityDataSource : TestDataSource +public sealed class TesthostCompatibilityDataSource : TestDataSource { private static XmlDocument? s_depsXml; private readonly string[] _runnerFrameworks; @@ -58,13 +58,18 @@ public override void CreateData(MethodInfo methodInfo) var testhostVersions = runner.StartsWith("net4") ? onlyLatest : _testhostVersions; foreach (var testhostVersion in testhostVersions) { - var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation, - DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); - - var vstestConsoleInfo = TranslationLayerCompatibilityDataSource.GetVSTestConsoleInfo(AcceptanceTestBase.LATEST, runnerInfo); - var testhostInfo = GetTesthostInfo(testhostVersion); - - AddData(runnerInfo, vstestConsoleInfo, testhostInfo); + var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation); + runnerInfo.DebugInfo = new DebugInfo + { + DebugDataCollector = DebugDataCollector, + DebugTesthost = DebugTesthost, + DebugVSTestConsole = DebugVSTestConsole, + NoDefaultBreakpoints = NoDefaultBreakpoints + }; + runnerInfo.VSTestConsoleInfo = TranslationLayerCompatibilityDataSource.GetVSTestConsoleInfo(AcceptanceTestBase.LATEST, runnerInfo); + runnerInfo.DllInfos.Add(GetNetTestSdkInfo(testhostVersion)); + + AddData(runnerInfo); } } } @@ -81,14 +86,18 @@ public override void CreateData(MethodInfo methodInfo) var consoleVersions = runner.StartsWith("net4") ? onlyLatest : _testhostVersions; foreach (var consoleVersion in consoleVersions) { - var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation, - DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); - - var vstestConsoleInfo = TranslationLayerCompatibilityDataSource.GetVSTestConsoleInfo(consoleVersion, runnerInfo); - // Generate only for latest testhsot, - var testhostInfo = GetTesthostInfo(AcceptanceTestBase.LATEST); - - AddData(runnerInfo, vstestConsoleInfo, testhostInfo); + var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation); + runnerInfo.DebugInfo = new DebugInfo + { + DebugDataCollector = DebugDataCollector, + DebugTesthost = DebugTesthost, + DebugVSTestConsole = DebugVSTestConsole, + NoDefaultBreakpoints = NoDefaultBreakpoints + }; + runnerInfo.VSTestConsoleInfo = TranslationLayerCompatibilityDataSource.GetVSTestConsoleInfo(consoleVersion, runnerInfo); + runnerInfo.DllInfos.Add(GetNetTestSdkInfo(AcceptanceTestBase.LATEST)); + + AddData(runnerInfo); } } } @@ -99,7 +108,7 @@ public string GetDisplayName(MethodInfo methodInfo, object[] data) return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); } - internal static TesthostInfo GetTesthostInfo(string testhostVersionType) + internal static NetTestSdkInfo GetNetTestSdkInfo(string testhostVersionType) { var depsXml = GetDependenciesXml(); @@ -119,7 +128,7 @@ internal static TesthostInfo GetTesthostInfo(string testhostVersionType) var slash = Path.DirectorySeparatorChar; var versionSpecificBinPath = $"{slash}bin{slash}NETTestSdk{testhostVersionType}-{version}{slash}"; - return new TesthostInfo(testhostVersionType, version, versionSpecificBinPath); + return new NetTestSdkInfo(testhostVersionType, version, versionSpecificBinPath); } private static XmlDocument GetDependenciesXml() diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs index d5e8d2ee77..50dfd86bbf 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs @@ -85,8 +85,14 @@ public override void CreateData(MethodInfo methodInfo) { foreach (var vstestConsoleVersion in _vstestConsoleVersions) { - var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation, - DebugVSTestConsole, DebugTesthost, DebugDataCollector, NoDefaultBreakpoints); + var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation); + runnerInfo.DebugInfo = new DebugInfo + { + DebugVSTestConsole = DebugVSTestConsole, + DebugTesthost = DebugTesthost, + DebugDataCollector = DebugDataCollector, + NoDefaultBreakpoints = NoDefaultBreakpoints, + }; var vsTestConsoleInfo = GetVSTestConsoleInfo(vstestConsoleVersion, runnerInfo); if (beforeVersion != null && vsTestConsoleInfo.Version > beforeVersion) @@ -156,7 +162,18 @@ private static XmlDocument GetDependenciesXml() } } -public readonly record struct Feature(string Version, string Issue); +public class Feature +{ + public Feature (string version, string issue) + { + Version = version; + Issue = issue; + } + + public string Version { get; } + public string Issue { get; } +} + public static class Features { @@ -164,6 +181,6 @@ public static class Features public static Dictionary Table { get; } = new Dictionary { - [ATTACH_DEBUGGER] = new(Version: "v16.7.0-preview-20200519-01", Issue: "https://github.com/microsoft/vstest/pull/2325") + [ATTACH_DEBUGGER] = new(version: "v16.7.0-preview-20200519-01", issue: "https://github.com/microsoft/vstest/pull/2325") }; } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs index f0524f871e..e6c341ca4f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs @@ -35,9 +35,9 @@ public void Cleanup() [TestMethod] [TranslationLayerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] - public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncher(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) + public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncher(RunnerInfo runnerInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo); _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); @@ -56,9 +56,9 @@ public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvided [TestMethod] [TranslationLayerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] - public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncherWhenITestHostLauncher2IsProvided(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) + public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncherWhenITestHostLauncher2IsProvided(RunnerInfo runnerInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo); _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); @@ -78,9 +78,9 @@ public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvided [TestMethod] [TranslationLayerCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER)] - public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvidedLauncher(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) + public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvidedLauncher(RunnerInfo runnerInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo); _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); @@ -103,9 +103,9 @@ public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvided + "It sees 6 but does not realize that the provided CustomTesthostLauncher is not supporting the new feature, it ends up calling back to EditoAttachDebugger" + "in translation layer, and that just silently skips the call.")] [TranslationLayerCompatibilityDataSource("net451", "net451", "Latest", AfterFeature = Features.ATTACH_DEBUGGER, DebugVSTestConsole = true, DebugTesthost=true)] - public void RunTestsWithCustomTestHostLauncherUsesLaunchWhenGivenAnOutdatedITestHostLauncher(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) + public void RunTestsWithCustomTestHostLauncherUsesLaunchWhenGivenAnOutdatedITestHostLauncher(RunnerInfo runnerInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo); _vstestConsoleWrapper = GetVsTestConsoleWrapper(); _runEventHandler = new RunEventHandler(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index 3a59d0de0f..c0a278cf14 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -41,9 +41,9 @@ public void Cleanup() [TestMethod] [TranslationLayerCompatibilityDataSource] - public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) + public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo); // Setup(); _discoveryEventHandler = new DiscoveryEventHandler(); @@ -58,9 +58,9 @@ public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo, VSTe [TestMethod] [TranslationLayerCompatibilityDataSource()] - public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedOut(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) + public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedOut(RunnerInfo runnerInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo); // Setup(); _discoveryEventHandler = new DiscoveryEventHandler(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs index d966022eeb..ed121eaa0e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs @@ -41,9 +41,9 @@ public void Cleanup() [TestMethod] [TranslationLayerCompatibilityDataSource] - public void RunAllTests(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) + public void RunAllTests(RunnerInfo runnerInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo); var vstestConsoleWrapper = GetVsTestConsoleWrapper(); var runEventHandler = new RunEventHandler(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs index 9edda9281e..9661f18242 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs @@ -37,9 +37,9 @@ public void Cleanup() [TestMethod] [TranslationLayerCompatibilityDataSource] - public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo, VSTestConsoleInfo vsTestConsoleInfo) + public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo) { - SetTestEnvironment(_testEnvironment, runnerInfo, vsTestConsoleInfo); + SetTestEnvironment(_testEnvironment, runnerInfo); // Setup(); _runEventHandler = new RunEventHandler(); diff --git a/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs new file mode 100644 index 0000000000..3869ccb2ea --- /dev/null +++ b/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; + +namespace Microsoft.TestPlatform.TestUtilities; + +[Serializable] +public class DebugInfo +{ + public bool DebugVSTestConsole { get; set; } + public bool DebugTesthost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; +} + + diff --git a/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs index e417ef9db7..8299661238 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs @@ -5,6 +5,7 @@ namespace Microsoft.TestPlatform.TestUtilities; +[Serializable] public abstract class DllInfo { protected DllInfo(string name, string propertyName, string versionType, string? version, string path) diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 6155bfa57a..2c80715014 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -222,26 +222,26 @@ public void InvokeVsTestForExecution(string testAssembly, { var arguments = PrepareArguments(testAssembly, testAdapterPath, runSettings, framework, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); - if (_testEnvironment.DebugVSTestConsole || _testEnvironment.DebugTesthost || _testEnvironment.DebugDataCollector) + if (_testEnvironment.DebugInfo.DebugVSTestConsole || _testEnvironment.DebugInfo.DebugTesthost || _testEnvironment.DebugInfo.DebugDataCollector) { environmentVariables ??= new Dictionary(); - if (_testEnvironment.DebugVSTestConsole) + if (_testEnvironment.DebugInfo.DebugVSTestConsole) { environmentVariables.Add("VSTEST_RUNNER_DEBUG_ATTACHVS", "1"); } - if (_testEnvironment.DebugTesthost) + if (_testEnvironment.DebugInfo.DebugTesthost) { environmentVariables.Add("VSTEST_HOST_DEBUG_ATTACHVS", "1"); } - if (_testEnvironment.DebugDataCollector) + if (_testEnvironment.DebugInfo.DebugDataCollector) { environmentVariables.Add("VSTEST_DATACOLLECTOR_DEBUG_ATTACHVS", "1"); } - if (_testEnvironment.NoDefaultBreakpoints) + if (_testEnvironment.DebugInfo.NoDefaultBreakpoints) { environmentVariables.Add("VSTEST_DEBUG_NOBP", "1"); } @@ -286,7 +286,7 @@ public void ExecuteNotSupportedRunnerFrameworkTests(string runnerFramework, stri public void ValidateSummaryStatus(int passed, int failed, int skipped) { // TODO: Switch on the actual version of vstest console when we have that set on test environment. - if (_testEnvironment.VSTestConsolePath.Contains($"{Path.DirectorySeparatorChar}15.")) + if (_testEnvironment.VSTestConsoleInfo != null && _testEnvironment.VSTestConsoleInfo.Path.Contains($"{Path.DirectorySeparatorChar}15.")) { ValidateSummaryStatusv15(passed, failed, skipped); return; @@ -603,9 +603,9 @@ public virtual string GetConsoleRunnerPath() if (IsDesktopRunner()) { - if (!string.IsNullOrWhiteSpace(_testEnvironment.VSTestConsolePath)) + if (!string.IsNullOrWhiteSpace(_testEnvironment?.VSTestConsoleInfo.Path)) { - consoleRunnerPath = _testEnvironment.VSTestConsolePath; + consoleRunnerPath = _testEnvironment.VSTestConsoleInfo.Path; } else { @@ -672,27 +672,27 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper() Console.WriteLine($"Console runner path: {consoleRunnerPath}"); VsTestConsoleWrapper vstestConsoleWrapper; - if (_testEnvironment.DebugVSTestConsole || _testEnvironment.DebugTesthost || _testEnvironment.DebugDataCollector) + if (_testEnvironment.DebugInfo.DebugVSTestConsole || _testEnvironment.DebugInfo.DebugTesthost || _testEnvironment.DebugInfo.DebugDataCollector) { var environmentVariables = new Dictionary(); Environment.GetEnvironmentVariables().OfType().ToList().ForEach(e => environmentVariables.Add(e.Key.ToString(), e.Value.ToString())); - if (_testEnvironment.DebugVSTestConsole) + if (_testEnvironment.DebugInfo.DebugVSTestConsole) { environmentVariables.Add("VSTEST_RUNNER_DEBUG_ATTACHVS", "1"); } - if (_testEnvironment.DebugTesthost) + if (_testEnvironment.DebugInfo.DebugTesthost) { environmentVariables.Add("VSTEST_HOST_DEBUG_ATTACHVS", "1"); } - if (_testEnvironment.DebugDataCollector) + if (_testEnvironment.DebugInfo.DebugDataCollector) { environmentVariables.Add("VSTEST_DATACOLLECTOR_DEBUG_ATTACHVS", "1"); } - if (_testEnvironment.NoDefaultBreakpoints) + if (_testEnvironment.DebugInfo.NoDefaultBreakpoints) { environmentVariables.Add("VSTEST_DEBUG_NOBP", "1"); } @@ -903,41 +903,22 @@ public static void CreateRunSettingsFile(string destinationRunsettingsPath, stri } protected string BuildMultipleAssemblyPath(params string[] assetNames) - { - var assetFullPath = new string[assetNames.Length]; - for (var i = 0; i < assetNames.Length; i++) - { - assetFullPath[i] = GetAssetFullPath(assetNames[i]).AddDoubleQuote(); - } - - return string.Join(" ", assetFullPath); - } - - protected string BuildMultipleAssemblyPath(DllInfo dllInfo, params string[] assetNames) { var assetFullPaths = new string[assetNames.Length]; for (var i = 0; i < assetNames.Length; i++) { var path = GetAssetFullPath(assetNames[i]); - var updatedPath = dllInfo.UpdatePath(path); - Assert.IsTrue(File.Exists(updatedPath), "GetTestAsset: Path not found: \"{0}\". Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", updatedPath); - - assetFullPaths[i] = updatedPath.AddDoubleQuote(); - } - - return string.Join(" ", assetFullPaths); - } + if (_testEnvironment.DllInfos.Count > 0) + { + foreach (var dllInfo in _testEnvironment.DllInfos) + { + path = dllInfo.UpdatePath(path); + } - protected string BuildMultipleAssemblyPath(TesthostInfo testhostInfo, DllInfo adapterInfo, params string[] assetNames) - { - var assetFullPaths = new string[assetNames.Length]; - for (var i = 0; i < assetNames.Length; i++) - { - var path = GetAssetFullPath(assetNames[i]); - var updatedPath = testhostInfo.UpdatePath(adapterInfo.UpdatePath(path)); - Assert.IsTrue(File.Exists(updatedPath), "GetTestAsset: Path not found: \"{0}\". Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", updatedPath); + Assert.IsTrue(File.Exists(path), "GetTestAsset: Path not found: \"{0}\". Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", path); + } - assetFullPaths[i] = updatedPath.AddDoubleQuote(); + assetFullPaths[i] = path.AddDoubleQuote(); } return string.Join(" ", assetFullPaths); @@ -979,5 +960,5 @@ protected static string GetDownloadedDotnetMuxerFromTools(string architecture) return path; } - protected string GetDotnetRunnerPath() => _testEnvironment.VSTestConsolePath ?? Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll"); + protected string GetDotnetRunnerPath() => _testEnvironment?.VSTestConsoleInfo.Path ?? Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll"); } diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs index 1762200b2f..b0af339915 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq.Expressions; using System.Xml; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -193,11 +194,9 @@ public string RunnerFramework // A known AzureDevOps env variable meaning we are running in CI. public static bool IsCI { get; } = Environment.GetEnvironmentVariable("TF_BUILD") == "True"; - public bool DebugVSTestConsole { get; set; } - public bool DebugTesthost { get; set; } - public bool DebugDataCollector { get; set; } - public bool NoDefaultBreakpoints { get; set; } - public string VSTestConsolePath { get; set; } + public DebugInfo DebugInfo { get; set; } + public VSTestConsoleInfo VSTestConsoleInfo { get; set; } + public List DllInfos { get; set; } /// /// Gets the full path to a test asset. diff --git a/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs index e4c128774e..6529493ab0 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs @@ -1,8 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System; + namespace Microsoft.TestPlatform.TestUtilities; +[Serializable] public class MSTestInfo : DllInfo { public MSTestInfo(string versionType, string? version, string path) diff --git a/test/Microsoft.TestPlatform.TestUtilities/TesthostInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/TesthostInfo.cs index f29006fd38..a9be09a3d8 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/TesthostInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/TesthostInfo.cs @@ -1,11 +1,14 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System; + namespace Microsoft.TestPlatform.TestUtilities; -public class TesthostInfo : DllInfo +[Serializable] +public class NetTestSdkInfo : DllInfo { - public TesthostInfo(string versionType, string? version, string path) + public NetTestSdkInfo(string versionType, string? version, string path) : base(name: "Testhost", propertyName: "VSTestConsole", versionType, version, path) { } diff --git a/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs index d88ad20375..41d441e3b8 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs @@ -1,8 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System; + namespace Microsoft.TestPlatform.TestUtilities; +[Serializable] public class VSTestConsoleInfo { public VSTestConsoleInfo(string versionType, string? version, string path) From 5216498a73defacbe18668d84cf798782881653e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 16 Mar 2022 19:22:24 +0100 Subject: [PATCH 37/64] More fun with sources. --- playground/TestPlatform.Playground/Program.cs | 3 +- .../TestPlatform.Playground.csproj | 2 +- scripts/build.ps1 | 72 ++-- sln.cmd | 14 - src/SettingsMigrator/ConsoleParameters.cs | 93 ----- .../AcceptanceTestBase.cs | 9 +- .../ExecutionTests.cs | 27 +- .../Extension/CompatibilityRowsBuilder.cs | 380 ++++++++++++++++++ .../Extension/Feature.cs | 16 + .../Extension/Features.cs | 23 ++ .../MSTestCompatibilityDataSource.cs | 152 +++---- .../Extension/NetCoreRunner.cs | 7 +- .../NetCoreTargetFrameworkDataSource.cs | 7 +- .../Extension/NetFrameworkRunner.cs | 7 +- .../NetFullTargetFrameworkDataSource.cs | 21 +- .../Extension/RunnnerInfo.cs | 17 +- .../Extension/TestDataSource.cs | 1 + .../TestPlatformCompatibilityDataSource.cs | 333 ++------------- .../TesthostCompatibilityDataSource.cs | 176 +++----- ...TranslationLayerCompatibilityDataSource.cs | 205 +++------- .../CustomTestHostTests.cs | 15 +- .../TranslationLayerTests/DiscoverTests.cs | 4 +- .../TranslationLayerTests/RunTests.cs | 2 +- .../RunTestsWithFilterTests.cs | 3 +- .../IntegrationTestBase.cs | 14 +- ...icrosoft.TestPlatform.TestUtilities.csproj | 1 + 26 files changed, 737 insertions(+), 867 deletions(-) delete mode 100644 sln.cmd delete mode 100644 src/SettingsMigrator/ConsoleParameters.cs create mode 100644 test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs create mode 100644 test/Microsoft.TestPlatform.AcceptanceTests/Extension/Feature.cs create mode 100644 test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs diff --git a/playground/TestPlatform.Playground/Program.cs b/playground/TestPlatform.Playground/Program.cs index 2b1fcb0a0c..5d46ab9328 100644 --- a/playground/TestPlatform.Playground/Program.cs +++ b/playground/TestPlatform.Playground/Program.cs @@ -55,7 +55,8 @@ static void Main(string[] args) "; var sources = new[] { - Path.Combine(playground, "MSTest1", "bin", "Debug", "net472", "MSTest1.dll") + @"C:\p\vstest\test\TestAssets\MSTestProject1\bin\NETTestSdkLatest-17.2.0-dev\MSTestLatestPreview-2.2.9-preview-20220210-07\Debug\net451\MSTestProject1.dll", + @"C:\p\vstest\test\TestAssets\MSTestProject2\bin\NETTestSdkLatest-17.2.0-dev\MSTestLatestPreview-2.2.9-preview-20220210-07\Debug\net451\MSTestProject2.dll" }; var options = new TestPlatformOptions(); diff --git a/playground/TestPlatform.Playground/TestPlatform.Playground.csproj b/playground/TestPlatform.Playground/TestPlatform.Playground.csproj index 77ccfd0b7e..7bdbeb915e 100644 --- a/playground/TestPlatform.Playground/TestPlatform.Playground.csproj +++ b/playground/TestPlatform.Playground/TestPlatform.Playground.csproj @@ -38,7 +38,7 @@ - + diff --git a/scripts/build.ps1 b/scripts/build.ps1 index 23c5872736..c5723cb933 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -276,7 +276,7 @@ function Invoke-TestAssetsBuild { $dirMSTestVersion = $mstestVersion -replace "\[|\]" $dirMSTestPropertyName = $propertyName -replace "Framework" -replace "Version" - Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration --no-restore -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:MSTestFrameworkVersion=$mstestVersion -p:MSTestAdapterVersion=$mstestVersion -p:NETTestSdkVersion=$netTestSdkVersion -p:BaseOutputPath=""bin\$dirNetTestSdkPropertyName-$dirNetTestSdkVersion\$dirMSTestPropertyName-$dirMSTestVersion\\"" -bl:""$env:TP_OUT_DIR\log\$Configuration\perm.binlog""" + Invoke-Exe $dotnetExe -Arguments "build $project --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:MSTestFrameworkVersion=$mstestVersion -p:MSTestAdapterVersion=$mstestVersion -p:NETTestSdkVersion=$netTestSdkVersion -p:BaseOutputPath=""bin\$dirNetTestSdkPropertyName-$dirNetTestSdkVersion\$dirMSTestPropertyName-$dirMSTestVersion\\"" -bl:""$env:TP_OUT_DIR\log\$Configuration\perm.binlog""" } } } @@ -1276,49 +1276,49 @@ if ($ProjectNamePatterns.Count -ne 0) { } # Execute build -# $timer = Start-Timer -# Write-Log "Build started: args = '$args'" -# Write-Log "Test platform environment variables: " -# Get-ChildItem env: | Where-Object -FilterScript { $_.Name.StartsWith("TP_") } | Format-Table -# Write-Log "Test platform build variables: " -# Get-Variable | Where-Object -FilterScript { $_.Name.StartsWith("TPB_") } | Format-Table +$timer = Start-Timer +Write-Log "Build started: args = '$args'" +Write-Log "Test platform environment variables: " +Get-ChildItem env: | Where-Object -FilterScript { $_.Name.StartsWith("TP_") } | Format-Table +Write-Log "Test platform build variables: " +Get-Variable | Where-Object -FilterScript { $_.Name.StartsWith("TPB_") } | Format-Table if ($Force -or $Steps -contains "InstallDotnet") { Install-DotNetCli } -# if ($Force -or $Steps -contains "Restore") { -# Clear-Package -# Restore-Package -# } - -# if ($Force -or $Steps -contains "UpdateLocalization") { -# Update-LocalizedResources -# } - -# if ($Force -or $Steps -contains "Build") { -# Invoke-Build -# } - -# if ($Force -or $Steps -contains "Publish") { -# Publish-Package -# Create-VsixPackage -# Create-NugetPackages -# } - -# if ($Force -or $Steps -contains "Publish" -or $Steps -contains "Manifest") { -# Generate-Manifest -PackageFolder $TPB_PackageOutDir -# if (Test-Path $TPB_SourceBuildPackageOutDir) -# { -# Generate-Manifest -PackageFolder $TPB_SourceBuildPackageOutDir -# } -# Copy-PackageIntoStaticDirectory -# } +if ($Force -or $Steps -contains "Restore") { + Clear-Package + Restore-Package +} + +if ($Force -or $Steps -contains "UpdateLocalization") { + Update-LocalizedResources +} + +if ($Force -or $Steps -contains "Build") { + Invoke-Build +} + +if ($Force -or $Steps -contains "Publish") { + Publish-Package + Create-VsixPackage + Create-NugetPackages +} + +if ($Force -or $Steps -contains "Publish" -or $Steps -contains "Manifest") { + Generate-Manifest -PackageFolder $TPB_PackageOutDir + if (Test-Path $TPB_SourceBuildPackageOutDir) + { + Generate-Manifest -PackageFolder $TPB_SourceBuildPackageOutDir + } + Copy-PackageIntoStaticDirectory +} if ($Force -or $Steps -contains "PrepareAcceptanceTests") { - #Publish-PatchedDotnet + Publish-PatchedDotnet Invoke-TestAssetsBuild - #Publish-Tests + Publish-Tests } if ($Script:ScriptFailed) { diff --git a/sln.cmd b/sln.cmd deleted file mode 100644 index 8ba8ab0bb9..0000000000 --- a/sln.cmd +++ /dev/null @@ -1,14 +0,0 @@ -@echo off - -REM Copyright (c) Microsoft. All rights reserved. - -REM set DOTNET_ROOT to point at the locally installed dotnet, -REM to avoid problems with .NET Core 2.1 that we run our tests and tools against, -REM but that is regularly corrupted. - -set DOTNET_ROOT=%~dp0tools\dotnet -set DOTNET_ROOT(x86)=%~dp0tools\dotnet_x86 - -start %~dp0TestPlatform.sln - -if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/src/SettingsMigrator/ConsoleParameters.cs b/src/SettingsMigrator/ConsoleParameters.cs deleted file mode 100644 index 80470abdff..0000000000 --- a/src/SettingsMigrator/ConsoleParameters.cs +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -#if NETFRAMEWORK -using System.Collections.Generic; -#endif -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; - -using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Extensions; -using Microsoft.VisualStudio.TestPlatform.ObjectModel; -using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers; -using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces; - -#nullable disable - -namespace Microsoft.TestPlatform.VsTestConsole.TranslationLayer; - -/// -/// Class which defines additional specifiable parameters for vstest.console.exe -/// -public class ConsoleParameters -{ - internal static readonly ConsoleParameters Default = new(); - - private string _logFilePath; - private readonly IFileHelper _fileHelper; - - /// - /// Create instance of - /// - public ConsoleParameters() : this(new FileHelper()) - { } - - /// - /// Create instance of - /// - /// Object of type - public ConsoleParameters(IFileHelper fileHelper) - { - _fileHelper = fileHelper; - } - - /// - /// Environment variables to be set for the process - /// - public Dictionary EnvironmentVariables { get; set; } - - /// - /// Trace level for logs. - /// - public TraceLevel TraceLevel { get; set; } = TraceLevel.Verbose; - - /// - /// Full path for the log file - /// - public string LogFilePath - { - get - { - return _logFilePath; - } - - set - { - ValidateArg.NotNullOrEmpty(value, "LogFilePath"); - var directoryPath = Path.GetDirectoryName(value); - if (!string.IsNullOrEmpty(directoryPath) && !_fileHelper.DirectoryExists(directoryPath)) - { - Directory.CreateDirectory(directoryPath); - } - - // Ensure path is double quoted. if path has white space then it can create problem. - _logFilePath = value.AddDoubleQuote(); - } - } - - /// - /// Port Number for communication - /// vstest.console will need this port number to communicate with this component - translation layer - /// Currently Internal as we are not intentionally exposing this to consumers of translation layer - /// - internal int PortNumber { get; set; } - - /// - /// Parent Process ID of the process whose lifetime should dictate the life time of vstest.console.exe - /// vstest.console will need this process ID to know when the process exits. - /// If parent process dies/crashes without invoking EndSession, vstest.console should exit immediately - /// Currently Internal as we are not intentionally exposing this to consumers of translation layer - /// - internal int ParentProcessId { get; set; } -} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs index c44ec9775e..574e742f2e 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs @@ -45,7 +45,7 @@ public class AcceptanceTestBase : IntegrationTestBase public const string DesktopRunnerTargetRuntime = "win7-x64"; public const string CoreRunnerTargetRuntime = ""; public const string InIsolation = "/InIsolation"; - + public const string NETFX452_48 = "net452;net461;net472;net48"; public const string NETFX451_48 = "net452;net461;net472;net48"; public const string NETCORE21_50 = "netcoreapp2.1;netcoreapp3.1;net5.0"; @@ -55,12 +55,13 @@ public class AcceptanceTestBase : IntegrationTestBase /// /// Our current defaults for .NET and .NET Framework. /// - public const string DEFAULT_RUNNER_NETFX_AND_NET = $"{DEFAULT_RUNNER_NETFX};netcoreapp2.1"; + public const string DEFAULT_RUNNER_NETFX_AND_NET = $"{DEFAULT_RUNNER_NETFX};netcoreapp2.1"; public const string DEFAULT_HOST_NETFX_AND_NET = "net451;netcoreapp2.1"; public const string LATEST_TO_LEGACY = "Latest;LatestPreview;LatestStable;RecentStable;MostDownloaded;PreviousStable;LegacyStable"; public const string LATESTPREVIEW_TO_LEGACY = "LatestPreview;LatestStable;RecentStable;MostDownloaded;PreviousStable;LegacyStable"; public const string LATEST = "Latest"; - internal const string MSTEST="MSTest"; + public const string LATESTSTABLE= "LatestStable"; + internal const string MSTEST = "MSTest"; public static string And(string left, string right) { @@ -74,7 +75,7 @@ protected static void SetTestEnvironment(IntegrationTestEnvironment testEnvironm testEnvironment.VSTestConsoleInfo = runnerInfo.VSTestConsoleInfo; testEnvironment.DllInfos = runnerInfo.DllInfos; testEnvironment.DebugInfo = runnerInfo.DebugInfo; - + testEnvironment.RunnerFramework = runnerInfo.RunnerFramework; testEnvironment.TargetFramework = runnerInfo.TargetFramework; testEnvironment.InIsolationValue = runnerInfo.InIsolationValue; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 3faa9178ab..b39e6b76a1 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -23,16 +23,18 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); + var assemblyPaths = BuildMultipleAssemblyPath("MSTestProject1.dll", "MSTestProject2.dll"); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); ValidateSummaryStatus(2, 2, 2); ExitCodeEquals(1); // failing tests + StdErrHasTestRunFailedMessageButNoOtherError(); + StdOutHasNoWarnings(); } [TestMethod] - [TestPlatformCompatibilityDataSource(WithInProcess = true, WithEveryVersionOfAdapter = false, WithEveryVersionOfHost = false, WithEveryVersionOfRunner = false, WithOlderConfigurations = false)] + [TestPlatformCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER, AfterAdapterFeature = Features.MSTEST_IFRAMEWORK_HANDLE_99)] public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo) { @@ -44,15 +46,32 @@ public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo) ValidateSummaryStatus(passed: 2, failed: 2, skipped: 2); ExitCodeEquals(1); // failing tests + StdErrHasTestRunFailedMessageButNoOtherError(); + StdOutHasNoWarnings(); } [TestMethod] - [TesthostCompatibilityDataSource] + [HostCompatibilityDataSource] public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); + var assemblyPaths = BuildMultipleAssemblyPath("MSTestProject1.dll", "MSTestProject2.dll"); + + InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); + + ValidateSummaryStatus(2, 2, 2); + ExitCodeEquals(1); // failing tests + } + + + [TestMethod] + [RunnerCompatibilityDataSource] + public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations2(RunnerInfo runnerInfo) + { + SetTestEnvironment(_testEnvironment, runnerInfo); + + var assemblyPaths = BuildMultipleAssemblyPath("MSTestProject1.dll", "MSTestProject2.dll"); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs new file mode 100644 index 0000000000..f4fa979dbd --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs @@ -0,0 +1,380 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml; + +using Microsoft.TestPlatform.TestUtilities; + +using Semver; + +namespace Microsoft.TestPlatform.AcceptanceTests; + +public class CompatibilityRowsBuilder +{ + private static XmlDocument? s_depsXml; + private readonly string[] _runnerFrameworks; + private readonly string[] _runnerVersions; + private readonly string[] _hostFrameworks; + private readonly string[] _adapterVersions; + private readonly string[] _adapters; + private readonly string[] _hostVersions; + + public CompatibilityRowsBuilder(string runnerFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string runnerVersions = AcceptanceTestBase.LATEST_TO_LEGACY, + string hostFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string hostVersions = AcceptanceTestBase.LATEST_TO_LEGACY, + string adapterVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY, + string adapters = AcceptanceTestBase.MSTEST) + { + _runnerFrameworks = runnerFrameworks.Split(';'); + _runnerVersions = runnerVersions.Split(';'); + _hostFrameworks = hostFrameworks.Split(';'); + _hostVersions = hostVersions.Split(';'); + _adapterVersions = adapterVersions.Split(';'); + _adapters = adapters.Split(';'); + } + + /// + /// Add run for in-process using the selected .NET Framework runners, and and all selected adapters. + /// + public bool WithInProcess { get; set; } = true; + public bool WithEveryVersionOfRunner { get; set; } = true; + public bool WithEveryVersionOfHost { get; set; } = true; + public bool WithEveryVersionOfAdapter { get; set; } = true; + public bool WithOlderConfigurations { get; set; } = true; + + public string? BeforeFeature { get; set; } + public string? AfterFeature { get; set; } + public string? BeforeAdapterFeature { get; set; } + public string? AfterAdapterFeature { get; set; } + + public bool DebugVSTestConsole { get; set; } + public bool DebugTesthost { get; set; } + public bool DebugDataCollector { get; set; } + public bool NoDefaultBreakpoints { get; set; } = true; + + + public List CreateData() + { + var dataRows = new List(); + + if (WithEveryVersionOfRunner) + AddEveryVersionOfRunner(dataRows); + + if (WithEveryVersionOfHost) + AddEveryVersionOfHost(dataRows); + + if (WithEveryVersionOfAdapter) + AddEveryVersionOfAdapter(dataRows); + + if (WithOlderConfigurations) + AddOlderConfigurations(dataRows); + + if (WithInProcess) + AddInProcess(dataRows); + + var minVersion = SemVersion.Parse("0.0.0-alpha.1"); + var maxVersion = SemVersion.Parse("9999.0.0"); + SemVersion? beforeVersion = maxVersion; + SemVersion? afterVersion = minVersion; + SemVersion? beforeAdapterVersion = maxVersion; + SemVersion? afterAdapterVersion = minVersion; + + if (BeforeFeature != null) + { + var feature = Features.TestPlatformFeatures[BeforeFeature]; + beforeVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + } + + if (AfterFeature != null) + { + var feature = Features.TestPlatformFeatures[AfterFeature]; + afterVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + } + + if (BeforeFeature != null) + { + var feature = Features.TestPlatformFeatures[BeforeFeature]; + beforeAdapterVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + } + + if (AfterAdapterFeature != null) + { + var feature = Features.AdapterFeatures[AfterAdapterFeature]; + afterAdapterVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + } + + var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); + // Run .NET Framework tests only on Windows. + Func filter = tfm => isWindows || !tfm.StartsWith("net4"); + + // TODO: maybe we should throw if we don't end up generating any data + // because none of the versions match, or some other way to identify tests that will never run because they are very outdated. + // We probably don't have that need right now, because legacy version is 15.x.x, which is very old, and we are still keeping + // compatibility. + + Func isInRange = (version, before, after) => version < before && after < version; + + var rows = dataRows.Where(r => r.VSTestConsoleInfo != null + && isInRange(r.VSTestConsoleInfo.Version, beforeVersion, afterVersion) + && r.DllInfos.All(d => d is NetTestSdkInfo ? isInRange(d.Version, beforeVersion, afterVersion) : isInRange(d.Version, beforeAdapterVersion, afterAdapterVersion))).ToList(); + + if (rows.Count == 0) + { + // TODO: This needs to be way more specific about what happened. + throw new InvalidOperationException("There were no rows that matched the specified criteria."); + } + + return rows; + } + + private void AddInProcess(List dataRows) + { + foreach (var runnerFramework in _runnerFrameworks) + { + if (!runnerFramework.StartsWith("net4")) + { + continue; + } + + foreach (var runnerVersion in _runnerVersions) + { + foreach (var adapter in _adapters) + { + foreach (var adapterVersion in _adapterVersions) + { + AddRow(dataRows, runnerVersion, runnerFramework, runnerVersion, runnerFramework, adapter, adapterVersion, inIsolation: false); + } + } + } + } + } + + private void AddOlderConfigurations(List dataRows) + { + // Older configurations where the runner, host and adapter version are the same. + // We already added the row where all are newest when adding combination with all runners. + foreach (var runnerVersion in _runnerVersions.Skip(1)) + { + foreach (var runnerFramework in _runnerFrameworks) + { + foreach (var hostFramework in _hostFrameworks) + { + var isNetFramework = hostFramework.StartsWith("net4"); + var hostVersion = runnerVersion; + foreach (var adapter in _adapters) + { + var adapterVersion = runnerVersion; + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); + } + } + } + } + } + + + private void AddEveryVersionOfAdapter(List dataRows) + { + var runnerVersion = _runnerVersions[0]; + foreach (var runnerFramework in _runnerFrameworks) + { + foreach (var hostFramework in _hostFrameworks) + { + var isNetFramework = hostFramework.StartsWith("net4"); + // .NET Framework testhost ships with the runner, and the version from the + // runner directory is always selected, otherwise select the newest version from _hostFrameworks. + var hostVersion = isNetFramework ? runnerVersion : _hostVersions[0]; + foreach (var adapter in _adapters) + { + // We already used the newest when adding combination with every runner + foreach (var adapterVersion in _adapterVersions.Skip(1)) + { + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); + } + } + } + } + } + + private void AddEveryVersionOfHost(List dataRows) + { + var runnerVersion = _runnerVersions[0]; + + foreach (var runnerFramework in _runnerFrameworks) + { + foreach (var hostFramework in _hostFrameworks) + { + var isNetFramework = hostFramework.StartsWith("net4"); + // .NET Framework testhost ships with the runner, and the version from the + // runner directory is always the same as the runner. There are no variations + // so we just need to add host versions for .NET testhosts. We also skip the + // newest version because we already added it when AddEveryVersionOfRunner + var hostVersions = isNetFramework ? Array.Empty() : _hostVersions.Skip(1).ToArray(); + foreach (var hostVersion in hostVersions) + { + foreach (var adapter in _adapters) + { + // use the newest + var adapterVersion = _adapterVersions[0]; + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); + } + } + } + } + } + + private void AddEveryVersionOfRunner(List dataRows) + { + foreach (var runnerVersion in _runnerVersions) + { + foreach (var runnerFramework in _runnerFrameworks) + { + foreach (var hostFramework in _hostFrameworks) + { + var isNetFramework = hostFramework.StartsWith("net4"); + // .NET Framework testhost ships with the runner, and the version from the + // runner directory is always selected, otherwise select the newest version from _hostFrameworks. + var hostVersion = isNetFramework ? runnerVersion : _hostVersions[0]; + foreach (var adapter in _adapters) + { + // use the newest + var adapterVersion = _adapterVersions[0]; + AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); + } + } + } + } + } + + private void AddRow(List dataRows, +string runnerVersion, string runnerFramework, string hostVersion, string hostFramework, string adapter, string adapterVersion, bool inIsolation) + { + RunnerInfo runnerInfo = GetRunnerInfo(runnerFramework, hostFramework, inIsolation); + runnerInfo.DebugInfo = GetDebugInfo(); + runnerInfo.VSTestConsoleInfo = GetVSTestConsoleInfo(runnerVersion, runnerInfo); + + // The order in which we add them matters. We end up both modifying the same path + // and adding to it. So the first one added will be later in the path. E.g.: + // Adding testSdk first: + // C:\p\vstest\test\TestAssets\MSTestProject1\bin\MSTestLatestPreview-2.2.9-preview-20220210-07\NETTestSdkLatest-17.2.0-dev\Debug\net451\MSTestProject1.dll + // versus adding testSdk second: + // C:\p\vstest\test\TestAssets\MSTestProject1\bin\NETTestSdkLatest-17.2.0-dev\MSTestLatestPreview-2.2.9-preview-20220210-07\Debug\net451\MSTestProject1.dll + runnerInfo.DllInfos.Add(GetMSTestInfo(adapterVersion)); + runnerInfo.DllInfos.Add(GetNetTestSdkInfo(hostVersion)); + dataRows.Add(runnerInfo); + } + + private DebugInfo GetDebugInfo() + { + return new DebugInfo + { + DebugDataCollector = DebugDataCollector, + DebugTesthost = DebugTesthost, + DebugVSTestConsole = DebugVSTestConsole, + NoDefaultBreakpoints = NoDefaultBreakpoints + }; + } + + + private RunnerInfo GetRunnerInfo(string runnerFramework, string hostFramework, bool inIsolation) + { + return new RunnerInfo + { + RunnerFramework = runnerFramework, + TargetFramework = hostFramework, + InIsolationValue = inIsolation ? AcceptanceTestBase.InIsolation : null + }; + } + + private MSTestInfo GetMSTestInfo(string msTestVersion) + { + var depsXml = GetDependenciesXml(); + + // It is okay when node is null, we check that Version has value when we update paths by using MSTestInfo, and throw. + // This way it throws in the body of the test which has better error reporting than throwing in the data source. + XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/MSTestFramework{msTestVersion}Version"); + var version = node?.InnerText.Replace("[", "").Replace("]", ""); + var slash = Path.DirectorySeparatorChar; + var versionSpecificBinPath = $"{slash}bin{slash}MSTest{msTestVersion}-{version}{slash}"; + + return new MSTestInfo(msTestVersion, version, versionSpecificBinPath); + } + + private static VSTestConsoleInfo GetVSTestConsoleInfo(string vstestConsoleVersion, RunnerInfo runnerInfo) + { + var depsXml = GetDependenciesXml(); + + // When version is Latest, we built it locally, but it gets restored into our nuget cache on build + // same as other versions, we just need to grab the version from a different property. + + var propertyName = vstestConsoleVersion == AcceptanceTestBase.LATEST + ? $"NETTestSdkVersion" + : $"VSTestConsole{vstestConsoleVersion}Version"; + + var packageName = runnerInfo.IsNetFrameworkRunner + ? "microsoft.testplatform" + : "microsoft.testplatform.cli"; + + // It is okay when node is null, we will fail to find the executable later, and throw. + // This way it throws in the body of the test which has better error reporting than throwing in the data source. + // And we can easily find out what is going on because --WRONG-VERSION-- sticks out, and is easy to find in the codebase. + XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/{propertyName}"); + var version = node?.InnerText.Replace("[", "").Replace("]", "") ?? "--WRONG-VERSION--"; + var vstestConsolePath = runnerInfo.IsNetFrameworkRunner + ? Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "packages", packageName, version, + "tools", "net451", "Common7", "IDE", "Extensions", "TestPlatform", "vstest.console.exe") + : Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "packages", packageName, version, + "contentFiles", "any", "netcoreapp2.1", "vstest.console.dll"); + + if (version.StartsWith("15.")) + { + vstestConsolePath = vstestConsolePath.Replace("netcoreapp2.1", "netcoreapp2.0"); + } + + return new VSTestConsoleInfo(vstestConsoleVersion, version, vstestConsolePath); + } + + private static NetTestSdkInfo GetNetTestSdkInfo(string testhostVersionType) + { + var depsXml = GetDependenciesXml(); + + // When version is Latest, we built it locally, but it gets restored into our nuget cache on build + // same as other versions, we just need to grab the version from a different property. + + var propertyName = testhostVersionType == AcceptanceTestBase.LATEST + ? $"NETTestSdkVersion" + : $"VSTestConsole{testhostVersionType}Version"; + + // It is okay when node is null, we check that Version has value when we update paths by using TesthostInfo, and throw. + // This way it throws in the body of the test which has better error reporting than throwing in the data source. + // + // We use the VSTestConsole properties to figure out testhost version, for now. + XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/{propertyName}"); + var version = node?.InnerText.Replace("[", "").Replace("]", ""); + var slash = Path.DirectorySeparatorChar; + var versionSpecificBinPath = $"{slash}bin{slash}NETTestSdk{testhostVersionType}-{version}{slash}"; + + return new NetTestSdkInfo(testhostVersionType, version, versionSpecificBinPath); + } + + + private static XmlDocument GetDependenciesXml() + { + if (s_depsXml != null) + return s_depsXml; + + var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); + var fileStream = File.OpenRead(depsXmlPath); + var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false }; + var depsXml = new XmlDocument(); + depsXml.Load(xmlTextReader); + + s_depsXml = depsXml; + return depsXml; + } +} + diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Feature.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Feature.cs new file mode 100644 index 0000000000..9014349f06 --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Feature.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.TestPlatform.AcceptanceTests; + +public class Feature +{ + public Feature(string version, string issue) + { + Version = version; + Issue = issue; + } + + public string Version { get; } + public string Issue { get; } +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs new file mode 100644 index 0000000000..e82389e404 --- /dev/null +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; + +namespace Microsoft.TestPlatform.AcceptanceTests; + +public static class Features +{ + public const string ATTACH_DEBUGGER = nameof(ATTACH_DEBUGGER); + public const string MSTEST_IFRAMEWORK_HANDLE_99 = nameof(MSTEST_IFRAMEWORK_HANDLE_99); + + + public static Dictionary TestPlatformFeatures { get; } = new Dictionary + { + [ATTACH_DEBUGGER] = new(version: "v16.7.0-preview-20200519-01", issue: "https://github.com/microsoft/vstest/pull/2325"), + }; + + public static Dictionary AdapterFeatures { get; internal set; } = new Dictionary + { + [MSTEST_IFRAMEWORK_HANDLE_99] = new("2.2.8", issue: "idk"), + }; +} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs index af3101c971..30875f6886 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs @@ -1,127 +1,73 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System; -using System.Globalization; -using System.IO; -using System.Linq; using System.Reflection; -using System.Xml; - -using Microsoft.TestPlatform.TestUtilities; namespace Microsoft.TestPlatform.AcceptanceTests; -public sealed class MSTestCompatibilityDataSource : TestDataSource + +public class MSTestCompatibilityDataSource : TestDataSource { - private static XmlDocument? s_depsXml; - private readonly string[] _runnerFrameworks; - private readonly string[] _targetFrameworks; - private readonly string[] _msTestVersions; + private readonly CompatibilityRowsBuilder _builder; - /// - /// Initializes a new instance. - /// - /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. - public MSTestCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT_RUNNER_NETFX_AND_NET, string targetFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, string msTestVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY) + public MSTestCompatibilityDataSource( + string runnerFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string hostFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string adapterVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY) { - _runnerFrameworks = runners.Split(';'); - _targetFrameworks = targetFrameworks.Split(';'); - _msTestVersions = msTestVersions.Split(';'); - - // Do not generate the data rows here, properties (e.g. InProcess) are not populated until after constructor is done. + // TODO: We actually don't generate values to use different translation layers, because we don't have a good way to do + // that right now. Translation layer is loaded directly into the acceptance test, and so we don't have easy way to substitute it. + + _builder = new CompatibilityRowsBuilder( + runnerFrameworks, + // runner versions + AcceptanceTestBase.LATEST_TO_LEGACY, + hostFrameworks, + // host versions + AcceptanceTestBase.LATEST_TO_LEGACY, + adapterVersions, + // adapters + AcceptanceTestBase.MSTEST); + + // Do not generate the data rows here, properties (e.g. DebugVSTestConsole) are not populated until after constructor is done. } - /// - /// Add also run for in-process using the runner. - /// - // TODO: Can we somehow assert that we actually ran in process? - public bool InProcess { get; set; } public bool DebugVSTestConsole { get; set; } public bool DebugTesthost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; - public override void CreateData(MethodInfo methodInfo) - { - var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); - // Run .NET Framework tests only on Windows. - Func filter = tfm => isWindows || !tfm.StartsWith("net4"); - - if (InProcess) - { - foreach (var msTestVersion in _msTestVersions) - { - // We run in the .NET Framework runner process, the runner and target framework must agree. - var runnerInfo = new RunnerInfo(AcceptanceTestBase.DEFAULT_RUNNER_NETFX, AcceptanceTestBase.DEFAULT_RUNNER_NETFX, inIsolationValue: null); - runnerInfo.DebugInfo = new DebugInfo - { - DebugVSTestConsole = DebugVSTestConsole, - DebugTesthost = DebugTesthost, - DebugDataCollector = DebugDataCollector, - NoDefaultBreakpoints = NoDefaultBreakpoints, - }; - runnerInfo.DllInfos.Add(GetMSTestInfo(msTestVersion)); - - AddData(runnerInfo); - } - } - - foreach (var runner in _runnerFrameworks.Where(filter)) - { - foreach (var fmw in _targetFrameworks.Where(filter)) - { - foreach (var msTestVersion in _msTestVersions) - { - var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation); - runnerInfo.DebugInfo = new DebugInfo - { - DebugVSTestConsole = DebugVSTestConsole, - DebugTesthost = DebugTesthost, - DebugDataCollector = DebugDataCollector, - NoDefaultBreakpoints = NoDefaultBreakpoints, - }; - runnerInfo.DllInfos.Add(GetMSTestInfo(msTestVersion)); - - AddData(runnerInfo); - } - } - } - } - - public string GetDisplayName(MethodInfo methodInfo, object[] data) - { - return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); - } - - private MSTestInfo GetMSTestInfo(string msTestVersion) - { - var depsXml = GetDependenciesXml(); + /// + /// Add run for in-process using the selected .NET Framework runners, and and all selected adapters. + /// + public bool InProcess { get; set; } - // It is okay when node is null, we check that Version has value when we update paths by using MSTestInfo, and throw. - // This way it throws in the body of the test which has better error reporting than throwing in the data source. - XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/MSTestFramework{msTestVersion}Version"); - var version = node?.InnerText.Replace("[", "").Replace("]", ""); - var slash = Path.DirectorySeparatorChar; - var versionSpecificBinPath = $"{slash}bin{slash}MSTest{msTestVersion}-{version}{slash}"; + public string? BeforeFeature { get; set; } + public string? AfterFeature { get; set; } - return new MSTestInfo(msTestVersion, version, versionSpecificBinPath); - } + public string? BeforeAdapterFeature { get; set; } + public string? AfterAdapterFeature { get; set; } - private static XmlDocument GetDependenciesXml() + public override void CreateData(MethodInfo methodInfo) { - if (s_depsXml != null) - return s_depsXml; - - var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); - var fileStream = File.OpenRead(depsXmlPath); - var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false }; - var depsXml = new XmlDocument(); - depsXml.Load(xmlTextReader); - - s_depsXml = depsXml; - return depsXml; + _builder.WithEveryVersionOfRunner = false; + _builder.WithEveryVersionOfHost = false; + _builder.WithEveryVersionOfAdapter = true; + _builder.WithOlderConfigurations = false; + _builder.WithInProcess = InProcess; + + _builder.BeforeFeature = BeforeFeature; + _builder.AfterFeature = AfterFeature; + _builder.BeforeAdapterFeature = BeforeAdapterFeature; + _builder.AfterAdapterFeature = AfterAdapterFeature; + + _builder.DebugDataCollector = DebugDataCollector; + _builder.DebugVSTestConsole = DebugVSTestConsole; + _builder.DebugTesthost = DebugTesthost; + _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; + + var data = _builder.CreateData(); + data.ForEach(AddData); } } - diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs index e12d7f6011..60d49baf1a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs @@ -47,7 +47,12 @@ public IEnumerable GetData(MethodInfo methodInfo) Func filter = tfm => isWindows || !tfm.StartsWith("net4"); foreach (var fmw in _targetFrameworks.Split(';').Where(filter)) { - var runnerInfo = new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, fmw, inIsolationValue: null); + var runnerInfo = new RunnerInfo + { + RunnerFramework = IntegrationTestBase.CoreRunnerFramework, + TargetFramework = fmw, + InIsolationValue = null + }; runnerInfo.DebugInfo = new DebugInfo { DebugVSTestConsole = DebugVSTestConsole, diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs index d5bfd3b4ce..e4d646a8b8 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs @@ -55,7 +55,12 @@ public NetCoreTargetFrameworkDataSource( private void AddRunnerDataRow(List dataRows, string runnerFramework, string targetFramework) { - var runnerInfo = new RunnerInfo(runnerFramework, targetFramework, inIsolationValue: null); + var runnerInfo = new RunnerInfo + { + RunnerFramework = runnerFramework, + TargetFramework = targetFramework, + InIsolationValue = null + }; runnerInfo.DebugInfo = new DebugInfo { DebugDataCollector = DebugDataCollector, diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs index 07f30ee823..c7be429345 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs @@ -49,7 +49,12 @@ public IEnumerable GetData(MethodInfo methodInfo) foreach (var fmw in _targetFrameworks.Split(';')) { - var runnerInfo = new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, fmw, AcceptanceTestBase.InIsolation); + var runnerInfo = new RunnerInfo + { + RunnerFramework = IntegrationTestBase.DesktopRunnerFramework, + TargetFramework = fmw, + InIsolationValue = AcceptanceTestBase.InIsolation + }; runnerInfo.DebugInfo = new DebugInfo { DebugVSTestConsole = DebugVSTestConsole, diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs index 3b45a19e74..468aded7f7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs @@ -53,7 +53,12 @@ public IEnumerable GetData(MethodInfo methodInfo) var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); if (_useCoreRunner && isWindows) { - var runnerInfo = new RunnerInfo(IntegrationTestBase.CoreRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, inIsolationValue: null); + var runnerInfo = new RunnerInfo + { + RunnerFramework = IntegrationTestBase.CoreRunnerFramework, + TargetFramework = AcceptanceTestBase.DesktopTargetFramework, + InIsolationValue = null + }; runnerInfo.DebugInfo = new DebugInfo { DebugVSTestConsole = DebugVSTestConsole, @@ -68,7 +73,12 @@ public IEnumerable GetData(MethodInfo methodInfo) { if (_inIsolation) { - var runnerInfo = new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, AcceptanceTestBase.InIsolation); + var runnerInfo = new RunnerInfo + { + RunnerFramework = IntegrationTestBase.DesktopRunnerFramework, + TargetFramework = AcceptanceTestBase.DesktopTargetFramework, + InIsolationValue = AcceptanceTestBase.InIsolation + }; runnerInfo.DebugInfo = new DebugInfo { DebugVSTestConsole = DebugVSTestConsole, @@ -81,7 +91,12 @@ public IEnumerable GetData(MethodInfo methodInfo) if (_inProcess) { - var runnerInfo = new RunnerInfo(IntegrationTestBase.DesktopRunnerFramework, AcceptanceTestBase.DesktopTargetFramework, inIsolationValue: null); + var runnerInfo = new RunnerInfo + { + RunnerFramework = IntegrationTestBase.DesktopRunnerFramework, + TargetFramework = AcceptanceTestBase.DesktopTargetFramework, + InIsolationValue = null + }; runnerInfo.DebugInfo = new DebugInfo { DebugVSTestConsole = DebugVSTestConsole, diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs index fcf9cca46e..bb30619479 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs @@ -17,19 +17,12 @@ namespace Microsoft.TestPlatform.AcceptanceTests; [Serializable] public class RunnerInfo { - public RunnerInfo(string runnerFramework, string targetFramework, string? inIsolationValue = "") - { - RunnerFramework = runnerFramework; - TargetFramework = targetFramework; - InIsolationValue = inIsolationValue; - } + public string? RunnerFramework { get; set; } - public string RunnerFramework { get; set; } + public VSTestConsoleInfo? VSTestConsoleInfo { get; set; } - public VSTestConsoleInfo VSTestConsoleInfo { get; set; } - - public string TargetFramework { get; set; } + public string? TargetFramework { get; set; } public string? InIsolationValue { get; set; } public DebugInfo? DebugInfo { get; set; } @@ -44,7 +37,7 @@ public RunnerInfo(string runnerFramework, string targetFramework, string? inIsol /// /// Is running via .NET Framework vstest.console? /// - public bool IsNetFrameworkRunner => RunnerFramework.StartsWith("net4", StringComparison.InvariantCultureIgnoreCase); + public bool IsNetFrameworkRunner => RunnerFramework!.StartsWith("net4", StringComparison.InvariantCultureIgnoreCase); /// /// Is running via .NET "Core" testhost? @@ -54,7 +47,7 @@ public RunnerInfo(string runnerFramework, string targetFramework, string? inIsol /// /// Is running via .NET Framework testhost? /// - public bool IsNetFrameworkTarget => TargetFramework.StartsWith("net4", StringComparison.InvariantCultureIgnoreCase); + public bool IsNetFrameworkTarget => TargetFramework!.StartsWith("net4", StringComparison.InvariantCultureIgnoreCase); public override string ToString() => $"Runner = {RunnerFramework}, TargetFramework = {TargetFramework}, {(string.IsNullOrEmpty(InIsolationValue) ? "InProcess" : "InIsolation")}, {VSTestConsoleInfo}, {string.Join(",", DllInfos)}"; } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs index abfead224e..d041d36e79 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestDataSource.cs @@ -106,6 +106,7 @@ public abstract class TestDataSource : Attribute, ITestDataSourc where T1 : notnull where T2 : notnull where T3 : notnull + where T4 : notnull { private readonly List _data = new(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs index eca9ac55e6..b607811116 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs @@ -1,29 +1,30 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; using System.Reflection; -using System.Xml; - -using Microsoft.TestPlatform.TestUtilities; - -using Semver; namespace Microsoft.TestPlatform.AcceptanceTests; -public sealed class TestPlatformCompatibilityDataSource : TestDataSource +/// +/// A data source that provides a huge mix of runners, hosts and mstest adapter, to add up >100 tests. +/// You can control which runner versions, host versions, and adapter versions will be used. This should be +/// used only to test the most common scenarios, or special configurations that are candidates for their own +/// specialized source. +/// +/// By default net451 and netcoreapp2.1 are used for both runner and host. (4 combinations) +/// Then run with every version of runner is added. +/// Then run with every version of test.sdk is added. +/// Then run with every combination of testhost and adapter is added. +/// And then run in process is added. +/// +/// All of those are filtered down to have no duplicates, and to pass the +/// Before and After platform version filters, and adapter filters. +/// +/// When that adds up to no configuration exception is thrown. +/// +public class TestPlatformCompatibilityDataSource : TestDataSource { - private static XmlDocument? s_depsXml; - private readonly string[] _runnerFrameworks; - private readonly string[] _runnerVersions; - private readonly string[] _hostFrameworks; - private readonly string[] _adapterVersions; - private readonly string[] _adapters; - private readonly string[] _hostVersions; + private readonly CompatibilityRowsBuilder _builder; public TestPlatformCompatibilityDataSource( string runnerFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, @@ -35,14 +36,8 @@ public TestPlatformCompatibilityDataSource( { // TODO: We actually don't generate values to use different translation layers, because we don't have a good way to do // that right now. Translation layer is loaded directly into the acceptance test, and so we don't have easy way to substitute it. - // I am keeping this source separate from vstest console compatibility data source, to be able to easily add this feature later. - _runnerFrameworks = runnerFrameworks.Split(';'); - _runnerVersions = runnerVersions.Split(';'); - _hostFrameworks = hostFrameworks.Split(';'); - _hostVersions = hostVersions.Split(';'); - _adapterVersions = adapterVersions.Split(';'); - _adapters = adapters.Split(';'); + _builder = new CompatibilityRowsBuilder(runnerFrameworks, runnerVersions, hostFrameworks, hostVersions, adapterVersions, adapters); // Do not generate the data rows here, properties (e.g. DebugVSTestConsole) are not populated until after constructor is done. } @@ -72,275 +67,23 @@ public TestPlatformCompatibilityDataSource( public override void CreateData(MethodInfo methodInfo) { - var dataRows = new List(); - - if (WithEveryVersionOfRunner) - AddEveryVersionOfRunner(dataRows); - - if (WithEveryVersionOfHost) - AddEveryVersionOfHost(dataRows); - - if (WithEveryVersionOfAdapter) - AddEveryVersionOfAdapter(dataRows); - - if (WithOlderConfigurations) - AddOlderConfigurations(dataRows); - - if (WithInProcess) - AddInProcess(dataRows); - - var c = dataRows.Count(); - - if (BeforeFeature != null && AfterFeature != null) - { - throw new InvalidOperationException($"You cannot specify {nameof(BeforeFeature)} and {nameof(AfterFeature)} at the same time"); - } - - var minVersion = SemVersion.Parse("0.0.0-alpha.1"); - SemVersion? beforeVersion = null; - SemVersion? afterVersion = null; - if (BeforeFeature != null) - { - var feature = Features.Table[BeforeFeature]; - beforeVersion = SemVersion.Parse(feature.Version.TrimStart('v')); - } - if (AfterFeature != null) - { - var feature = Features.Table[AfterFeature]; - afterVersion = SemVersion.Parse(feature.Version.TrimStart('v')); - } - - var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); - // Run .NET Framework tests only on Windows. - Func filter = tfm => isWindows || !tfm.StartsWith("net4"); - - // TODO: maybe we should throw if we don't end up generating any data - // because none of the versions match, or some other way to identify tests that will never run because they are very outdated. - // We probably don't have that need right now, because legacy version is 15.x.x, which is very old, and we are still keeping - // compatibility. - - foreach (var dataRow in dataRows) - { - AddData(dataRow); - } - } - - private void AddInProcess(List dataRows) - { - foreach (var runnerFramework in _runnerFrameworks) - { - if (!runnerFramework.StartsWith("net4")) - { - continue; - } - - foreach (var runnerVersion in _runnerVersions) - { - foreach (var adapter in _adapters) - { - foreach (var adapterVersion in _adapterVersions) - { - AddRow(dataRows, runnerVersion, runnerFramework, runnerVersion, runnerFramework, adapter, adapterVersion, inIsolation: false); - } - } - } - } - } - - private void AddOlderConfigurations(List dataRows) - { - // Older configurations where the runner, host and adapter version are the same. - // We already added the row where all are newest when adding combination with all runners. - foreach (var runnerVersion in _runnerVersions.Skip(1)) - { - foreach (var runnerFramework in _runnerFrameworks) - { - foreach (var hostFramework in _hostFrameworks) - { - var isNetFramework = hostFramework.StartsWith("net4"); - var hostVersion = runnerVersion; - foreach (var adapter in _adapters) - { - var adapterVersion = runnerVersion; - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); - } - } - } - } - } - - private void AddEveryVersionOfAdapter(List dataRows) - { - var runnerVersion = _runnerVersions[0]; - foreach (var runnerFramework in _runnerFrameworks) - { - foreach (var hostFramework in _hostFrameworks) - { - var isNetFramework = hostFramework.StartsWith("net4"); - // .NET Framework testhost ships with the runner, and the version from the - // runner directory is always selected, otherwise select the newest version from _hostFrameworks. - var hostVersion = isNetFramework ? runnerVersion : _hostVersions[0]; - foreach (var adapter in _adapters) - { - // We already used the newest when adding combination with every runner - foreach (var adapterVersion in _adapterVersions.Skip(1)) - { - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); - } - } - } - } - } - - private void AddEveryVersionOfHost(List dataRows) - { - var runnerVersion = _runnerVersions[0]; - - foreach (var runnerFramework in _runnerFrameworks) - { - foreach (var hostFramework in _hostFrameworks) - { - var isNetFramework = hostFramework.StartsWith("net4"); - // .NET Framework testhost ships with the runner, and the version from the - // runner directory is always the same as the runner. There are no variations - // so we just need to add host versions for .NET testhosts. We also skip the - // newest version because we already added it when AddEveryVersionOfRunner - var hostVersions = isNetFramework ? Array.Empty() : _hostVersions.Skip(1).ToArray(); - foreach (var hostVersion in hostVersions) - { - foreach (var adapter in _adapters) - { - // use the newest - var adapterVersion = _adapterVersions[0]; - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); - } - } - } - } - } - - private void AddEveryVersionOfRunner(List dataRows) - { - foreach (var runnerVersion in _runnerVersions) - { - foreach (var runnerFramework in _runnerFrameworks) - { - foreach (var hostFramework in _hostFrameworks) - { - var isNetFramework = hostFramework.StartsWith("net4"); - // .NET Framework testhost ships with the runner, and the version from the - // runner directory is always selected, otherwise select the newest version from _hostFrameworks. - var hostVersion = isNetFramework ? runnerVersion : _hostVersions[0]; - foreach (var adapter in _adapters) - { - // use the newest - var adapterVersion = _adapterVersions[0]; - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); - } - } - } - } - } - - private void AddRow(List dataRows, - string runnerVersion, string runnerFramework, string hostVersion, string hostFramework, string adapter, string adapterVersion, bool inIsolation) - { - RunnerInfo runnerInfo = GetRunnerInfo(runnerFramework, hostFramework, inIsolation); - runnerInfo.DebugInfo = GetDebugInfo(); - runnerInfo.VSTestConsoleInfo = GetVSTestConsoleInfo(runnerVersion, runnerInfo); - - // The order in which we add them matters. We end up both modifying the same path - // and adding to it. So the first one added will be later in the path. E.g.: - // Adding testSdk first: - // C:\p\vstest\test\TestAssets\MSTestProject1\bin\MSTestLatestPreview-2.2.9-preview-20220210-07\NETTestSdkLatest-17.2.0-dev\Debug\net451\MSTestProject1.dll - // versus adding testSdk second: - // C:\p\vstest\test\TestAssets\MSTestProject1\bin\NETTestSdkLatest-17.2.0-dev\MSTestLatestPreview-2.2.9-preview-20220210-07\Debug\net451\MSTestProject1.dll - runnerInfo.DllInfos.Add(GetMSTestInfo(adapterVersion)); - runnerInfo.DllInfos.Add(TesthostCompatibilityDataSource.GetNetTestSdkInfo(hostVersion)); - dataRows.Add(runnerInfo); - } - - private DebugInfo GetDebugInfo() - { - return new DebugInfo - { - DebugDataCollector = DebugDataCollector, - DebugTesthost = DebugTesthost, - DebugVSTestConsole = DebugVSTestConsole, - NoDefaultBreakpoints = NoDefaultBreakpoints - }; - } - - private RunnerInfo GetRunnerInfo(string runnerFramework, string hostFramework, bool inIsolation) - { - return new RunnerInfo(runnerFramework, hostFramework, inIsolation ? AcceptanceTestBase.InIsolation : null); - } - - public string GetDisplayName(MethodInfo methodInfo, object[] data) - { - return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); - } - - private MSTestInfo GetMSTestInfo(string msTestVersion) - { - var depsXml = GetDependenciesXml(); - - // It is okay when node is null, we check that Version has value when we update paths by using MSTestInfo, and throw. - // This way it throws in the body of the test which has better error reporting than throwing in the data source. - XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/MSTestFramework{msTestVersion}Version"); - var version = node?.InnerText.Replace("[", "").Replace("]", ""); - var slash = Path.DirectorySeparatorChar; - var versionSpecificBinPath = $"{slash}bin{slash}MSTest{msTestVersion}-{version}{slash}"; - - return new MSTestInfo(msTestVersion, version, versionSpecificBinPath); - } - - internal static VSTestConsoleInfo GetVSTestConsoleInfo(string vstestConsoleVersion, RunnerInfo runnerInfo) - { - var depsXml = GetDependenciesXml(); - - // When version is Latest, we built it locally, but it gets restored into our nuget cache on build - // same as other versions, we just need to grab the version from a different property. - - var propertyName = vstestConsoleVersion == AcceptanceTestBase.LATEST - ? $"NETTestSdkVersion" - : $"VSTestConsole{vstestConsoleVersion}Version"; - - var packageName = runnerInfo.IsNetFrameworkRunner - ? "microsoft.testplatform" - : "microsoft.testplatform.cli"; - - // It is okay when node is null, we will fail to find the executable later, and throw. - // This way it throws in the body of the test which has better error reporting than throwing in the data source. - // And we can easily find out what is going on because --WRONG-VERSION-- sticks out, and is easy to find in the codebase. - XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/{propertyName}"); - var version = node?.InnerText.Replace("[", "").Replace("]", "") ?? "--WRONG-VERSION--"; - var vstestConsolePath = runnerInfo.IsNetFrameworkRunner - ? Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "packages", packageName, version, - "tools", "net451", "Common7", "IDE", "Extensions", "TestPlatform", "vstest.console.exe") - : Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "packages", packageName, version, - "contentFiles", "any", "netcoreapp2.1", "vstest.console.dll"); - - if (version.StartsWith("15.")) - { - vstestConsolePath = vstestConsolePath.Replace("netcoreapp2.1", "netcoreapp2.0"); - } - - return new VSTestConsoleInfo(vstestConsoleVersion, version, vstestConsolePath); - } - - private static XmlDocument GetDependenciesXml() - { - if (s_depsXml != null) - return s_depsXml; - - var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); - var fileStream = File.OpenRead(depsXmlPath); - var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false }; - var depsXml = new XmlDocument(); - depsXml.Load(xmlTextReader); - - s_depsXml = depsXml; - return depsXml; + _builder.WithEveryVersionOfRunner = WithEveryVersionOfRunner; + _builder.WithEveryVersionOfHost = WithEveryVersionOfHost; + _builder.WithEveryVersionOfAdapter = WithEveryVersionOfAdapter; + _builder.WithOlderConfigurations = WithOlderConfigurations; + _builder.WithInProcess = WithInProcess; + + _builder.BeforeFeature = BeforeFeature; + _builder.AfterFeature = AfterFeature; + _builder.BeforeAdapterFeature = BeforeAdapterFeature; + _builder.AfterAdapterFeature = AfterAdapterFeature; + + _builder.DebugDataCollector = DebugDataCollector; + _builder.DebugVSTestConsole = DebugVSTestConsole; + _builder.DebugTesthost = DebugTesthost; + _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; + + var data = _builder.CreateData(); + data.ForEach(AddData); } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs index a74c45adc9..e32c32aa87 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs @@ -1,37 +1,39 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System; -using System.Globalization; -using System.IO; -using System.Linq; using System.Reflection; -using System.Xml; - -using Microsoft.TestPlatform.TestUtilities; - namespace Microsoft.TestPlatform.AcceptanceTests; -public sealed class TesthostCompatibilityDataSource : TestDataSource +/// +/// A data source that provides every testhost. +/// +/// When that adds up to no configuration exception is thrown. +/// +public class HostCompatibilityDataSource : TestDataSource { - private static XmlDocument? s_depsXml; - private readonly string[] _runnerFrameworks; - private readonly string[] _targetFrameworks; - private readonly string[] _testhostVersions; + private readonly CompatibilityRowsBuilder _builder; - /// - /// Initializes a new instance. - /// - /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. - public TesthostCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAULT_RUNNER_NETFX_AND_NET, string targetFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, - string runnerVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY, string testhostVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY) + public HostCompatibilityDataSource( + string runnerFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string hostFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string hostVersions = AcceptanceTestBase.LATEST_TO_LEGACY) { - _runnerFrameworks = runners.Split(';'); - _targetFrameworks = targetFrameworks.Split(';'); - _testhostVersions = testhostVersions.Split(';'); - - // Do not generate the data rows here, properties (e.g. InProcess) are not populated until after constructor is done. + // TODO: We actually don't generate values to use different translation layers, because we don't have a good way to do + // that right now. Translation layer is loaded directly into the acceptance test, and so we don't have easy way to substitute it. + + _builder = new CompatibilityRowsBuilder( + runnerFrameworks, + // runner versions + AcceptanceTestBase.LATEST, + hostFrameworks, + hostVersions, + // adapter versions + AcceptanceTestBase.LATESTSTABLE, + // adapter + AcceptanceTestBase.MSTEST); + + // Do not generate the data rows here, properties (e.g. DebugVSTestConsole) are not populated until after constructor is done. } public bool DebugVSTestConsole { get; set; } @@ -39,110 +41,30 @@ public TesthostCompatibilityDataSource(string runners = AcceptanceTestBase.DEFAU public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; - public override void CreateData(MethodInfo methodInfo) - { - var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); - // Run .NET Framework tests only on Windows. - Func filter = tfm => isWindows || tfm.StartsWith("net4"); - var onlyLatest = new[] { AcceptanceTestBase.LATEST }; - - // Test different versions of vstest console together with latest testhost. - // There is no point in proving that old testhost does not work with old vstest.console - // when we can patch neither. - foreach (var runner in _runnerFrameworks.Where(filter)) - { - foreach (var fmw in _targetFrameworks.Where(filter)) - { - // For .NET Framework generate only latest console with latest testhost, - // we cannot control the version of testhost because it is shipped with the console. - var testhostVersions = runner.StartsWith("net4") ? onlyLatest : _testhostVersions; - foreach (var testhostVersion in testhostVersions) - { - var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation); - runnerInfo.DebugInfo = new DebugInfo - { - DebugDataCollector = DebugDataCollector, - DebugTesthost = DebugTesthost, - DebugVSTestConsole = DebugVSTestConsole, - NoDefaultBreakpoints = NoDefaultBreakpoints - }; - runnerInfo.VSTestConsoleInfo = TranslationLayerCompatibilityDataSource.GetVSTestConsoleInfo(AcceptanceTestBase.LATEST, runnerInfo); - runnerInfo.DllInfos.Add(GetNetTestSdkInfo(testhostVersion)); + public string? BeforeFeature { get; set; } + public string? AfterFeature { get; set; } + public string? BeforeAdapterFeature { get; set; } + public string? AfterAdapterFeature { get; set; } - AddData(runnerInfo); - } - } - } - - // Test different versions of vstest console together with latest vstest.console. - // There is no point in proving that old testhost does not work with old vstest.console - // when we can patch neither. - foreach (var runner in _runnerFrameworks.Where(filter)) - { - foreach (var fmw in _targetFrameworks.Where(filter)) - { - // For .NET Framework generate only latest console with latest testhost, - // we cannot control the version of testhost because it is shipped with the console. - var consoleVersions = runner.StartsWith("net4") ? onlyLatest : _testhostVersions; - foreach (var consoleVersion in consoleVersions) - { - var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation); - runnerInfo.DebugInfo = new DebugInfo - { - DebugDataCollector = DebugDataCollector, - DebugTesthost = DebugTesthost, - DebugVSTestConsole = DebugVSTestConsole, - NoDefaultBreakpoints = NoDefaultBreakpoints - }; - runnerInfo.VSTestConsoleInfo = TranslationLayerCompatibilityDataSource.GetVSTestConsoleInfo(consoleVersion, runnerInfo); - runnerInfo.DllInfos.Add(GetNetTestSdkInfo(AcceptanceTestBase.LATEST)); - - AddData(runnerInfo); - } - } - } - } - - public string GetDisplayName(MethodInfo methodInfo, object[] data) - { - return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); - } - - internal static NetTestSdkInfo GetNetTestSdkInfo(string testhostVersionType) - { - var depsXml = GetDependenciesXml(); - - // When version is Latest, we built it locally, but it gets restored into our nuget cache on build - // same as other versions, we just need to grab the version from a different property. - - var propertyName = testhostVersionType == AcceptanceTestBase.LATEST - ? $"NETTestSdkVersion" - : $"VSTestConsole{testhostVersionType}Version"; - - // It is okay when node is null, we check that Version has value when we update paths by using TesthostInfo, and throw. - // This way it throws in the body of the test which has better error reporting than throwing in the data source. - // - // We use the VSTestConsole properties to figure out testhost version, for now. - XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/{propertyName}"); - var version = node?.InnerText.Replace("[", "").Replace("]", ""); - var slash = Path.DirectorySeparatorChar; - var versionSpecificBinPath = $"{slash}bin{slash}NETTestSdk{testhostVersionType}-{version}{slash}"; - - return new NetTestSdkInfo(testhostVersionType, version, versionSpecificBinPath); - } - - private static XmlDocument GetDependenciesXml() + public override void CreateData(MethodInfo methodInfo) { - if (s_depsXml != null) - return s_depsXml; - - var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); - var fileStream = File.OpenRead(depsXmlPath); - var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false }; - var depsXml = new XmlDocument(); - depsXml.Load(xmlTextReader); - - s_depsXml = depsXml; - return depsXml; + _builder.WithEveryVersionOfRunner = false; + _builder.WithEveryVersionOfHost = true; + _builder.WithEveryVersionOfAdapter = false; + _builder.WithOlderConfigurations = false; + _builder.WithInProcess = false; + + _builder.BeforeFeature = BeforeFeature; + _builder.AfterFeature = AfterFeature; + _builder.BeforeAdapterFeature = BeforeAdapterFeature; + _builder.AfterAdapterFeature = AfterAdapterFeature; + + _builder.DebugDataCollector = DebugDataCollector; + _builder.DebugVSTestConsole = DebugVSTestConsole; + _builder.DebugTesthost = DebugTesthost; + _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; + + var data = _builder.CreateData(); + data.ForEach(AddData); } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs index 50dfd86bbf..d55cfe0c62 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs @@ -1,43 +1,38 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; using System.Reflection; -using System.Xml; - -using Microsoft.TestPlatform.TestUtilities; - -using Semver; namespace Microsoft.TestPlatform.AcceptanceTests; -public sealed class TranslationLayerCompatibilityDataSource : TestDataSource + +/// +/// A data source that provides every version of runner. +/// +/// When that adds up to no configuration exception is thrown. +/// +public class RunnerCompatibilityDataSource : TestDataSource { - private static XmlDocument? s_depsXml; - private readonly string[] _runnerFrameworks; - private readonly string[] _targetFrameworks; - // private readonly string[] _translationLayerVersions; - private readonly string[] _vstestConsoleVersions; + private readonly CompatibilityRowsBuilder _builder; - /// - /// Initializes a new instance. - /// - /// To run tests with desktop runner(vstest.console.exe), use AcceptanceTestBase.Net452TargetFramework or alike values. - public TranslationLayerCompatibilityDataSource( - string runners = AcceptanceTestBase.DEFAULT_RUNNER_NETFX_AND_NET, - string targetFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, - string vstestConsoleVersions = AcceptanceTestBase.LATEST_TO_LEGACY) + public RunnerCompatibilityDataSource( + string runnerFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string runnerVersions = AcceptanceTestBase.LATEST_TO_LEGACY, + string hostFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET) { // TODO: We actually don't generate values to use different translation layers, because we don't have a good way to do // that right now. Translation layer is loaded directly into the acceptance test, and so we don't have easy way to substitute it. - // I am keeping this source separate from vstest console compatibility data source, to be able to easily add this feature later. - _runnerFrameworks = runners.Split(';'); - _targetFrameworks = targetFrameworks.Split(';'); - _vstestConsoleVersions = vstestConsoleVersions.Split(';'); + + _builder = new CompatibilityRowsBuilder( + runnerFrameworks, + runnerVersions, + hostFrameworks, + // host versions + AcceptanceTestBase.LATEST, + // adapter versions + AcceptanceTestBase.LATESTSTABLE, + // adapters + AcceptanceTestBase.MSTEST); // Do not generate the data rows here, properties (e.g. DebugVSTestConsole) are not populated until after constructor is done. } @@ -47,140 +42,36 @@ public TranslationLayerCompatibilityDataSource( public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; + /// + /// Add run for in-process using the selected .NET Framework runners, and and all selected adapters. + /// + public bool InProcess { get; set; } + public string? BeforeFeature { get; set; } public string? AfterFeature { get; set; } + public string? BeforeAdapterFeature { get; set; } + public string? AfterAdapterFeature { get; set; } public override void CreateData(MethodInfo methodInfo) { - if (BeforeFeature != null && AfterFeature != null) - { - throw new InvalidOperationException($"You cannot specify {nameof(BeforeFeature)} and {nameof(AfterFeature)} at the same time"); - } - - var minVersion = SemVersion.Parse("0.0.0-alpha.1"); - SemVersion? beforeVersion = null; - SemVersion? afterVersion = null; - if (BeforeFeature != null) - { - var feature = Features.Table[BeforeFeature]; - beforeVersion = SemVersion.Parse(feature.Version.TrimStart('v')); - } - if (AfterFeature != null) - { - var feature = Features.Table[AfterFeature]; - afterVersion = SemVersion.Parse(feature.Version.TrimStart('v')); - } - - var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); - // Run .NET Framework tests only on Windows. - Func filter = tfm => isWindows || !tfm.StartsWith("net4"); - - // TODO: maybe we should throw if we don't end up generating any data - // because none of the versions match, or some other way to identify tests that will never run because they are very outdated. - // We probably don't have that need right now, because legacy version is 15.x.x, which is very old, and we are still keeping - // compatibility. - foreach (var runner in _runnerFrameworks.Where(filter)) - { - foreach (var fmw in _targetFrameworks.Where(filter)) - { - foreach (var vstestConsoleVersion in _vstestConsoleVersions) - { - var runnerInfo = new RunnerInfo(runner, fmw, AcceptanceTestBase.InIsolation); - runnerInfo.DebugInfo = new DebugInfo - { - DebugVSTestConsole = DebugVSTestConsole, - DebugTesthost = DebugTesthost, - DebugDataCollector = DebugDataCollector, - NoDefaultBreakpoints = NoDefaultBreakpoints, - }; - var vsTestConsoleInfo = GetVSTestConsoleInfo(vstestConsoleVersion, runnerInfo); - - if (beforeVersion != null && vsTestConsoleInfo.Version > beforeVersion) - continue; - - if (afterVersion != null && vsTestConsoleInfo.Version < afterVersion) - continue; - - AddData(runnerInfo, vsTestConsoleInfo); - } - } - } - } - - public string GetDisplayName(MethodInfo methodInfo, object[] data) - { - return string.Format(CultureInfo.CurrentCulture, "{0} ({1})", methodInfo.Name, string.Join(",", data)); - } - - internal static VSTestConsoleInfo GetVSTestConsoleInfo(string vstestConsoleVersion, RunnerInfo runnerInfo) - { - var depsXml = GetDependenciesXml(); - - // When version is Latest, we built it locally, but it gets restored into our nuget cache on build - // same as other versions, we just need to grab the version from a different property. - - var propertyName = vstestConsoleVersion == AcceptanceTestBase.LATEST - ? $"NETTestSdkVersion" - : $"VSTestConsole{vstestConsoleVersion}Version"; - - var packageName = runnerInfo.IsNetFrameworkRunner - ? "microsoft.testplatform" - : "microsoft.testplatform.cli"; - - // It is okay when node is null, we will fail to find the executable later, and throw. - // This way it throws in the body of the test which has better error reporting than throwing in the data source. - // And we can easily find out what is going on because --WRONG-VERSION-- sticks out, and is easy to find in the codebase. - XmlNode? node = depsXml.DocumentElement?.SelectSingleNode($"PropertyGroup/{propertyName}"); - var version = node?.InnerText.Replace("[", "").Replace("]", "") ?? "--WRONG-VERSION--"; - var vstestConsolePath = runnerInfo.IsNetFrameworkRunner - ? Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "packages", packageName, version, - "tools", "net451", "Common7", "IDE", "Extensions", "TestPlatform", "vstest.console.exe") - : Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "packages", packageName, version, - "contentFiles", "any", "netcoreapp2.1", "vstest.console.dll"); - - if (version.StartsWith("15.")) - { - vstestConsolePath = vstestConsolePath.Replace("netcoreapp2.1", "netcoreapp2.0"); - } - - return new VSTestConsoleInfo(vstestConsoleVersion, version, vstestConsolePath); - } - - private static XmlDocument GetDependenciesXml() - { - if (s_depsXml != null) - return s_depsXml; - - var depsXmlPath = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "scripts", "build", "TestPlatform.Dependencies.props"); - var fileStream = File.OpenRead(depsXmlPath); - var xmlTextReader = new XmlTextReader(fileStream) { Namespaces = false }; - var depsXml = new XmlDocument(); - depsXml.Load(xmlTextReader); - - s_depsXml = depsXml; - return depsXml; + _builder.WithEveryVersionOfRunner = true; + _builder.WithEveryVersionOfHost = false; + _builder.WithEveryVersionOfAdapter = false; + _builder.WithOlderConfigurations = false; + _builder.WithInProcess = InProcess; + + _builder.BeforeFeature = BeforeFeature; + _builder.AfterFeature = AfterFeature; + _builder.BeforeAdapterFeature = BeforeAdapterFeature; + _builder.AfterAdapterFeature = AfterAdapterFeature; + + _builder.DebugDataCollector = DebugDataCollector; + _builder.DebugVSTestConsole = DebugVSTestConsole; + _builder.DebugTesthost = DebugTesthost; + _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; + + var data = _builder.CreateData(); + data.ForEach(AddData); } } -public class Feature -{ - public Feature (string version, string issue) - { - Version = version; - Issue = issue; - } - - public string Version { get; } - public string Issue { get; } -} - - -public static class Features -{ - public const string ATTACH_DEBUGGER = nameof(ATTACH_DEBUGGER); - - public static Dictionary Table { get; } = new Dictionary - { - [ATTACH_DEBUGGER] = new(version: "v16.7.0-preview-20200519-01", issue: "https://github.com/microsoft/vstest/pull/2325") - }; -} diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs index e6c341ca4f..996adfec65 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs @@ -8,7 +8,6 @@ using FluentAssertions; -using Microsoft.TestPlatform.TestUtilities; using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.Interfaces; @@ -34,7 +33,7 @@ public void Cleanup() } [TestMethod] - [TranslationLayerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] + [RunnerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncher(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -55,7 +54,7 @@ public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvided } [TestMethod] - [TranslationLayerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] + [RunnerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncherWhenITestHostLauncher2IsProvided(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -77,7 +76,7 @@ public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvided } [TestMethod] - [TranslationLayerCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER)] + [RunnerCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER)] public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvidedLauncher(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -99,10 +98,10 @@ public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvided } [TestMethod] - [Ignore("This is not working. The compatibility code only checks the protocol version (in handler), which is dictated by testhost. " - + "It sees 6 but does not realize that the provided CustomTesthostLauncher is not supporting the new feature, it ends up calling back to EditoAttachDebugger" + - "in translation layer, and that just silently skips the call.")] - [TranslationLayerCompatibilityDataSource("net451", "net451", "Latest", AfterFeature = Features.ATTACH_DEBUGGER, DebugVSTestConsole = true, DebugTesthost=true)] + //[Ignore("This is not working. The compatibility code only checks the protocol version (in handler), which is dictated by testhost. " + // + "It sees 6 but does not realize that the provided CustomTesthostLauncher is not supporting the new feature, it ends up calling back to EditoAttachDebugger" + + // "in translation layer, and that just silently skips the call.")] + [RunnerCompatibilityDataSource("net451", "net451", "Latest", AfterFeature = Features.ATTACH_DEBUGGER, DebugVSTestConsole = true, DebugTesthost=true)] public void RunTestsWithCustomTestHostLauncherUsesLaunchWhenGivenAnOutdatedITestHostLauncher(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index c0a278cf14..38ec639a3f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -40,7 +40,7 @@ public void Cleanup() } [TestMethod] - [TranslationLayerCompatibilityDataSource] + [RunnerCompatibilityDataSource] public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -57,7 +57,7 @@ public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo) } [TestMethod] - [TranslationLayerCompatibilityDataSource()] + [RunnerCompatibilityDataSource()] public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedOut(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs index ed121eaa0e..ffe2449e15 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs @@ -40,7 +40,7 @@ public void Cleanup() } [TestMethod] - [TranslationLayerCompatibilityDataSource] + [RunnerCompatibilityDataSource] public void RunAllTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs index 9661f18242..ea4c38ae0f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using System.Linq; -using Microsoft.TestPlatform.TestUtilities; using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client; @@ -36,7 +35,7 @@ public void Cleanup() } [TestMethod] - [TranslationLayerCompatibilityDataSource] + [RunnerCompatibilityDataSource] public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 2c80715014..dbafe748fb 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -11,6 +11,8 @@ using System.Text.RegularExpressions; using System.Xml; +using FluentAssertions; + using Microsoft.TestPlatform.VsTestConsole.TranslationLayer; using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Extensions; @@ -960,5 +962,15 @@ protected static string GetDownloadedDotnetMuxerFromTools(string architecture) return path; } - protected string GetDotnetRunnerPath() => _testEnvironment?.VSTestConsoleInfo.Path ?? Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll"); + protected string GetDotnetRunnerPath() => _testEnvironment.VSTestConsoleInfo?.Path ?? Path.Combine(_testEnvironment.PublishDirectory, "vstest.console.dll"); + + protected void StdOutHasNoWarnings() + { + StdOut.Should().NotContainEquivalentOf("warning"); + } + + protected void StdErrHasTestRunFailedMessageButNoOtherError() + { + StdErr?.Trim().Should().Be("Test Run Failed."); + } } diff --git a/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj b/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj index 7ed26529d4..baea232289 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj +++ b/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj @@ -19,6 +19,7 @@ + From 0f087c0968ccbd79117ee5c032040bd63b4ae64d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Thu, 17 Mar 2022 16:08:33 +0100 Subject: [PATCH 38/64] Tests tests. --- TestPlatform.sln | 22 ++-- scripts/build/TestPlatform.Dependencies.props | 2 +- .../DesignMode/DesignModeClient.cs | 2 +- .../AcceptanceTestBase.cs | 40 ++++-- .../AppDomainTests.cs | 2 +- .../ArgumentProcessorTests.cs | 2 +- .../BlameDataCollectorTests.cs | 14 +-- .../CodeCoverageTests.cs | 2 +- .../DataCollectionTests.cs | 4 +- .../DeprecateExtensionsPathWarningTests.cs | 2 +- .../DifferentTestFrameworkSimpleTests.cs | 2 +- .../DiscoveryTests.cs | 6 +- .../DotnetArchitectureSwitchTests.cs | 8 +- .../ExecutionTests.cs | 7 +- .../Extension/CompatibilityRowsBuilder.cs | 117 ++++++++++++------ .../Extension/Features.cs | 4 +- .../MSTestCompatibilityDataSource.cs | 19 ++- .../Extension/NetCoreRunner.cs | 4 +- .../NetCoreTargetFrameworkDataSource.cs | 4 +- .../Extension/NetFrameworkRunner.cs | 4 +- .../NetFullTargetFrameworkDataSource.cs | 8 +- ...ce.cs => RunnerCompatibilityDataSource.cs} | 16 ++- .../Extension/RunnnerInfo.cs | 24 ++-- .../TestPlatformCompatibilityDataSource.cs | 19 ++- .../TesthostCompatibilityDataSource.cs | 16 +-- .../FilePatternParserTests.cs | 6 +- .../FrameworkTests.cs | 8 +- .../LoggerTests.cs | 24 ++-- .../PlatformTests.cs | 2 +- .../PortableNugetPackageTests.cs | 2 +- .../ProcessesInteractionTests.cs | 2 +- .../ResultsDirectoryTests.cs | 4 +- .../RunsettingsTests.cs | 16 +-- .../TelemetryTests.cs | 4 +- .../TestCaseFilterTests.cs | 14 +-- .../CodeCoverageTests.cs | 2 +- .../CustomTestHostTests.cs | 105 +++++++++------- .../DataCollectorAttachmentProcessor.cs | 2 +- .../DifferentTestFrameworkSimpleTests.cs | 8 +- .../TranslationLayerTests/DiscoverTests.cs | 22 ++-- .../LiveUnitTestingTests.cs | 4 +- .../TranslationLayerTests/RunSelectedTests.cs | 12 +- .../TranslationLayerTests/RunTests.cs | 20 +-- ...RunTestsWithDifferentConfigurationTests.cs | 12 +- .../RunTestsWithFilterTests.cs | 8 +- .../DiaSessionTests.cs | 10 +- .../PerformanceTests.cs | 10 +- .../DiscoveryTests.cs | 2 +- .../ExecutionTests.cs | 4 +- .../DebugInfo.cs | 7 +- .../DllInfo.cs | 27 ++-- .../IntegrationTestBase.cs | 39 ++---- .../IntegrationTestEnvironment.cs | 12 +- .../MSTestInfo.cs | 15 --- .../{TesthostInfo.cs => NetTestSdkInfo.cs} | 5 +- .../VSTestConsoleInfo.cs | 15 +-- 56 files changed, 417 insertions(+), 356 deletions(-) rename test/Microsoft.TestPlatform.AcceptanceTests/Extension/{TranslationLayerCompatibilityDataSource.cs => RunnerCompatibilityDataSource.cs} (86%) delete mode 100644 test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs rename test/Microsoft.TestPlatform.TestUtilities/{TesthostInfo.cs => NetTestSdkInfo.cs} (64%) diff --git a/TestPlatform.sln b/TestPlatform.sln index 05b0609b3c..f4b952a31f 100644 --- a/TestPlatform.sln +++ b/TestPlatform.sln @@ -181,21 +181,13 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AttachmentProcessorDataColl EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "vstest.ProgrammerTests", "test\vstest.ProgrammerTests\vstest.ProgrammerTests.csproj", "{B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Intent", "test\Intent\Intent.csproj", "{BFBB35C9-6437-480A-8DCC-AE3700110E7D}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Intent", "test\Intent\Intent.csproj", "{BFBB35C9-6437-480A-8DCC-AE3700110E7D}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Intent.Primitives", "test\Intent.Primitives\Intent.Primitives.csproj", "{29270853-90DC-4C39-9621-F47AE40A79B6}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Intent.Primitives", "test\Intent.Primitives\Intent.Primitives.csproj", "{29270853-90DC-4C39-9621-F47AE40A79B6}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "testhost.arm64", "src\testhost.arm64\testhost.arm64.csproj", "{186069FE-E1E8-4DE1-BEA4-0FF1484D22D1}" EndProject Global - GlobalSection(SharedMSBuildProjectFiles) = preSolution - src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{10b6ade1-f808-4612-801d-4452f5b52242}*SharedItemsImports = 5 - src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{186069fe-e1e8-4de1-bea4-0ff1484d22d1}*SharedItemsImports = 5 - src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{27dfbd04-64b2-4f1b-82b2-006620cca6f8}*SharedItemsImports = 5 - src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{2c7ce1f8-e73e-4987-8023-b5a0ebac86e8}*SharedItemsImports = 5 - src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{71cb42ff-e750-4a3b-9c3a-ac938853cc89}*SharedItemsImports = 5 - src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{7f26eda3-c8c4-4b7f-a9b6-d278c2f40a13}*SharedItemsImports = 13 - EndGlobalSection GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU Debug|x64 = Debug|x64 @@ -955,7 +947,7 @@ Global {61F7F446-9EF3-4768-B33A-4D75F60E1059} = {7D4082EA-7AC9-4DFB-98E8-C5E08BDC0EC3} {D5296435-3A3F-4B1A-81D1-434EC9E97DEF} = {5E7F18A8-F843-4C8A-AB02-4C7D9205C6CF} {790B8030-00C2-4121-B125-EDC4CE329BA3} = {ED0C35EB-7F31-4841-A24F-8EB708FFA959} - {27DFBD04-64B2-4F1B-82B2-006620CCA6F8} = {ED0C35EB-7F31-4841-A24F-8EB708FFA959} + {27DFBD04-64B2-4F1B-82B2-006620CCA6F8} = {D9A30E32-D466-4EC5-B4F2-62E17562279B} {71CB42FF-E750-4A3B-9C3A-AC938853CC89} = {ED0C35EB-7F31-4841-A24F-8EB708FFA959} {10B6ADE1-F808-4612-801D-4452F5B52242} = {ED0C35EB-7F31-4841-A24F-8EB708FFA959} {46250E12-4CF1-4051-B4A7-80C8C06E0068} = {B27FAFDF-2DBA-4AB0-BA85-FD5F21D359D6} @@ -1020,4 +1012,12 @@ Global GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {0541B30C-FF51-4E28-B172-83F5F3934BCD} EndGlobalSection + GlobalSection(SharedMSBuildProjectFiles) = preSolution + src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{10b6ade1-f808-4612-801d-4452f5b52242}*SharedItemsImports = 5 + src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{186069fe-e1e8-4de1-bea4-0ff1484d22d1}*SharedItemsImports = 5 + src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{27dfbd04-64b2-4f1b-82b2-006620cca6f8}*SharedItemsImports = 5 + src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{2c7ce1f8-e73e-4987-8023-b5a0ebac86e8}*SharedItemsImports = 5 + src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{71cb42ff-e750-4a3b-9c3a-ac938853cc89}*SharedItemsImports = 5 + src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{7f26eda3-c8c4-4b7f-a9b6-d278c2f40a13}*SharedItemsImports = 13 + EndGlobalSection EndGlobal diff --git a/scripts/build/TestPlatform.Dependencies.props b/scripts/build/TestPlatform.Dependencies.props index 03fd5958bd..d57a941d3d 100644 --- a/scripts/build/TestPlatform.Dependencies.props +++ b/scripts/build/TestPlatform.Dependencies.props @@ -49,7 +49,7 @@ [17.2.0-preview-20220131-20] [17.1.0] [17.0.0] - [16.9.4] + [16.6.1] [16.11.0] [15.9.2] diff --git a/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs b/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs index 9bbbca6b84..fc251b1c00 100644 --- a/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs +++ b/src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs @@ -357,7 +357,7 @@ public bool AttachDebuggerToProcess(int pid, CancellationToken cancellationToken var ackPayload = _dataSerializer.DeserializePayload(ackMessage); if (!ackPayload.Attached) { - EqtTrace.Warning(ackPayload.ErrorMessage); + EqtTrace.Warning($"DesignModeClient.AttachDebuggerToProcess: Attaching to process failed: {ackPayload.ErrorMessage}"); } return ackPayload.Attached; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs index 574e742f2e..bd7fe5afcd 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +using System.Linq; using Microsoft.TestPlatform.TestUtilities; @@ -73,7 +74,8 @@ public static string And(string left, string right) protected static void SetTestEnvironment(IntegrationTestEnvironment testEnvironment, RunnerInfo runnerInfo) { testEnvironment.VSTestConsoleInfo = runnerInfo.VSTestConsoleInfo; - testEnvironment.DllInfos = runnerInfo.DllInfos; + // The order here matters, it changes how the resulting path is built when we resolve test dlls and other assets. + testEnvironment.DllInfos = new[] { runnerInfo.AdapterInfo, runnerInfo.TestHostInfo }.Where(d => d != null).ToList(); testEnvironment.DebugInfo = runnerInfo.DebugInfo; testEnvironment.RunnerFramework = runnerInfo.RunnerFramework; @@ -111,17 +113,37 @@ protected string GetTargetFramworkForRunsettings() } /// - /// Default RunSettings + /// Empty runsettings, just with the RunSettings tag that we require. /// /// - public string GetDefaultRunSettings() + public string GetEmptyRunsettings() { - string runSettingsXml = $@" - - - {FrameworkArgValue} - - "; + return ""; + } + + /// + /// Almost empty runsettings, just specifying the target framework from the currently set test environment. + /// + public string GetRunSettingsWithCurrentTargetFramework() + { + return GetRunSettingsWithTargetFramework(FrameworkArgValue); + } + + /// + /// Almost empty runsettings, just specifying the given target framework. + /// Use the overload without any parameters to get the target framework from the currently set test environment. + /// + /// + public string GetRunSettingsWithTargetFramework(string targetFramework) + { + string runSettingsXml = + $@" + + + {targetFramework} + + "; + return runSettingsXml; } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs index 17e8f88423..a413e8d7d6 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs @@ -38,7 +38,7 @@ public void RunTestExecutionWithDisableAppDomain(RunnerInfo runnerInfo) var runsettingsFilePath = GetInProcDataCollectionRunsettingsFile(true, TempDirectory); var arguments = PrepareArguments( - GetSampleTestAssembly(), + GetSampleTestDll(), GetTestAdapterPath(), runsettingsFilePath, FrameworkArgValue, diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs index 44a4a39dbd..8b9dd95d40 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs @@ -40,7 +40,7 @@ public void PassingInvalidArgumentsToVsTestConsoleShouldNotPrintHelpMessage(Runn { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /badArgument"); InvokeVsTest(arguments); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs index e9e6af25ee..8df3e8bc6a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs @@ -46,7 +46,7 @@ public BlameDataCollectorTests() public void BlameDataCollectorShouldGiveCorrectTestCaseName(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetAssetFullPath("BlameUnitTestProject.dll"); + var assemblyPaths = GetTestDll("BlameUnitTestProject.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $" /Blame"); arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); @@ -137,7 +137,7 @@ public void BlameDataCollectorShouldOutputDumpFileWhenNoCrashOccursButCollectAlw public void HangDumpOnTimeout(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetAssetFullPath("timeout.dll"); + var assemblyPaths = GetTestDll("timeout.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectHangDump;HangDumpType=full;TestTimeout=3s"""); @@ -160,7 +160,7 @@ public void HangDumpOnTimeout(RunnerInfo runnerInfo) public void CrashDumpWhenThereIsNoTimeout(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetAssetFullPath("timeout.dll"); + var assemblyPaths = GetTestDll("timeout.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full;CollectAlways=true;CollectHangDump"""); @@ -183,7 +183,7 @@ public void CrashDumpWhenThereIsNoTimeout(RunnerInfo runnerInfo) public void CrashDumpOnExit(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetAssetFullPath("timeout.dll"); + var assemblyPaths = GetTestDll("timeout.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full;CollectAlways=true"""); @@ -204,7 +204,7 @@ public void CrashDumpOnExit(RunnerInfo runnerInfo) public void CrashDumpOnStackOverflow(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetAssetFullPath("crash.dll"); + var assemblyPaths = GetTestDll("crash.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full"""); @@ -225,7 +225,7 @@ public void CrashDumpOnStackOverflow(RunnerInfo runnerInfo) public void CrashDumpChildProcesses(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetAssetFullPath("child-crash.dll"); + var assemblyPaths = GetTestDll("child-crash.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full"""); InvokeVsTest(arguments); @@ -240,7 +240,7 @@ public void CrashDumpChildProcesses(RunnerInfo runnerInfo) public void HangDumpChildProcesses(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetAssetFullPath("child-hang.dll"); + var assemblyPaths = GetTestDll("child-hang.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectHangDump;HangDumpType=full;TestTimeout=15s"""); InvokeVsTest(arguments); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs index b426fc7fd1..c7901f2326 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs @@ -251,7 +251,7 @@ private string CreateArguments( TestParameters testParameters, out string trxFilePath) { - var assemblyPaths = GetAssetFullPath(testParameters.AssemblyName); + var assemblyPaths = GetTestDll(testParameters.AssemblyName); string traceDataCollectorDir = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "Microsoft.CodeCoverage"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs index 2486a605ff..1308f4345a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs @@ -87,7 +87,7 @@ public void DataCollectorAssemblyLoadingShouldNotThrowErrorForNetCore(RunnerInfo { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetAssetFullPath("AppDomainGetAssembliesTestProject.dll", "netcoreapp2.1"), string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetTestDllForFramework("AppDomainGetAssembliesTestProject.dll", "netcoreapp2.1"), string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); @@ -100,7 +100,7 @@ public void DataCollectorAssemblyLoadingShouldNotThrowErrorForFullFramework(Runn { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetAssetFullPath("AppDomainGetAssembliesTestProject.dll"), string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetTestDll("AppDomainGetAssembliesTestProject.dll"), string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs index 4c10549775..d33857347a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs @@ -61,7 +61,7 @@ public void CopyAdapterToExtensions() [TestMethod] public void VerifyDeprecatedWarningIsThrownWhenAdaptersPickedFromExtensionDirectory() { - var arguments = PrepareArguments(GetSampleTestAssembly(), null, null, FrameworkArgValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), null, null, FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); StdOutputContains("Adapter lookup is being changed, please follow"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs index faaeb1fd7e..8f112a83e5 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs @@ -153,7 +153,7 @@ public void NUnitRunAllTestExecution(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetAssetFullPath("NUTestProject.dll"), + GetTestDll("NUTestProject.dll"), GetTestAdapterPath(UnitTestFramework.NUnit), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs index fa9bb8627b..444cc873ec 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs @@ -25,7 +25,7 @@ public void DiscoverAllTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - InvokeVsTestForDiscovery(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue); + InvokeVsTestForDiscovery(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue); var listOfTests = new[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", "SampleUnitTestProject.UnitTest1.SkippingTest" }; ValidateDiscoveredTests(listOfTests); @@ -65,7 +65,7 @@ public void DiscoverFullyQualifiedTests(RunnerInfo runnerInfo) var listOfTests = new[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", "SampleUnitTestProject.UnitTest1.SkippingTest" }; - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /ListFullyQualifiedTests", " /ListTestsTargetPath:\"" + dummyFilePath + "\""); InvokeVsTest(arguments); @@ -80,7 +80,7 @@ public void DiscoverTestsShouldShowProperWarningIfNoTestsOnTestCaseFilter(Runner { SetTestEnvironment(_testEnvironment, runnerInfo); - var assetFullPath = GetAssetFullPath("SimpleTestProject2.dll"); + var assetFullPath = GetTestDll("SimpleTestProject2.dll"); var arguments = PrepareArguments(assetFullPath, GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /listtests"); arguments = string.Concat(arguments, " /testcasefilter:NonExistTestCaseName"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs index e60f46ed94..5ff96f6cd2 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.cs @@ -78,7 +78,7 @@ public void GlobalInstallation() AssertSwitch(stdOut); // Verify switch using test container - var buildAssemblyPath = GetAssetFullPath("ArchitectureSwitch.dll", "net6.0"); + var buildAssemblyPath = GetTestDllForFramework("ArchitectureSwitch.dll", "net6.0"); ExecuteApplication(GetDefaultDotnetMuxerLocation, $"test {buildAssemblyPath} --arch x64", out stdOut, out _, out _, env, projectDirectory); AssertSwitch(stdOut); @@ -148,7 +148,7 @@ public void DOTNET_ROOTS_EnvironmentVariables(bool dotnetRoot, bool dotnetRootX6 AssertSwitch(stdOut); // Verify switch using test container - var buildAssemblyPath = GetAssetFullPath("ArchitectureSwitch.dll", "net6.0"); + var buildAssemblyPath = GetTestDllForFramework("ArchitectureSwitch.dll", "net6.0"); ExecuteApplication($"{s_privateX64Installation}/{GetMuxerName}", $"test {buildAssemblyPath} --framework net6.0 --arch x64", out stdOut, out _, out _, env, projectDirectory); AssertSwitch(stdOut); @@ -192,7 +192,7 @@ public void PrivateX64BuildToGlobalArmInstallation() AssertSwitch(stdOut); // Verify switch using test container - var buildAssemblyPath = GetAssetFullPath("ArchitectureSwitch.dll", "net6.0"); + var buildAssemblyPath = GetTestDllForFramework("ArchitectureSwitch.dll", "net6.0"); ExecuteApplication($"{s_privateX64Installation}/{GetMuxerName}", $"test {buildAssemblyPath} --framework net6.0 --arch arm64", out stdOut, out _, out _, env, projectDirectory); AssertSwitch(stdOut); @@ -249,7 +249,7 @@ public void PrivateX64BuildToDOTNET_ROOTS_EnvironmentVariables(bool dotnetRoot, AssertSwitch(stdOut); // Verify switch using test container - var buildAssemblyPath = GetAssetFullPath("ArchitectureSwitch.dll", "net6.0"); + var buildAssemblyPath = GetTestDllForFramework("ArchitectureSwitch.dll", "net6.0"); ExecuteApplication($"{s_privateX64Installation}/{GetMuxerName}", $"test {buildAssemblyPath} --framework net6.0 --arch arm64", out stdOut, out _, out _, env, projectDirectory); AssertSwitch(stdOut); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index b39e6b76a1..60b60036ed 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -34,7 +34,8 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) } [TestMethod] - [TestPlatformCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER, AfterAdapterFeature = Features.MSTEST_IFRAMEWORK_HANDLE_99)] + [TestPlatformCompatibilityDataSource()] + //[TestPlatformCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER, AfterAdapterFeature = Features.MSTEST_IFRAMEWORK_HANDLE_99)] public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo) { @@ -51,7 +52,7 @@ public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo) } [TestMethod] - [HostCompatibilityDataSource] + [TestHostCompatibilityDataSource] public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -107,7 +108,7 @@ public void RunMultipleTestAssembliesWithoutTestAdapterPath(RunnerInfo runnerInf [TestCategory("Windows")] [TestMethod] [MSTestCompatibilityDataSource] - public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo, MSTestInfo msTestInfo) + public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); using var tempDir = new TempDirectory(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs index f4fa979dbd..dccbd03525 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs @@ -47,13 +47,15 @@ public CompatibilityRowsBuilder(string runnerFrameworks = AcceptanceTestBase.DEF public bool WithEveryVersionOfAdapter { get; set; } = true; public bool WithOlderConfigurations { get; set; } = true; - public string? BeforeFeature { get; set; } - public string? AfterFeature { get; set; } + public string? BeforeRunnerFeature { get; set; } + public string? AfterRunnerFeature { get; set; } + public string? BeforeTestHostFeature { get; set; } + public string? AfterTestHostFeature { get; set; } public string? BeforeAdapterFeature { get; set; } public string? AfterAdapterFeature { get; set; } public bool DebugVSTestConsole { get; set; } - public bool DebugTesthost { get; set; } + public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; @@ -79,26 +81,40 @@ public List CreateData() var minVersion = SemVersion.Parse("0.0.0-alpha.1"); var maxVersion = SemVersion.Parse("9999.0.0"); - SemVersion? beforeVersion = maxVersion; - SemVersion? afterVersion = minVersion; + SemVersion? beforeRunnerVersion = maxVersion; + SemVersion? afterRunnerVersion = minVersion; + SemVersion? beforeTestHostVersion = maxVersion; + SemVersion? afterTestHostVersion = minVersion; SemVersion? beforeAdapterVersion = maxVersion; SemVersion? afterAdapterVersion = minVersion; - if (BeforeFeature != null) + if (BeforeRunnerFeature != null) { - var feature = Features.TestPlatformFeatures[BeforeFeature]; - beforeVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + var feature = Features.TestPlatformFeatures[BeforeRunnerFeature]; + beforeRunnerVersion = SemVersion.Parse(feature.Version.TrimStart('v')); } - if (AfterFeature != null) + if (AfterRunnerFeature != null) { - var feature = Features.TestPlatformFeatures[AfterFeature]; - afterVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + var feature = Features.TestPlatformFeatures[AfterRunnerFeature]; + afterRunnerVersion = SemVersion.Parse(feature.Version.TrimStart('v')); } - if (BeforeFeature != null) + if (BeforeTestHostFeature != null) { - var feature = Features.TestPlatformFeatures[BeforeFeature]; + var feature = Features.TestPlatformFeatures[BeforeTestHostFeature]; + beforeTestHostVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + } + + if (AfterTestHostFeature != null) + { + var feature = Features.TestPlatformFeatures[AfterTestHostFeature]; + afterTestHostVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + } + + if (BeforeAdapterFeature != null) + { + var feature = Features.TestPlatformFeatures[BeforeAdapterFeature]; beforeAdapterVersion = SemVersion.Parse(feature.Version.TrimStart('v')); } @@ -120,16 +136,21 @@ public List CreateData() Func isInRange = (version, before, after) => version < before && after < version; var rows = dataRows.Where(r => r.VSTestConsoleInfo != null - && isInRange(r.VSTestConsoleInfo.Version, beforeVersion, afterVersion) - && r.DllInfos.All(d => d is NetTestSdkInfo ? isInRange(d.Version, beforeVersion, afterVersion) : isInRange(d.Version, beforeAdapterVersion, afterAdapterVersion))).ToList(); + && isInRange(r.VSTestConsoleInfo.Version, beforeRunnerVersion, afterRunnerVersion) + && r.TestHostInfo != null && isInRange(r.TestHostInfo.Version, beforeTestHostVersion, afterTestHostVersion) + && r.AdapterInfo != null && isInRange(r.AdapterInfo.Version, beforeAdapterVersion, afterAdapterVersion)).ToList(); - if (rows.Count == 0) + // We use ToString to determine which values are unique. Not great solution, but works better than using records. + var distinctRows = new Dictionary(); + rows.ForEach(r => distinctRows[r.ToString()] = r); + + if (distinctRows.Count == 0) { - // TODO: This needs to be way more specific about what happened. + // TODO: This needs to be way more specific about what happened. And possibly propagate as inconclusive state if we decide to update versions automatically? throw new InvalidOperationException("There were no rows that matched the specified criteria."); } - return rows; + return distinctRows.Values.ToList(); } private void AddInProcess(List dataRows) @@ -147,7 +168,7 @@ private void AddInProcess(List dataRows) { foreach (var adapterVersion in _adapterVersions) { - AddRow(dataRows, runnerVersion, runnerFramework, runnerVersion, runnerFramework, adapter, adapterVersion, inIsolation: false); + AddRow(dataRows, "In process", runnerVersion, runnerFramework, runnerVersion, runnerFramework, adapter, adapterVersion, inIsolation: false); } } } @@ -158,7 +179,7 @@ private void AddOlderConfigurations(List dataRows) { // Older configurations where the runner, host and adapter version are the same. // We already added the row where all are newest when adding combination with all runners. - foreach (var runnerVersion in _runnerVersions.Skip(1)) + foreach (var runnerVersion in _runnerVersions) { foreach (var runnerFramework in _runnerFrameworks) { @@ -168,8 +189,8 @@ private void AddOlderConfigurations(List dataRows) var hostVersion = runnerVersion; foreach (var adapter in _adapters) { - var adapterVersion = runnerVersion; - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); + var adapterVersion = _adapterVersions[0]; + AddRow(dataRows, "Older", runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); } } } @@ -190,10 +211,9 @@ private void AddEveryVersionOfAdapter(List dataRows) var hostVersion = isNetFramework ? runnerVersion : _hostVersions[0]; foreach (var adapter in _adapters) { - // We already used the newest when adding combination with every runner - foreach (var adapterVersion in _adapterVersions.Skip(1)) + foreach (var adapterVersion in _adapterVersions) { - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); + AddRow(dataRows, "Every adapter", runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); } } } @@ -211,16 +231,15 @@ private void AddEveryVersionOfHost(List dataRows) var isNetFramework = hostFramework.StartsWith("net4"); // .NET Framework testhost ships with the runner, and the version from the // runner directory is always the same as the runner. There are no variations - // so we just need to add host versions for .NET testhosts. We also skip the - // newest version because we already added it when AddEveryVersionOfRunner - var hostVersions = isNetFramework ? Array.Empty() : _hostVersions.Skip(1).ToArray(); + // so we just need to add host versions for .NET testhosts. + var hostVersions = isNetFramework ? Array.Empty() : _hostVersions.ToArray(); foreach (var hostVersion in hostVersions) { foreach (var adapter in _adapters) { // use the newest var adapterVersion = _adapterVersions[0]; - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); + AddRow(dataRows, "Every host", runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); } } } @@ -243,17 +262,17 @@ private void AddEveryVersionOfRunner(List dataRows) { // use the newest var adapterVersion = _adapterVersions[0]; - AddRow(dataRows, runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); + AddRow(dataRows, "Every runner", runnerVersion, runnerFramework, hostVersion, hostFramework, adapter, adapterVersion, inIsolation: true); } } } } } - private void AddRow(List dataRows, + private void AddRow(List dataRows, string batch, string runnerVersion, string runnerFramework, string hostVersion, string hostFramework, string adapter, string adapterVersion, bool inIsolation) { - RunnerInfo runnerInfo = GetRunnerInfo(runnerFramework, hostFramework, inIsolation); + RunnerInfo runnerInfo = GetRunnerInfo(batch, runnerFramework, hostFramework, inIsolation); runnerInfo.DebugInfo = GetDebugInfo(); runnerInfo.VSTestConsoleInfo = GetVSTestConsoleInfo(runnerVersion, runnerInfo); @@ -263,8 +282,8 @@ private void AddRow(List dataRows, // C:\p\vstest\test\TestAssets\MSTestProject1\bin\MSTestLatestPreview-2.2.9-preview-20220210-07\NETTestSdkLatest-17.2.0-dev\Debug\net451\MSTestProject1.dll // versus adding testSdk second: // C:\p\vstest\test\TestAssets\MSTestProject1\bin\NETTestSdkLatest-17.2.0-dev\MSTestLatestPreview-2.2.9-preview-20220210-07\Debug\net451\MSTestProject1.dll - runnerInfo.DllInfos.Add(GetMSTestInfo(adapterVersion)); - runnerInfo.DllInfos.Add(GetNetTestSdkInfo(hostVersion)); + runnerInfo.TestHostInfo = GetNetTestSdkInfo(hostVersion); + runnerInfo.AdapterInfo = GetMSTestInfo(adapterVersion); dataRows.Add(runnerInfo); } @@ -273,24 +292,25 @@ private DebugInfo GetDebugInfo() return new DebugInfo { DebugDataCollector = DebugDataCollector, - DebugTesthost = DebugTesthost, + DebugTestHost = DebugTestHost, DebugVSTestConsole = DebugVSTestConsole, NoDefaultBreakpoints = NoDefaultBreakpoints }; } - private RunnerInfo GetRunnerInfo(string runnerFramework, string hostFramework, bool inIsolation) + private RunnerInfo GetRunnerInfo(string batch, string runnerFramework, string hostFramework, bool inIsolation) { return new RunnerInfo { + Batch = batch, RunnerFramework = runnerFramework, TargetFramework = hostFramework, InIsolationValue = inIsolation ? AcceptanceTestBase.InIsolation : null }; } - private MSTestInfo GetMSTestInfo(string msTestVersion) + private DllInfo GetMSTestInfo(string msTestVersion) { var depsXml = GetDependenciesXml(); @@ -301,7 +321,14 @@ private MSTestInfo GetMSTestInfo(string msTestVersion) var slash = Path.DirectorySeparatorChar; var versionSpecificBinPath = $"{slash}bin{slash}MSTest{msTestVersion}-{version}{slash}"; - return new MSTestInfo(msTestVersion, version, versionSpecificBinPath); + return new DllInfo + { + Name = "MSTest", + PropertyName = "MSTest", + VersionType = msTestVersion, + Version = version, + Path = versionSpecificBinPath, + }; } private static VSTestConsoleInfo GetVSTestConsoleInfo(string vstestConsoleVersion, RunnerInfo runnerInfo) @@ -335,7 +362,12 @@ private static VSTestConsoleInfo GetVSTestConsoleInfo(string vstestConsoleVersio vstestConsolePath = vstestConsolePath.Replace("netcoreapp2.1", "netcoreapp2.0"); } - return new VSTestConsoleInfo(vstestConsoleVersion, version, vstestConsolePath); + return new VSTestConsoleInfo + { + VersionType = vstestConsoleVersion, + Version = version, + Path = vstestConsolePath, + }; } private static NetTestSdkInfo GetNetTestSdkInfo(string testhostVersionType) @@ -358,7 +390,12 @@ private static NetTestSdkInfo GetNetTestSdkInfo(string testhostVersionType) var slash = Path.DirectorySeparatorChar; var versionSpecificBinPath = $"{slash}bin{slash}NETTestSdk{testhostVersionType}-{version}{slash}"; - return new NetTestSdkInfo(testhostVersionType, version, versionSpecificBinPath); + return new NetTestSdkInfo + { + VersionType = testhostVersionType, + Version = version, + Path = versionSpecificBinPath + }; } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs index e82389e404..b966b9daa3 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs @@ -7,13 +7,13 @@ namespace Microsoft.TestPlatform.AcceptanceTests; public static class Features { - public const string ATTACH_DEBUGGER = nameof(ATTACH_DEBUGGER); + public const string ATTACH_DEBUGGER_FLOW = nameof(ATTACH_DEBUGGER_FLOW); public const string MSTEST_IFRAMEWORK_HANDLE_99 = nameof(MSTEST_IFRAMEWORK_HANDLE_99); public static Dictionary TestPlatformFeatures { get; } = new Dictionary { - [ATTACH_DEBUGGER] = new(version: "v16.7.0-preview-20200519-01", issue: "https://github.com/microsoft/vstest/pull/2325"), + [ATTACH_DEBUGGER_FLOW] = new(version: "v16.7.0-preview-20200519-01", issue: "https://github.com/microsoft/vstest/pull/2325"), }; public static Dictionary AdapterFeatures { get; internal set; } = new Dictionary diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs index 30875f6886..33c53f8a5d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs @@ -34,7 +34,7 @@ public MSTestCompatibilityDataSource( } public bool DebugVSTestConsole { get; set; } - public bool DebugTesthost { get; set; } + public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; @@ -43,8 +43,11 @@ public MSTestCompatibilityDataSource( /// public bool InProcess { get; set; } - public string? BeforeFeature { get; set; } - public string? AfterFeature { get; set; } + public string? BeforeRunnerFeature { get; set; } + public string? AfterRunnerFeature { get; set; } + + public string? BeforeTestHostFeature { get; set; } + public string? AfterTestHostFeature { get; set; } public string? BeforeAdapterFeature { get; set; } public string? AfterAdapterFeature { get; set; } @@ -57,14 +60,18 @@ public override void CreateData(MethodInfo methodInfo) _builder.WithOlderConfigurations = false; _builder.WithInProcess = InProcess; - _builder.BeforeFeature = BeforeFeature; - _builder.AfterFeature = AfterFeature; + _builder.BeforeRunnerFeature = BeforeRunnerFeature; + _builder.AfterRunnerFeature = AfterRunnerFeature; + + _builder.BeforeTestHostFeature = BeforeTestHostFeature; + _builder.AfterTestHostFeature = AfterTestHostFeature; + _builder.BeforeAdapterFeature = BeforeAdapterFeature; _builder.AfterAdapterFeature = AfterAdapterFeature; _builder.DebugDataCollector = DebugDataCollector; _builder.DebugVSTestConsole = DebugVSTestConsole; - _builder.DebugTesthost = DebugTesthost; + _builder.DebugTestHost = DebugTestHost; _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; var data = _builder.CreateData(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs index 60d49baf1a..71db23bdee 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs @@ -35,7 +35,7 @@ public NetCoreRunner(string targetFrameworks = AcceptanceTestBase.NETFX452_NET50 } public bool DebugVSTestConsole { get; set; } - public bool DebugTesthost { get; set; } + public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; @@ -56,7 +56,7 @@ public IEnumerable GetData(MethodInfo methodInfo) runnerInfo.DebugInfo = new DebugInfo { DebugVSTestConsole = DebugVSTestConsole, - DebugTesthost = DebugTesthost, + DebugTestHost = DebugTestHost, DebugDataCollector = DebugDataCollector, NoDefaultBreakpoints = NoDefaultBreakpoints, }; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs index e4d646a8b8..56ff01036b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreTargetFrameworkDataSource.cs @@ -49,7 +49,7 @@ public NetCoreTargetFrameworkDataSource( } public bool DebugVSTestConsole { get; set; } - public bool DebugTesthost { get; set; } + public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; @@ -64,7 +64,7 @@ private void AddRunnerDataRow(List dataRows, string runnerFramework, s runnerInfo.DebugInfo = new DebugInfo { DebugDataCollector = DebugDataCollector, - DebugTesthost = DebugTesthost, + DebugTestHost = DebugTestHost, DebugVSTestConsole = DebugVSTestConsole, NoDefaultBreakpoints = NoDefaultBreakpoints, }; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs index c7be429345..0f1faa65f5 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFrameworkRunner.cs @@ -32,7 +32,7 @@ public NetFrameworkRunner(string targetFrameworks = AcceptanceTestBase.NETFX452_ } public bool DebugVSTestConsole { get; set; } - public bool DebugTesthost { get; set; } + public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; @@ -58,7 +58,7 @@ public IEnumerable GetData(MethodInfo methodInfo) runnerInfo.DebugInfo = new DebugInfo { DebugVSTestConsole = DebugVSTestConsole, - DebugTesthost = DebugTesthost, + DebugTestHost = DebugTestHost, DebugDataCollector = DebugDataCollector, NoDefaultBreakpoints = NoDefaultBreakpoints, }; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs index 468aded7f7..b809da7799 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetFullTargetFrameworkDataSource.cs @@ -43,7 +43,7 @@ public NetFullTargetFrameworkDataSource(bool inIsolation = true, bool inProcess } public bool DebugVSTestConsole { get; set; } - public bool DebugTesthost { get; set; } + public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; @@ -62,7 +62,7 @@ public IEnumerable GetData(MethodInfo methodInfo) runnerInfo.DebugInfo = new DebugInfo { DebugVSTestConsole = DebugVSTestConsole, - DebugTesthost = DebugTesthost, + DebugTestHost = DebugTestHost, DebugDataCollector = DebugDataCollector, NoDefaultBreakpoints = NoDefaultBreakpoints, }; @@ -82,7 +82,7 @@ public IEnumerable GetData(MethodInfo methodInfo) runnerInfo.DebugInfo = new DebugInfo { DebugVSTestConsole = DebugVSTestConsole, - DebugTesthost = DebugTesthost, + DebugTestHost = DebugTestHost, DebugDataCollector = DebugDataCollector, NoDefaultBreakpoints = NoDefaultBreakpoints, }; @@ -100,7 +100,7 @@ public IEnumerable GetData(MethodInfo methodInfo) runnerInfo.DebugInfo = new DebugInfo { DebugVSTestConsole = DebugVSTestConsole, - DebugTesthost = DebugTesthost, + DebugTestHost = DebugTestHost, DebugDataCollector = DebugDataCollector, NoDefaultBreakpoints = NoDefaultBreakpoints, }; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnerCompatibilityDataSource.cs similarity index 86% rename from test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs rename to test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnerCompatibilityDataSource.cs index d55cfe0c62..3b7b9d10ca 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TranslationLayerCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnerCompatibilityDataSource.cs @@ -38,7 +38,7 @@ public RunnerCompatibilityDataSource( } public bool DebugVSTestConsole { get; set; } - public bool DebugTesthost { get; set; } + public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; @@ -49,6 +49,10 @@ public RunnerCompatibilityDataSource( public string? BeforeFeature { get; set; } public string? AfterFeature { get; set; } + + //public string? BeforeTestHostFeature { get; set; } + //public string? AfterTestHostFeature { get; set; } + public string? BeforeAdapterFeature { get; set; } public string? AfterAdapterFeature { get; set; } @@ -60,14 +64,18 @@ public override void CreateData(MethodInfo methodInfo) _builder.WithOlderConfigurations = false; _builder.WithInProcess = InProcess; - _builder.BeforeFeature = BeforeFeature; - _builder.AfterFeature = AfterFeature; + _builder.BeforeRunnerFeature = BeforeFeature; + _builder.AfterRunnerFeature = AfterFeature; + + //_builder.BeforeTestHostFeature = BeforeTestHostFeature; + //_builder.AfterTestHostFeature = AfterTestHostFeature; + _builder.BeforeAdapterFeature = BeforeAdapterFeature; _builder.AfterAdapterFeature = AfterAdapterFeature; _builder.DebugDataCollector = DebugDataCollector; _builder.DebugVSTestConsole = DebugVSTestConsole; - _builder.DebugTesthost = DebugTesthost; + _builder.DebugTestHost = DebugTestHost; _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; var data = _builder.CreateData(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs index bb30619479..88f1404005 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs @@ -2,7 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; -using System.Collections.Generic; +using System.Linq; using Microsoft.TestPlatform.TestUtilities; @@ -18,16 +18,14 @@ namespace Microsoft.TestPlatform.AcceptanceTests; public class RunnerInfo { public string? RunnerFramework { get; set; } - public VSTestConsoleInfo? VSTestConsoleInfo { get; set; } - - public string? TargetFramework { get; set; } public string? InIsolationValue { get; set; } - public DebugInfo? DebugInfo { get; set; } + public NetTestSdkInfo? TestHostInfo { get; set; } + public DllInfo? AdapterInfo { get; set; } - public List DllInfos { get; set; } = new(); + public string? Batch { get; set; } /// /// Is running via .NET "Core" vstest.console? @@ -49,5 +47,17 @@ public class RunnerInfo /// public bool IsNetFrameworkTarget => TargetFramework!.StartsWith("net4", StringComparison.InvariantCultureIgnoreCase); - public override string ToString() => $"Runner = {RunnerFramework}, TargetFramework = {TargetFramework}, {(string.IsNullOrEmpty(InIsolationValue) ? "InProcess" : "InIsolation")}, {VSTestConsoleInfo}, {string.Join(",", DllInfos)}"; + public override string ToString() + { + return string.Join(", ", new[] + { + Batch != null ? $"{Batch}" : null, + $"Runner = {RunnerFramework}", + $"TargetFramework = {TargetFramework}", + string.IsNullOrEmpty(InIsolationValue) ? "InProcess" : "InIsolation", + VSTestConsoleInfo == null ? null : VSTestConsoleInfo.ToString(), + TestHostInfo == null ? null : string.Join(",", TestHostInfo), + AdapterInfo == null ? null : string.Join(",", AdapterInfo) + }.Where(s => s != null)); + } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs index b607811116..5f117fff0b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TestPlatformCompatibilityDataSource.cs @@ -42,7 +42,7 @@ public TestPlatformCompatibilityDataSource( } public bool DebugVSTestConsole { get; set; } - public bool DebugTesthost { get; set; } + public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; @@ -59,9 +59,12 @@ public TestPlatformCompatibilityDataSource( public bool WithOlderConfigurations { get; set; } = true; + public string? BeforeRunnerFeature { get; set; } + public string? AfterRunnerFeature { get; set; } + + public string? BeforeTestHostFeature { get; set; } + public string? AfterTestHostFeature { get; set; } - public string? BeforeFeature { get; set; } - public string? AfterFeature { get; set; } public string? BeforeAdapterFeature { get; set; } public string? AfterAdapterFeature { get; set; } @@ -73,14 +76,18 @@ public override void CreateData(MethodInfo methodInfo) _builder.WithOlderConfigurations = WithOlderConfigurations; _builder.WithInProcess = WithInProcess; - _builder.BeforeFeature = BeforeFeature; - _builder.AfterFeature = AfterFeature; + _builder.BeforeRunnerFeature = BeforeRunnerFeature; + _builder.AfterRunnerFeature = AfterRunnerFeature; + + _builder.BeforeTestHostFeature = BeforeTestHostFeature; + _builder.AfterTestHostFeature = AfterTestHostFeature; + _builder.BeforeAdapterFeature = BeforeAdapterFeature; _builder.AfterAdapterFeature = AfterAdapterFeature; _builder.DebugDataCollector = DebugDataCollector; _builder.DebugVSTestConsole = DebugVSTestConsole; - _builder.DebugTesthost = DebugTesthost; + _builder.DebugTestHost = DebugTestHost; _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; var data = _builder.CreateData(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs index e32c32aa87..4bcbba2f93 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/TesthostCompatibilityDataSource.cs @@ -10,11 +10,11 @@ namespace Microsoft.TestPlatform.AcceptanceTests; /// /// When that adds up to no configuration exception is thrown. /// -public class HostCompatibilityDataSource : TestDataSource +public class TestHostCompatibilityDataSource : TestDataSource { private readonly CompatibilityRowsBuilder _builder; - public HostCompatibilityDataSource( + public TestHostCompatibilityDataSource( string runnerFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, string hostFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, string hostVersions = AcceptanceTestBase.LATEST_TO_LEGACY) @@ -37,14 +37,12 @@ public HostCompatibilityDataSource( } public bool DebugVSTestConsole { get; set; } - public bool DebugTesthost { get; set; } + public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; public string? BeforeFeature { get; set; } public string? AfterFeature { get; set; } - public string? BeforeAdapterFeature { get; set; } - public string? AfterAdapterFeature { get; set; } public override void CreateData(MethodInfo methodInfo) { @@ -54,14 +52,12 @@ public override void CreateData(MethodInfo methodInfo) _builder.WithOlderConfigurations = false; _builder.WithInProcess = false; - _builder.BeforeFeature = BeforeFeature; - _builder.AfterFeature = AfterFeature; - _builder.BeforeAdapterFeature = BeforeAdapterFeature; - _builder.AfterAdapterFeature = AfterAdapterFeature; + _builder.BeforeTestHostFeature = BeforeFeature; + _builder.AfterTestHostFeature = AfterFeature; _builder.DebugDataCollector = DebugDataCollector; _builder.DebugVSTestConsole = DebugVSTestConsole; - _builder.DebugTesthost = DebugTesthost; + _builder.DebugTestHost = DebugTestHost; _builder.NoDefaultBreakpoints = NoDefaultBreakpoints; var data = _builder.CreateData(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs index 3d8f36218b..4e31163543 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs @@ -20,7 +20,7 @@ public void WildCardPatternShouldCorrectlyWorkOnFiles(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssembly = GetSampleTestAssembly(); + var testAssembly = GetSampleTestDll(); testAssembly = testAssembly.Replace("SimpleTestProject.dll", "*TestProj*.dll"); var arguments = PrepareArguments( @@ -40,7 +40,7 @@ public void WildCardPatternShouldCorrectlyWorkOnArbitraryDepthDirectories(Runner { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssembly = GetSampleTestAssembly(); + var testAssembly = GetSampleTestDll(); // Add one more directory to the temp path, so we can substitute it with ** // and copy then whole directory there. @@ -66,7 +66,7 @@ public void WildCardPatternShouldCorrectlyWorkForRelativeAssemblyPath(RunnerInfo { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssembly = GetSampleTestAssembly(); + var testAssembly = GetSampleTestDll(); testAssembly = testAssembly.Replace("SimpleTestProject.dll", "*TestProj*.dll"); var wildCardIndex = testAssembly.IndexOfAny(new char[] { '*' }); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs index 394db87a80..9a00c24cb8 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs @@ -19,7 +19,7 @@ public void FrameworkArgumentShouldWork(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " ", $"/Framework:{FrameworkArgValue}"); InvokeVsTest(arguments); @@ -33,7 +33,7 @@ public void FrameworkShortNameArgumentShouldWork(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " ", $"/Framework:{_testEnvironment.TargetFramework}"); InvokeVsTest(arguments); @@ -49,7 +49,7 @@ public void OnWrongFrameworkPassedTestRunShouldNotRun(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); if (runnerInfo.TargetFramework.Contains("netcore")) { arguments = string.Concat(arguments, " ", "/Framework:Framework45"); @@ -80,7 +80,7 @@ public void RunSpecificTestsShouldWorkWithFrameworkInCompatibleWarning(RunnerInf { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " ", "/tests:PassingTest"); arguments = string.Concat(arguments, " ", "/Framework:Framework40"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs index e9e30d8fdc..8e79cc7589 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs @@ -23,12 +23,12 @@ public void TrxLoggerWithFriendlyNameShouldProperlyOverwriteFile(RunnerInfo runn { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var trxFileName = "TestResults.trx"; arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); @@ -44,12 +44,12 @@ public void HtmlLoggerWithFriendlyNameShouldProperlyOverwriteFile(RunnerInfo run { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var htmlFileName = "TestResults.html"; arguments = string.Concat(arguments, $" /logger:\"html;LogFileName={htmlFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"html;LogFileName={htmlFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); @@ -64,12 +64,12 @@ public void TrxLoggerWithExecutorUriShouldProperlyOverwriteFile(RunnerInfo runne { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var trxFileName = "TestResults.trx"; arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFileName={trxFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFileName={trxFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); @@ -86,11 +86,11 @@ public void TrxLoggerWithLogFilePrefixShouldGenerateMultipleTrx(RunnerInfo runne SetTestEnvironment(_testEnvironment, runnerInfo); var trxFileNamePattern = "TestResults"; - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFilePrefix={trxFileNamePattern}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFilePrefix={trxFileNamePattern}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); @@ -105,12 +105,12 @@ public void HtmlLoggerWithExecutorUriShouldProperlyOverwriteFile(RunnerInfo runn { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var htmlFileName = "TestResults.html"; arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/htmlLogger/v1;LogFileName{htmlFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/htmlLogger/v1;LogFileName={htmlFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); @@ -126,7 +126,7 @@ public void TrxLoggerResultSummaryOutcomeValueShouldBeFailedIfNoTestsExecutedAnd { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); var trxFilePath = Path.Combine(TempDirectory.Path, "TrxLogger.trx"); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFilePath}\""); @@ -149,7 +149,7 @@ public void TrxLoggerResultSummaryOutcomeValueShouldNotChangeIfNoTestsExecutedAn { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); var trxFilePath = Path.Combine(TempDirectory.Path, "TrxLogger.trx"); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFilePath}\""); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs index b149fa2994..289363a269 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs @@ -45,7 +45,7 @@ private void RunTestExecutionWithPlatform(string platformArg, string testhostPro { var arguments = PrepareArguments( - GetSampleTestAssembly(), + GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs index d5f227fccf..be24f8ff38 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs @@ -58,7 +58,7 @@ public void DiscoverAllTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - InvokeVsTestForDiscovery(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue); + InvokeVsTestForDiscovery(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue); var listOfTests = new[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", "SampleUnitTestProject.UnitTest1.SkippingTest" }; ValidateDiscoveredTests(listOfTests); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs index e82ddc9520..13cd916a56 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ProcessesInteractionTests.cs @@ -22,7 +22,7 @@ public void WhenTestHostProcessExitsBecauseTheTargetedRuntimeIsNoFoundThenTheMes // Arrange SetTestEnvironment(_testEnvironment, runnerInfo); const string testAssetProjectName = "SimpleTestProjectMessedUpTargetFramework"; - var assemblyPath = GetAssetFullPath(testAssetProjectName + ".dll", Core21TargetFramework); + var assemblyPath = GetTestDllForFramework(testAssetProjectName + ".dll", Core21TargetFramework); UpdateRuntimeConfigJsonWithInvalidFramework(assemblyPath, testAssetProjectName); // Act diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs index a7b02265a7..4b6d60ca87 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs @@ -20,7 +20,7 @@ public void TrxFileShouldBeCreatedInResultsDirectory(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); var trxFileName = "TestResults.trx"; var trxFilePath = Path.Combine(TempDirectory.Path, trxFileName); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); @@ -41,7 +41,7 @@ public void ResultsDirectoryRelativePathShouldWork(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); var trxFileName = "TestResults.trx"; var relativeDirectory = @"relative\directory"; var resultsDirectory = Path.Combine(Directory.GetCurrentDirectory(), relativeDirectory); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs index 3d5dbcbea7..d1c8084875 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs @@ -246,7 +246,7 @@ public void RunSettingsWithInvalidValueShouldLogError(RunnerInfo runnerInfo) }; var runsettingsFilePath = GetRunsettingsFilePath(runConfigurationDictionary, TempDirectory); var arguments = PrepareArguments( - GetSampleTestAssembly(), + GetSampleTestDll(), string.Empty, runsettingsFilePath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -267,7 +267,7 @@ public void TestAdapterPathFromRunSettings(RunnerInfo runnerInfo) }; var runsettingsFilePath = GetRunsettingsFilePath(runConfigurationDictionary, TempDirectory); var arguments = PrepareArguments( - GetSampleTestAssembly(), + GetSampleTestDll(), string.Empty, runsettingsFilePath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -284,7 +284,7 @@ public void LegacySettingsWithPlatform(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); + var testAssemblyPath = GetTestDll("LegacySettingsUnitTestProject.dll"); _ = Path.GetDirectoryName(testAssemblyPath); var runsettingsXml = @" @@ -315,7 +315,7 @@ public void LegacySettingsWithScripts(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); + var testAssemblyPath = GetTestDll("LegacySettingsUnitTestProject.dll"); // Create the script files var guid = Guid.NewGuid(); @@ -361,7 +361,7 @@ public void LegacySettingsWithDeploymentItem(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); + var testAssemblyPath = GetTestDll("LegacySettingsUnitTestProject.dll"); var testAssemblyDirectory = Path.GetDirectoryName(testAssemblyPath); var deploymentItem = Path.Combine(testAssemblyDirectory, "Deployment", "DeploymentFile.xml"); @@ -397,7 +397,7 @@ public void LegacySettingsTestTimeout(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); + var testAssemblyPath = GetTestDll("LegacySettingsUnitTestProject.dll"); var runsettingsXml = @" true @@ -424,7 +424,7 @@ public void LegacySettingsAssemblyResolution(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); + var testAssemblyPath = GetTestDll("LegacySettingsUnitTestProject.dll"); var runsettingsFormat = @" true @@ -466,7 +466,7 @@ public void EnvironmentVariablesSettingsShouldSetEnvironmentVariables(RunnerInfo { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetAssetFullPath("EnvironmentVariablesTestProject.dll"); + var testAssemblyPath = GetTestDll("EnvironmentVariablesTestProject.dll"); var runsettingsXml = @" diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs index fdf49097ce..1b519c1d9f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs @@ -49,7 +49,7 @@ private void RunTests(RunnerInfo runnerInfo) return; } - var assemblyPaths = GetAssetFullPath("SimpleTestProject2.dll"); + var assemblyPaths = GetTestDll("SimpleTestProject2.dll"); var env = new Dictionary { @@ -70,7 +70,7 @@ private void DiscoverTests(RunnerInfo runnerInfo) return; } - var assemblyPaths = GetAssetFullPath("SimpleTestProject2.dll"); + var assemblyPaths = GetTestDll("SimpleTestProject2.dll"); var env = new Dictionary { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs index c4363cf825..d235068166 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs @@ -21,7 +21,7 @@ public void RunSelectedTestsWithAndOperatorTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestAssembly(), + GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -38,7 +38,7 @@ public void RunSelectedTestsWithCategoryTraitInMixCase(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestAssembly(), + GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -55,7 +55,7 @@ public void RunSelectedTestsWithClassNameTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestAssembly(), + GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -72,7 +72,7 @@ public void RunSelectedTestsWithFullyQualifiedNameTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestAssembly(), + GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -91,7 +91,7 @@ public void RunSelectedTestsWithNameTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestAssembly(), + GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -108,7 +108,7 @@ public void RunSelectedTestsWithOrOperatorTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestAssembly(), + GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -125,7 +125,7 @@ public void RunSelectedTestsWithPriorityTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestAssembly(), + GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs index c77faab8ab..a5f5efb6eb 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs @@ -437,7 +437,7 @@ public async Task EndSessionShouldEnsureVstestConsoleProcessDies(RunnerInfo runn private IList GetTestAssemblies() { - return GetProjects().Select(p => GetAssetFullPath(p)).ToList(); + return GetProjects().Select(p => GetTestDll(p)).ToList(); } private IList GetProjects() diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs index 996adfec65..d3730b7378 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Threading; @@ -24,7 +23,6 @@ namespace Microsoft.TestPlatform.AcceptanceTests.TranslationLayerTests; public class CustomTestHostTests : AcceptanceTestBase { private IVsTestConsoleWrapper _vstestConsoleWrapper; - private RunEventHandler _runEventHandler; [TestCleanup] public void Cleanup() @@ -33,109 +31,120 @@ public void Cleanup() } [TestMethod] - [RunnerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] + [RunnerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER_FLOW)] + [TestHostCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER_FLOW)] public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncher(RunnerInfo runnerInfo) { + // Pins the existing functionality. + + // Arrange SetTestEnvironment(_testEnvironment, runnerInfo); + _vstestConsoleWrapper = GetVsTestConsoleWrapper(); - _runEventHandler = new RunEventHandler(); + var runEventHandler = new RunEventHandler(); + // Act var customTestHostLauncher = new TestHostLauncherV1(); - _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler, customTestHostLauncher); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetRunSettingsWithCurrentTargetFramework(), runEventHandler, customTestHostLauncher); // Assert + EnsureTestsRunWithoutErrors(runEventHandler, passed: 2, failed: 2, skipped: 2); + + // Ensure we tried to launch testhost process. customTestHostLauncher.Should().BeAssignableTo(); customTestHostLauncher.LaunchProcessProcessId.Should().NotBeNull("we should launch some real process and save the pid of it"); - - _runEventHandler.TestResults.Should().HaveCount(6); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed).Should().Be(2); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed).Should().Be(2); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped).Should().Be(2); } [TestMethod] - [RunnerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER)] + // [RunnerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER_FLOW)] + [TestHostCompatibilityDataSource("net451", "netcoreapp2.1", "LegacyStable", BeforeFeature = Features.ATTACH_DEBUGGER_FLOW, DebugVSTestConsole = true)] + [Ignore("This is not working for any testhost prior 16.7.0 where the change was introduced. The launch testhost flow was replaced with AttachDebugger in runner, and the new callback to AttachDebugger happens in testhost." + + "But any testhost prior 16.7.0 where the change was introduced does not know to call back AttachDebugger, and the call never happens.")] + // You can confirm that the functionality broke between runner and testhost, past this point by using newer runners, against older testhosts. + // [TestPlatformCompatibilityDataSource(AfterRunnerFeature = Features.ATTACH_DEBUGGER_FLOW, BeforeTestHostFeature = Features.ATTACH_DEBUGGER_FLOW)] public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncherWhenITestHostLauncher2IsProvided(RunnerInfo runnerInfo) { + // Ensures compatibility with testhost and runners that were created before 16.3.0. It makes sure that even if user provides + // an implementation of the ITestHostLauncher2 interface, then testhost expecting ITestHostLauncher still works correctly. + + // Arrange SetTestEnvironment(_testEnvironment, runnerInfo); _vstestConsoleWrapper = GetVsTestConsoleWrapper(); - _runEventHandler = new RunEventHandler(); + var runEventHandler = new RunEventHandler(); + // Act var customTestHostLauncher = new TestHostLauncherV2(); - _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler, customTestHostLauncher); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetRunSettingsWithCurrentTargetFramework(), runEventHandler, customTestHostLauncher); // Assert + EnsureTestsRunWithoutErrors(runEventHandler, passed: 2, failed: 2, skipped: 2); + customTestHostLauncher.Should().BeAssignableTo(); customTestHostLauncher.LaunchProcessProcessId.Should().NotBeNull("we should launch some real process and save the pid of it"); customTestHostLauncher.AttachDebuggerProcessId.Should().BeNull("we should not be asked to attach to a debugger, that flow is not used when vstest.console does not support it yet, even when it is given ITestHostLauncher2"); - - _runEventHandler.TestResults.Should().HaveCount(6); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed).Should().Be(2); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed).Should().Be(2); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped).Should().Be(2); } [TestMethod] - [RunnerCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER)] + [RunnerCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER_FLOW)] + // [TestHostCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER_FLOW)] + [TestHostCompatibilityDataSource()] public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvidedLauncher(RunnerInfo runnerInfo) { + + // Arrange SetTestEnvironment(_testEnvironment, runnerInfo); _vstestConsoleWrapper = GetVsTestConsoleWrapper(); - _runEventHandler = new RunEventHandler(); + var runEventHandler = new RunEventHandler(); + // Act var customTestHostLauncher = new TestHostLauncherV2(); - _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler, customTestHostLauncher); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetRunSettingsWithCurrentTargetFramework(), runEventHandler, customTestHostLauncher); // Assert + EnsureTestsRunWithoutErrors(runEventHandler, passed: 2, failed: 2, skipped: 2); + customTestHostLauncher.Should().BeAssignableTo(); customTestHostLauncher.AttachDebuggerProcessId.Should().NotBeNull("we should be asked to attach a debugger to some process and save the pid of the process"); customTestHostLauncher.LaunchProcessProcessId.Should().BeNull("we should not be asked to launch some real process, that flow is not used when vstest.console supports it and is given ITestHostLauncher2"); - - _runEventHandler.TestResults.Should().HaveCount(6); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed).Should().Be(2); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed).Should().Be(2); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped).Should().Be(2); } [TestMethod] - //[Ignore("This is not working. The compatibility code only checks the protocol version (in handler), which is dictated by testhost. " - // + "It sees 6 but does not realize that the provided CustomTesthostLauncher is not supporting the new feature, it ends up calling back to EditoAttachDebugger" + - // "in translation layer, and that just silently skips the call.")] - [RunnerCompatibilityDataSource("net451", "net451", "Latest", AfterFeature = Features.ATTACH_DEBUGGER, DebugVSTestConsole = true, DebugTesthost=true)] + [Ignore("This is not working. The compatibility code only checks the protocol version (in handler), which is dictated by testhost. " + + "It sees 6 but does not realize that the provided CustomTesthostLauncher is not supporting the new feature, it ends up calling back to EditoAttachDebugger" + + "in translation layer, and that just silently skips the call.")] + [RunnerCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER_FLOW)] + [TestHostCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER_FLOW)] public void RunTestsWithCustomTestHostLauncherUsesLaunchWhenGivenAnOutdatedITestHostLauncher(RunnerInfo runnerInfo) { + // Arrange SetTestEnvironment(_testEnvironment, runnerInfo); _vstestConsoleWrapper = GetVsTestConsoleWrapper(); - _runEventHandler = new RunEventHandler(); + var runEventHandler = new RunEventHandler(); + // Act var customTestHostLauncher = new TestHostLauncherV1(); - _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler, customTestHostLauncher); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetRunSettingsWithCurrentTargetFramework(), runEventHandler, customTestHostLauncher); // Assert + EnsureTestsRunWithoutErrors(runEventHandler, passed: 2, failed: 2, skipped: 2); + customTestHostLauncher.Should().NotBeAssignableTo(); customTestHostLauncher.LaunchProcessProcessId.Should().NotBeNull("we should launch some real process and save the pid of it"); - - _runEventHandler.TestResults.Should().HaveCount(6); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed).Should().Be(2); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed).Should().Be(2); - _runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped).Should().Be(2); } - private IList GetTestAssemblies() + private static void EnsureTestsRunWithoutErrors(RunEventHandler runEventHandler, int passed, int failed, int skipped) { - var testAssemblies = new List - { - GetAssetFullPath("SimpleTestProject.dll"), - GetAssetFullPath("SimpleTestProject2.dll") - }; - - return testAssemblies; + runEventHandler.Errors.Should().BeEmpty(); + runEventHandler.TestResults.Should().HaveCount(passed + failed + skipped); + runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Passed).Should().Be(passed); + runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Failed).Should().Be(failed); + runEventHandler.TestResults.Count(t => t.Outcome == TestOutcome.Skipped).Should().Be(skipped); } /// /// The custom test host launcher implementing ITestHostLauncher. /// - public class TestHostLauncherV1 : ITestHostLauncher + private class TestHostLauncherV1 : ITestHostLauncher { public int? LaunchProcessProcessId { get; private set; } @@ -168,9 +177,9 @@ public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo, Cancell } /// - /// The custom test host launcher implementing ITestHostLauncher2. + /// The custom test host launcher implementing ITestHostLauncher2, and through that also ITestHostLauncher. /// - public class TestHostLauncherV2 : TestHostLauncherV1, ITestHostLauncher2 + private class TestHostLauncherV2 : TestHostLauncherV1, ITestHostLauncher2 { public int? AttachDebuggerProcessId { get; private set; } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DataCollectorAttachmentProcessor.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DataCollectorAttachmentProcessor.cs index 16708093ba..4a38f260e7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DataCollectorAttachmentProcessor.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DataCollectorAttachmentProcessor.cs @@ -124,5 +124,5 @@ private static void CreateDataCollectionRunSettingsFile(string destinationRunset } private IList GetTestAssemblies() - => new List { "SimpleTestProject.dll", "SimpleTestProject2.dll" }.Select(p => GetAssetFullPath(p)).ToList(); + => new List { "SimpleTestProject.dll", "SimpleTestProject2.dll" }.Select(p => GetTestDll(p)).ToList(); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs index b08763da05..af2456f1d8 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs @@ -47,12 +47,12 @@ public void RunTestsWithNunitAdapter(RunnerInfo runnerInfo) var sources = new List { - GetAssetFullPath("NUTestProject.dll") + GetTestDll("NUTestProject.dll") }; _vstestConsoleWrapper.RunTests( sources, - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), _runEventHandler); var testCase = @@ -94,7 +94,7 @@ public void RunTestsWithXunitAdapter(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( sources, - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), _runEventHandler); var testCase = @@ -134,7 +134,7 @@ public void RunTestsWithChutzpahAdapter(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( sources, - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), _runEventHandler); var testCase = diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index 38ec639a3f..b09ce2cc0f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -50,7 +50,7 @@ public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo) _discoveryEventHandler2 = new DiscoveryEventHandler2(); var vstestConsoleWrapper = GetVsTestConsoleWrapper(); - vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), _discoveryEventHandler); + vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), _discoveryEventHandler); // Assert. Assert.AreEqual(6, _discoveryEventHandler.DiscoveredTestCases.Count); @@ -69,7 +69,7 @@ public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedOut(RunnerI var vstestConsoleWrapper = GetVsTestConsoleWrapper(); vstestConsoleWrapper.DiscoverTests( GetTestAssemblies(), - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), new TestPlatformOptions() { CollectMetrics = false }, _discoveryEventHandler2); @@ -86,7 +86,7 @@ public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedIn(RunnerIn SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), new TestPlatformOptions() { CollectMetrics = true }, _discoveryEventHandler2); + _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), new TestPlatformOptions() { CollectMetrics = true }, _discoveryEventHandler2); // Assert. Assert.AreEqual(6, _discoveryEventHandler2.DiscoveredTestCases.Count); @@ -164,7 +164,7 @@ public void DisoverTestUsingEventHandler2ShouldContainAllSourcesAsFullyDiscovere _vstestConsoleWrapper.DiscoverTests( GetTestAssemblies(), - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), null, eventHandler2); @@ -182,7 +182,7 @@ public void DiscoverTestsUsingSourceNavigation(RunnerInfo runnerInfo) _vstestConsoleWrapper.DiscoverTests( GetTestAssemblies(), - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), _discoveryEventHandler); // Assert. @@ -209,9 +209,9 @@ public void CancelTestDiscovery(RunnerInfo runnerInfo) // Setup var testAssemblies = new List { - GetAssetFullPath("DiscoveryTestProject.dll"), - GetAssetFullPath("SimpleTestProject.dll"), - GetAssetFullPath("SimpleTestProject2.dll") + GetTestDll("DiscoveryTestProject.dll"), + GetTestDll("SimpleTestProject.dll"), + GetTestDll("SimpleTestProject2.dll") }; SetTestEnvironment(_testEnvironment, runnerInfo); @@ -223,7 +223,7 @@ public void CancelTestDiscovery(RunnerInfo runnerInfo) ((IEnumerable testcases) => discoveredTests.AddRange(testcases)); // Act - var discoveryTask = Task.Run(() => _vstestConsoleWrapper.DiscoverTests(testAssemblies, GetDefaultRunSettings(), discoveryEvents.Object)); + var discoveryTask = Task.Run(() => _vstestConsoleWrapper.DiscoverTests(testAssemblies, GetRunSettingsWithCurrentTargetFramework(), discoveryEvents.Object)); Task.Delay(2000).Wait(); _vstestConsoleWrapper.CancelDiscovery(); @@ -238,8 +238,8 @@ private IList GetTestAssemblies() { var testAssemblies = new List { - GetAssetFullPath("SimpleTestProject.dll"), - GetAssetFullPath("SimpleTestProject2.dll") + GetTestDll("SimpleTestProject.dll"), + GetTestDll("SimpleTestProject2.dll") }; return testAssemblies; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs index 0beac2e85c..01b4367468 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs @@ -90,8 +90,8 @@ private IList GetTestAssemblies() { var testAssemblies = new List { - GetAssetFullPath("SimpleTestProject.dll"), - GetAssetFullPath("SimpleTestProject2.dll") + GetTestDll("SimpleTestProject.dll"), + GetTestDll("SimpleTestProject2.dll") }; return testAssemblies; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs index 73b07981b6..f2722cc154 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs @@ -42,10 +42,10 @@ public void RunSelectedTestsWithoutTestPlatformOptions(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), _discoveryEventHandler); + _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), _discoveryEventHandler); var testCases = _discoveryEventHandler.DiscoveredTestCases; - _vstestConsoleWrapper.RunTests(testCases, GetDefaultRunSettings(), _runEventHandler); + _vstestConsoleWrapper.RunTests(testCases, GetRunSettingsWithCurrentTargetFramework(), _runEventHandler); // Assert Assert.AreEqual(6, _runEventHandler.TestResults.Count); @@ -62,12 +62,12 @@ public void RunSelectedTestsWithTestPlatformOptions(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), _discoveryEventHandler); + _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), _discoveryEventHandler); var testCases = _discoveryEventHandler.DiscoveredTestCases; _vstestConsoleWrapper.RunTests( testCases, - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), new TestPlatformOptions() { CollectMetrics = true }, _runEventHandler); @@ -85,8 +85,8 @@ private IList GetTestAssemblies() { var testAssemblies = new List { - GetAssetFullPath("SimpleTestProject.dll"), - GetAssetFullPath("SimpleTestProject2.dll") + GetTestDll("SimpleTestProject.dll"), + GetTestDll("SimpleTestProject2.dll") }; return testAssemblies; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs index ffe2449e15..168120af82 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs @@ -47,7 +47,7 @@ public void RunAllTests(RunnerInfo runnerInfo) var vstestConsoleWrapper = GetVsTestConsoleWrapper(); var runEventHandler = new RunEventHandler(); - vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetDefaultRunSettings(), runEventHandler); + vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), runEventHandler); // Assert Assert.AreEqual(6, runEventHandler.TestResults.Count); @@ -67,7 +67,7 @@ public void EndSessionShouldEnsureVstestConsoleProcessDies(RunnerInfo runnerInfo SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - _vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler); + _vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), _runEventHandler); _vstestConsoleWrapper?.EndSession(); // Assert @@ -89,7 +89,7 @@ public void RunTestsWithTelemetryOptedIn(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( GetTestAssemblies(), - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), new TestPlatformOptions() { CollectMetrics = true }, _runEventHandler); @@ -113,7 +113,7 @@ public void RunTestsWithTelemetryOptedOut(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( GetTestAssemblies(), - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), new TestPlatformOptions() { CollectMetrics = false }, _runEventHandler); @@ -137,11 +137,11 @@ public void RunTestsShouldThrowOnStackOverflowException(RunnerInfo runnerInfo) return; } - var source = new[] { GetAssetFullPath("SimpleTestProject3.dll") }; + var source = new[] { GetTestDll("SimpleTestProject3.dll") }; _vstestConsoleWrapper.RunTests( source, - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), new TestPlatformOptions() { TestCaseFilter = "ExitWithStackoverFlow" }, _runEventHandler); @@ -162,7 +162,7 @@ public void RunTestsShouldShowProperWarningOnNoTestsForTestCaseFilter(RunnerInfo Setup(); var testAssemblyName = "SimpleTestProject2.dll"; - var source = new List() { GetAssetFullPath(testAssemblyName) }; + var source = new List() { GetTestDll(testAssemblyName) }; var veryLongTestCaseFilter = "FullyQualifiedName=VeryLongTestCaseNameeeeeeeeeeeeee" + @@ -173,7 +173,7 @@ public void RunTestsShouldShowProperWarningOnNoTestsForTestCaseFilter(RunnerInfo _vstestConsoleWrapper.RunTests( source, - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), new TestPlatformOptions() { TestCaseFilter = veryLongTestCaseFilter }, _runEventHandler); @@ -190,8 +190,8 @@ private IList GetTestAssemblies() { return new List { - GetAssetFullPath("SimpleTestProject.dll"), - GetAssetFullPath("SimpleTestProject2.dll") + GetTestDll("SimpleTestProject.dll"), + GetTestDll("SimpleTestProject2.dll") }; } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs index 4f6e641fb2..de916c635d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs @@ -56,7 +56,7 @@ public void RunTestsWithTestAdapterPath(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( GetTestAssemblies(), - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), _runEventHandler); // Assert @@ -115,7 +115,7 @@ public void RunTestsWithTestSettings(RunnerInfo runnerInfo) var runSettings = $"{FrameworkArgValue}" + testsettingsFile + ""; var sources = new List { - GetAssetFullPath("MstestV1UnitTestProject.dll") + GetTestDll("MstestV1UnitTestProject.dll") }; _vstestConsoleWrapper.RunTests( @@ -140,7 +140,7 @@ public void RunTestsWithX64Source(RunnerInfo runnerInfo) var sources = new List { - GetAssetFullPath("SimpleTestProject3.dll") + GetTestDll("SimpleTestProject3.dll") }; @@ -149,7 +149,7 @@ public void RunTestsWithX64Source(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( sources, - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), new TestPlatformOptions() { TestCaseFilter = "FullyQualifiedName = SampleUnitTestProject3.UnitTest1.WorkingDirectoryTest" }, _runEventHandler); @@ -163,8 +163,8 @@ private IList GetTestAssemblies() { var testAssemblies = new List { - GetAssetFullPath("SimpleTestProject.dll"), - GetAssetFullPath("SimpleTestProject2.dll") + GetTestDll("SimpleTestProject.dll"), + GetTestDll("SimpleTestProject2.dll") }; return testAssemblies; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs index ea4c38ae0f..bfc394614c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs @@ -44,11 +44,11 @@ public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo) _runEventHandler = new RunEventHandler(); var vstestConsoleWrapper = GetVsTestConsoleWrapper(); - var sources = new List { GetAssetFullPath("SimpleTestProject.dll") }; + var sources = new List { GetTestDll("SimpleTestProject.dll") }; vstestConsoleWrapper.RunTests( sources, - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), new TestPlatformOptions() { TestCaseFilter = "FullyQualifiedName=SampleUnitTestProject.UnitTest1.PassingTest" }, _runEventHandler); @@ -65,11 +65,11 @@ public void RunTestsWithFastFilter(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - var sources = new List { GetAssetFullPath("SimpleTestProject.dll") }; + var sources = new List { GetTestDll("SimpleTestProject.dll") }; _vstestConsoleWrapper.RunTests( sources, - GetDefaultRunSettings(), + GetRunSettingsWithCurrentTargetFramework(), new TestPlatformOptions() { TestCaseFilter = "FullyQualifiedName=SampleUnitTestProject.UnitTest1.PassingTest | FullyQualifiedName=SampleUnitTestProject.UnitTest1.FailingTest" }, _runEventHandler); diff --git a/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/DiaSessionTests.cs b/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/DiaSessionTests.cs index 4925635fe3..c7880824e9 100644 --- a/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/DiaSessionTests.cs +++ b/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/DiaSessionTests.cs @@ -34,7 +34,7 @@ public static string GetAndSetTargetFrameWork(IntegrationTestEnvironment testEnv public void GetNavigationDataShouldReturnCorrectFileNameAndLineNumber() { var currentTargetFrameWork = GetAndSetTargetFrameWork(_testEnvironment); - var assemblyPath = GetAssetFullPath("SimpleClassLibrary.dll"); + var assemblyPath = GetTestDll("SimpleClassLibrary.dll"); var diaSession = new DiaSession(assemblyPath); DiaNavigationData diaNavigationData = diaSession.GetNavigationData("SimpleClassLibrary.Class1", "PassingTest"); @@ -52,7 +52,7 @@ public void GetNavigationDataShouldReturnCorrectFileNameAndLineNumber() public void GetNavigationDataShouldReturnCorrectDataForAsyncMethod() { var currentTargetFrameWork = GetAndSetTargetFrameWork(_testEnvironment); - var assemblyPath = GetAssetFullPath("SimpleClassLibrary.dll"); + var assemblyPath = GetTestDll("SimpleClassLibrary.dll"); var diaSession = new DiaSession(assemblyPath); DiaNavigationData diaNavigationData = diaSession.GetNavigationData("SimpleClassLibrary.Class1+d__1", "MoveNext"); @@ -70,7 +70,7 @@ public void GetNavigationDataShouldReturnCorrectDataForAsyncMethod() public void GetNavigationDataShouldReturnCorrectDataForOverLoadedMethod() { var currentTargetFrameWork = GetAndSetTargetFrameWork(_testEnvironment); - var assemblyPath = GetAssetFullPath("SimpleClassLibrary.dll"); + var assemblyPath = GetTestDll("SimpleClassLibrary.dll"); var diaSession = new DiaSession(assemblyPath); DiaNavigationData diaNavigationData = diaSession.GetNavigationData("SimpleClassLibrary.Class1", "OverLoadedMethod"); @@ -89,7 +89,7 @@ public void GetNavigationDataShouldReturnCorrectDataForOverLoadedMethod() public void GetNavigationDataShouldReturnNullForNotExistMethodNameOrNotExistTypeName() { var currentTargetFrameWork = GetAndSetTargetFrameWork(_testEnvironment); - var assemblyPath = GetAssetFullPath("SimpleClassLibrary.dll"); + var assemblyPath = GetTestDll("SimpleClassLibrary.dll"); var diaSession = new DiaSession(assemblyPath); @@ -108,7 +108,7 @@ public void GetNavigationDataShouldReturnNullForNotExistMethodNameOrNotExistType public void DiaSessionPerfTest() { var currentTargetFrameWork = GetAndSetTargetFrameWork(_testEnvironment); - var assemblyPath = GetAssetFullPath("SimpleClassLibrary.dll"); + var assemblyPath = GetTestDll("SimpleClassLibrary.dll"); var watch = Stopwatch.StartNew(); var diaSession = new DiaSession(assemblyPath); diff --git a/test/Microsoft.TestPlatform.PerformanceTests/PerformanceTests.cs b/test/Microsoft.TestPlatform.PerformanceTests/PerformanceTests.cs index 7db4ae1e30..f87afc7f58 100644 --- a/test/Microsoft.TestPlatform.PerformanceTests/PerformanceTests.cs +++ b/test/Microsoft.TestPlatform.PerformanceTests/PerformanceTests.cs @@ -18,7 +18,7 @@ public class PerformanceTests : PerformanceTestBase [TestMethod] public void ExecutionPerformanceTest() { - RunExecutionPerformanceTests(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty); + RunExecutionPerformanceTests(GetSampleTestDll(), GetTestAdapterPath(), string.Empty); ValidateSummaryStatus(1, 1, 1); ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); @@ -35,7 +35,7 @@ public void ExecutionPerformanceTest() [TestMethod] public void DiscoveryPerformanceTest() { - RunDiscoveryPerformanceTests(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty); + RunDiscoveryPerformanceTests(GetSampleTestDll(), GetTestAdapterPath(), string.Empty); ValidateDiscoveredTests( "SampleUnitTestProject.UnitTest1.PassingTest", @@ -52,7 +52,7 @@ public void DiscoveryPerformanceTest() [TestMethod] public void VsTestConsolePerformanceTest() { - RunExecutionPerformanceTests(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty); + RunExecutionPerformanceTests(GetSampleTestDll(), GetTestAdapterPath(), string.Empty); ValidateSummaryStatus(1, 1, 1); ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); @@ -69,7 +69,7 @@ public void VsTestConsolePerformanceTest() [TestMethod] public void TestHostPerformanceTest() { - RunExecutionPerformanceTests(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty); + RunExecutionPerformanceTests(GetSampleTestDll(), GetTestAdapterPath(), string.Empty); ValidateSummaryStatus(1, 1, 1); ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); @@ -86,7 +86,7 @@ public void TestHostPerformanceTest() [TestMethod] public void MsTestV2AdapterPerformanceTest() { - RunExecutionPerformanceTests(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty); + RunExecutionPerformanceTests(GetSampleTestDll(), GetTestAdapterPath(), string.Empty); ValidateSummaryStatus(1, 1, 1); ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); diff --git a/test/Microsoft.TestPlatform.SmokeTests/DiscoveryTests.cs b/test/Microsoft.TestPlatform.SmokeTests/DiscoveryTests.cs index c74c041202..83a3a2b71d 100644 --- a/test/Microsoft.TestPlatform.SmokeTests/DiscoveryTests.cs +++ b/test/Microsoft.TestPlatform.SmokeTests/DiscoveryTests.cs @@ -15,7 +15,7 @@ public class DiscoveryTests : IntegrationTestBase [TestMethod] public void DiscoverAllTests() { - InvokeVsTestForDiscovery(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, ".NETFramework,Version=v4.5.1"); + InvokeVsTestForDiscovery(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, ".NETFramework,Version=v4.5.1"); var listOfTests = new string[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", "SampleUnitTestProject.UnitTest1.SkippingTest" }; ValidateDiscoveredTests(listOfTests); } diff --git a/test/Microsoft.TestPlatform.SmokeTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.SmokeTests/ExecutionTests.cs index a12a33d98b..b74d83b531 100644 --- a/test/Microsoft.TestPlatform.SmokeTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.SmokeTests/ExecutionTests.cs @@ -15,7 +15,7 @@ public class ExecutionTests : IntegrationTestBase [TestMethod] public void RunAllTestExecution() { - InvokeVsTestForExecution(GetSampleTestAssembly(), GetTestAdapterPath(), ".NETFramework,Version=v4.5.1"); + InvokeVsTestForExecution(GetSampleTestDll(), GetTestAdapterPath(), ".NETFramework,Version=v4.5.1"); ValidateSummaryStatus(1, 1, 1); ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); ValidateFailedTests("SampleUnitTestProject.UnitTest1.FailingTest"); @@ -26,7 +26,7 @@ public void RunAllTestExecution() public void RunSelectedTests() { using var resultsDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, ".NETFramework,Version=v4.5.1", resultsDirectory: resultsDir.Path); + var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, ".NETFramework,Version=v4.5.1", resultsDirectory: resultsDir.Path); arguments = string.Concat(arguments, " /Tests:PassingTest"); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); diff --git a/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs index 3869ccb2ea..995855ca1d 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs @@ -6,10 +6,13 @@ namespace Microsoft.TestPlatform.TestUtilities; [Serializable] -public class DebugInfo +// For data source to serialize correctly to enable splitting testcases to one per test in VS, +// this must be serializable. This is sealed because the exact type must be used, not any child type. +// Otherwise it works, but silently does not split the test cases anymore. +public sealed class DebugInfo { public bool DebugVSTestConsole { get; set; } - public bool DebugTesthost { get; set; } + public bool DebugTestHost { get; set; } public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; } diff --git a/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs index 8299661238..b5d0590b65 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/DllInfo.cs @@ -5,25 +5,18 @@ namespace Microsoft.TestPlatform.TestUtilities; +// For data source to serialize correctly to enable splitting testcases to one per test in VS, +// this must be serializable. This is NOT sealed because we need this for adapters and testSdk. +// But be aware that the exact type must be used, not any child type for the data on data source object (RunnerInfo). +// Otherwise it works, but silently does not split the test cases anymore. [Serializable] -public abstract class DllInfo +public class DllInfo { - protected DllInfo(string name, string propertyName, string versionType, string? version, string path) - { - Name = name; - PropertyName = propertyName; - VersionType = versionType; - // Version can be null when we fail to find the respective propertin in TestPlatform.Dependencies.props - // when that happens we throw when we try to update the path. - Version = version; - Path = path; - } - - public string Name { get; } - public string PropertyName { get; } - public string VersionType { get; } - public string? Version { get; } - public string Path { get; } + public string? Name { get; set; } + public string? PropertyName { get; set; } + public string? VersionType { get; set; } + public string? Version { get; set; } + public string? Path { get; set; } public override string ToString() => $" {Name} = {Version} [{VersionType}]"; diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index dbafe748fb..16d5d9e0bc 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -224,7 +224,7 @@ public void InvokeVsTestForExecution(string testAssembly, { var arguments = PrepareArguments(testAssembly, testAdapterPath, runSettings, framework, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); - if (_testEnvironment.DebugInfo.DebugVSTestConsole || _testEnvironment.DebugInfo.DebugTesthost || _testEnvironment.DebugInfo.DebugDataCollector) + if (_testEnvironment.DebugInfo.DebugVSTestConsole || _testEnvironment.DebugInfo.DebugTestHost || _testEnvironment.DebugInfo.DebugDataCollector) { environmentVariables ??= new Dictionary(); @@ -233,7 +233,7 @@ public void InvokeVsTestForExecution(string testAssembly, environmentVariables.Add("VSTEST_RUNNER_DEBUG_ATTACHVS", "1"); } - if (_testEnvironment.DebugInfo.DebugTesthost) + if (_testEnvironment.DebugInfo.DebugTestHost) { environmentVariables.Add("VSTEST_HOST_DEBUG_ATTACHVS", "1"); } @@ -522,27 +522,27 @@ public void ValidateFullyQualifiedDiscoveredTests(string filePath, params string } } - protected string GetSampleTestAssembly() + protected string GetSampleTestDll() { - return GetAssetFullPath("SimpleTestProject.dll"); + return GetTestDll("SimpleTestProject.dll"); } - protected string GetAssetFullPath(string assetName) + protected string GetTestDll(string assetName) { return _testEnvironment.GetTestAsset(assetName); } - protected string GetAssetFullPath(string assetName, string targetFramework) + protected string GetTestDllForFramework(string assetName, string targetFramework) { return _testEnvironment.GetTestAsset(assetName, targetFramework); } - protected List GetAssetFullPath(DllInfo dllInfo, params string[] assetNames) + protected List GetTestDlls(params string[] assetNames) { var assets = new List(); foreach (var assetName in assetNames) { - assets.Add(dllInfo.UpdatePath(GetAssetFullPath(assetName))); + assets.Add(GetTestDll(assetName)); } return assets; @@ -674,7 +674,7 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper() Console.WriteLine($"Console runner path: {consoleRunnerPath}"); VsTestConsoleWrapper vstestConsoleWrapper; - if (_testEnvironment.DebugInfo.DebugVSTestConsole || _testEnvironment.DebugInfo.DebugTesthost || _testEnvironment.DebugInfo.DebugDataCollector) + if (_testEnvironment.DebugInfo.DebugVSTestConsole || _testEnvironment.DebugInfo.DebugTestHost || _testEnvironment.DebugInfo.DebugDataCollector) { var environmentVariables = new Dictionary(); Environment.GetEnvironmentVariables().OfType().ToList().ForEach(e => environmentVariables.Add(e.Key.ToString(), e.Value.ToString())); @@ -684,7 +684,7 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper() environmentVariables.Add("VSTEST_RUNNER_DEBUG_ATTACHVS", "1"); } - if (_testEnvironment.DebugInfo.DebugTesthost) + if (_testEnvironment.DebugInfo.DebugTestHost) { environmentVariables.Add("VSTEST_HOST_DEBUG_ATTACHVS", "1"); } @@ -906,24 +906,7 @@ public static void CreateRunSettingsFile(string destinationRunsettingsPath, stri protected string BuildMultipleAssemblyPath(params string[] assetNames) { - var assetFullPaths = new string[assetNames.Length]; - for (var i = 0; i < assetNames.Length; i++) - { - var path = GetAssetFullPath(assetNames[i]); - if (_testEnvironment.DllInfos.Count > 0) - { - foreach (var dllInfo in _testEnvironment.DllInfos) - { - path = dllInfo.UpdatePath(path); - } - - Assert.IsTrue(File.Exists(path), "GetTestAsset: Path not found: \"{0}\". Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", path); - } - - assetFullPaths[i] = path.AddDoubleQuote(); - } - - return string.Join(" ", assetFullPaths); + return string.Join(" ", GetTestDlls(assetNames)); } protected static string GetDiagArg(string rootDir) diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs index b0af339915..01c3069ee1 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestEnvironment.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq.Expressions; using System.Xml; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -196,7 +195,7 @@ public string RunnerFramework public static bool IsCI { get; } = Environment.GetEnvironmentVariable("TF_BUILD") == "True"; public DebugInfo DebugInfo { get; set; } public VSTestConsoleInfo VSTestConsoleInfo { get; set; } - public List DllInfos { get; set; } + public List DllInfos { get; set; } = new(); /// /// Gets the full path to a test asset. @@ -239,6 +238,15 @@ public string GetTestAsset(string assetName, string targetFramework) targetFramework, assetName); + // Update the path to be taken from the compatibility matrix instead of from the root folder. + if (DllInfos.Count > 0) + { + foreach (var dllInfo in DllInfos) + { + assetPath = dllInfo.UpdatePath(assetPath); + } + } + Assert.IsTrue(File.Exists(assetPath), "GetTestAsset: Path not found: \"{0}\". Most likely you need to build using build.cmd -s PrepareAcceptanceTests.", assetPath); // If you are thinking about wrapping the path in double quotes here, diff --git a/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs deleted file mode 100644 index 6529493ab0..0000000000 --- a/test/Microsoft.TestPlatform.TestUtilities/MSTestInfo.cs +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System; - -namespace Microsoft.TestPlatform.TestUtilities; - -[Serializable] -public class MSTestInfo : DllInfo -{ - public MSTestInfo(string versionType, string? version, string path) - : base(name: "MSTest", propertyName: "MSTest", versionType, version, path) - { - } -} diff --git a/test/Microsoft.TestPlatform.TestUtilities/TesthostInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/NetTestSdkInfo.cs similarity index 64% rename from test/Microsoft.TestPlatform.TestUtilities/TesthostInfo.cs rename to test/Microsoft.TestPlatform.TestUtilities/NetTestSdkInfo.cs index a9be09a3d8..c16d2a3f44 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/TesthostInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/NetTestSdkInfo.cs @@ -8,8 +8,9 @@ namespace Microsoft.TestPlatform.TestUtilities; [Serializable] public class NetTestSdkInfo : DllInfo { - public NetTestSdkInfo(string versionType, string? version, string path) - : base(name: "Testhost", propertyName: "VSTestConsole", versionType, version, path) + public NetTestSdkInfo() { + Name = "Testhost"; + PropertyName = "VSTestConsole"; } } diff --git a/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs index 41d441e3b8..e55503afbf 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs @@ -8,18 +8,9 @@ namespace Microsoft.TestPlatform.TestUtilities; [Serializable] public class VSTestConsoleInfo { - public VSTestConsoleInfo(string versionType, string? version, string path) - { - VersionType = versionType; - // Version can be null when we fail to find the respective propertin in TestPlatform.Dependencies.props - // when that happens we throw when we try to update the path. - Version = version; - Path = path; - } - - public string VersionType { get; } - public string? Version { get; } - public string Path { get; } + public string? VersionType { get; set; } + public string? Version { get; set; } + public string? Path { get; set; } public override string ToString() => $" vstest.console = {Version} [{VersionType}]"; } From 1a45042ba7ea4a0b6b5e0de81f79145eb8cf9dff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 21 Mar 2022 16:19:17 +0100 Subject: [PATCH 39/64] merge --- .../TranslationLayerTests/DiscoverTests.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index 00a8805769..e7db8258ce 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -208,8 +208,8 @@ public async Task CancelTestDiscovery(RunnerInfo runnerInfo) // Setup var testAssemblies = new List { - GetAssetFullPath("DiscoveryTestProject.dll"), - GetAssetFullPath("SimpleTestProject.dll"), + GetTestDll("DiscoveryTestProject.dll"), + GetTestDll("SimpleTestProject.dll"), }; SetTestEnvironment(_testEnvironment, runnerInfo); @@ -235,7 +235,7 @@ public async Task CancelTestDiscovery(RunnerInfo runnerInfo) }); // Act - await Task.Run(() => _vstestConsoleWrapper.DiscoverTests(testAssemblies, GetDefaultRunSettings(), discoveryEvents.Object)); + await Task.Run(() => _vstestConsoleWrapper.DiscoverTests(testAssemblies, GetRunSettingsWithCurrentTargetFramework(), discoveryEvents.Object)); // Assert. Assert.IsTrue(isTestCancelled); From 6e841581ee31ec6b505ebd8b1edc19b14d88c97f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 21 Mar 2022 16:31:42 +0100 Subject: [PATCH 40/64] Revert unnecessary renames --- .../AcceptanceTestBase.cs | 2 +- .../AppDomainTests.cs | 2 +- .../ArgumentProcessorTests.cs | 2 +- .../BlameDataCollectorTests.cs | 14 +++++------ .../CodeCoverageTests.cs | 2 +- .../DataCollectionTests.cs | 2 +- .../DeprecateExtensionsPathWarningTests.cs | 2 +- .../DifferentTestFrameworkSimpleTests.cs | 2 +- .../DiscoveryTests.cs | 6 ++--- .../FilePatternParserTests.cs | 6 ++--- .../FrameworkTests.cs | 8 +++---- .../LoggerTests.cs | 24 +++++++++---------- .../PlatformTests.cs | 2 +- .../PortableNugetPackageTests.cs | 2 +- .../ResultsDirectoryTests.cs | 4 ++-- .../RunsettingsTests.cs | 16 ++++++------- .../TelemetryTests.cs | 4 ++-- .../TestCaseFilterTests.cs | 14 +++++------ .../CodeCoverageTests.cs | 2 +- .../CustomTestHostTests.cs | 8 +++---- .../DataCollectorAttachmentProcessor.cs | 2 +- .../DifferentTestFrameworkSimpleTests.cs | 8 +++---- .../TranslationLayerTests/DiscoverTests.cs | 20 ++++++++-------- .../LiveUnitTestingTests.cs | 4 ++-- .../TranslationLayerTests/RunSelectedTests.cs | 12 +++++----- .../TranslationLayerTests/RunTests.cs | 20 ++++++++-------- ...RunTestsWithDifferentConfigurationTests.cs | 12 +++++----- .../RunTestsWithFilterTests.cs | 8 +++---- .../DiaSessionTests.cs | 10 ++++---- .../PerformanceTests.cs | 10 ++++---- .../DiscoveryTests.cs | 2 +- .../ExecutionTests.cs | 4 ++-- .../IntegrationTestBase.cs | 8 +++---- 33 files changed, 122 insertions(+), 122 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs index bd7fe5afcd..934b7de8e6 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs @@ -124,7 +124,7 @@ public string GetEmptyRunsettings() /// /// Almost empty runsettings, just specifying the target framework from the currently set test environment. /// - public string GetRunSettingsWithCurrentTargetFramework() + public string GetDefaultRunSettings() { return GetRunSettingsWithTargetFramework(FrameworkArgValue); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs index a413e8d7d6..17e8f88423 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs @@ -38,7 +38,7 @@ public void RunTestExecutionWithDisableAppDomain(RunnerInfo runnerInfo) var runsettingsFilePath = GetInProcDataCollectionRunsettingsFile(true, TempDirectory); var arguments = PrepareArguments( - GetSampleTestDll(), + GetSampleTestAssembly(), GetTestAdapterPath(), runsettingsFilePath, FrameworkArgValue, diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs index 8b9dd95d40..44a4a39dbd 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs @@ -40,7 +40,7 @@ public void PassingInvalidArgumentsToVsTestConsoleShouldNotPrintHelpMessage(Runn { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /badArgument"); InvokeVsTest(arguments); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs index 8df3e8bc6a..e9e6af25ee 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs @@ -46,7 +46,7 @@ public BlameDataCollectorTests() public void BlameDataCollectorShouldGiveCorrectTestCaseName(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetTestDll("BlameUnitTestProject.dll"); + var assemblyPaths = GetAssetFullPath("BlameUnitTestProject.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $" /Blame"); arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); @@ -137,7 +137,7 @@ public void BlameDataCollectorShouldOutputDumpFileWhenNoCrashOccursButCollectAlw public void HangDumpOnTimeout(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetTestDll("timeout.dll"); + var assemblyPaths = GetAssetFullPath("timeout.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectHangDump;HangDumpType=full;TestTimeout=3s"""); @@ -160,7 +160,7 @@ public void HangDumpOnTimeout(RunnerInfo runnerInfo) public void CrashDumpWhenThereIsNoTimeout(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetTestDll("timeout.dll"); + var assemblyPaths = GetAssetFullPath("timeout.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full;CollectAlways=true;CollectHangDump"""); @@ -183,7 +183,7 @@ public void CrashDumpWhenThereIsNoTimeout(RunnerInfo runnerInfo) public void CrashDumpOnExit(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetTestDll("timeout.dll"); + var assemblyPaths = GetAssetFullPath("timeout.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full;CollectAlways=true"""); @@ -204,7 +204,7 @@ public void CrashDumpOnExit(RunnerInfo runnerInfo) public void CrashDumpOnStackOverflow(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetTestDll("crash.dll"); + var assemblyPaths = GetAssetFullPath("crash.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full"""); @@ -225,7 +225,7 @@ public void CrashDumpOnStackOverflow(RunnerInfo runnerInfo) public void CrashDumpChildProcesses(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetTestDll("child-crash.dll"); + var assemblyPaths = GetAssetFullPath("child-crash.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full"""); InvokeVsTest(arguments); @@ -240,7 +240,7 @@ public void CrashDumpChildProcesses(RunnerInfo runnerInfo) public void HangDumpChildProcesses(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = GetTestDll("child-hang.dll"); + var assemblyPaths = GetAssetFullPath("child-hang.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); arguments = string.Concat(arguments, $@" /Blame:""CollectHangDump;HangDumpType=full;TestTimeout=15s"""); InvokeVsTest(arguments); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs index c7901f2326..b426fc7fd1 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs @@ -251,7 +251,7 @@ private string CreateArguments( TestParameters testParameters, out string trxFilePath) { - var assemblyPaths = GetTestDll(testParameters.AssemblyName); + var assemblyPaths = GetAssetFullPath(testParameters.AssemblyName); string traceDataCollectorDir = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory, "artifacts", IntegrationTestEnvironment.BuildConfiguration, "Microsoft.CodeCoverage"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs index 1308f4345a..a600a7bf0c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DataCollectionTests.cs @@ -100,7 +100,7 @@ public void DataCollectorAssemblyLoadingShouldNotThrowErrorForFullFramework(Runn { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetTestDll("AppDomainGetAssembliesTestProject.dll"), string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetAssetFullPath("AppDomainGetAssembliesTestProject.dll"), string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs index d33857347a..4c10549775 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DeprecateExtensionsPathWarningTests.cs @@ -61,7 +61,7 @@ public void CopyAdapterToExtensions() [TestMethod] public void VerifyDeprecatedWarningIsThrownWhenAdaptersPickedFromExtensionDirectory() { - var arguments = PrepareArguments(GetSampleTestDll(), null, null, FrameworkArgValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), null, null, FrameworkArgValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); StdOutputContains("Adapter lookup is being changed, please follow"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs index 10d59d084b..70de29e59c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DifferentTestFrameworkSimpleTests.cs @@ -155,7 +155,7 @@ public void NUnitRunAllTestExecution(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetTestDll("NUTestProject.dll"), + GetAssetFullPath("NUTestProject.dll"), GetTestAdapterPath(UnitTestFramework.NUnit), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs index 444cc873ec..fa9bb8627b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DiscoveryTests.cs @@ -25,7 +25,7 @@ public void DiscoverAllTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - InvokeVsTestForDiscovery(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue); + InvokeVsTestForDiscovery(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue); var listOfTests = new[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", "SampleUnitTestProject.UnitTest1.SkippingTest" }; ValidateDiscoveredTests(listOfTests); @@ -65,7 +65,7 @@ public void DiscoverFullyQualifiedTests(RunnerInfo runnerInfo) var listOfTests = new[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", "SampleUnitTestProject.UnitTest1.SkippingTest" }; - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /ListFullyQualifiedTests", " /ListTestsTargetPath:\"" + dummyFilePath + "\""); InvokeVsTest(arguments); @@ -80,7 +80,7 @@ public void DiscoverTestsShouldShowProperWarningIfNoTestsOnTestCaseFilter(Runner { SetTestEnvironment(_testEnvironment, runnerInfo); - var assetFullPath = GetTestDll("SimpleTestProject2.dll"); + var assetFullPath = GetAssetFullPath("SimpleTestProject2.dll"); var arguments = PrepareArguments(assetFullPath, GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /listtests"); arguments = string.Concat(arguments, " /testcasefilter:NonExistTestCaseName"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs index 4e31163543..3d8f36218b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/FilePatternParserTests.cs @@ -20,7 +20,7 @@ public void WildCardPatternShouldCorrectlyWorkOnFiles(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssembly = GetSampleTestDll(); + var testAssembly = GetSampleTestAssembly(); testAssembly = testAssembly.Replace("SimpleTestProject.dll", "*TestProj*.dll"); var arguments = PrepareArguments( @@ -40,7 +40,7 @@ public void WildCardPatternShouldCorrectlyWorkOnArbitraryDepthDirectories(Runner { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssembly = GetSampleTestDll(); + var testAssembly = GetSampleTestAssembly(); // Add one more directory to the temp path, so we can substitute it with ** // and copy then whole directory there. @@ -66,7 +66,7 @@ public void WildCardPatternShouldCorrectlyWorkForRelativeAssemblyPath(RunnerInfo { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssembly = GetSampleTestDll(); + var testAssembly = GetSampleTestAssembly(); testAssembly = testAssembly.Replace("SimpleTestProject.dll", "*TestProj*.dll"); var wildCardIndex = testAssembly.IndexOfAny(new char[] { '*' }); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs index 9a00c24cb8..394db87a80 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/FrameworkTests.cs @@ -19,7 +19,7 @@ public void FrameworkArgumentShouldWork(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " ", $"/Framework:{FrameworkArgValue}"); InvokeVsTest(arguments); @@ -33,7 +33,7 @@ public void FrameworkShortNameArgumentShouldWork(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " ", $"/Framework:{_testEnvironment.TargetFramework}"); InvokeVsTest(arguments); @@ -49,7 +49,7 @@ public void OnWrongFrameworkPassedTestRunShouldNotRun(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); if (runnerInfo.TargetFramework.Contains("netcore")) { arguments = string.Concat(arguments, " ", "/Framework:Framework45"); @@ -80,7 +80,7 @@ public void RunSpecificTestsShouldWorkWithFrameworkInCompatibleWarning(RunnerInf { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " ", "/tests:PassingTest"); arguments = string.Concat(arguments, " ", "/Framework:Framework40"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs index 8e79cc7589..e9e30d8fdc 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs @@ -23,12 +23,12 @@ public void TrxLoggerWithFriendlyNameShouldProperlyOverwriteFile(RunnerInfo runn { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var trxFileName = "TestResults.trx"; arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); @@ -44,12 +44,12 @@ public void HtmlLoggerWithFriendlyNameShouldProperlyOverwriteFile(RunnerInfo run { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var htmlFileName = "TestResults.html"; arguments = string.Concat(arguments, $" /logger:\"html;LogFileName={htmlFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"html;LogFileName={htmlFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); @@ -64,12 +64,12 @@ public void TrxLoggerWithExecutorUriShouldProperlyOverwriteFile(RunnerInfo runne { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var trxFileName = "TestResults.trx"; arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFileName={trxFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFileName={trxFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); @@ -86,11 +86,11 @@ public void TrxLoggerWithLogFilePrefixShouldGenerateMultipleTrx(RunnerInfo runne SetTestEnvironment(_testEnvironment, runnerInfo); var trxFileNamePattern = "TestResults"; - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFilePrefix={trxFileNamePattern}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/TrxLogger/v1;LogFilePrefix={trxFileNamePattern}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); @@ -105,12 +105,12 @@ public void HtmlLoggerWithExecutorUriShouldProperlyOverwriteFile(RunnerInfo runn { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); var htmlFileName = "TestResults.html"; arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/htmlLogger/v1;LogFileName{htmlFileName}\""); InvokeVsTest(arguments); - arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); + arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, TempDirectory.Path); arguments = string.Concat(arguments, $" /logger:\"logger://Microsoft/TestPlatform/htmlLogger/v1;LogFileName={htmlFileName}\""); arguments = string.Concat(arguments, " /testcasefilter:Name~Pass"); InvokeVsTest(arguments); @@ -126,7 +126,7 @@ public void TrxLoggerResultSummaryOutcomeValueShouldBeFailedIfNoTestsExecutedAnd { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); var trxFilePath = Path.Combine(TempDirectory.Path, "TrxLogger.trx"); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFilePath}\""); @@ -149,7 +149,7 @@ public void TrxLoggerResultSummaryOutcomeValueShouldNotChangeIfNoTestsExecutedAn { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); var trxFilePath = Path.Combine(TempDirectory.Path, "TrxLogger.trx"); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFilePath}\""); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs index 289363a269..b149fa2994 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/PlatformTests.cs @@ -45,7 +45,7 @@ private void RunTestExecutionWithPlatform(string platformArg, string testhostPro { var arguments = PrepareArguments( - GetSampleTestDll(), + GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs index be24f8ff38..d5f227fccf 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/PortableNugetPackageTests.cs @@ -58,7 +58,7 @@ public void DiscoverAllTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - InvokeVsTestForDiscovery(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue); + InvokeVsTestForDiscovery(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue); var listOfTests = new[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", "SampleUnitTestProject.UnitTest1.SkippingTest" }; ValidateDiscoveredTests(listOfTests); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs index 4b6d60ca87..a7b02265a7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ResultsDirectoryTests.cs @@ -20,7 +20,7 @@ public void TrxFileShouldBeCreatedInResultsDirectory(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); var trxFileName = "TestResults.trx"; var trxFilePath = Path.Combine(TempDirectory.Path, trxFileName); arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); @@ -41,7 +41,7 @@ public void ResultsDirectoryRelativePathShouldWork(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue); var trxFileName = "TestResults.trx"; var relativeDirectory = @"relative\directory"; var resultsDirectory = Path.Combine(Directory.GetCurrentDirectory(), relativeDirectory); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs index d1c8084875..3d5dbcbea7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/RunsettingsTests.cs @@ -246,7 +246,7 @@ public void RunSettingsWithInvalidValueShouldLogError(RunnerInfo runnerInfo) }; var runsettingsFilePath = GetRunsettingsFilePath(runConfigurationDictionary, TempDirectory); var arguments = PrepareArguments( - GetSampleTestDll(), + GetSampleTestAssembly(), string.Empty, runsettingsFilePath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -267,7 +267,7 @@ public void TestAdapterPathFromRunSettings(RunnerInfo runnerInfo) }; var runsettingsFilePath = GetRunsettingsFilePath(runConfigurationDictionary, TempDirectory); var arguments = PrepareArguments( - GetSampleTestDll(), + GetSampleTestAssembly(), string.Empty, runsettingsFilePath, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -284,7 +284,7 @@ public void LegacySettingsWithPlatform(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetTestDll("LegacySettingsUnitTestProject.dll"); + var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); _ = Path.GetDirectoryName(testAssemblyPath); var runsettingsXml = @" @@ -315,7 +315,7 @@ public void LegacySettingsWithScripts(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetTestDll("LegacySettingsUnitTestProject.dll"); + var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); // Create the script files var guid = Guid.NewGuid(); @@ -361,7 +361,7 @@ public void LegacySettingsWithDeploymentItem(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetTestDll("LegacySettingsUnitTestProject.dll"); + var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); var testAssemblyDirectory = Path.GetDirectoryName(testAssemblyPath); var deploymentItem = Path.Combine(testAssemblyDirectory, "Deployment", "DeploymentFile.xml"); @@ -397,7 +397,7 @@ public void LegacySettingsTestTimeout(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetTestDll("LegacySettingsUnitTestProject.dll"); + var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); var runsettingsXml = @" true @@ -424,7 +424,7 @@ public void LegacySettingsAssemblyResolution(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetTestDll("LegacySettingsUnitTestProject.dll"); + var testAssemblyPath = GetAssetFullPath("LegacySettingsUnitTestProject.dll"); var runsettingsFormat = @" true @@ -466,7 +466,7 @@ public void EnvironmentVariablesSettingsShouldSetEnvironmentVariables(RunnerInfo { SetTestEnvironment(_testEnvironment, runnerInfo); - var testAssemblyPath = GetTestDll("EnvironmentVariablesTestProject.dll"); + var testAssemblyPath = GetAssetFullPath("EnvironmentVariablesTestProject.dll"); var runsettingsXml = @" diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs index 1b519c1d9f..fdf49097ce 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TelemetryTests.cs @@ -49,7 +49,7 @@ private void RunTests(RunnerInfo runnerInfo) return; } - var assemblyPaths = GetTestDll("SimpleTestProject2.dll"); + var assemblyPaths = GetAssetFullPath("SimpleTestProject2.dll"); var env = new Dictionary { @@ -70,7 +70,7 @@ private void DiscoverTests(RunnerInfo runnerInfo) return; } - var assemblyPaths = GetTestDll("SimpleTestProject2.dll"); + var assemblyPaths = GetAssetFullPath("SimpleTestProject2.dll"); var env = new Dictionary { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs index d235068166..c4363cf825 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TestCaseFilterTests.cs @@ -21,7 +21,7 @@ public void RunSelectedTestsWithAndOperatorTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestDll(), + GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -38,7 +38,7 @@ public void RunSelectedTestsWithCategoryTraitInMixCase(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestDll(), + GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -55,7 +55,7 @@ public void RunSelectedTestsWithClassNameTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestDll(), + GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -72,7 +72,7 @@ public void RunSelectedTestsWithFullyQualifiedNameTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestDll(), + GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -91,7 +91,7 @@ public void RunSelectedTestsWithNameTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestDll(), + GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -108,7 +108,7 @@ public void RunSelectedTestsWithOrOperatorTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestDll(), + GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); @@ -125,7 +125,7 @@ public void RunSelectedTestsWithPriorityTrait(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var arguments = PrepareArguments( - GetSampleTestDll(), + GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs index a5f5efb6eb..c77faab8ab 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs @@ -437,7 +437,7 @@ public async Task EndSessionShouldEnsureVstestConsoleProcessDies(RunnerInfo runn private IList GetTestAssemblies() { - return GetProjects().Select(p => GetTestDll(p)).ToList(); + return GetProjects().Select(p => GetAssetFullPath(p)).ToList(); } private IList GetProjects() diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs index d3730b7378..f045bf159d 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs @@ -45,7 +45,7 @@ public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvided // Act var customTestHostLauncher = new TestHostLauncherV1(); - _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetRunSettingsWithCurrentTargetFramework(), runEventHandler, customTestHostLauncher); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetDefaultRunSettings(), runEventHandler, customTestHostLauncher); // Assert EnsureTestsRunWithoutErrors(runEventHandler, passed: 2, failed: 2, skipped: 2); @@ -74,7 +74,7 @@ public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvided // Act var customTestHostLauncher = new TestHostLauncherV2(); - _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetRunSettingsWithCurrentTargetFramework(), runEventHandler, customTestHostLauncher); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetDefaultRunSettings(), runEventHandler, customTestHostLauncher); // Assert EnsureTestsRunWithoutErrors(runEventHandler, passed: 2, failed: 2, skipped: 2); @@ -98,7 +98,7 @@ public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvided // Act var customTestHostLauncher = new TestHostLauncherV2(); - _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetRunSettingsWithCurrentTargetFramework(), runEventHandler, customTestHostLauncher); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetDefaultRunSettings(), runEventHandler, customTestHostLauncher); // Assert EnsureTestsRunWithoutErrors(runEventHandler, passed: 2, failed: 2, skipped: 2); @@ -123,7 +123,7 @@ public void RunTestsWithCustomTestHostLauncherUsesLaunchWhenGivenAnOutdatedITest // Act var customTestHostLauncher = new TestHostLauncherV1(); - _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetRunSettingsWithCurrentTargetFramework(), runEventHandler, customTestHostLauncher); + _vstestConsoleWrapper.RunTestsWithCustomTestHost(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetDefaultRunSettings(), runEventHandler, customTestHostLauncher); // Assert EnsureTestsRunWithoutErrors(runEventHandler, passed: 2, failed: 2, skipped: 2); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DataCollectorAttachmentProcessor.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DataCollectorAttachmentProcessor.cs index 4a38f260e7..16708093ba 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DataCollectorAttachmentProcessor.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DataCollectorAttachmentProcessor.cs @@ -124,5 +124,5 @@ private static void CreateDataCollectionRunSettingsFile(string destinationRunset } private IList GetTestAssemblies() - => new List { "SimpleTestProject.dll", "SimpleTestProject2.dll" }.Select(p => GetTestDll(p)).ToList(); + => new List { "SimpleTestProject.dll", "SimpleTestProject2.dll" }.Select(p => GetAssetFullPath(p)).ToList(); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs index af2456f1d8..b08763da05 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs @@ -47,12 +47,12 @@ public void RunTestsWithNunitAdapter(RunnerInfo runnerInfo) var sources = new List { - GetTestDll("NUTestProject.dll") + GetAssetFullPath("NUTestProject.dll") }; _vstestConsoleWrapper.RunTests( sources, - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), _runEventHandler); var testCase = @@ -94,7 +94,7 @@ public void RunTestsWithXunitAdapter(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( sources, - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), _runEventHandler); var testCase = @@ -134,7 +134,7 @@ public void RunTestsWithChutzpahAdapter(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( sources, - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), _runEventHandler); var testCase = diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index e7db8258ce..519c8686ce 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -50,7 +50,7 @@ public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo) _discoveryEventHandler2 = new DiscoveryEventHandler2(); var vstestConsoleWrapper = GetVsTestConsoleWrapper(); - vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), _discoveryEventHandler); + vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), _discoveryEventHandler); // Assert. Assert.AreEqual(6, _discoveryEventHandler.DiscoveredTestCases.Count); @@ -69,7 +69,7 @@ public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedOut(RunnerI var vstestConsoleWrapper = GetVsTestConsoleWrapper(); vstestConsoleWrapper.DiscoverTests( GetTestAssemblies(), - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), new TestPlatformOptions() { CollectMetrics = false }, _discoveryEventHandler2); @@ -86,7 +86,7 @@ public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedIn(RunnerIn SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), new TestPlatformOptions() { CollectMetrics = true }, _discoveryEventHandler2); + _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), new TestPlatformOptions() { CollectMetrics = true }, _discoveryEventHandler2); // Assert. Assert.AreEqual(6, _discoveryEventHandler2.DiscoveredTestCases.Count); @@ -164,7 +164,7 @@ public void DisoverTestUsingEventHandler2ShouldContainAllSourcesAsFullyDiscovere _vstestConsoleWrapper.DiscoverTests( GetTestAssemblies(), - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), null, eventHandler2); @@ -182,7 +182,7 @@ public void DiscoverTestsUsingSourceNavigation(RunnerInfo runnerInfo) _vstestConsoleWrapper.DiscoverTests( GetTestAssemblies(), - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), _discoveryEventHandler); // Assert. @@ -208,8 +208,8 @@ public async Task CancelTestDiscovery(RunnerInfo runnerInfo) // Setup var testAssemblies = new List { - GetTestDll("DiscoveryTestProject.dll"), - GetTestDll("SimpleTestProject.dll"), + GetAssetFullPath("DiscoveryTestProject.dll"), + GetAssetFullPath("SimpleTestProject.dll"), }; SetTestEnvironment(_testEnvironment, runnerInfo); @@ -235,7 +235,7 @@ public async Task CancelTestDiscovery(RunnerInfo runnerInfo) }); // Act - await Task.Run(() => _vstestConsoleWrapper.DiscoverTests(testAssemblies, GetRunSettingsWithCurrentTargetFramework(), discoveryEvents.Object)); + await Task.Run(() => _vstestConsoleWrapper.DiscoverTests(testAssemblies, GetDefaultRunSettings(), discoveryEvents.Object)); // Assert. Assert.IsTrue(isTestCancelled); @@ -247,8 +247,8 @@ private IList GetTestAssemblies() { var testAssemblies = new List { - GetTestDll("SimpleTestProject.dll"), - GetTestDll("SimpleTestProject2.dll") + GetAssetFullPath("SimpleTestProject.dll"), + GetAssetFullPath("SimpleTestProject2.dll") }; return testAssemblies; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs index 01b4367468..0beac2e85c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/LiveUnitTestingTests.cs @@ -90,8 +90,8 @@ private IList GetTestAssemblies() { var testAssemblies = new List { - GetTestDll("SimpleTestProject.dll"), - GetTestDll("SimpleTestProject2.dll") + GetAssetFullPath("SimpleTestProject.dll"), + GetAssetFullPath("SimpleTestProject2.dll") }; return testAssemblies; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs index f2722cc154..73b07981b6 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunSelectedTests.cs @@ -42,10 +42,10 @@ public void RunSelectedTestsWithoutTestPlatformOptions(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), _discoveryEventHandler); + _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), _discoveryEventHandler); var testCases = _discoveryEventHandler.DiscoveredTestCases; - _vstestConsoleWrapper.RunTests(testCases, GetRunSettingsWithCurrentTargetFramework(), _runEventHandler); + _vstestConsoleWrapper.RunTests(testCases, GetDefaultRunSettings(), _runEventHandler); // Assert Assert.AreEqual(6, _runEventHandler.TestResults.Count); @@ -62,12 +62,12 @@ public void RunSelectedTestsWithTestPlatformOptions(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), _discoveryEventHandler); + _vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), _discoveryEventHandler); var testCases = _discoveryEventHandler.DiscoveredTestCases; _vstestConsoleWrapper.RunTests( testCases, - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), new TestPlatformOptions() { CollectMetrics = true }, _runEventHandler); @@ -85,8 +85,8 @@ private IList GetTestAssemblies() { var testAssemblies = new List { - GetTestDll("SimpleTestProject.dll"), - GetTestDll("SimpleTestProject2.dll") + GetAssetFullPath("SimpleTestProject.dll"), + GetAssetFullPath("SimpleTestProject2.dll") }; return testAssemblies; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs index 168120af82..ffe2449e15 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs @@ -47,7 +47,7 @@ public void RunAllTests(RunnerInfo runnerInfo) var vstestConsoleWrapper = GetVsTestConsoleWrapper(); var runEventHandler = new RunEventHandler(); - vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), runEventHandler); + vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetDefaultRunSettings(), runEventHandler); // Assert Assert.AreEqual(6, runEventHandler.TestResults.Count); @@ -67,7 +67,7 @@ public void EndSessionShouldEnsureVstestConsoleProcessDies(RunnerInfo runnerInfo SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - _vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetRunSettingsWithCurrentTargetFramework(), _runEventHandler); + _vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetDefaultRunSettings(), _runEventHandler); _vstestConsoleWrapper?.EndSession(); // Assert @@ -89,7 +89,7 @@ public void RunTestsWithTelemetryOptedIn(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( GetTestAssemblies(), - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), new TestPlatformOptions() { CollectMetrics = true }, _runEventHandler); @@ -113,7 +113,7 @@ public void RunTestsWithTelemetryOptedOut(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( GetTestAssemblies(), - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), new TestPlatformOptions() { CollectMetrics = false }, _runEventHandler); @@ -137,11 +137,11 @@ public void RunTestsShouldThrowOnStackOverflowException(RunnerInfo runnerInfo) return; } - var source = new[] { GetTestDll("SimpleTestProject3.dll") }; + var source = new[] { GetAssetFullPath("SimpleTestProject3.dll") }; _vstestConsoleWrapper.RunTests( source, - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), new TestPlatformOptions() { TestCaseFilter = "ExitWithStackoverFlow" }, _runEventHandler); @@ -162,7 +162,7 @@ public void RunTestsShouldShowProperWarningOnNoTestsForTestCaseFilter(RunnerInfo Setup(); var testAssemblyName = "SimpleTestProject2.dll"; - var source = new List() { GetTestDll(testAssemblyName) }; + var source = new List() { GetAssetFullPath(testAssemblyName) }; var veryLongTestCaseFilter = "FullyQualifiedName=VeryLongTestCaseNameeeeeeeeeeeeee" + @@ -173,7 +173,7 @@ public void RunTestsShouldShowProperWarningOnNoTestsForTestCaseFilter(RunnerInfo _vstestConsoleWrapper.RunTests( source, - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), new TestPlatformOptions() { TestCaseFilter = veryLongTestCaseFilter }, _runEventHandler); @@ -190,8 +190,8 @@ private IList GetTestAssemblies() { return new List { - GetTestDll("SimpleTestProject.dll"), - GetTestDll("SimpleTestProject2.dll") + GetAssetFullPath("SimpleTestProject.dll"), + GetAssetFullPath("SimpleTestProject2.dll") }; } } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs index de916c635d..4f6e641fb2 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithDifferentConfigurationTests.cs @@ -56,7 +56,7 @@ public void RunTestsWithTestAdapterPath(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( GetTestAssemblies(), - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), _runEventHandler); // Assert @@ -115,7 +115,7 @@ public void RunTestsWithTestSettings(RunnerInfo runnerInfo) var runSettings = $"{FrameworkArgValue}" + testsettingsFile + ""; var sources = new List { - GetTestDll("MstestV1UnitTestProject.dll") + GetAssetFullPath("MstestV1UnitTestProject.dll") }; _vstestConsoleWrapper.RunTests( @@ -140,7 +140,7 @@ public void RunTestsWithX64Source(RunnerInfo runnerInfo) var sources = new List { - GetTestDll("SimpleTestProject3.dll") + GetAssetFullPath("SimpleTestProject3.dll") }; @@ -149,7 +149,7 @@ public void RunTestsWithX64Source(RunnerInfo runnerInfo) _vstestConsoleWrapper.RunTests( sources, - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), new TestPlatformOptions() { TestCaseFilter = "FullyQualifiedName = SampleUnitTestProject3.UnitTest1.WorkingDirectoryTest" }, _runEventHandler); @@ -163,8 +163,8 @@ private IList GetTestAssemblies() { var testAssemblies = new List { - GetTestDll("SimpleTestProject.dll"), - GetTestDll("SimpleTestProject2.dll") + GetAssetFullPath("SimpleTestProject.dll"), + GetAssetFullPath("SimpleTestProject2.dll") }; return testAssemblies; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs index bfc394614c..ea4c38ae0f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs @@ -44,11 +44,11 @@ public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo) _runEventHandler = new RunEventHandler(); var vstestConsoleWrapper = GetVsTestConsoleWrapper(); - var sources = new List { GetTestDll("SimpleTestProject.dll") }; + var sources = new List { GetAssetFullPath("SimpleTestProject.dll") }; vstestConsoleWrapper.RunTests( sources, - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), new TestPlatformOptions() { TestCaseFilter = "FullyQualifiedName=SampleUnitTestProject.UnitTest1.PassingTest" }, _runEventHandler); @@ -65,11 +65,11 @@ public void RunTestsWithFastFilter(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - var sources = new List { GetTestDll("SimpleTestProject.dll") }; + var sources = new List { GetAssetFullPath("SimpleTestProject.dll") }; _vstestConsoleWrapper.RunTests( sources, - GetRunSettingsWithCurrentTargetFramework(), + GetDefaultRunSettings(), new TestPlatformOptions() { TestCaseFilter = "FullyQualifiedName=SampleUnitTestProject.UnitTest1.PassingTest | FullyQualifiedName=SampleUnitTestProject.UnitTest1.FailingTest" }, _runEventHandler); diff --git a/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/DiaSessionTests.cs b/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/DiaSessionTests.cs index c7880824e9..4925635fe3 100644 --- a/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/DiaSessionTests.cs +++ b/test/Microsoft.TestPlatform.ObjectModel.PlatformTests/DiaSessionTests.cs @@ -34,7 +34,7 @@ public static string GetAndSetTargetFrameWork(IntegrationTestEnvironment testEnv public void GetNavigationDataShouldReturnCorrectFileNameAndLineNumber() { var currentTargetFrameWork = GetAndSetTargetFrameWork(_testEnvironment); - var assemblyPath = GetTestDll("SimpleClassLibrary.dll"); + var assemblyPath = GetAssetFullPath("SimpleClassLibrary.dll"); var diaSession = new DiaSession(assemblyPath); DiaNavigationData diaNavigationData = diaSession.GetNavigationData("SimpleClassLibrary.Class1", "PassingTest"); @@ -52,7 +52,7 @@ public void GetNavigationDataShouldReturnCorrectFileNameAndLineNumber() public void GetNavigationDataShouldReturnCorrectDataForAsyncMethod() { var currentTargetFrameWork = GetAndSetTargetFrameWork(_testEnvironment); - var assemblyPath = GetTestDll("SimpleClassLibrary.dll"); + var assemblyPath = GetAssetFullPath("SimpleClassLibrary.dll"); var diaSession = new DiaSession(assemblyPath); DiaNavigationData diaNavigationData = diaSession.GetNavigationData("SimpleClassLibrary.Class1+d__1", "MoveNext"); @@ -70,7 +70,7 @@ public void GetNavigationDataShouldReturnCorrectDataForAsyncMethod() public void GetNavigationDataShouldReturnCorrectDataForOverLoadedMethod() { var currentTargetFrameWork = GetAndSetTargetFrameWork(_testEnvironment); - var assemblyPath = GetTestDll("SimpleClassLibrary.dll"); + var assemblyPath = GetAssetFullPath("SimpleClassLibrary.dll"); var diaSession = new DiaSession(assemblyPath); DiaNavigationData diaNavigationData = diaSession.GetNavigationData("SimpleClassLibrary.Class1", "OverLoadedMethod"); @@ -89,7 +89,7 @@ public void GetNavigationDataShouldReturnCorrectDataForOverLoadedMethod() public void GetNavigationDataShouldReturnNullForNotExistMethodNameOrNotExistTypeName() { var currentTargetFrameWork = GetAndSetTargetFrameWork(_testEnvironment); - var assemblyPath = GetTestDll("SimpleClassLibrary.dll"); + var assemblyPath = GetAssetFullPath("SimpleClassLibrary.dll"); var diaSession = new DiaSession(assemblyPath); @@ -108,7 +108,7 @@ public void GetNavigationDataShouldReturnNullForNotExistMethodNameOrNotExistType public void DiaSessionPerfTest() { var currentTargetFrameWork = GetAndSetTargetFrameWork(_testEnvironment); - var assemblyPath = GetTestDll("SimpleClassLibrary.dll"); + var assemblyPath = GetAssetFullPath("SimpleClassLibrary.dll"); var watch = Stopwatch.StartNew(); var diaSession = new DiaSession(assemblyPath); diff --git a/test/Microsoft.TestPlatform.PerformanceTests/PerformanceTests.cs b/test/Microsoft.TestPlatform.PerformanceTests/PerformanceTests.cs index f87afc7f58..7db4ae1e30 100644 --- a/test/Microsoft.TestPlatform.PerformanceTests/PerformanceTests.cs +++ b/test/Microsoft.TestPlatform.PerformanceTests/PerformanceTests.cs @@ -18,7 +18,7 @@ public class PerformanceTests : PerformanceTestBase [TestMethod] public void ExecutionPerformanceTest() { - RunExecutionPerformanceTests(GetSampleTestDll(), GetTestAdapterPath(), string.Empty); + RunExecutionPerformanceTests(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty); ValidateSummaryStatus(1, 1, 1); ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); @@ -35,7 +35,7 @@ public void ExecutionPerformanceTest() [TestMethod] public void DiscoveryPerformanceTest() { - RunDiscoveryPerformanceTests(GetSampleTestDll(), GetTestAdapterPath(), string.Empty); + RunDiscoveryPerformanceTests(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty); ValidateDiscoveredTests( "SampleUnitTestProject.UnitTest1.PassingTest", @@ -52,7 +52,7 @@ public void DiscoveryPerformanceTest() [TestMethod] public void VsTestConsolePerformanceTest() { - RunExecutionPerformanceTests(GetSampleTestDll(), GetTestAdapterPath(), string.Empty); + RunExecutionPerformanceTests(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty); ValidateSummaryStatus(1, 1, 1); ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); @@ -69,7 +69,7 @@ public void VsTestConsolePerformanceTest() [TestMethod] public void TestHostPerformanceTest() { - RunExecutionPerformanceTests(GetSampleTestDll(), GetTestAdapterPath(), string.Empty); + RunExecutionPerformanceTests(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty); ValidateSummaryStatus(1, 1, 1); ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); @@ -86,7 +86,7 @@ public void TestHostPerformanceTest() [TestMethod] public void MsTestV2AdapterPerformanceTest() { - RunExecutionPerformanceTests(GetSampleTestDll(), GetTestAdapterPath(), string.Empty); + RunExecutionPerformanceTests(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty); ValidateSummaryStatus(1, 1, 1); ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); diff --git a/test/Microsoft.TestPlatform.SmokeTests/DiscoveryTests.cs b/test/Microsoft.TestPlatform.SmokeTests/DiscoveryTests.cs index 83a3a2b71d..c74c041202 100644 --- a/test/Microsoft.TestPlatform.SmokeTests/DiscoveryTests.cs +++ b/test/Microsoft.TestPlatform.SmokeTests/DiscoveryTests.cs @@ -15,7 +15,7 @@ public class DiscoveryTests : IntegrationTestBase [TestMethod] public void DiscoverAllTests() { - InvokeVsTestForDiscovery(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, ".NETFramework,Version=v4.5.1"); + InvokeVsTestForDiscovery(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, ".NETFramework,Version=v4.5.1"); var listOfTests = new string[] { "SampleUnitTestProject.UnitTest1.PassingTest", "SampleUnitTestProject.UnitTest1.FailingTest", "SampleUnitTestProject.UnitTest1.SkippingTest" }; ValidateDiscoveredTests(listOfTests); } diff --git a/test/Microsoft.TestPlatform.SmokeTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.SmokeTests/ExecutionTests.cs index b74d83b531..a12a33d98b 100644 --- a/test/Microsoft.TestPlatform.SmokeTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.SmokeTests/ExecutionTests.cs @@ -15,7 +15,7 @@ public class ExecutionTests : IntegrationTestBase [TestMethod] public void RunAllTestExecution() { - InvokeVsTestForExecution(GetSampleTestDll(), GetTestAdapterPath(), ".NETFramework,Version=v4.5.1"); + InvokeVsTestForExecution(GetSampleTestAssembly(), GetTestAdapterPath(), ".NETFramework,Version=v4.5.1"); ValidateSummaryStatus(1, 1, 1); ValidatePassedTests("SampleUnitTestProject.UnitTest1.PassingTest"); ValidateFailedTests("SampleUnitTestProject.UnitTest1.FailingTest"); @@ -26,7 +26,7 @@ public void RunAllTestExecution() public void RunSelectedTests() { using var resultsDir = new TempDirectory(); - var arguments = PrepareArguments(GetSampleTestDll(), GetTestAdapterPath(), string.Empty, ".NETFramework,Version=v4.5.1", resultsDirectory: resultsDir.Path); + var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, ".NETFramework,Version=v4.5.1", resultsDirectory: resultsDir.Path); arguments = string.Concat(arguments, " /Tests:PassingTest"); InvokeVsTest(arguments); ValidateSummaryStatus(1, 0, 0); diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index e68e8fec05..a3084ede30 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -522,12 +522,12 @@ public void ValidateFullyQualifiedDiscoveredTests(string filePath, params string } } - protected string GetSampleTestDll() + protected string GetSampleTestAssembly() { - return GetTestDll("SimpleTestProject.dll"); + return GetAssetFullPath("SimpleTestProject.dll"); } - protected string GetTestDll(string assetName) + protected string GetAssetFullPath(string assetName) { return _testEnvironment.GetTestAsset(assetName); } @@ -542,7 +542,7 @@ protected List GetTestDlls(params string[] assetNames) var assets = new List(); foreach (var assetName in assetNames) { - assets.Add(GetTestDll(assetName)); + assets.Add(GetAssetFullPath(assetName)); } return assets; From 2b3e039f525713af91722d128b8f587bf35d960c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 21 Mar 2022 16:48:05 +0100 Subject: [PATCH 41/64] Ignore strong naming in tests --- playground/TestPlatform.Playground/Program.cs | 3 +-- .../TestPlatform.Playground/TestPlatform.Playground.csproj | 2 +- scripts/build/TestPlatform.Settings.targets | 3 ++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/playground/TestPlatform.Playground/Program.cs b/playground/TestPlatform.Playground/Program.cs index 5d46ab9328..2b1fcb0a0c 100644 --- a/playground/TestPlatform.Playground/Program.cs +++ b/playground/TestPlatform.Playground/Program.cs @@ -55,8 +55,7 @@ static void Main(string[] args) "; var sources = new[] { - @"C:\p\vstest\test\TestAssets\MSTestProject1\bin\NETTestSdkLatest-17.2.0-dev\MSTestLatestPreview-2.2.9-preview-20220210-07\Debug\net451\MSTestProject1.dll", - @"C:\p\vstest\test\TestAssets\MSTestProject2\bin\NETTestSdkLatest-17.2.0-dev\MSTestLatestPreview-2.2.9-preview-20220210-07\Debug\net451\MSTestProject2.dll" + Path.Combine(playground, "MSTest1", "bin", "Debug", "net472", "MSTest1.dll") }; var options = new TestPlatformOptions(); diff --git a/playground/TestPlatform.Playground/TestPlatform.Playground.csproj b/playground/TestPlatform.Playground/TestPlatform.Playground.csproj index 7bdbeb915e..77ccfd0b7e 100644 --- a/playground/TestPlatform.Playground/TestPlatform.Playground.csproj +++ b/playground/TestPlatform.Playground/TestPlatform.Playground.csproj @@ -38,7 +38,7 @@ - + diff --git a/scripts/build/TestPlatform.Settings.targets b/scripts/build/TestPlatform.Settings.targets index b297c84ab9..3c1e0d7ef9 100644 --- a/scripts/build/TestPlatform.Settings.targets +++ b/scripts/build/TestPlatform.Settings.targets @@ -77,7 +77,8 @@ false - $(NoWarn);MSB3270 + + $(NoWarn);MSB3270;CS8002 From 598739127683b0613022a8dba4c7f0d256e44dbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 21 Mar 2022 17:17:10 +0100 Subject: [PATCH 42/64] Add tags --- .../Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs | 4 ++++ .../Extension/MSTestCompatibilityDataSource.cs | 2 -- .../TranslationLayerTests/CustomTestHostTests.cs | 5 ++++- .../TranslationLayerTests/DiscoverTests.cs | 2 ++ 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 60b60036ed..c9676ae547 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -18,6 +18,7 @@ public class ExecutionTests : AcceptanceTestBase { // TODO: It looks like the first 3 tests would be useful to multiply by all 3 test frameworks, should we make the test even more generic, or duplicate them? [TestMethod] + [TestCategory("Windows-Review")] [MSTestCompatibilityDataSource(InProcess = true)] public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) { @@ -34,6 +35,7 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) } [TestMethod] + [TestCategory("Windows-Review")] [TestPlatformCompatibilityDataSource()] //[TestPlatformCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER, AfterAdapterFeature = Features.MSTEST_IFRAMEWORK_HANDLE_99)] @@ -52,6 +54,7 @@ public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo) } [TestMethod] + [TestCategory("Windows-Review")] [TestHostCompatibilityDataSource] public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(RunnerInfo runnerInfo) { @@ -67,6 +70,7 @@ public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(Ru [TestMethod] + [TestCategory("Windows-Review")] [RunnerCompatibilityDataSource] public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations2(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs index 33c53f8a5d..f6d20b577c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/MSTestCompatibilityDataSource.cs @@ -3,10 +3,8 @@ using System.Reflection; - namespace Microsoft.TestPlatform.AcceptanceTests; - public class MSTestCompatibilityDataSource : TestDataSource { private readonly CompatibilityRowsBuilder _builder; diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs index f045bf159d..7783efc554 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs @@ -31,6 +31,7 @@ public void Cleanup() } [TestMethod] + [TestCategory("Windows-Review")] [RunnerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER_FLOW)] [TestHostCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER_FLOW)] public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncher(RunnerInfo runnerInfo) @@ -56,6 +57,7 @@ public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvided } [TestMethod] + [TestCategory("Windows-Review")] // [RunnerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER_FLOW)] [TestHostCompatibilityDataSource("net451", "netcoreapp2.1", "LegacyStable", BeforeFeature = Features.ATTACH_DEBUGGER_FLOW, DebugVSTestConsole = true)] [Ignore("This is not working for any testhost prior 16.7.0 where the change was introduced. The launch testhost flow was replaced with AttachDebugger in runner, and the new callback to AttachDebugger happens in testhost." @@ -85,9 +87,9 @@ public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvided } [TestMethod] + [TestCategory("Windows-Review")] [RunnerCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER_FLOW)] // [TestHostCompatibilityDataSource(AfterFeature = Features.ATTACH_DEBUGGER_FLOW)] - [TestHostCompatibilityDataSource()] public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvidedLauncher(RunnerInfo runnerInfo) { @@ -109,6 +111,7 @@ public void RunTestsWithCustomTestHostLauncherAttachesToDebuggerUsingTheProvided } [TestMethod] + [TestCategory("Windows-Review")] [Ignore("This is not working. The compatibility code only checks the protocol version (in handler), which is dictated by testhost. " + "It sees 6 but does not realize that the provided CustomTesthostLauncher is not supporting the new feature, it ends up calling back to EditoAttachDebugger" + "in translation layer, and that just silently skips the call.")] diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index 519c8686ce..9280597fc6 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -40,6 +40,7 @@ public void Cleanup() } [TestMethod] + [TestCategory("Windows-Review")] [RunnerCompatibilityDataSource] public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo) { @@ -57,6 +58,7 @@ public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo) } [TestMethod] + [TestCategory("Windows-Review")] [RunnerCompatibilityDataSource()] public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedOut(RunnerInfo runnerInfo) { From 0e4ce5a3d63e731d448abf16f96bbd10866b276f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 21 Mar 2022 17:59:18 +0100 Subject: [PATCH 43/64] Fix console path --- .../TranslationLayerTests/CustomTestHostTests.cs | 5 ++++- .../IntegrationTestBase.cs | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs index 7783efc554..ac6b33001a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CustomTestHostTests.cs @@ -33,7 +33,10 @@ public void Cleanup() [TestMethod] [TestCategory("Windows-Review")] [RunnerCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER_FLOW)] - [TestHostCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER_FLOW)] + // This does not work with testhosts that are earlier than when the feature was introduced, + // when latest runner is used, because the latest runner does not downgrade the messages when + // older testhost launcher is used. + // [TestHostCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER_FLOW)] public void RunTestsWithCustomTestHostLauncherLaunchesTheProcessUsingTheProvidedLauncher(RunnerInfo runnerInfo) { // Pins the existing functionality. diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index a3084ede30..0b93cfcfae 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -605,7 +605,7 @@ public virtual string GetConsoleRunnerPath() if (IsDesktopRunner()) { - if (!string.IsNullOrWhiteSpace(_testEnvironment?.VSTestConsoleInfo.Path)) + if (!string.IsNullOrWhiteSpace(_testEnvironment.VSTestConsoleInfo?.Path)) { consoleRunnerPath = _testEnvironment.VSTestConsoleInfo.Path; } From 969c2b3fd8efeeee3516840741073f7aab1ff4ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Tue, 22 Mar 2022 11:53:38 +0100 Subject: [PATCH 44/64] fix --- .../TestPluginDiscoverer.cs | 2 +- .../DotnetTestTests.cs | 2 +- .../DifferentTestFrameworkSimpleTests.cs | 16 +++--- .../BannedSymbols.txt | 3 ++ .../IntegrationTestBase.cs | 54 ++++++++++--------- ...icrosoft.TestPlatform.TestUtilities.csproj | 10 ++++ .../TempDirectory.cs | 39 +++++++++++++- 7 files changed, 91 insertions(+), 35 deletions(-) create mode 100644 test/Microsoft.TestPlatform.TestUtilities/BannedSymbols.txt diff --git a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs index e6fb7315ca..4401f39cec 100644 --- a/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs +++ b/src/Microsoft.TestPlatform.Common/ExtensionFramework/TestPluginDiscoverer.cs @@ -190,7 +190,7 @@ private void GetTestExtensionsFromAssembly(Assembly ass if (e.Types?.Length > 0) { - types.AddRange(e.Types.Where(type => type.GetTypeInfo().IsClass && !type.GetTypeInfo().IsAbstract)); + types.AddRange(e.Types.Where(type => type != null && type.GetTypeInfo().IsClass && !type.GetTypeInfo().IsAbstract)); } if (e.LoaderExceptions != null) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetTestTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetTestTests.cs index afd985065d..cd5515701a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetTestTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetTestTests.cs @@ -31,7 +31,7 @@ public void RunDotnetTestWithCsproj(RunnerInfo runnerInfo) [TestMethod] // patched dotnet is not published on non-windows systems [TestCategory("Windows-Review")] - [NetCoreTargetFrameworkDataSource] + [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)] public void RunDotnetTestWithDll(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs index b08763da05..cfae534f62 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DifferentTestFrameworkSimpleTests.cs @@ -124,21 +124,23 @@ public void RunTestsWithChutzpahAdapter(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); - var sources = new List - { - Path.Combine(_testEnvironment.TestAssetsPath, "test.js") - }; + var jsSource = Path.Combine(_testEnvironment.TestAssetsPath, "test.js"); + + // Chuzpah adapter creates _Chutzpah temp files, to give data back from the runner. + // But when cleaning up it deletes all the _Chutzpah files, not just the one it owns, + // so when we run in parallel, the slower process will never find it's own file, because it was already deleted: + // https://github.com/mmanela/chutzpah/issues/812 + var jsInTemp = TempDirectory.CopyFile(jsSource); var testAdapterPath = Directory.EnumerateFiles(GetTestAdapterPath(UnitTestFramework.Chutzpah), "*.TestAdapter.dll").ToList(); _vstestConsoleWrapper.InitializeExtensions(new List() { testAdapterPath.FirstOrDefault() }); _vstestConsoleWrapper.RunTests( - sources, + new[] { jsInTemp }, GetDefaultRunSettings(), _runEventHandler); - var testCase = - _runEventHandler.TestResults.Where(tr => tr.TestCase.DisplayName.Equals("TestMethod1")); + var testCase = _runEventHandler.TestResults.Where(tr => tr.TestCase.DisplayName.Equals("TestMethod1")); // Assert Assert.AreEqual(2, _runEventHandler.TestResults.Count); diff --git a/test/Microsoft.TestPlatform.TestUtilities/BannedSymbols.txt b/test/Microsoft.TestPlatform.TestUtilities/BannedSymbols.txt new file mode 100644 index 0000000000..c8eaef56e1 --- /dev/null +++ b/test/Microsoft.TestPlatform.TestUtilities/BannedSymbols.txt @@ -0,0 +1,3 @@ +M:System.IO.Path.GetTempPath(); Use 'IntegrationTestBase.GetTempPath()' instead +M:System.Environment.SetEnvironmentVariable(System.String,System.String); Use one of the overload accepting a dictionary of environment variables instead +M:System.Environment.SetEnvironmentVariable(System.String,System.String,System.EnvironmentVariableTarget); Use one of the overload accepting a dictionary of environment variables instead \ No newline at end of file diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 0b93cfcfae..6e835eedd0 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -189,24 +189,14 @@ public void InvokeVsTest(string arguments, Dictionary environmen public void InvokeDotnetTest(string arguments, Dictionary environmentVariables = null) { var vstestConsolePath = GetDotnetRunnerPath(); - var env = "VSTEST_CONSOLE_PATH"; - var originalVstestConsolePath = Environment.GetEnvironmentVariable(env); - - try - { - Environment.SetEnvironmentVariable(env, vstestConsolePath); - if (arguments.Contains(".csproj")) - { - arguments = $@"-p:VsTestConsolePath=""{vstestConsolePath}"" " + arguments; - } - - ExecutePatchedDotnet("test", arguments, out _standardTestOutput, out _standardTestError, out _runnerExitCode, environmentVariables); - FormatStandardOutCome(); - } - finally + + if (arguments.Contains(".csproj")) { - Environment.SetEnvironmentVariable(env, originalVstestConsolePath); + arguments = $@"-p:VsTestConsolePath=""{vstestConsolePath}"" " + arguments; } + + ExecutePatchedDotnet("test", arguments, out _standardTestOutput, out _standardTestError, out _runnerExitCode, environmentVariables); + FormatStandardOutCome(); } /// @@ -223,33 +213,44 @@ public void InvokeVsTestForExecution(string testAssembly, Dictionary environmentVariables = null) { var arguments = PrepareArguments(testAssembly, testAdapterPath, runSettings, framework, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path); + InvokeVsTest(arguments, AddDebugEnvVariables(environmentVariables)); + } - if (_testEnvironment.DebugInfo.DebugVSTestConsole || _testEnvironment.DebugInfo.DebugTestHost || _testEnvironment.DebugInfo.DebugDataCollector) + private Dictionary AddDebugEnvVariables(Dictionary environmentVariables) + { + var debugVariables = new Dictionary(); + if (environmentVariables != null) { - environmentVariables ??= new Dictionary(); + foreach (var pair in environmentVariables) + { + debugVariables.Add(pair.Key, pair.Value); + } + } + if (_testEnvironment.DebugInfo != null) + { if (_testEnvironment.DebugInfo.DebugVSTestConsole) { - environmentVariables.Add("VSTEST_RUNNER_DEBUG_ATTACHVS", "1"); + debugVariables.Add("VSTEST_RUNNER_DEBUG_ATTACHVS", "1"); } if (_testEnvironment.DebugInfo.DebugTestHost) { - environmentVariables.Add("VSTEST_HOST_DEBUG_ATTACHVS", "1"); + debugVariables.Add("VSTEST_HOST_DEBUG_ATTACHVS", "1"); } if (_testEnvironment.DebugInfo.DebugDataCollector) { - environmentVariables.Add("VSTEST_DATACOLLECTOR_DEBUG_ATTACHVS", "1"); + debugVariables.Add("VSTEST_DATACOLLECTOR_DEBUG_ATTACHVS", "1"); } if (_testEnvironment.DebugInfo.NoDefaultBreakpoints) { - environmentVariables.Add("VSTEST_DEBUG_NOBP", "1"); + debugVariables.Add("VSTEST_DEBUG_NOBP", "1"); } } - InvokeVsTest(arguments, environmentVariables); + return debugVariables; } /// @@ -674,7 +675,10 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper() Console.WriteLine($"Console runner path: {consoleRunnerPath}"); VsTestConsoleWrapper vstestConsoleWrapper; - if (_testEnvironment.DebugInfo.DebugVSTestConsole || _testEnvironment.DebugInfo.DebugTestHost || _testEnvironment.DebugInfo.DebugDataCollector) + if (_testEnvironment.DebugInfo != null + && (_testEnvironment.DebugInfo.DebugVSTestConsole + || _testEnvironment.DebugInfo.DebugTestHost + || _testEnvironment.DebugInfo.DebugDataCollector)) { var environmentVariables = new Dictionary(); Environment.GetEnvironmentVariables().OfType().ToList().ForEach(e => environmentVariables.Add(e.Key.ToString(), e.Value.ToString())); @@ -908,7 +912,7 @@ public static void CreateRunSettingsFile(string destinationRunsettingsPath, stri protected string BuildMultipleAssemblyPath(params string[] assetNames) { - return string.Join(" ", GetTestDlls(assetNames)); + return @$"""{string.Join(@""" """, GetTestDlls(assetNames))}"""; } protected static string GetDiagArg(string rootDir) diff --git a/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj b/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj index baea232289..527a9a44b3 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj +++ b/test/Microsoft.TestPlatform.TestUtilities/Microsoft.TestPlatform.TestUtilities.csproj @@ -10,6 +10,12 @@ netcoreapp3.1 false + + + + + + @@ -22,6 +28,10 @@ + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/test/Microsoft.TestPlatform.TestUtilities/TempDirectory.cs b/test/Microsoft.TestPlatform.TestUtilities/TempDirectory.cs index d005672347..f7353ace6c 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/TempDirectory.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/TempDirectory.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +using System.Collections.Generic; using System.IO; using IO = System.IO; @@ -51,6 +52,36 @@ public void CopyDirectory(DirectoryInfo source, DirectoryInfo target) } } + /// + /// Copy given files into the TempDirectory and return the updated paths that are pointing to TempDirectory. + /// + /// + /// + public string[] CopyFile(params string[] filePaths) + { + var paths = new List(filePaths.Length); + foreach (var filePath in filePaths) + { + var destination = IO.Path.Combine(Path, IO.Path.GetFileName(filePath)); + File.Copy(filePath, destination); + paths.Add(destination); + } + + return paths.ToArray(); + } + + /// + /// Copy given file into TempDirectory and return the updated path. + /// + /// + /// + public string CopyFile(string filePath) + { + var destination = IO.Path.Combine(Path, IO.Path.GetFileName(filePath)); + File.Copy(filePath, destination); + return destination; + } + /// /// Creates an unique temporary directory. /// @@ -71,7 +102,13 @@ private static string GetTempPath() // AGENT_TEMPDIRECTORY is AzureDevops variable, which is set to path // that is cleaned up after every job. This is preferable to use over // just the normal TEMP, because that is not cleaned up for every run. - return Environment.GetEnvironmentVariable("AGENT_TEMPDIRECTORY") ?? IO.Path.GetTempPath(); + // + // System.IO.Path.GetTempPath is banned from the rest of the code. This is the only + // place we we are allowed to use it. All other methods should use our GetTempPath (this method). +#pragma warning disable RS0030 // Do not used banned APIs + return Environment.GetEnvironmentVariable("AGENT_TEMPDIRECTORY") + ?? IO.Path.GetTempPath(); +#pragma warning restore RS0030 // Do not used banned APIs } public static void TryRemoveDirectory(string directory) From 9894fa311d1d7764fbe542c667cfab32c2e4601a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Tue, 22 Mar 2022 14:59:07 +0100 Subject: [PATCH 45/64] CI cleanup always --- .../IntegrationTestBase.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index e3aebd8afe..bb5d07d65e 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -82,8 +82,11 @@ public IntegrationTestBase() [TestCleanup] public void TempDirectoryCleanup() { - // Delete the directory only when the test succeeded, so we can look at results and logs of failed tests. - if (TestContext.CurrentTestOutcome == UnitTestOutcome.Passed) + // In CI always delete the results, because we have limited disk space there. + // + // Locally delete the directory only when the test succeeded, so we can look + // at results and logs of failed tests. + if (IsCI || TestContext.CurrentTestOutcome == UnitTestOutcome.Passed) { TempDirectory.Dispose(); } From 5af74378ecfa71928a1b6b55ddbbbbfbd6477514 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 23 Mar 2022 13:14:27 +0100 Subject: [PATCH 46/64] Print free disk space --- .../IntegrationTestBase.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index bb5d07d65e..39b9099afe 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -62,6 +62,10 @@ public IntegrationTestBase() _testEnvironment = new IntegrationTestEnvironment(); BuildConfiguration = IntegrationTestEnvironment.BuildConfiguration; TempDirectory = new TempDirectory(); + + var drive = new DriveInfo(Directory.GetDirectoryRoot(TempDirectory.Path)); + Console.WriteLine($"Available space for TEMP: {drive.Name} {drive.AvailableFreeSpace / (1024 * 1024)} MB"); + IsCI = IntegrationTestEnvironment.IsCI; } From 81c0b1b97ba6548964e2332e942ee85ce7fdb7ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 23 Mar 2022 14:26:47 +0100 Subject: [PATCH 47/64] Whitespace and so on --- .../ExecutionTests.cs | 8 ++------ .../Extension/CompatibilityRowsBuilder.cs | 19 ++++++++----------- .../Extension/Features.cs | 14 +++++++------- .../Extension/NetCoreRunner.cs | 2 +- .../RunnerCompatibilityDataSource.cs | 1 - .../Extension/RunnnerInfo.cs | 2 +- .../DebugInfo.cs | 2 -- .../NetTestSdkInfo.cs | 2 +- .../VSTestConsoleInfo.cs | 2 +- 9 files changed, 21 insertions(+), 31 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index c9676ae547..5f5fc4ead8 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -38,7 +38,6 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) [TestCategory("Windows-Review")] [TestPlatformCompatibilityDataSource()] //[TestPlatformCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER, AfterAdapterFeature = Features.MSTEST_IFRAMEWORK_HANDLE_99)] - public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -68,7 +67,6 @@ public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations(Ru ExitCodeEquals(1); // failing tests } - [TestMethod] [TestCategory("Windows-Review")] [RunnerCompatibilityDataSource] @@ -84,7 +82,6 @@ public void RunMultipleMSTestAssembliesOnVstestConsoleAndTesthostCombinations2(R ExitCodeEquals(1); // failing tests } - // TODO: This one mixes different frameworks, I can make it work, but it is worth it? We are going to test // the two respective versions together (e.g. latest xunit and latest mstest), but does using two different test // frameworks have any added value over using 2 mstest dlls? @@ -115,13 +112,12 @@ public void RunMultipleTestAssembliesWithoutTestAdapterPath(RunnerInfo runnerInf public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, testAdapterPath: null, runSettings: null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); arguments = string.Concat(arguments, " /Parallel"); arguments = string.Concat(arguments, " /Platform:x86"); - arguments += GetDiagArg(tempDir.Path); + arguments += GetDiagArg(TempDirectory.Path); // for the desktop we will run testhost.x86 in two copies, but for core // we will run a combination of testhost.x86 and dotnet, where the dotnet will be @@ -134,7 +130,7 @@ public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) InvokeVsTest(arguments); - AssertExpectedNumberOfHostProcesses(expectedNumOfProcessCreated, tempDir.Path, testHostProcessNames); + AssertExpectedNumberOfHostProcesses(expectedNumOfProcessCreated, TempDirectory.Path, testHostProcessNames); ValidateSummaryStatus(2, 2, 2); ExitCodeEquals(1); // failing tests } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs index dccbd03525..3a4184bc9c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs @@ -24,11 +24,11 @@ public class CompatibilityRowsBuilder private readonly string[] _hostVersions; public CompatibilityRowsBuilder(string runnerFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, - string runnerVersions = AcceptanceTestBase.LATEST_TO_LEGACY, - string hostFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, - string hostVersions = AcceptanceTestBase.LATEST_TO_LEGACY, - string adapterVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY, - string adapters = AcceptanceTestBase.MSTEST) + string runnerVersions = AcceptanceTestBase.LATEST_TO_LEGACY, + string hostFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET, + string hostVersions = AcceptanceTestBase.LATEST_TO_LEGACY, + string adapterVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY, + string adapters = AcceptanceTestBase.MSTEST) { _runnerFrameworks = runnerFrameworks.Split(';'); _runnerVersions = runnerVersions.Split(';'); @@ -197,7 +197,6 @@ private void AddOlderConfigurations(List dataRows) } } - private void AddEveryVersionOfAdapter(List dataRows) { var runnerVersion = _runnerVersions[0]; @@ -270,7 +269,7 @@ private void AddEveryVersionOfRunner(List dataRows) } private void AddRow(List dataRows, string batch, -string runnerVersion, string runnerFramework, string hostVersion, string hostFramework, string adapter, string adapterVersion, bool inIsolation) + string runnerVersion, string runnerFramework, string hostVersion, string hostFramework, string adapter, string adapterVersion, bool inIsolation) { RunnerInfo runnerInfo = GetRunnerInfo(batch, runnerFramework, hostFramework, inIsolation); runnerInfo.DebugInfo = GetDebugInfo(); @@ -298,7 +297,6 @@ private DebugInfo GetDebugInfo() }; } - private RunnerInfo GetRunnerInfo(string batch, string runnerFramework, string hostFramework, bool inIsolation) { return new RunnerInfo @@ -339,7 +337,7 @@ private static VSTestConsoleInfo GetVSTestConsoleInfo(string vstestConsoleVersio // same as other versions, we just need to grab the version from a different property. var propertyName = vstestConsoleVersion == AcceptanceTestBase.LATEST - ? $"NETTestSdkVersion" + ? "NETTestSdkVersion" : $"VSTestConsole{vstestConsoleVersion}Version"; var packageName = runnerInfo.IsNetFrameworkRunner @@ -378,7 +376,7 @@ private static NetTestSdkInfo GetNetTestSdkInfo(string testhostVersionType) // same as other versions, we just need to grab the version from a different property. var propertyName = testhostVersionType == AcceptanceTestBase.LATEST - ? $"NETTestSdkVersion" + ? "NETTestSdkVersion" : $"VSTestConsole{testhostVersionType}Version"; // It is okay when node is null, we check that Version has value when we update paths by using TesthostInfo, and throw. @@ -398,7 +396,6 @@ private static NetTestSdkInfo GetNetTestSdkInfo(string testhostVersionType) }; } - private static XmlDocument GetDependenciesXml() { if (s_depsXml != null) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs index b966b9daa3..0be9ac104c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/Features.cs @@ -2,22 +2,22 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Collections.Generic; +using System.Collections.Immutable; namespace Microsoft.TestPlatform.AcceptanceTests; public static class Features { public const string ATTACH_DEBUGGER_FLOW = nameof(ATTACH_DEBUGGER_FLOW); - public const string MSTEST_IFRAMEWORK_HANDLE_99 = nameof(MSTEST_IFRAMEWORK_HANDLE_99); + public const string MSTEST_EXAMPLE_FEATURE = nameof(MSTEST_EXAMPLE_FEATURE); - - public static Dictionary TestPlatformFeatures { get; } = new Dictionary + public static IImmutableDictionary TestPlatformFeatures { get; } = new Dictionary { [ATTACH_DEBUGGER_FLOW] = new(version: "v16.7.0-preview-20200519-01", issue: "https://github.com/microsoft/vstest/pull/2325"), - }; + }.ToImmutableDictionary(); - public static Dictionary AdapterFeatures { get; internal set; } = new Dictionary + public static IImmutableDictionary AdapterFeatures { get; internal set; } = new Dictionary { - [MSTEST_IFRAMEWORK_HANDLE_99] = new("2.2.8", issue: "idk"), - }; + [MSTEST_EXAMPLE_FEATURE] = new("2.2.8", issue: "This feature does not actually exist."), + }.ToImmutableDictionary(); } diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs index 71db23bdee..202264d0c7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/NetCoreRunner.cs @@ -51,7 +51,7 @@ public IEnumerable GetData(MethodInfo methodInfo) { RunnerFramework = IntegrationTestBase.CoreRunnerFramework, TargetFramework = fmw, - InIsolationValue = null + InIsolationValue = null, }; runnerInfo.DebugInfo = new DebugInfo { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnerCompatibilityDataSource.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnerCompatibilityDataSource.cs index 3b7b9d10ca..357dcd1070 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnerCompatibilityDataSource.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnerCompatibilityDataSource.cs @@ -5,7 +5,6 @@ namespace Microsoft.TestPlatform.AcceptanceTests; - /// /// A data source that provides every version of runner. /// diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs index 88f1404005..6e1fefc769 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/RunnnerInfo.cs @@ -14,7 +14,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests; /// /// /// Supported value = /InIsolation. -[Serializable] +[Serializable] // Type should be serializable to allow the tree-view behavior of test discovery in Test Explorer public class RunnerInfo { public string? RunnerFramework { get; set; } diff --git a/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs index 995855ca1d..f9ac35d770 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/DebugInfo.cs @@ -16,5 +16,3 @@ public sealed class DebugInfo public bool DebugDataCollector { get; set; } public bool NoDefaultBreakpoints { get; set; } = true; } - - diff --git a/test/Microsoft.TestPlatform.TestUtilities/NetTestSdkInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/NetTestSdkInfo.cs index c16d2a3f44..00deda8823 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/NetTestSdkInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/NetTestSdkInfo.cs @@ -5,7 +5,7 @@ namespace Microsoft.TestPlatform.TestUtilities; -[Serializable] +[Serializable] // Type should be serializable to allow the tree-view behavior of test discovery in Test Explorer public class NetTestSdkInfo : DllInfo { public NetTestSdkInfo() diff --git a/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs b/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs index e55503afbf..3ba8deabc9 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/VSTestConsoleInfo.cs @@ -5,7 +5,7 @@ namespace Microsoft.TestPlatform.TestUtilities; -[Serializable] +[Serializable] // Type should be serializable to allow the tree-view behavior of test discovery in Test Explorer public class VSTestConsoleInfo { public string? VersionType { get; set; } From eff69def0b3da3fe50f593c25152f4803a306e0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 23 Mar 2022 14:29:46 +0100 Subject: [PATCH 48/64] Missing temp dir --- test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 5f5fc4ead8..ead92079dc 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -114,7 +114,7 @@ public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); - var arguments = PrepareArguments(assemblyPaths, testAdapterPath: null, runSettings: null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, testAdapterPath: null, runSettings: null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /Parallel"); arguments = string.Concat(arguments, " /Platform:x86"); arguments += GetDiagArg(TempDirectory.Path); From 5f3f5c2836adfcf03180623c42b1e3db4e84381b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 23 Mar 2022 15:29:04 +0100 Subject: [PATCH 49/64] Fix env variables --- .../TranslationLayerTests/CodeCoverageTests.cs | 1 + .../IntegrationTestBase.cs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs index c77faab8ab..0b56578e3f 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/CodeCoverageTests.cs @@ -183,6 +183,7 @@ await _vstestConsoleWrapper.ProcessTestRunAttachmentsAsync( [NetCoreTargetFrameworkDataSource] public async Task TestRunWithCodeCoverageAndAttachmentsProcessingNoMetrics(RunnerInfo runnerInfo) { + // System.Environment.SetEnvironmentVariable("VSTEST_RUNNER_DEBUG_ATTACHVS", "1"); // arrange SetTestEnvironment(_testEnvironment, runnerInfo); Setup(); diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 39b9099afe..013cad3096 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -696,14 +696,14 @@ public IVsTestConsoleWrapper GetVsTestConsoleWrapper() Environment.GetEnvironmentVariables().OfType().ToList().ForEach(e => environmentVariables.Add(e.Key.ToString(), e.Value.ToString())); foreach (var pair in debugEnvironmentVariables) { - environmentVariables.Add(pair.Key, pair.Value); + environmentVariables[pair.Key] = pair.Value; } } if (environmentVariables.Count > 0) { // This clears all variables, so we copy all environment variables, and add the debug ones to them. - vstestConsoleWrapper = new VsTestConsoleWrapper(consoleRunnerPath, dotnetPath, new ConsoleParameters() { LogFilePath = logFilePath, EnvironmentVariables = debugEnvironmentVariables }); + vstestConsoleWrapper = new VsTestConsoleWrapper(consoleRunnerPath, dotnetPath, new ConsoleParameters() { LogFilePath = logFilePath, EnvironmentVariables = environmentVariables }); } else { From a1a461c6a1b7f660dcb2cbbb8a9da0f13d242a46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 23 Mar 2022 16:04:58 +0100 Subject: [PATCH 50/64] Fixes I feel like I did before --- .../TranslationLayerTests/RunTests.cs | 1 + .../TranslationLayerTests/RunTestsWithFilterTests.cs | 1 + .../Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs index ffe2449e15..62830b4d29 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs @@ -40,6 +40,7 @@ public void Cleanup() } [TestMethod] + [TestCategory("Windows-Review")] [RunnerCompatibilityDataSource] public void RunAllTests(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs index ea4c38ae0f..78039d80d3 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs @@ -35,6 +35,7 @@ public void Cleanup() } [TestMethod] + [TestCategory("Windows-Review")] [RunnerCompatibilityDataSource] public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo) { diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 013cad3096..3bf607b9a1 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -911,7 +911,7 @@ public static void CreateRunSettingsFile(string destinationRunsettingsPath, stri protected string BuildMultipleAssemblyPath(params string[] assetNames) { - return string.Join(" ", GetTestDlls(assetNames)); + return $"\"{string.Join("\" \"", GetTestDlls(assetNames))}\""; } protected static string GetDiagArg(string rootDir) From 5a19d194cf193adf5e91e17c77e2054d2e4da5ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Wed, 23 Mar 2022 16:37:09 +0100 Subject: [PATCH 51/64] Touch --- test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index ead92079dc..f3b7645b4c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -16,7 +16,7 @@ namespace Microsoft.TestPlatform.AcceptanceTests; [TestClass] public class ExecutionTests : AcceptanceTestBase { - // TODO: It looks like the first 3 tests would be useful to multiply by all 3 test frameworks, should we make the test even more generic, or duplicate them? + //TODO: It looks like the first 3 tests would be useful to multiply by all 3 test frameworks, should we make the test even more generic, or duplicate them? [TestMethod] [TestCategory("Windows-Review")] [MSTestCompatibilityDataSource(InProcess = true)] From 0075b0bf901f37282b45d30898f72c6ca696ceb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Thu, 24 Mar 2022 13:59:24 +0100 Subject: [PATCH 52/64] Remove comment, and revert back to using TempDirectory --- .../ExecutionTests.cs | 41 +++++++------------ 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index f3b7645b4c..bf79c780ae 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -37,7 +37,6 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo) [TestMethod] [TestCategory("Windows-Review")] [TestPlatformCompatibilityDataSource()] - //[TestPlatformCompatibilityDataSource(BeforeFeature = Features.ATTACH_DEBUGGER, AfterAdapterFeature = Features.MSTEST_IFRAMEWORK_HANDLE_99)] public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); @@ -141,11 +140,10 @@ public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) public void TestSessionTimeOutTests(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /TestCaseFilter:TestSessionTimeoutTest"); // set TestSessionTimeOut = 7 sec @@ -163,10 +161,9 @@ public void TestSessionTimeOutTests(RunnerInfo runnerInfo) public void TestPlatformShouldBeCompatibleWithOldTestHost(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SampleProjectWithOldTestHost.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); @@ -180,10 +177,9 @@ public void TestPlatformShouldBeCompatibleWithOldTestHost(RunnerInfo runnerInfo) public void WorkingDirectoryIsSourceDirectory(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /tests:WorkingDirectoryTest"); InvokeVsTest(arguments); @@ -198,7 +194,6 @@ public void WorkingDirectoryIsSourceDirectory(RunnerInfo runnerInfo) public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); if (IntegrationTestEnvironment.BuildConfiguration.Equals("release", StringComparison.OrdinalIgnoreCase)) { @@ -207,11 +202,11 @@ public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerIn return; } - var diagLogFilePath = Path.Combine(tempDir.Path, $"std_error_log_{Guid.NewGuid()}.txt"); + var diagLogFilePath = Path.Combine(TempDirectory.Path, $"std_error_log_{Guid.NewGuid()}.txt"); File.Delete(diagLogFilePath); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:ExitWithStackoverFlow"); arguments = string.Concat(arguments, $" /diag:{diagLogFilePath}"); @@ -235,14 +230,13 @@ public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerIn public void UnhandleExceptionExceptionShouldBeLoggedToDiagLogFile(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); - var diagLogFilePath = Path.Combine(tempDir.Path, $"std_error_log_{Guid.NewGuid()}.txt"); + var diagLogFilePath = Path.Combine(TempDirectory.Path, $"std_error_log_{Guid.NewGuid()}.txt"); File.Delete(diagLogFilePath); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:ExitwithUnhandleException"); arguments = string.Concat(arguments, $" /diag:{diagLogFilePath}"); @@ -259,12 +253,11 @@ public void UnhandleExceptionExceptionShouldBeLoggedToDiagLogFile(RunnerInfo run public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProject3.dll is built for Framework .NETFramework,Version=v4.5.1 and Platform X64"; var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll", "SimpleTestProjectx86.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /testcasefilter:PassingTestx86"); InvokeVsTest(arguments); @@ -282,12 +275,11 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo r public void NoIncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProjectx86 is built for Framework .NETFramework,Version=v4.5.1 and Platform X86"; var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProjectx86.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); @@ -303,12 +295,11 @@ public void NoIncompatibleSourcesWarningShouldBeDisplayedInTheConsole(RunnerInfo public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsoleOnlyWhenRunningIn32BitOS(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProject2.dll is built for Framework .NETFramework,Version=v4.5.1 and Platform X64"; var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); InvokeVsTest(arguments); @@ -333,11 +324,10 @@ public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsoleOnlyWhenRunni public void ExitCodeShouldReturnOneWhenTreatNoTestsAsErrorParameterSetToTrueAndNoTestMatchesFilter(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); // Setting /TestCaseFilter to the test name, which does not exists in the assembly, so we will have 0 tests executed arguments = string.Concat(arguments, " /TestCaseFilter:TestNameThatMatchesNoTestInTheAssembly"); @@ -354,10 +344,9 @@ public void ExitCodeShouldReturnOneWhenTreatNoTestsAsErrorParameterSetToTrueAndN public void ExitCodeShouldReturnZeroWhenTreatNoTestsAsErrorParameterSetToFalseAndNoTestMatchesFilter(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); // Setting /TestCaseFilter to the test name, which does not exists in the assembly, so we will have 0 tests executed arguments = string.Concat(arguments, " /TestCaseFilter:TestNameThatMatchesNoTestInTheAssembly"); @@ -374,11 +363,10 @@ public void ExitCodeShouldReturnZeroWhenTreatNoTestsAsErrorParameterSetToFalseAn public void ExitCodeShouldNotDependOnTreatNoTestsAsErrorTrueValueWhenThereAreAnyTestsToRun(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " -- RunConfiguration.TreatNoTestsAsError=true"); InvokeVsTest(arguments); @@ -393,10 +381,9 @@ public void ExitCodeShouldNotDependOnTreatNoTestsAsErrorTrueValueWhenThereAreAny public void ExitCodeShouldNotDependOnFailTreatNoTestsAsErrorFalseValueWhenThereAreAnyTestsToRun(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - using var tempDir = new TempDirectory(); var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject2.dll"); - var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: tempDir.Path); + var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " -- RunConfiguration.TreatNoTestsAsError=false"); InvokeVsTest(arguments); From 85085dc2f0c0ff2f51906acb961597da19116b09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Thu, 24 Mar 2022 14:09:39 +0100 Subject: [PATCH 53/64] Fix quoting --- test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs | 3 ++- .../IntegrationTestBase.cs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index bf79c780ae..57dc40b3a8 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -6,6 +6,7 @@ using Microsoft.TestPlatform.TestUtilities; using Microsoft.VisualStudio.TestTools.UnitTesting; +using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Extensions; using TestPlatform.TestUtilities; @@ -96,7 +97,7 @@ public void RunMultipleTestAssembliesWithoutTestAdapterPath(RunnerInfo runnerInf _testEnvironment.GetTestAsset("XUTestProject.dll", "net46") : _testEnvironment.GetTestAsset("XUTestProject.dll"); - assemblyPaths = string.Concat(assemblyPaths, "\" \"", xunitAssemblyPath); + assemblyPaths = string.Join(" ", assemblyPaths, xunitAssemblyPath.AddDoubleQuote()); InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty); ValidateSummaryStatus(2, 2, 1); diff --git a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs index 3bf607b9a1..ae05021e2a 100644 --- a/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs +++ b/test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs @@ -911,7 +911,8 @@ public static void CreateRunSettingsFile(string destinationRunsettingsPath, stri protected string BuildMultipleAssemblyPath(params string[] assetNames) { - return $"\"{string.Join("\" \"", GetTestDlls(assetNames))}\""; + // Double quoted sources sepearated by space. + return string.Join(" ", GetTestDlls(assetNames).Select(a => a.AddDoubleQuote())); } protected static string GetDiagArg(string rootDir) From efc810fb10aa98743cf87ccd94f1f50eb0c22a28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Thu, 24 Mar 2022 14:38:33 +0100 Subject: [PATCH 54/64] Use Nuget.Versioning rather than semver, and revert strong name warning ignore for test projects --- scripts/build/TestPlatform.Settings.targets | 3 +- .../Extension/CompatibilityRowsBuilder.cs | 39 +++++++++---------- ...rosoft.TestPlatform.AcceptanceTests.csproj | 2 +- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/scripts/build/TestPlatform.Settings.targets b/scripts/build/TestPlatform.Settings.targets index 3c1e0d7ef9..b297c84ab9 100644 --- a/scripts/build/TestPlatform.Settings.targets +++ b/scripts/build/TestPlatform.Settings.targets @@ -77,8 +77,7 @@ false - - $(NoWarn);MSB3270;CS8002 + $(NoWarn);MSB3270 diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs index 3a4184bc9c..157b2a14b0 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Extension/CompatibilityRowsBuilder.cs @@ -6,11 +6,10 @@ using System.IO; using System.Linq; using System.Xml; +using NuGet.Versioning; using Microsoft.TestPlatform.TestUtilities; -using Semver; - namespace Microsoft.TestPlatform.AcceptanceTests; public class CompatibilityRowsBuilder @@ -79,49 +78,49 @@ public List CreateData() if (WithInProcess) AddInProcess(dataRows); - var minVersion = SemVersion.Parse("0.0.0-alpha.1"); - var maxVersion = SemVersion.Parse("9999.0.0"); - SemVersion? beforeRunnerVersion = maxVersion; - SemVersion? afterRunnerVersion = minVersion; - SemVersion? beforeTestHostVersion = maxVersion; - SemVersion? afterTestHostVersion = minVersion; - SemVersion? beforeAdapterVersion = maxVersion; - SemVersion? afterAdapterVersion = minVersion; + var minVersion = SemanticVersion.Parse("0.0.0-alpha.1"); + var maxVersion = SemanticVersion.Parse("9999.0.0"); + SemanticVersion? beforeRunnerVersion = maxVersion; + SemanticVersion? afterRunnerVersion = minVersion; + SemanticVersion? beforeTestHostVersion = maxVersion; + SemanticVersion? afterTestHostVersion = minVersion; + SemanticVersion? beforeAdapterVersion = maxVersion; + SemanticVersion? afterAdapterVersion = minVersion; if (BeforeRunnerFeature != null) { var feature = Features.TestPlatformFeatures[BeforeRunnerFeature]; - beforeRunnerVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + beforeRunnerVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); } if (AfterRunnerFeature != null) { var feature = Features.TestPlatformFeatures[AfterRunnerFeature]; - afterRunnerVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + afterRunnerVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); } if (BeforeTestHostFeature != null) { var feature = Features.TestPlatformFeatures[BeforeTestHostFeature]; - beforeTestHostVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + beforeTestHostVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); } if (AfterTestHostFeature != null) { var feature = Features.TestPlatformFeatures[AfterTestHostFeature]; - afterTestHostVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + afterTestHostVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); } if (BeforeAdapterFeature != null) { var feature = Features.TestPlatformFeatures[BeforeAdapterFeature]; - beforeAdapterVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + beforeAdapterVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); } if (AfterAdapterFeature != null) { var feature = Features.AdapterFeatures[AfterAdapterFeature]; - afterAdapterVersion = SemVersion.Parse(feature.Version.TrimStart('v')); + afterAdapterVersion = SemanticVersion.Parse(feature.Version.TrimStart('v')); } var isWindows = Environment.OSVersion.Platform.ToString().StartsWith("Win"); @@ -133,12 +132,12 @@ public List CreateData() // We probably don't have that need right now, because legacy version is 15.x.x, which is very old, and we are still keeping // compatibility. - Func isInRange = (version, before, after) => version < before && after < version; + Func isInRange = (version, before, after) => version < before && after < version; var rows = dataRows.Where(r => r.VSTestConsoleInfo != null - && isInRange(r.VSTestConsoleInfo.Version, beforeRunnerVersion, afterRunnerVersion) - && r.TestHostInfo != null && isInRange(r.TestHostInfo.Version, beforeTestHostVersion, afterTestHostVersion) - && r.AdapterInfo != null && isInRange(r.AdapterInfo.Version, beforeAdapterVersion, afterAdapterVersion)).ToList(); + && isInRange(SemanticVersion.Parse(r.VSTestConsoleInfo.Version), beforeRunnerVersion, afterRunnerVersion) + && r.TestHostInfo != null && isInRange(SemanticVersion.Parse(r.TestHostInfo.Version), beforeTestHostVersion, afterTestHostVersion) + && r.AdapterInfo != null && isInRange(SemanticVersion.Parse(r.AdapterInfo.Version), beforeAdapterVersion, afterAdapterVersion)).ToList(); // We use ToString to determine which values are unique. Not great solution, but works better than using records. var distinctRows = new Dictionary(); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj index c33244d042..ef1e78ee2c 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj @@ -36,7 +36,7 @@ - + From 02c7d41917ecffb378cb6e103a76fdea88a99c8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 28 Mar 2022 14:59:40 +0200 Subject: [PATCH 55/64] Fix tests --- test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs | 2 +- .../TranslationLayerTests/DiscoverTests.cs | 4 ++-- .../TranslationLayerTests/RunTests.cs | 2 +- .../TranslationLayerTests/RunTestsWithFilterTests.cs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs index 57dc40b3a8..037ec5deda 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs @@ -113,7 +113,7 @@ public void RunMultipleTestAssembliesInParallel(RunnerInfo runnerInfo) { SetTestEnvironment(_testEnvironment, runnerInfo); - var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll", "SimpleTestProject2.dll"); + var assemblyPaths = BuildMultipleAssemblyPath("MSTestProject1.dll", "MSTestProject2.dll"); var arguments = PrepareArguments(assemblyPaths, testAdapterPath: null, runSettings: null, FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: TempDirectory.Path); arguments = string.Concat(arguments, " /Parallel"); arguments = string.Concat(arguments, " /Platform:x86"); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index 9280597fc6..a227871e1b 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -51,7 +51,7 @@ public void DiscoverTestsUsingDiscoveryEventHandler1(RunnerInfo runnerInfo) _discoveryEventHandler2 = new DiscoveryEventHandler2(); var vstestConsoleWrapper = GetVsTestConsoleWrapper(); - vstestConsoleWrapper.DiscoverTests(GetTestAssemblies(), GetDefaultRunSettings(), _discoveryEventHandler); + vstestConsoleWrapper.DiscoverTests(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetDefaultRunSettings(), _discoveryEventHandler); // Assert. Assert.AreEqual(6, _discoveryEventHandler.DiscoveredTestCases.Count); @@ -70,7 +70,7 @@ public void DiscoverTestsUsingDiscoveryEventHandler2AndTelemetryOptedOut(RunnerI var vstestConsoleWrapper = GetVsTestConsoleWrapper(); vstestConsoleWrapper.DiscoverTests( - GetTestAssemblies(), + GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetDefaultRunSettings(), new TestPlatformOptions() { CollectMetrics = false }, _discoveryEventHandler2); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs index 62830b4d29..b17f17b57a 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTests.cs @@ -48,7 +48,7 @@ public void RunAllTests(RunnerInfo runnerInfo) var vstestConsoleWrapper = GetVsTestConsoleWrapper(); var runEventHandler = new RunEventHandler(); - vstestConsoleWrapper.RunTests(GetTestAssemblies(), GetDefaultRunSettings(), runEventHandler); + vstestConsoleWrapper.RunTests(GetTestDlls("MSTestProject1.dll", "MSTestProject2.dll"), GetDefaultRunSettings(), runEventHandler); // Assert Assert.AreEqual(6, runEventHandler.TestResults.Count); diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs index 78039d80d3..6c1b41fac1 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs @@ -45,7 +45,7 @@ public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo) _runEventHandler = new RunEventHandler(); var vstestConsoleWrapper = GetVsTestConsoleWrapper(); - var sources = new List { GetAssetFullPath("SimpleTestProject.dll") }; + var sources = new List { GetAssetFullPath("MSTestProject1.dll") }; vstestConsoleWrapper.RunTests( sources, From 57be8db00b5f540f81a99c488fd9b1099b072ad0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 28 Mar 2022 16:16:02 +0200 Subject: [PATCH 56/64] Fix test --- .../TranslationLayerTests/RunTestsWithFilterTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs index 78039d80d3..6c1b41fac1 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs @@ -45,7 +45,7 @@ public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo) _runEventHandler = new RunEventHandler(); var vstestConsoleWrapper = GetVsTestConsoleWrapper(); - var sources = new List { GetAssetFullPath("SimpleTestProject.dll") }; + var sources = new List { GetAssetFullPath("MSTestProject1.dll") }; vstestConsoleWrapper.RunTests( sources, From ab3093f71aaed62df068d39a9b19508958ab94a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 28 Mar 2022 16:18:18 +0200 Subject: [PATCH 57/64] Fix --- .../TranslationLayerTests/RunTestsWithFilterTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs index 6c1b41fac1..162ed10bde 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/RunTestsWithFilterTests.cs @@ -50,7 +50,7 @@ public void RunTestsWithTestCaseFilter(RunnerInfo runnerInfo) vstestConsoleWrapper.RunTests( sources, GetDefaultRunSettings(), - new TestPlatformOptions() { TestCaseFilter = "FullyQualifiedName=SampleUnitTestProject.UnitTest1.PassingTest" }, + new TestPlatformOptions() { TestCaseFilter = "FullyQualifiedName=MSTestProject1.UnitTest1.PassingTest" }, _runEventHandler); // Assert From 2ac14cb6b950b89a2e18df16bc904d833c981080 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 28 Mar 2022 17:08:35 +0200 Subject: [PATCH 58/64] Free space --- azure-pipelines.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8df66a8969..0deed3301f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,6 +8,9 @@ jobs: variables: buildConfiguration: 'Release' steps: + - checkout: self + fetchDepth: 1 + clean: true - task: BatchScript@1 displayName: 'Run script build.cmd' inputs: @@ -57,6 +60,9 @@ jobs: variables: buildConfiguration: 'Release' steps: + - checkout: self + fetchDepth: 1 + clean: true - task: DownloadPipelineArtifact@2 inputs: buildType: 'current' @@ -145,6 +151,9 @@ jobs: buildConfiguration: 'Release' VSTestRTMBuild: 'false' steps: + - checkout: self + fetchDepth: 1 + clean: true - script: chmod +x ./scripts/vsts-prebuild.sh displayName: 'Preparing for set version' From 020abf0b074944cb050fcca87570ca39451de08e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 28 Mar 2022 17:17:28 +0200 Subject: [PATCH 59/64] Free space --- azure-pipelines.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0deed3301f..96c3602587 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -63,12 +63,24 @@ jobs: - checkout: self fetchDepth: 1 clean: true + - task: PowerShell@2 + displayName: 'Free space' + inputs: + targetType: 'inline' + script: 'Get-PSDrive -PSProvider FileSystem | Out-String' + - task: DownloadPipelineArtifact@2 inputs: buildType: 'current' artifactName: 'testArtifacts' targetPath: '$(Build.SourcesDirectory)\artifacts' + - task: PowerShell@2 + displayName: 'Free space' + inputs: + targetType: 'inline' + script: 'Get-PSDrive -PSProvider FileSystem | Out-String' + - task: PowerShell@2 inputs: targetType: 'inline' @@ -90,6 +102,12 @@ jobs: targetType: 'inline' script: 'ls "$(Build.SourcesDirectory)\artifacts\Release\packages"' + - task: PowerShell@2 + displayName: 'Free space' + inputs: + targetType: 'inline' + script: 'Get-PSDrive -PSProvider FileSystem | Out-String' + - task: PowerShell@2 displayName: 'Disable Strong Name Validation' inputs: From 55e8714e583ba071ad9a9a82516686c20603f909 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 28 Mar 2022 17:24:26 +0200 Subject: [PATCH 60/64] Add results directory to blame --- .../BlameDataCollectorTests.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs index e9e6af25ee..f740998847 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs @@ -139,6 +139,7 @@ public void HangDumpOnTimeout(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = GetAssetFullPath("timeout.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, $@" /Blame:""CollectHangDump;HangDumpType=full;TestTimeout=3s"""); var env = new Dictionary @@ -162,6 +163,7 @@ public void CrashDumpWhenThereIsNoTimeout(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = GetAssetFullPath("timeout.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full;CollectAlways=true;CollectHangDump"""); var env = new Dictionary @@ -185,6 +187,7 @@ public void CrashDumpOnExit(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = GetAssetFullPath("timeout.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full;CollectAlways=true"""); var env = new Dictionary @@ -206,6 +209,7 @@ public void CrashDumpOnStackOverflow(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = GetAssetFullPath("crash.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full"""); var env = new Dictionary @@ -227,6 +231,7 @@ public void CrashDumpChildProcesses(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = GetAssetFullPath("child-crash.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, $@" /Blame:""CollectDump;DumpType=full"""); InvokeVsTest(arguments); @@ -242,6 +247,7 @@ public void HangDumpChildProcesses(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var assemblyPaths = GetAssetFullPath("child-hang.dll"); var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue); + arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}"); arguments = string.Concat(arguments, $@" /Blame:""CollectHangDump;HangDumpType=full;TestTimeout=15s"""); InvokeVsTest(arguments); From 23b5f663b7609e0e31573041e3335346a6d61f17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Mon, 28 Mar 2022 16:56:11 +0200 Subject: [PATCH 61/64] Fix build --- scripts/build.ps1 | 35 +++------------------------- scripts/common.lib.ps1 | 2 +- src/package/external/external.csproj | 2 +- test/TestAssets/TestAssets.sln | 16 ++++++++++++- 4 files changed, 20 insertions(+), 35 deletions(-) diff --git a/scripts/build.ps1 b/scripts/build.ps1 index 685f3907a6..cff737dc9c 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -197,40 +197,11 @@ function Invoke-TestAssetsBuild { throw "VSTestConsoleVersion for $propertyName is empty." } - $packages = @( - "Microsoft.TestPlatform" - "Microsoft.TestPlatform.CLI", - "Microsoft.TestPlatform.TranslationLayer" - "Microsoft.NET.Test.SDK" - ) - - foreach ($package in $packages) { - $packagePath = "$env:TP_ROOT_DIR\packages\$($package.ToLower())" - $cachePath = "$packagePath\$vstestConsoleVersion" - - if ((Test-Path -Path $cachePath) -and (Get-ChildItem $cachePath)) { - "Package $package $vsTestConsoleVersion is already in nuget cache at $cachePath." - continue - } - - Invoke-Exe $nugetExe -Arguments "install $package -Version $vsTestConsoleVersion -OutputDirectory $packagePath -ConfigFile ""$nugetConfig""" - - # Install puts it in packages/microsoft.testplatform/Microsoft.TestPlatform.17.1.0, - # because we use that as our output folder. And it also caches it in packages/microsoft.testplatform/17.1.0 - # unless the package is from local source, then it does not do that. So we need to rename the folder and remove - # the original one. - if (-not (Test-Path -Path $cachePath) -or -not (Get-ChildItem $cachePath)) { - Rename-Item "$packagePath\$package.$vsTestConsoleVersion" $cachePath - # Nuget locks the locally copied package from time to time. - Start-Sleep -Milliseconds 300 - } - if (Test-Path "$packagePath\$package.$vsTestConsoleVersion") { - Remove-Item -Recurse -Force "$packagePath\$package.$vsTestConsoleVersion" - } - } + # We don't use the results of this build anywhere, we just use them to restore the packages to nuget cache + # because using nuget.exe install errors out in various weird ways. + Invoke-Exe $dotnetExe -Arguments "build $env:TP_ROOT_DIR\test\TestAssets\Tools\Tools.csproj --configuration $TPB_Configuration -v:minimal -p:CIBuild=$TPB_CIBuild -p:LocalizedBuild=$TPB_LocalizedBuild -p:NETTestSdkVersion=$vsTestConsoleVersion" } - # Build with multiple versions of MSTest. The projects are directly in the root. # The folder structure in VS is not echoed in the TestAssets directory. $projects = @( diff --git a/scripts/common.lib.ps1 b/scripts/common.lib.ps1 index 34fd49a1e8..38b98bd805 100644 --- a/scripts/common.lib.ps1 +++ b/scripts/common.lib.ps1 @@ -47,7 +47,7 @@ Write-Verbose "Setup dotnet configuration." $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = 1 # Dotnet build doesn't support --packages yet. See https://github.com/dotnet/cli/issues/2712 $env:NUGET_PACKAGES = $env:TP_PACKAGES_DIR -$env:NUGET_EXE_Version = "5.8.1" +$env:NUGET_EXE_Version = "6.0.0" $env:DOTNET_CLI_VERSION = $GlobalJson.tools.dotnet # $env:DOTNET_RUNTIME_VERSION = "LATEST" $env:VSWHERE_VERSION = "2.0.2" diff --git a/src/package/external/external.csproj b/src/package/external/external.csproj index ebead5a9dc..4a0e70027d 100644 --- a/src/package/external/external.csproj +++ b/src/package/external/external.csproj @@ -33,7 +33,7 @@ - + diff --git a/test/TestAssets/TestAssets.sln b/test/TestAssets/TestAssets.sln index 834852d277..e349d2626f 100644 --- a/test/TestAssets/TestAssets.sln +++ b/test/TestAssets/TestAssets.sln @@ -116,10 +116,12 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "XUnit1000Passing", "perform EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MSTestProject2", "MSTestProject2\MSTestProject2.csproj", "{10AA955C-B412-41A8-899F-8609AAE19F61}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MSTestProject1", "MSTestProject1\MSTestProject1.csproj", "{E166D337-4033-4209-863F-8F77675EAEE8}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MSTestProject1", "MSTestProject1\MSTestProject1.csproj", "{E166D337-4033-4209-863F-8F77675EAEE8}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "basic", "basic", "{2633D125-64A7-456C-AD37-F8A6B56C2403}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tools", "Tools\Tools.csproj", "{85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -778,6 +780,18 @@ Global {E166D337-4033-4209-863F-8F77675EAEE8}.Release|x64.Build.0 = Release|Any CPU {E166D337-4033-4209-863F-8F77675EAEE8}.Release|x86.ActiveCfg = Release|Any CPU {E166D337-4033-4209-863F-8F77675EAEE8}.Release|x86.Build.0 = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|x64.ActiveCfg = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|x64.Build.0 = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|x86.ActiveCfg = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Debug|x86.Build.0 = Debug|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|Any CPU.Build.0 = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|x64.ActiveCfg = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|x64.Build.0 = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|x86.ActiveCfg = Release|Any CPU + {85F9F2A8-D2A5-4EA1-8BE0-06CCE141EC7A}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 1d9003c64a481e859280245df34ba6763dd80f7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Tue, 29 Mar 2022 09:11:51 +0200 Subject: [PATCH 62/64] Add tools and fix gitingore --- .gitignore | 12 ++++++------ test/TestAssets/Tools/Program.cs | 7 +++++++ test/TestAssets/Tools/Tools.csproj | 23 +++++++++++++++++++++++ 3 files changed, 36 insertions(+), 6 deletions(-) create mode 100644 test/TestAssets/Tools/Program.cs create mode 100644 test/TestAssets/Tools/Tools.csproj diff --git a/.gitignore b/.gitignore index d13211f267..0d22715429 100644 --- a/.gitignore +++ b/.gitignore @@ -104,14 +104,14 @@ ehthumbs.db # =========================== # Custom ignores # =========================== -packages -artifacts -[tT]ools +/packages/ +/artifacts/ +/[tT]ools/ *.dmp *.nupkg *.zip -src/package/sign/sign.nuget.targets +/src/package/sign/sign.nuget.targets # =========================== # Localized resx files @@ -140,5 +140,5 @@ src/package/sign/sign.nuget.targets # =========================== logs/ -.fake -.ionide +.fake/ +.ionide/ diff --git a/test/TestAssets/Tools/Program.cs b/test/TestAssets/Tools/Program.cs new file mode 100644 index 0000000000..94783ab8a9 --- /dev/null +++ b/test/TestAssets/Tools/Program.cs @@ -0,0 +1,7 @@ +// See https://aka.ms/new-console-template for more information +using System; + +// This project is used to restore TestPlatform and TestPlatform.CLI tools packages +// for testing with older versions. + +Console.WriteLine("Hello, World!"); diff --git a/test/TestAssets/Tools/Tools.csproj b/test/TestAssets/Tools/Tools.csproj new file mode 100644 index 0000000000..5981f1c93c --- /dev/null +++ b/test/TestAssets/Tools/Tools.csproj @@ -0,0 +1,23 @@ + + + + ..\..\..\ + true + true + + + + + + net5.0 + Exe + hanging_child + + + + + + + + + From 25c58025a91670eea1fe4dac099a7c7426ef571b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Tue, 29 Mar 2022 11:25:22 +0200 Subject: [PATCH 63/64] Make Cancel Discovery faster and more reliable --- .../TranslationLayerTests/DiscoverTests.cs | 30 +++- .../LongDiscoveryTestClass.cs | 135 +++++++++++++++++- 2 files changed, 158 insertions(+), 7 deletions(-) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs index 6c5d6fea09..b982311927 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/TranslationLayerTests/DiscoverTests.cs @@ -3,9 +3,13 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading.Tasks; +using FluentAssertions; +using FluentAssertions.Extensions; + using Microsoft.TestPlatform.TestUtilities; using Microsoft.TestPlatform.VsTestConsole.TranslationLayer.Interfaces; using Microsoft.VisualStudio.TestPlatform.Common.Telemetry; @@ -200,6 +204,7 @@ public void DiscoverTestsUsingSourceNavigation(RunnerInfo runnerInfo) [NetCoreTargetFrameworkDataSource] public async Task CancelTestDiscovery(RunnerInfo runnerInfo) { + var sw = Stopwatch.StartNew(); // Setup var testAssemblies = new List { @@ -212,11 +217,20 @@ public async Task CancelTestDiscovery(RunnerInfo runnerInfo) var discoveredTests = new List(); var discoveryEvents = new Mock(); + var alreadyCancelled = false; + TimeSpan cancellationCalled = TimeSpan.Zero; discoveryEvents.Setup(events => events.HandleDiscoveredTests(It.IsAny>())) .Callback((IEnumerable testcases) => { + // As soon as we get first test call cancel. That way we know there is discovery in progress. discoveredTests.AddRange(testcases); - _vstestConsoleWrapper.CancelDiscovery(); + if (!alreadyCancelled) + { + cancellationCalled = sw.Elapsed; + // Calling cancel many times crashes. https://github.com/microsoft/vstest/issues/3526 + alreadyCancelled = true; + _vstestConsoleWrapper.CancelDiscovery(); + } }); var isTestCancelled = false; discoveryEvents.Setup(events => events.HandleDiscoveryComplete(It.IsAny(), It.IsAny>(), It.IsAny())) @@ -229,11 +243,23 @@ public async Task CancelTestDiscovery(RunnerInfo runnerInfo) } }); + string runSettingsXml = + $@" + + + {FrameworkArgValue} + 1 + + "; + // Act - await Task.Run(() => _vstestConsoleWrapper.DiscoverTests(testAssemblies, GetDefaultRunSettings(), discoveryEvents.Object)); + await Task.Run(() => _vstestConsoleWrapper.DiscoverTests(testAssemblies, runSettingsXml, discoveryEvents.Object)); // Assert. Assert.IsTrue(isTestCancelled); + var done = sw.Elapsed; + var timeTillCancelled = done - cancellationCalled; + timeTillCancelled.Should().BeLessThan(2.Seconds()); int discoveredSourcesCount = discoveredTests.Select(testcase => testcase.Source).Distinct().Count(); Assert.AreNotEqual(testAssemblies.Count, discoveredSourcesCount, "All test assemblies discovered"); } diff --git a/test/TestAssets/DiscoveryTestProject/LongDiscoveryTestClass.cs b/test/TestAssets/DiscoveryTestProject/LongDiscoveryTestClass.cs index 26549f1961..4969cc84ed 100644 --- a/test/TestAssets/DiscoveryTestProject/LongDiscoveryTestClass.cs +++ b/test/TestAssets/DiscoveryTestProject/LongDiscoveryTestClass.cs @@ -10,18 +10,143 @@ namespace DiscoveryTestProject3 [TestClass] public class LongDiscoveryTestClass { - [MyTestMethod] - public void CustomTestMethod() + // This is for discovery cancellation test. + // 20 tests below to be discovered until we reach the X_ Y_ Z_LongDiscoveryTestMethod which haver attribute that + // takes a very long time to create, which prolongs the discovery time and keeps us discovering while we + // are cancelling the discovery from the test. + #region 20 empty tests + + [TestMethod] + public void TestMethod1() + { + } + + [TestMethod] + public void TestMethod2() + { + } + + [TestMethod] + public void TestMethod3() + { + } + + [TestMethod] + public void TestMethod4() + { + } + + [TestMethod] + public void TestMethod5() + { + } + + [TestMethod] + public void TestMethod6() + { + } + + [TestMethod] + public void TestMethod7() + { + } + + [TestMethod] + public void TestMethod8() + { + } + + [TestMethod] + public void TestMethod9() + { + } + + [TestMethod] + public void TestMethod10() + { + } + + [TestMethod] + public void TestMethod11() + { + } + + [TestMethod] + public void TestMethod12() + { + } + + [TestMethod] + public void TestMethod13() + { + } + + [TestMethod] + public void TestMethod14() + { + } + + [TestMethod] + public void TestMethod15() + { + } + + [TestMethod] + public void TestMethod16() + { + } + + [TestMethod] + public void TestMethod17() + { + } + + [TestMethod] + public void TestMethod18() + { + } + + [TestMethod] + public void TestMethod19() + { + } + + [TestMethod] + public void TestMethod20() + { + } + + #endregion + + // X_ to make it discover last. + [TestMethodWithDelay] + public void X_LongDiscoveryTestMethod() + { + + } + + // Y_ to make it discover last. + [TestMethodWithDelay] + public void Y_LongDiscoveryTestMethod() + { + + } + + // Z_ to make it discover last. + [TestMethodWithDelay] + public void Z_LongDiscoveryTestMethod() { } } - internal class MyTestMethodAttribute : TestMethodAttribute + internal class TestMethodWithDelayAttribute : TestMethodAttribute { - public MyTestMethodAttribute() + public TestMethodWithDelayAttribute() { - Thread.Sleep(10000); + // This will be multiplied by 3 because the framework will internally create this + // attribute 3 times. And by another 3 because we have 3 slow tests. + Thread.Sleep(100); } } } From 4dd10bc90e164a577aaf64f67f7cf930ab506ca8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Tue, 29 Mar 2022 12:14:41 +0200 Subject: [PATCH 64/64] Add fluent assertions --- .../Microsoft.TestPlatform.AcceptanceTests.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj index c5a1f80019..543b5b5acc 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj +++ b/test/Microsoft.TestPlatform.AcceptanceTests/Microsoft.TestPlatform.AcceptanceTests.csproj @@ -28,6 +28,7 @@ + all runtime; build; native; contentfiles; analyzers; buildtransitive