Test matrix

TestPlatform.sln

@@ -181,21 +181,13 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AttachmentProcessorDataColl
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "vstest.ProgrammerTests", "test\vstest.ProgrammerTests\vstest.ProgrammerTests.csproj", "{B1F84FD8-6150-4ECA-9AD7-C316E04E17D8}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Intent", "test\Intent\Intent.csproj", "{BFBB35C9-6437-480A-8DCC-AE3700110E7D}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Intent", "test\Intent\Intent.csproj", "{BFBB35C9-6437-480A-8DCC-AE3700110E7D}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Intent.Primitives", "test\Intent.Primitives\Intent.Primitives.csproj", "{29270853-90DC-4C39-9621-F47AE40A79B6}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Intent.Primitives", "test\Intent.Primitives\Intent.Primitives.csproj", "{29270853-90DC-4C39-9621-F47AE40A79B6}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "testhost.arm64", "src\testhost.arm64\testhost.arm64.csproj", "{186069FE-E1E8-4DE1-BEA4-0FF1484D22D1}"
 EndProject
 Global
-	GlobalSection(SharedMSBuildProjectFiles) = preSolution
-		src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{10b6ade1-f808-4612-801d-4452f5b52242}*SharedItemsImports = 5
-		src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{186069fe-e1e8-4de1-bea4-0ff1484d22d1}*SharedItemsImports = 5
-		src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{27dfbd04-64b2-4f1b-82b2-006620cca6f8}*SharedItemsImports = 5
-		src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{2c7ce1f8-e73e-4987-8023-b5a0ebac86e8}*SharedItemsImports = 5
-		src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{71cb42ff-e750-4a3b-9c3a-ac938853cc89}*SharedItemsImports = 5
-		src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{7f26eda3-c8c4-4b7f-a9b6-d278c2f40a13}*SharedItemsImports = 13
-	EndGlobalSection
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
 		Debug|x64 = Debug|x64
@@ -955,7 +947,7 @@ Global
 		{61F7F446-9EF3-4768-B33A-4D75F60E1059} = {7D4082EA-7AC9-4DFB-98E8-C5E08BDC0EC3}
 		{D5296435-3A3F-4B1A-81D1-434EC9E97DEF} = {5E7F18A8-F843-4C8A-AB02-4C7D9205C6CF}
 		{790B8030-00C2-4121-B125-EDC4CE329BA3} = {ED0C35EB-7F31-4841-A24F-8EB708FFA959}
-		{27DFBD04-64B2-4F1B-82B2-006620CCA6F8} = {ED0C35EB-7F31-4841-A24F-8EB708FFA959}
+		{27DFBD04-64B2-4F1B-82B2-006620CCA6F8} = {D9A30E32-D466-4EC5-B4F2-62E17562279B}
 		{71CB42FF-E750-4A3B-9C3A-AC938853CC89} = {ED0C35EB-7F31-4841-A24F-8EB708FFA959}
 		{10B6ADE1-F808-4612-801D-4452F5B52242} = {ED0C35EB-7F31-4841-A24F-8EB708FFA959}
 		{46250E12-4CF1-4051-B4A7-80C8C06E0068} = {B27FAFDF-2DBA-4AB0-BA85-FD5F21D359D6}
@@ -1020,4 +1012,12 @@ Global
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {0541B30C-FF51-4E28-B172-83F5F3934BCD}
 	EndGlobalSection
+	GlobalSection(SharedMSBuildProjectFiles) = preSolution
+		src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{10b6ade1-f808-4612-801d-4452f5b52242}*SharedItemsImports = 5
+		src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{186069fe-e1e8-4de1-bea4-0ff1484d22d1}*SharedItemsImports = 5
+		src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{27dfbd04-64b2-4f1b-82b2-006620cca6f8}*SharedItemsImports = 5
+		src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{2c7ce1f8-e73e-4987-8023-b5a0ebac86e8}*SharedItemsImports = 5
+		src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{71cb42ff-e750-4a3b-9c3a-ac938853cc89}*SharedItemsImports = 5
+		src\Microsoft.TestPlatform.Execution.Shared\Microsoft.TestPlatform.Execution.Shared.projitems*{7f26eda3-c8c4-4b7f-a9b6-d278c2f40a13}*SharedItemsImports = 13
+	EndGlobalSection
 EndGlobal

build.cmd

@@ -3,4 +3,5 @@
 REM Copyright (c) Microsoft. All rights reserved.
 
 powershell -ExecutionPolicy Bypass -NoProfile -NoLogo -Command "%~dp0scripts\build.ps1 %*; exit $LastExitCode;" 
+
 if %errorlevel% neq 0 exit /b %errorlevel% 

scripts/build/TestPlatform.Dependencies.props

@@ -13,20 +13,47 @@
     from a build parameter would not be available, so I am writing this version from the build.ps1 script to keep it in sync -->
     <NETTestSdkVersion>17.2.0-dev</NETTestSdkVersion>
 
+    <!-- These versions are used for running unit tests, and running acceptance tests. They are also used as the default version for projects
+    in TestAssets.sln to allow running and debugging tests in that solution directly in VS without having to run them via AcceptanceTests. -->
     <MSTestFrameworkVersion>2.2.8</MSTestFrameworkVersion>
     <MSTestAdapterVersion>2.2.8</MSTestAdapterVersion>
     <MSTestAssertExtensionVersion>1.0.3-preview</MSTestAssertExtensionVersion>
 
+
     <XUnitFrameworkVersion>2.3.1</XUnitFrameworkVersion>
     <XUnitAdapterVersion>2.3.1</XUnitAdapterVersion>
     <XUnitConsoleRunnerVersion>2.3.1</XUnitConsoleRunnerVersion>
 
     <NUnit3FrameworkVersion>3.10.1</NUnit3FrameworkVersion>
-    <NUnit3AdapterVersion>3.10.0</NUnit3AdapterVersion>
+    <NUnit3AdapterVersion>3.11.0</NUnit3AdapterVersion>
     <NUnitConsoleRunnerVersion>3.8.0</NUnitConsoleRunnerVersion>
 
     <ChutzpahAdapterVersion>4.4.12</ChutzpahAdapterVersion>
 
+    <!-- Versions that are used when building projects from TestAssets.sln for compatibility tests. See Invoke-TestAssetsBuild in scripts/build.ps1. 
+    Exact versions are used to avoid Nuget substituting them by closest match, if we make a typo. 
+    These versions need to be "statically" readable because we read this file as xml in our build and tests. -->
+    <!-- <MSTestFrameworkLatestVersion></MSTestFrameworkLatestVersion> is not here, because we don't build MSTest locally, so we don't have access to the latest version. -->
+    <MSTestFrameworkLatestPreviewVersion>[2.2.9-preview-20220210-07]</MSTestFrameworkLatestPreviewVersion>
+    <MSTestFrameworkLatestStableVersion>[2.2.8]</MSTestFrameworkLatestStableVersion>
+    <MSTestFrameworkRecentStableVersion>[2.2.7]</MSTestFrameworkRecentStableVersion>
+    <MSTestFrameworkMostDownloadedVersion>[2.1.0]</MSTestFrameworkMostDownloadedVersion>
+    <MSTestFrameworkPreviousStableVersion>[2.1.0]</MSTestFrameworkPreviousStableVersion>
+    <MSTestFrameworkLegacyStableVersion>[1.4.0]</MSTestFrameworkLegacyStableVersion>
+
+
+    <!-- Versions that are used to restore previous versions of console, translation layer, and test.sdk for compatibility tests. 
+    See Invoke-TestAssetsBuild in scripts/build.ps1. Exact versions are used to avoid Nuget substituting them by closest match, if we make a typo. 
+    These versions need to be "statically" readable because we read this file as xml in our build and tests. -->
+    <!-- <VSTestConsoleLatestVersion></VSTestConsoleLatestVersion> is not here, NETTestSdkVersion is used instead, because that is the version of the locally built latest package.  -->
+    <VSTestConsoleLatestPreviewVersion>[17.2.0-preview-20220131-20]</VSTestConsoleLatestPreviewVersion>
+    <VSTestConsoleLatestStableVersion>[17.1.0]</VSTestConsoleLatestStableVersion>
+    <VSTestConsoleRecentStableVersion>[17.0.0]</VSTestConsoleRecentStableVersion>
+    <VSTestConsoleMostDownloadedVersion>[16.6.1]</VSTestConsoleMostDownloadedVersion>
+    <VSTestConsolePreviousStableVersion>[16.11.0]</VSTestConsolePreviousStableVersion>
+    <VSTestConsoleLegacyStableVersion>[15.9.2]</VSTestConsoleLegacyStableVersion>
+
+
     <!-- This version also needs to be updated in src\package\nuspec\TestPlatform.ObjectModel.nuspec -->
     <NuGetFrameworksVersion>5.11.0</NuGetFrameworksVersion>
     <ILAsmPackageVersion>5.0.0</ILAsmPackageVersion>

scripts/build/TestPlatform.Settings.targets

@@ -77,7 +77,8 @@
       <PropertyGroup>
         <GenerateDocumentationFile>false</GenerateDocumentationFile>
         <!-- Suppress warnings about testhost being x64 (AMD64)/x86 when imported into AnyCPU (MSIL) test projects. -->
-        <NoWarn>$(NoWarn);MSB3270</NoWarn>
+        <!-- Suppress warnings about no strong-named nuget packages. -->
+        <NoWarn>$(NoWarn);MSB3270;CS8002</NoWarn>
       </PropertyGroup>
 
       <!-- Test projects are not discovered in test window without test container capability -->

scripts/verify-nupkgs.ps1

@@ -56,7 +56,9 @@ function Verify-Nuget-Packages($packageDirectory, $version)
             Write-Error "Number of files are not equal $unzipNugetPackageDir, expected: $($expectedNumOfFiles[$packageKey]) actual: $actualNumOfFiles"
         }
 
-        Remove-Item -Force -Recurse $unzipNugetPackageDir | Out-Null
+        # Don't remove the directories after you unpacked them
+        # they are useful for reviewing what is in the package.
+        # Remove-Item -Force -Recurse $unzipNugetPackageDir | Out-Null
     }
 
     Write-Log "Completed Verify-Nuget-Packages."

src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs

@@ -357,7 +357,7 @@ public bool AttachDebuggerToProcess(int pid, CancellationToken cancellationToken
             var ackPayload = _dataSerializer.DeserializePayload<EditorAttachDebuggerAckPayload>(ackMessage);
             if (!ackPayload.Attached)
             {
-                EqtTrace.Warning(ackPayload.ErrorMessage);
+                EqtTrace.Warning($"DesignModeClient.AttachDebuggerToProcess: Attaching to process failed: {ackPayload.ErrorMessage}");
             }
 
             return ackPayload.Attached;

src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/ConsoleParameters.cs

@@ -1,9 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
-#if NETFRAMEWORK
 using System.Collections.Generic;
-#endif
 using System.Diagnostics;
 using System.IO;
 
@@ -41,16 +39,11 @@ public ConsoleParameters(IFileHelper fileHelper)
         _fileHelper = fileHelper;
     }
 
-#if NETFRAMEWORK
-
     /// <summary>
-    /// TODO: Remove the #if when project is targeted to netstandard2.0
     /// Environment variables to be set for the process
     /// </summary>
     public Dictionary<string, string> EnvironmentVariables { get; set; }
 
-#endif
-
     /// <summary>
     /// Trace level for logs.
     /// </summary>

src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/PublicAPI.Shipped.txt

@@ -196,4 +196,6 @@ static Microsoft.VisualStudio.TestPlatform.VsTestConsole.TranslationLayer.Resour
 static Microsoft.VisualStudio.TestPlatform.VsTestConsole.TranslationLayer.Resources.Resources.InvalidFilePath.get -> string
 static Microsoft.VisualStudio.TestPlatform.VsTestConsole.TranslationLayer.Resources.Resources.ResourceManager.get -> System.Resources.ResourceManager
 static Microsoft.VisualStudio.TestPlatform.VsTestConsole.TranslationLayer.Resources.Resources.VsTestProcessExitedAbnormally.get -> string
-virtual Microsoft.TestPlatform.VsTestConsole.TranslationLayer.TestSession.Dispose(bool disposing) -> void
+virtual Microsoft.TestPlatform.VsTestConsole.TranslationLayer.TestSession.Dispose(bool disposing) -> void
+Microsoft.TestPlatform.VsTestConsole.TranslationLayer.ConsoleParameters.EnvironmentVariables.get -> System.Collections.Generic.Dictionary<string, string>
+Microsoft.TestPlatform.VsTestConsole.TranslationLayer.ConsoleParameters.EnvironmentVariables.set -> void

src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/PublicAPI/net451/PublicAPI.Shipped.txt

@@ -1,2 +0,0 @@
-Microsoft.TestPlatform.VsTestConsole.TranslationLayer.ConsoleParameters.EnvironmentVariables.get -> System.Collections.Generic.Dictionary<string, string>
-Microsoft.TestPlatform.VsTestConsole.TranslationLayer.ConsoleParameters.EnvironmentVariables.set -> void

src/Microsoft.TestPlatform.VsTestConsole.TranslationLayer/VsTestConsoleProcessManager.cs

@@ -121,7 +121,6 @@ public void StartProcess(ConsoleParameters consoleParameters)
 
         EqtTrace.Verbose("VsTestCommandLineWrapper: Process Start Info {0} {1}", info.FileName, info.Arguments);
 
-#if NETFRAMEWORK
         if (consoleParameters.EnvironmentVariables != null)
         {
             info.EnvironmentVariables.Clear();
@@ -133,7 +132,7 @@ public void StartProcess(ConsoleParameters consoleParameters)
                 }
             }
         }
-#endif
+
         try
         {
             _process = Process.Start(info);

test/Microsoft.TestPlatform.AcceptanceTests/AcceptanceTestBase.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System;
+using System.Linq;
 
 using Microsoft.TestPlatform.TestUtilities;
 
@@ -51,6 +52,17 @@ public class AcceptanceTestBase : IntegrationTestBase
     public const string NETCORE21_50 = "netcoreapp2.1;netcoreapp3.1;net5.0";
     public const string NETFX452_NET50 = "net452;net461;net472;net48;netcoreapp2.1;netcoreapp3.1;net5.0";
     public const string NETFX452_NET31 = "net452;net461;net472;net48;netcoreapp2.1;netcoreapp3.1";
+    public const string DEFAULT_RUNNER_NETFX = "net451";
+    /// <summary>
+    /// Our current defaults for .NET and .NET Framework.
+    /// </summary>
+    public const string DEFAULT_RUNNER_NETFX_AND_NET = $"{DEFAULT_RUNNER_NETFX};netcoreapp2.1";
+    public const string DEFAULT_HOST_NETFX_AND_NET = "net451;netcoreapp2.1";
+    public const string LATEST_TO_LEGACY = "Latest;LatestPreview;LatestStable;RecentStable;MostDownloaded;PreviousStable;LegacyStable";
+    public const string LATESTPREVIEW_TO_LEGACY = "LatestPreview;LatestStable;RecentStable;MostDownloaded;PreviousStable;LegacyStable";
+    public const string LATEST = "Latest";
+    public const string LATESTSTABLE= "LatestStable";
+    internal const string MSTEST = "MSTest";
 
     public static string And(string left, string right)
     {
@@ -61,6 +73,11 @@ public static string And(string left, string right)
 
     protected static void SetTestEnvironment(IntegrationTestEnvironment testEnvironment, RunnerInfo runnerInfo)
     {
+        testEnvironment.VSTestConsoleInfo = runnerInfo.VSTestConsoleInfo;
+        // The order here matters, it changes how the resulting path is built when we resolve test dlls and other assets.
+        testEnvironment.DllInfos = new[] { runnerInfo.AdapterInfo, runnerInfo.TestHostInfo }.Where(d => d != null).ToList();
+        testEnvironment.DebugInfo = runnerInfo.DebugInfo;
+
         testEnvironment.RunnerFramework = runnerInfo.RunnerFramework;
         testEnvironment.TargetFramework = runnerInfo.TargetFramework;
         testEnvironment.InIsolationValue = runnerInfo.InIsolationValue;
@@ -96,17 +113,37 @@ protected string GetTargetFramworkForRunsettings()
     }
 
     /// <summary>
-    /// Default RunSettings
+    /// Empty runsettings, just with the RunSettings tag that we require.
     /// </summary>
     /// <returns></returns>
+    public string GetEmptyRunsettings()
+    {
+        return "<RunSettings></RunSettings>";
+    }
+
+    /// <summary>
+    /// Almost empty runsettings, just specifying the target framework from the currently set test environment.
+    /// </summary>
     public string GetDefaultRunSettings()
     {
-        string runSettingsXml = $@"<?xml version=""1.0"" encoding=""utf-8""?>
-                                    <RunSettings>
-                                        <RunConfiguration>
-                                        <TargetFrameworkVersion>{FrameworkArgValue}</TargetFrameworkVersion>
-                                        </RunConfiguration>
-                                    </RunSettings>";
+        return GetRunSettingsWithTargetFramework(FrameworkArgValue);
+    }
+
+    /// <summary>
+    /// Almost empty runsettings, just specifying the given target framework.
+    /// Use the overload without any parameters to get the target framework from the currently set test environment.
+    /// </summary>
+    /// <returns></returns>
+    public string GetRunSettingsWithTargetFramework(string targetFramework)
+    {
+        string runSettingsXml =
+            $@"<?xml version=""1.0"" encoding=""utf-8""?>
+            <RunSettings>
+                <RunConfiguration>
+                    <TargetFrameworkVersion>{targetFramework}</TargetFrameworkVersion>
+                </RunConfiguration>
+            </RunSettings>";
+
         return runSettingsXml;
     }
 }

test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs

@@ -64,7 +64,7 @@ public void BlameDataCollectorShouldOutputDumpFile(RunnerInfo runnerInfo)
     {
 
         SetTestEnvironment(_testEnvironment, runnerInfo);
-        var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"');
+        var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll");
         var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue);
         arguments = string.Concat(arguments, $" /Blame:CollectDump");
         arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}");
@@ -89,7 +89,7 @@ public void BlameDataCollectorShouldNotOutputDumpFileWhenNoCrashOccurs(RunnerInf
     {
 
         SetTestEnvironment(_testEnvironment, runnerInfo);
-        var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"');
+        var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll");
         var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue);
         arguments = string.Concat(arguments, $" /Blame:CollectDump");
         arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}");
@@ -114,7 +114,7 @@ public void BlameDataCollectorShouldOutputDumpFileWhenNoCrashOccursButCollectAlw
     {
 
         SetTestEnvironment(_testEnvironment, runnerInfo);
-        var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"');
+        var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll");
         var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue);
         arguments = string.Concat(arguments, $" /Blame:CollectDump;CollectAlways=True");
         arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}");