Skip sources when runtime provider is not found

playground/TestPlatform.Playground/Environment.cs

@@ -11,7 +11,7 @@ internal class EnvironmentVariables
     {
         ["VSTEST_CONNECTION_TIMEOUT"] = "999",
         ["VSTEST_DEBUG_NOBP"] = "1",
-        ["VSTEST_RUNNER_DEBUG_ATTACHVS"] = "0",
+        ["VSTEST_RUNNER_DEBUG_ATTACHVS"] = "1",
         ["VSTEST_HOST_DEBUG_ATTACHVS"] = "0",
         ["VSTEST_DATACOLLECTOR_DEBUG_ATTACHVS"] = "0",
     };

playground/TestPlatform.Playground/Program.cs

@@ -43,13 +43,14 @@ static void Main(string[] args)
                     <RunConfiguration>
                         <InIsolation>true</InIsolation>
                         <MaxCpuCount>0</MaxCpuCount>
+<DesignMode>False</DesignMode>
                     </RunConfiguration>
                 </RunSettings>
             ";
 
         var sources = new[] {
-            Path.Combine(playground, "MSTest1", "bin", "Debug", "net472", "MSTest1.dll"),
-            Path.Combine(playground, "MSTest1", "bin", "Debug", "net5.0", "MSTest1.dll"),
+         @"S:\t\mstest-x64\bin\Debug\net472\Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.dll",
+         @"S:\t\mstest-x64\bin\Debug\net472\mstest-x64.dll",
         };
 
         // console mode
@@ -60,7 +61,7 @@ static void Main(string[] args)
             var processStartInfo = new ProcessStartInfo
             {
                 FileName = console,
-                Arguments = $"{string.Join(" ", sources)} --settings:{settingsFile} --listtests",
+                Arguments = $"{string.Join(" ", sources)} --settings:{settingsFile}",
                 UseShellExecute = false,
             };
             EnvironmentVariables.Variables.ToList().ForEach(processStartInfo.Environment.Add);
@@ -105,7 +106,7 @@ static void Main(string[] args)
         //// Run with test cases and without custom testhost launcher
         //r.RunTests(discoveryHandler.TestCases, sourceSettings, options, sessionHandler.TestSessionInfo, new TestRunHandler());
         //// Run with sources and custom testhost launcher
-        //r.RunTestsWithCustomTestHost(sources, sourceSettings, options, sessionHandler.TestSessionInfo, new TestRunHandler(), new DebuggerTestHostLauncher());
+        r.RunTestsWithCustomTestHost(sources, sourceSettings, options, sessionHandler.TestSessionInfo, new TestRunHandler(), new DebuggerTestHostLauncher());
         //// Run with sources
         //r.RunTests(sources, sourceSettings, options, sessionHandler.TestSessionInfo, new TestRunHandler());
         var rd = sw.ElapsedMilliseconds;
@@ -163,9 +164,9 @@ private static string WriteTests(IEnumerable<TestCase> testCases)
                 ? "\t" + string.Join("\n\t", testCases.Select(r => r.Source + " " + r.DisplayName))
                 : "\t<empty>";
 
-        private static string WriteSources(IEnumerable<string> sources)
-            => sources?.Any() == true
-                ? "\t" + string.Join("\n\t", sources)
+        private static string WriteSources(IEnumerable<DiscoveredSource> sources)
+            => sources?.Select(s => s.Source).Any() == true
+                ? "\t" + string.Join("\n\t", sources.Select(s => s.Source))
                 : "\t<empty>";
     }
 

src/Microsoft.TestPlatform.Client/DesignMode/DesignModeTestEventsRegistrar.cs

@@ -63,3 +63,4 @@ public void LogWarning(string? message)
         _designModeClient.SendTestMessage(TestMessageLevel.Warning, message);
     }
 }
+

src/Microsoft.TestPlatform.Client/TestPlatform.cs

@@ -80,7 +80,8 @@ public IDiscoveryRequest CreateDiscoveryRequest(
         IRequestData requestData,
         DiscoveryCriteria discoveryCriteria,
         TestPlatformOptions? options,
-        Dictionary<string, SourceDetail> sourceToSourceDetailMap)
+        Dictionary<string, SourceDetail> sourceToSourceDetailMap,
+        IWarningLogger warningLogger)
     {
         ValidateArg.NotNull(discoveryCriteria, nameof(discoveryCriteria));
 
@@ -90,7 +91,7 @@ public IDiscoveryRequest CreateDiscoveryRequest(
         ITestLoggerManager loggerManager = _testEngine.GetLoggerManager(requestData);
         loggerManager.Initialize(discoveryCriteria.RunSettings);
 
-        IProxyDiscoveryManager discoveryManager = _testEngine.GetDiscoveryManager(requestData, discoveryCriteria, sourceToSourceDetailMap);
+        IProxyDiscoveryManager discoveryManager = _testEngine.GetDiscoveryManager(requestData, discoveryCriteria, sourceToSourceDetailMap, warningLogger);
         discoveryManager.Initialize(options?.SkipDefaultAdapters ?? false);
 
         return new DiscoveryRequest(requestData, discoveryCriteria, discoveryManager, loggerManager);
@@ -101,7 +102,8 @@ public ITestRunRequest CreateTestRunRequest(
         IRequestData requestData,
         TestRunCriteria testRunCriteria,
         TestPlatformOptions? options,
-        Dictionary<string, SourceDetail> sourceToSourceDetailMap)
+        Dictionary<string, SourceDetail> sourceToSourceDetailMap,
+        IWarningLogger warningLogger)
     {
         ValidateArg.NotNull(testRunCriteria, nameof(testRunCriteria));
 
@@ -112,7 +114,7 @@ public ITestRunRequest CreateTestRunRequest(
         ITestLoggerManager loggerManager = _testEngine.GetLoggerManager(requestData);
         loggerManager.Initialize(testRunCriteria.TestRunSettings);
 
-        IProxyExecutionManager executionManager = _testEngine.GetExecutionManager(requestData, testRunCriteria, sourceToSourceDetailMap);
+        IProxyExecutionManager executionManager = _testEngine.GetExecutionManager(requestData, testRunCriteria, sourceToSourceDetailMap, warningLogger);
         executionManager.Initialize(options?.SkipDefaultAdapters ?? false);
 
         return new TestRunRequest(requestData, testRunCriteria, executionManager, loggerManager);
@@ -123,7 +125,8 @@ public bool StartTestSession(
         IRequestData requestData,
         StartTestSessionCriteria testSessionCriteria,
         ITestSessionEventsHandler eventsHandler,
-        Dictionary<string, SourceDetail> sourceToSourceDetailMap)
+        Dictionary<string, SourceDetail> sourceToSourceDetailMap,
+        IWarningLogger warningLogger)
     {
         ValidateArg.NotNull(testSessionCriteria, nameof(testSessionCriteria));
 
@@ -137,7 +140,7 @@ public bool StartTestSession(
             return false;
         }
 
-        IProxyTestSessionManager? testSessionManager = _testEngine.GetTestSessionManager(requestData, testSessionCriteria, sourceToSourceDetailMap);
+        IProxyTestSessionManager? testSessionManager = _testEngine.GetTestSessionManager(requestData, testSessionCriteria, sourceToSourceDetailMap, warningLogger);
         if (testSessionManager == null)
         {
             // The test session manager is null because the combination of runsettings and

src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/DiscoveryStatus.cs

@@ -22,4 +22,9 @@ public enum DiscoveryStatus
     /// Indicates that source was fully discovered.
     /// </summary>
     FullyDiscovered,
+
+    /// <summary>
+    /// Indicates that source was skipped in discovery.
+    /// </summary>
+    SkippedDiscovery
 }

src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/ITestEngine.cs

@@ -29,7 +29,8 @@ public interface ITestEngine
     IProxyDiscoveryManager GetDiscoveryManager(
         IRequestData requestData,
         DiscoveryCriteria discoveryCriteria,
-        IDictionary<string, SourceDetail> sourceToSourceDetailMap);
+        IDictionary<string, SourceDetail> sourceToSourceDetailMap,
+        IWarningLogger warningLogger);
 
     /// <summary>
     /// Fetches the ExecutionManager for this engine. This manager would provide all
@@ -46,7 +47,8 @@ IProxyDiscoveryManager GetDiscoveryManager(
     IProxyExecutionManager GetExecutionManager(
         IRequestData requestData,
         TestRunCriteria testRunCriteria,
-        IDictionary<string, SourceDetail> sourceToSourceDetailMap);
+        IDictionary<string, SourceDetail> sourceToSourceDetailMap,
+        IWarningLogger warningLogger);
 
     /// <summary>
     /// Fetches the TestSessionManager for this engine. This manager would provide all
@@ -64,7 +66,8 @@ IProxyExecutionManager GetExecutionManager(
     IProxyTestSessionManager GetTestSessionManager(
         IRequestData requestData,
         StartTestSessionCriteria testSessionCriteria,
-        IDictionary<string, SourceDetail> sourceToSourceDetailMap);
+        IDictionary<string, SourceDetail> sourceToSourceDetailMap,
+        IWarningLogger warningLogger);
 
     /// <summary>
     /// Fetches the extension manager for this engine. This manager would provide extensibility

src/Microsoft.TestPlatform.Common/PublicAPI/PublicAPI.Shipped.txt

@@ -251,13 +251,14 @@ Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus.Partially
 Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus.FullyDiscovered = 2 -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus
 Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IParallelOperationManager
 Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IParallelOperationManager.UpdateParallelLevel(int parallelLevel) -> void
-Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine.GetDiscoveryManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCriteria discoveryCriteria, System.Collections.Generic.IDictionary<string, Microsoft.VisualStudio.TestPlatform.ObjectModel.SourceDetail> sourceToSourceDetailMap) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyDiscoveryManager
-Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine.GetExecutionManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCriteria testRunCriteria, System.Collections.Generic.IDictionary<string, Microsoft.VisualStudio.TestPlatform.ObjectModel.SourceDetail> sourceToSourceDetailMap) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyExecutionManager
-Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine.GetTestSessionManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCriteria testSessionCriteria, System.Collections.Generic.IDictionary<string, Microsoft.VisualStudio.TestPlatform.ObjectModel.SourceDetail> sourceToSourceDetailMap) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyTestSessionManager
 Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyExecutionManager.Abort(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IInternalTestRunEventsHandler eventHandler) -> void
 Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyExecutionManager.Cancel(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IInternalTestRunEventsHandler eventHandler) -> void
 Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyExecutionManager.StartTestRun(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCriteria testRunCriteria, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IInternalTestRunEventsHandler eventHandler) -> int
 Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.TesthostProtocol.IExecutionManager.Abort(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IInternalTestRunEventsHandler testRunEventsHandler) -> void
 Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.TesthostProtocol.IExecutionManager.Cancel(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IInternalTestRunEventsHandler testRunEventsHandler) -> void
 Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.TesthostProtocol.IExecutionManager.StartTestRun(System.Collections.Generic.Dictionary<string, System.Collections.Generic.IEnumerable<string>> adapterSourceMap, string package, string runSettings, Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ClientProtocol.TestExecutionContext testExecutionContext, Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestCaseEventsHandler testCaseEvents, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IInternalTestRunEventsHandler eventHandler) -> void
 Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.TesthostProtocol.IExecutionManager.StartTestRun(System.Collections.Generic.IEnumerable<Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase> tests, string package, string runSettings, Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ClientProtocol.TestExecutionContext testExecutionContext, Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestCaseEventsHandler testCaseEvents, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IInternalTestRunEventsHandler eventHandler) -> void
+Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus.SkippedDiscovery = 3 -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.DiscoveryStatus
+Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine.GetDiscoveryManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveryCriteria discoveryCriteria, System.Collections.Generic.IDictionary<string, Microsoft.VisualStudio.TestPlatform.ObjectModel.SourceDetail> sourceToSourceDetailMap, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IWarningLogger warningLogger) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyDiscoveryManager
+Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine.GetExecutionManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.TestRunCriteria testRunCriteria, System.Collections.Generic.IDictionary<string, Microsoft.VisualStudio.TestPlatform.ObjectModel.SourceDetail> sourceToSourceDetailMap, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IWarningLogger warningLogger) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyExecutionManager
+Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.ITestEngine.GetTestSessionManager(Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IRequestData requestData, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.StartTestSessionCriteria testSessionCriteria, System.Collections.Generic.IDictionary<string, Microsoft.VisualStudio.TestPlatform.ObjectModel.SourceDetail> sourceToSourceDetailMap, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IWarningLogger warningLogger) -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine.IProxyTestSessionManager

src/Microsoft.TestPlatform.CommunicationUtilities/Messages/DiscoveryCompletePayload.cs

@@ -4,6 +4,7 @@
 using System.Collections.Generic;
 
 using Microsoft.VisualStudio.TestPlatform.ObjectModel;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
 
 #nullable disable
 
@@ -37,17 +38,17 @@ public class DiscoveryCompletePayload
     /// <summary>
     /// Gets or sets list of sources which were fully discovered.
     /// </summary>
-    public IList<string> FullyDiscoveredSources { get; set; } = new List<string>();
+    public IList<DiscoveredSource> FullyDiscoveredSources { get; set; } = new List<DiscoveredSource>();
 
     /// <summary>
     /// Gets or sets list of sources which were partially discovered (started discover tests, but then discovery aborted).
     /// </summary>
-    public IList<string> PartiallyDiscoveredSources { get; set; } = new List<string>();
+    public IList<DiscoveredSource> PartiallyDiscoveredSources { get; set; } = new List<DiscoveredSource>();
 
     /// <summary>
     /// Gets or sets list of sources which were not discovered at all.
     /// </summary>
-    public IList<string> NotDiscoveredSources { get; set; } = new List<string>();
+    public IList<DiscoveredSource> NotDiscoveredSources { get; set; } = new List<DiscoveredSource>();
 
     /// <summary>
     /// Gets or sets the collection of discovered extensions.

src/Microsoft.TestPlatform.CommunicationUtilities/PublicAPI/PublicAPI.Shipped.txt

@@ -372,11 +372,8 @@ static Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Resources.Reso
 static Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Resources.Resources.VersionCheckFailed.get -> string
 static Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Resources.Resources.VersionCheckTimedout.get -> string
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces.ITestRequestSender.SendDiscoveryAbort() -> void
-Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.FullyDiscoveredSources.get -> System.Collections.Generic.IList<string>
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.FullyDiscoveredSources.set -> void
-Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.NotDiscoveredSources.get -> System.Collections.Generic.IList<string>
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.NotDiscoveredSources.set -> void
-Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.PartiallyDiscoveredSources.get -> System.Collections.Generic.IList<string>
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.PartiallyDiscoveredSources.set -> void
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.TestRequestSender.SendDiscoveryAbort() -> void
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.DiscoveredExtensions.get -> System.Collections.Generic.Dictionary<string, System.Collections.Generic.HashSet<string>>
@@ -389,3 +386,6 @@ Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces.ITestReque
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Interfaces.ITestRequestSender.StartTestRun(Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.TestRunCriteriaWithTests runCriteria, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IInternalTestRunEventsHandler eventHandler) -> void
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.TestRequestSender.StartTestRun(Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.TestRunCriteriaWithSources runCriteria, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IInternalTestRunEventsHandler eventHandler) -> void
 Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.TestRequestSender.StartTestRun(Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.TestRunCriteriaWithTests runCriteria, Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.IInternalTestRunEventsHandler eventHandler) -> void
+Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.FullyDiscoveredSources.get -> System.Collections.Generic.IList<Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveredSource>
+Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.NotDiscoveredSources.get -> System.Collections.Generic.IList<Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveredSource>
+Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.ObjectModel.DiscoveryCompletePayload.PartiallyDiscoveredSources.get -> System.Collections.Generic.IList<Microsoft.VisualStudio.TestPlatform.ObjectModel.Client.DiscoveredSource>

src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/DiscoveryDataAggregator.cs

@@ -212,7 +212,7 @@ public void MarkSourcesWithStatus(IEnumerable<string?>? sources, DiscoveryStatus
                 (_, previousStatus) =>
                 {
                     if (previousStatus == DiscoveryStatus.FullyDiscovered && status != DiscoveryStatus.FullyDiscovered
-                        || previousStatus == DiscoveryStatus.PartiallyDiscovered && status == DiscoveryStatus.NotDiscovered)
+                        || previousStatus == DiscoveryStatus.PartiallyDiscovered && (status == DiscoveryStatus.NotDiscovered || status == DiscoveryStatus.SkippedDiscovery))
                     {
                         EqtTrace.Warning($"DiscoveryDataAggregator.MarkSourcesWithStatus: Downgrading source {source} status from '{previousStatus}' to '{status}'.");
                     }