Skip to content

Commit

Permalink
Refined runsettings matching criteria for test sessions (#3610)
Browse files Browse the repository at this point in the history 
Refined runsettings matching criteria for test sessions (#3610)
  • Loading branch information
@cvpoienaru
cvpoienaru committed Apr 29, 2022
1 parent 16423fa commit 9cbcd99
Show file tree
Hide file tree
Showing 2 changed files with 189 additions and 7 deletions.
41 changes: 37 additions & 4 deletions src/Microsoft.TestPlatform.CrossPlatEngine/TestSession/ProxyTestSessionManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,16 @@
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.Linq;
using System.Threading.Tasks;

using Microsoft.VisualStudio.TestPlatform.Common.Telemetry;
using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client;
using Microsoft.VisualStudio.TestPlatform.ObjectModel;
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine;
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities;
using Microsoft.VisualStudio.TestPlatform.Utilities;

using CrossPlatResources = Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Resources.Resources;

Expand Down Expand Up @@ -42,6 +45,22 @@ private enum TestSessionState
private readonly IList<ProxyOperationManagerContainer> _proxyContainerList;
private readonly IDictionary<string, int> _proxyMap;
private readonly Stopwatch _testSessionStopwatch;
private IDictionary<string, string> _testSessionEnvironmentVariables = new Dictionary<string, string>();

private IDictionary<string, string> TestSessionEnvironmentVariables
{
get
{
if (_testSessionEnvironmentVariables.Count == 0)
{
_testSessionEnvironmentVariables = InferRunSettingsHelper.GetEnvironmentVariables(
_testSessionCriteria.RunSettings)
?? _testSessionEnvironmentVariables;
}

return _testSessionEnvironmentVariables;
}
}

/// <summary>
/// Initializes a new instance of the <see cref="ProxyTestSessionManager"/> class.
Expand Down Expand Up @@ -227,10 +246,7 @@ public virtual ProxyOperationManager DequeueProxy(string source, string runSetti
// We must ensure the current run settings match the run settings from when the
// testhost was started. If not, throw an exception to force the caller to create
// its own proxy instead.
//
// TODO (copoiena): This run settings match is rudimentary. We should refine the
// match criteria in the future.
if (!_testSessionCriteria.RunSettings.Equals(runSettings))
if (!CheckRunSettingsAreCompatible(runSettings))
{
throw new InvalidOperationException(
string.Format(
Expand Down Expand Up @@ -382,6 +398,23 @@ private void DisposeProxies()
_proxyMap.Clear();
}
}

private bool CheckRunSettingsAreCompatible(string requestRunSettings)
{
// Environment variable sets should be identical, otherwise it's not safe to reuse the
// already running testhosts.
var requestEnvironmentVariables = InferRunSettingsHelper.GetEnvironmentVariables(requestRunSettings);
if (requestEnvironmentVariables != null
&& TestSessionEnvironmentVariables != null
&& (requestEnvironmentVariables.Count != TestSessionEnvironmentVariables.Count
|| requestEnvironmentVariables.Except(TestSessionEnvironmentVariables).Any()))
{
return false;
}

// Data collection is not supported for test sessions yet.
return !XmlRunSettingsUtilities.IsDataCollectionEnabled(requestRunSettings);
}
}

/// <summary>
Expand Down
155 changes: 152 additions & 3 deletions test/Microsoft.TestPlatform.CrossPlatEngine.UnitTests/Client/ProxyTestSessionManagerTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,42 @@ public class ProxyTestSessionManagerTests
@"C:\temp\FakeTestAsset7.dll",
@"C:\temp\FakeTestAsset8.dll",
};
private readonly string _runSettingsNoEnvVars = @"<?xml version=""1.0"" encoding=""utf-8""?>
<RunSettings>
<RunConfiguration>
</RunConfiguration>
</RunSettings>";
private readonly string _runSettingsOneEnvVar = @"<?xml version=""1.0"" encoding=""utf-8""?>
<RunSettings>
<RunConfiguration>
<EnvironmentVariables>
<AAA>Test1</AAA>
</EnvironmentVariables>
</RunConfiguration>
</RunSettings>";
private readonly string _runSettingsTwoEnvVars = @"<?xml version=""1.0"" encoding=""utf-8""?>
<RunSettings>
<RunConfiguration>
<EnvironmentVariables>
<AAA>Test1</AAA>
<BBB>Test2</BBB>
</EnvironmentVariables>
</RunConfiguration>
</RunSettings>";
private readonly string _runSettingsTwoEnvVarsAndDataCollectors = @"<?xml version=""1.0"" encoding=""utf-8""?>
<RunSettings>
<RunConfiguration>
<EnvironmentVariables>
<AAA>Test1</AAA>
<BBB>Test2</BBB>
</EnvironmentVariables>
</RunConfiguration>
<DataCollectionRunSettings>
<DataCollectors>
<DataCollector friendlyName=""blame"" enabled=""True""></DataCollector>
</DataCollectors>
</DataCollectionRunSettings>
</RunSettings>";
private readonly string _fakeRunSettings = "FakeRunSettings";
private readonly ProtocolConfig _protocolConfig = new() { Version = 1 };
private Mock<ITestSessionEventsHandler> _mockEventsHandler;
Expand Down Expand Up @@ -281,7 +317,7 @@ public void DequeueProxyShouldSucceedIfIdentificationCriteriaAreMet()
mockProxyOperationManager.Setup(pom => pom.SetupChannel(It.IsAny<IEnumerable<string>>(), It.IsAny<string>()))
.Returns(true);

var testSessionCriteria = CreateTestSession(_fakeTestSources, _fakeRunSettings);
var testSessionCriteria = CreateTestSession(_fakeTestSources, _runSettingsNoEnvVars);
var proxyManager = CreateProxy(testSessionCriteria, mockProxyOperationManager.Object);

// StartSession should succeed.
Expand All @@ -302,7 +338,7 @@ public void DequeueProxyShouldSucceedIfIdentificationCriteriaAreMet()
// Second call to DequeueProxy fails because of runsettings mismatch.
Assert.ThrowsException<InvalidOperationException>(() => proxyManager.DequeueProxy(
testSessionCriteria.Sources[0],
"DummyRunSettings"));
_runSettingsOneEnvVar));

// Third call to DequeueProxy succeeds.
Assert.AreEqual(proxyManager.DequeueProxy(
Expand All @@ -316,14 +352,127 @@ public void DequeueProxyShouldSucceedIfIdentificationCriteriaAreMet()
testSessionCriteria.RunSettings));
}

[TestMethod]
public void DequeueProxyTwoConsecutiveTimesWithEnqueueShouldBeSuccessful()
{
var mockProxyOperationManager = new Mock<ProxyOperationManager>(null, null, null);
mockProxyOperationManager.Setup(pom => pom.SetupChannel(It.IsAny<IEnumerable<string>>(), It.IsAny<string>()))
.Returns(true);

var testSessionCriteria = CreateTestSession(_fakeTestSources, _runSettingsTwoEnvVars);
var proxyManager = CreateProxy(testSessionCriteria, mockProxyOperationManager.Object);

// StartSession should succeed.
Assert.IsTrue(proxyManager.StartSession(_mockEventsHandler.Object, _mockRequestData.Object));
mockProxyOperationManager.Verify(pom => pom.SetupChannel(
It.IsAny<IEnumerable<string>>(),
testSessionCriteria.RunSettings),
Times.Exactly(testSessionCriteria.Sources.Count));
_mockEventsHandler.Verify(eh => eh.HandleStartTestSessionComplete(
It.IsAny<StartTestSessionCompleteEventArgs>()),
Times.Once);

// Call to DequeueProxy succeeds.
Assert.AreEqual(proxyManager.DequeueProxy(
testSessionCriteria.Sources[0],
testSessionCriteria.RunSettings),
mockProxyOperationManager.Object);

Assert.AreEqual(proxyManager.EnqueueProxy(mockProxyOperationManager.Object.Id), true);

// Call to DequeueProxy succeeds when called with the same runsettings as before.
Assert.AreEqual(proxyManager.DequeueProxy(
testSessionCriteria.Sources[0],
testSessionCriteria.RunSettings),
mockProxyOperationManager.Object);
}

[TestMethod]
public void DequeueProxyShouldFailIfRunSettingsMatchingFails()
{
var mockProxyOperationManager = new Mock<ProxyOperationManager>(null, null, null);
mockProxyOperationManager.Setup(pom => pom.SetupChannel(It.IsAny<IEnumerable<string>>(), It.IsAny<string>()))
.Returns(true);

var testSessionCriteria = CreateTestSession(_fakeTestSources, _runSettingsOneEnvVar);
var proxyManager = CreateProxy(testSessionCriteria, mockProxyOperationManager.Object);

// StartSession should succeed.
Assert.IsTrue(proxyManager.StartSession(_mockEventsHandler.Object, _mockRequestData.Object));
mockProxyOperationManager.Verify(pom => pom.SetupChannel(
It.IsAny<IEnumerable<string>>(),
testSessionCriteria.RunSettings),
Times.Exactly(testSessionCriteria.Sources.Count));
_mockEventsHandler.Verify(eh => eh.HandleStartTestSessionComplete(
It.IsAny<StartTestSessionCompleteEventArgs>()),
Times.Once);

// This call to DequeueProxy fails because of runsettings mismatch.
Assert.ThrowsException<InvalidOperationException>(() => proxyManager.DequeueProxy(
testSessionCriteria.Sources[0],
_runSettingsTwoEnvVars));
}

[TestMethod]
public void DequeueProxyShouldFailIfRunSettingsMatchingFailsFor2EnvVariables()
{
var mockProxyOperationManager = new Mock<ProxyOperationManager>(null, null, null);
mockProxyOperationManager.Setup(pom => pom.SetupChannel(It.IsAny<IEnumerable<string>>(), It.IsAny<string>()))
.Returns(true);

var testSessionCriteria = CreateTestSession(_fakeTestSources, _runSettingsTwoEnvVars);
var proxyManager = CreateProxy(testSessionCriteria, mockProxyOperationManager.Object);

// StartSession should succeed.
Assert.IsTrue(proxyManager.StartSession(_mockEventsHandler.Object, _mockRequestData.Object));
mockProxyOperationManager.Verify(pom => pom.SetupChannel(
It.IsAny<IEnumerable<string>>(),
testSessionCriteria.RunSettings),
Times.Exactly(testSessionCriteria.Sources.Count));
_mockEventsHandler.Verify(eh => eh.HandleStartTestSessionComplete(
It.IsAny<StartTestSessionCompleteEventArgs>()),
Times.Once);

// This call to DequeueProxy fails because of runsettings mismatch.
Assert.ThrowsException<InvalidOperationException>(() => proxyManager.DequeueProxy(
testSessionCriteria.Sources[0],
_runSettingsOneEnvVar));
}

[TestMethod]
public void DequeueProxyShouldFailIfRunSettingsMatchingFailsForDataCollectors()
{
var mockProxyOperationManager = new Mock<ProxyOperationManager>(null, null, null);
mockProxyOperationManager.Setup(pom => pom.SetupChannel(It.IsAny<IEnumerable<string>>(), It.IsAny<string>()))
.Returns(true);

var testSessionCriteria = CreateTestSession(_fakeTestSources, _runSettingsTwoEnvVars);
var proxyManager = CreateProxy(testSessionCriteria, mockProxyOperationManager.Object);

// StartSession should succeed.
Assert.IsTrue(proxyManager.StartSession(_mockEventsHandler.Object, _mockRequestData.Object));
mockProxyOperationManager.Verify(pom => pom.SetupChannel(
It.IsAny<IEnumerable<string>>(),
testSessionCriteria.RunSettings),
Times.Exactly(testSessionCriteria.Sources.Count));
_mockEventsHandler.Verify(eh => eh.HandleStartTestSessionComplete(
It.IsAny<StartTestSessionCompleteEventArgs>()),
Times.Once);

// This call to DequeueProxy fails because of runsettings mismatch.
Assert.ThrowsException<InvalidOperationException>(() => proxyManager.DequeueProxy(
testSessionCriteria.Sources[0],
_runSettingsTwoEnvVarsAndDataCollectors));
}

[TestMethod]
public void EnqueueProxyShouldSucceedIfIdentificationCriteriaAreMet()
{
var mockProxyOperationManager = new Mock<ProxyOperationManager>(null, null, null);
mockProxyOperationManager.Setup(pom => pom.SetupChannel(It.IsAny<IEnumerable<string>>(), It.IsAny<string>()))
.Returns(true);

var testSessionCriteria = CreateTestSession(_fakeTestSources, _fakeRunSettings);
var testSessionCriteria = CreateTestSession(_fakeTestSources, _runSettingsNoEnvVars);
var proxyManager = CreateProxy(testSessionCriteria, mockProxyOperationManager.Object);

// Validate sanity checks.
Expand Down

0 comments on commit 9cbcd99

Please sign in to comment.