diff --git a/src/Microsoft.TestPlatform.Common/Utilities/RunSettingsUtilities.cs b/src/Microsoft.TestPlatform.Common/Utilities/RunSettingsUtilities.cs index e9b655bd1c..28f037c4c9 100644 --- a/src/Microsoft.TestPlatform.Common/Utilities/RunSettingsUtilities.cs +++ b/src/Microsoft.TestPlatform.Common/Utilities/RunSettingsUtilities.cs @@ -131,20 +131,20 @@ public static int GetMaxCpuCount(RunConfiguration runConfiguration) return cpuCount; } /// - /// Gets the value of FailWhenNoTestsFound parameter from runsettings file + /// Gets the value of TreatNoTestsAsError parameter from runsettings file /// /// Runsetting string value - /// The value of FailWhenNoTestsFound + /// The value of TreatNoTestsAsError public static bool GetTreatNoTestsAsError(string runSettings) { - bool failWhenNoTestFound = false; + bool treatNoTestsAsError = false; if (runSettings != null) { try { RunConfiguration runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runSettings); - failWhenNoTestFound = GetTreatNoTestsAsError(runConfiguration); + treatNoTestsAsError = GetTreatNoTestsAsError(runConfiguration); } catch (SettingsException se) { @@ -155,7 +155,7 @@ public static bool GetTreatNoTestsAsError(string runSettings) } } - return failWhenNoTestFound; + return treatNoTestsAsError; } private static bool GetTreatNoTestsAsError(RunConfiguration runConfiguration) diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/TestLoggerManager.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/TestLoggerManager.cs index 80c8c6fcd4..2e3eb9a108 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/Client/TestLoggerManager.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/Client/TestLoggerManager.cs @@ -22,7 +22,7 @@ namespace Microsoft.VisualStudio.TestPlatform.CrossPlatEngine.Client using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities; using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions; using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces; - using CommonResources = Microsoft.VisualStudio.TestPlatform.Common.Resources.Resources; + using CommonResources = Common.Resources.Resources; /// /// Responsible for managing logger extensions and broadcasting results @@ -52,6 +52,11 @@ internal class TestLoggerManager : ITestLoggerManager /// private string targetFramework; + /// + /// TreatNoTestsAsError value; + /// + private bool treatNoTestsAsError; + /// /// Test Logger Events instance which will be passed to loggers when they are initialized. /// @@ -145,6 +150,7 @@ public void Initialize(string runSettings) // Store test run directory. This runsettings is the final runsettings merging CLI args and runsettings. this.testRunDirectory = GetResultsDirectory(runSettings); this.targetFramework = GetTargetFramework(runSettings)?.Name; + this.treatNoTestsAsError = GetTreatNoTestsAsError(runSettings); var loggers = XmlRunSettingsUtilities.GetLoggerRunSettings(runSettings); @@ -487,6 +493,16 @@ internal Framework GetTargetFramework(string runSettings) return targetFramework; } + /// + /// Get TreatNoTestsAsError value of the test run + /// + /// + /// + internal bool GetTreatNoTestsAsError(string runSettings) + { + return RunSettingsUtilities.GetTreatNoTestsAsError(runSettings); + } + /// /// Enables sending of events to the loggers which are registered. /// @@ -648,6 +664,13 @@ private Dictionary UpdateLoggerParameters(Dictionary /// Logger for Generating TRX @@ -65,13 +65,13 @@ internal TrxLogger(IFileHelper fileHelper, TrxFileHelper trxFileHelper) // The converter class private Converter converter; - private TrxLoggerObjectModel.TestRun testRun; - private ConcurrentDictionary results; - private ConcurrentDictionary testElements; + private TestRun testRun; + private ConcurrentDictionary results; + private ConcurrentDictionary testElements; private ConcurrentDictionary entries; // Caching results and inner test entries for constant time lookup for inner parents. - private ConcurrentDictionary innerResults; + private ConcurrentDictionary innerResults; private ConcurrentDictionary innerTestEntries; private readonly TrxFileHelper trxFileHelper; @@ -82,7 +82,7 @@ internal TrxLogger(IFileHelper fileHelper, TrxFileHelper trxFileHelper) private StringBuilder runLevelStdOut; // List of run level errors and warnings generated. These are logged in the Trx in the Results Summary. - private List runLevelErrorsAndWarnings; + private List runLevelErrorsAndWarnings; private TrxLoggerObjectModel.TestOutcome testRunOutcome = TrxLoggerObjectModel.TestOutcome.Passed; @@ -230,10 +230,11 @@ internal TrxLoggerObjectModel.TestOutcome TestResultOutcome /// internal void TestMessageHandler(object sender, TestRunMessageEventArgs e) { - ValidateArg.NotNull(sender, "sender"); - ValidateArg.NotNull(e, "e"); + ValidateArg.NotNull(sender, "sender"); + ValidateArg.NotNull(e, "e"); RunInfo runMessage; + switch (e.Level) { case TestMessageLevel.Informational: @@ -263,7 +264,7 @@ internal void TestMessageHandler(object sender, TestRunMessageEventArgs e) /// /// The eventArgs. /// - internal void TestResultHandler(object sender, ObjectModel.Logging.TestResultEventArgs e) + internal void TestResultHandler(object sender, TestResultEventArgs e) { // Create test run if (this.testRun == null) @@ -363,6 +364,8 @@ internal void TestRunCompleteHandler(object sender, TestRunCompleteEventArgs e) this.testRunOutcome = TrxLoggerObjectModel.TestOutcome.Completed; } + testRunOutcome = changeTestOutcomeIfNecessary(testRunOutcome); + List errorMessages = new List(); List collectorEntries = this.converter.ToCollectionEntries(e.AttachmentSets, this.testRun, this.testResultsDirPath); IList resultFiles = this.converter.ToResultFiles(e.AttachmentSets, this.testRun, this.testResultsDirPath, errorMessages); @@ -760,6 +763,19 @@ private TestEntry GetTestEntry(Guid executionId) return testEntry; } + private TrxLoggerObjectModel.TestOutcome changeTestOutcomeIfNecessary (TrxLoggerObjectModel.TestOutcome outcome) + { + // If no tests discovered/executed and TreatNoTestsAsError was set to True + // We will return ResultSummary as Failed + // Note : we only send the value of TreatNoTestsAsError if it is "True" + if (totalTests == 0 && parametersDictionary.ContainsKey(ObjectModelConstants.TreatNoTestsAsError)) + { + outcome = TrxLoggerObjectModel.TestOutcome.Failed; + } + + return outcome; + } + #endregion } } diff --git a/src/Microsoft.TestPlatform.ObjectModel/Constants.cs b/src/Microsoft.TestPlatform.ObjectModel/Constants.cs index 8a209fb9ad..fd6cef5fb2 100644 --- a/src/Microsoft.TestPlatform.ObjectModel/Constants.cs +++ b/src/Microsoft.TestPlatform.ObjectModel/Constants.cs @@ -115,6 +115,11 @@ public static class Constants /// public const string LoggerConfigurationNameLower = "configuration"; + /// + /// Name of TreatNoTestsAsError parameter + /// + public const string TreatNoTestsAsError = "TreatNoTestsAsError"; + /// /// Name of RunConfiguration settings node in RunSettings. /// diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs b/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs index 8ac1128a56..ddaaa544de 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/LoggerTests.cs @@ -116,6 +116,54 @@ public void HtmlLoggerWithExecutorUriShouldProperlyOverwriteFile(RunnerInfo runn IsFileAndContentEqual(htmlLogFilePath); } + [TestMethod] + [TestCategory("Windows-Review")] + [NetFullTargetFrameworkDataSource] + public void TrxLoggerResultSummaryOutcomeValueShouldBeFailedIfNoTestsExecutedAndTreatNoTestsAsErrorIsTrue(RunnerInfo runnerInfo) + { + SetTestEnvironment(this.testEnvironment, runnerInfo); + + var arguments = PrepareArguments(this.GetSampleTestAssembly(), this.GetTestAdapterPath(), string.Empty, this.FrameworkArgValue, runnerInfo.InIsolationValue); + var trxFileName = "TrxLogger.trx"; + + arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); + + // Setting /TestCaseFilter to the test name, which does not exists in the assembly, so we will have 0 tests executed + arguments = string.Concat(arguments, " /TestCaseFilter:TestNameThatMatchesNoTestInTheAssembly"); + arguments = string.Concat(arguments, " -- RunConfiguration.TreatNoTestsAsError=true"); + + this.InvokeVsTest(arguments); + + var trxLogFilePath = Path.Combine(Directory.GetCurrentDirectory(), "TestResults", trxFileName); + string outcomeValue = GetElementAtributeValueFromTrx(trxLogFilePath, "ResultSummary", "outcome"); + + Assert.AreEqual("Failed", outcomeValue); + } + + [TestMethod] + [TestCategory("Windows-Review")] + [NetFullTargetFrameworkDataSource] + public void TrxLoggerResultSummaryOutcomeValueShouldNotChangeIfNoTestsExecutedAndTreatNoTestsAsErrorIsFalse(RunnerInfo runnerInfo) + { + SetTestEnvironment(this.testEnvironment, runnerInfo); + + var arguments = PrepareArguments(this.GetSampleTestAssembly(), this.GetTestAdapterPath(), string.Empty, this.FrameworkArgValue, runnerInfo.InIsolationValue); + var trxFileName = "TrxLogger.trx"; + + arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\""); + + // Setting /TestCaseFilter to the test name, which does not exists in the assembly, so we will have 0 tests executed + arguments = string.Concat(arguments, " /TestCaseFilter:TestNameThatMatchesNoTestInTheAssembly"); + arguments = string.Concat(arguments, " -- RunConfiguration.TreatNoTestsAsError=false"); + + this.InvokeVsTest(arguments); + + var trxLogFilePath = Path.Combine(Directory.GetCurrentDirectory(), "TestResults", trxFileName); + string outcomeValue = GetElementAtributeValueFromTrx(trxLogFilePath, "ResultSummary", "outcome"); + + Assert.AreEqual("Completed", outcomeValue); + } + private bool IsValidXml(string xmlFilePath) { var reader = System.Xml.XmlReader.Create(File.OpenRead(xmlFilePath)); @@ -146,5 +194,22 @@ private void IsFileAndContentEqual(string filePath) StringAssert.Contains(filePathContent, str); } } + + private static string GetElementAtributeValueFromTrx(string trxFileName, string fieldName, string attributeName) + { + using (FileStream file = File.OpenRead(trxFileName)) + using (XmlReader reader = XmlReader.Create(file)) + { + while (reader.Read()) + { + if (reader.Name.Equals(fieldName) && reader.NodeType == XmlNodeType.Element && reader.HasAttributes) + { + return reader.GetAttribute(attributeName); + } + } + } + + return null; + } } }