backport: cctest TAP output changes

.gitignore

@@ -85,7 +85,7 @@ deps/npm/node_modules/.bin/
 # test artifacts
 tools/faketime
 icu_config.gypi
-test.tap
+*.tap
 
 # Xcode workspaces and project folders
 *.xcodeproj

Makefile

@@ -192,6 +192,7 @@ test-ci-js:
 		$(TEST_CI_ARGS) $(CI_JS_SUITES)
 
 test-ci: | build-addons
+	out/Release/cctest --gtest_output=tap:cctest.tap
 	$(PYTHON) tools/test.py -p tap --logfile test.tap --mode=release --flaky-tests=$(FLAKY_TESTS) \
 		$(TEST_CI_ARGS) $(CI_NATIVE_SUITES) $(CI_JS_SUITES)
 

deps/gtest/src/gtest.cc

@@ -3498,6 +3498,127 @@ std::string XmlUnitTestResultPrinter::RemoveInvalidXmlCharacters(
 //   </testsuite>
 // </testsuites>
 
+class TapUnitTestResultPrinter : public EmptyTestEventListener {
+ public:
+  TapUnitTestResultPrinter();
+  explicit TapUnitTestResultPrinter(const char* output_file);
+  virtual void OnTestIterationEnd(const UnitTest& unit_test, int iteration);
+
+ private:
+  static void PrintTapUnitTest(::std::ostream* stream,
+                               const UnitTest& unit_test);
+  static void PrintTapTestCase(int* count,
+                               ::std::ostream* stream,
+                               const TestCase& test_case);
+  static void OutputTapTestInfo(int* count,
+                                ::std::ostream* stream,
+                                const char* test_case_name,
+                                const TestInfo& test_info);
+  static void OutputTapComment(::std::ostream* stream, const char* comment);
+
+  const std::string output_file_;
+  GTEST_DISALLOW_COPY_AND_ASSIGN_(TapUnitTestResultPrinter);
+};
+
+TapUnitTestResultPrinter::TapUnitTestResultPrinter() {}
+
+TapUnitTestResultPrinter::TapUnitTestResultPrinter(const char* output_file)
+    : output_file_(output_file) {
+  if (output_file_.c_str() == NULL || output_file_.empty()) {
+    fprintf(stderr, "TAP output file may not be null\n");
+    fflush(stderr);
+    exit(EXIT_FAILURE);
+  }
+}
+
+void TapUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test,
+                                                  int /*iteration*/) {
+  FILE* tapout = stdout;
+
+  if (!output_file_.empty()) {
+    FilePath output_file(output_file_);
+    FilePath output_dir(output_file.RemoveFileName());
+
+    tapout = NULL;
+    if (output_dir.CreateDirectoriesRecursively())
+      tapout = posix::FOpen(output_file_.c_str(), "w");
+
+    if (tapout == NULL) {
+      fprintf(stderr, "Unable to open file \"%s\"\n", output_file_.c_str());
+      fflush(stderr);
+      exit(EXIT_FAILURE);
+    }
+  }
+
+  std::stringstream stream;
+  PrintTapUnitTest(&stream, unit_test);
+  fprintf(tapout, "%s", StringStreamToString(&stream).c_str());
+  fflush(tapout);
+
+  if (tapout != stdout)
+    fclose(tapout);
+}
+
+void TapUnitTestResultPrinter::PrintTapUnitTest(std::ostream* stream,
+                                                const UnitTest& unit_test) {
+  *stream << "TAP version 13\n";
+  *stream << "1.." << unit_test.reportable_test_count() << "\n";
+
+  int count = 1;
+  for (int i = 0; i < unit_test.total_test_case_count(); ++i) {
+    const TestCase& test_case = *unit_test.GetTestCase(i);
+    if (test_case.reportable_test_count() > 0)
+      PrintTapTestCase(&count, stream, test_case);
+  }
+
+  *stream << "# failures: " << unit_test.failed_test_count() << "\n";
+}
+
+void TapUnitTestResultPrinter::PrintTapTestCase(int* count,
+                                                std::ostream* stream,
+                                                const TestCase& test_case) {
+  for (int i = 0; i < test_case.total_test_count(); ++i) {
+    const TestInfo& test_info = *test_case.GetTestInfo(i);
+    if (test_info.is_reportable())
+      OutputTapTestInfo(count, stream, test_case.name(), test_info);
+  }
+}
+
+void TapUnitTestResultPrinter::OutputTapTestInfo(int* count,
+                                                 ::std::ostream* stream,
+                                                 const char* test_case_name,
+                                                 const TestInfo& test_info) {
+  const TestResult& result = *test_info.result();
+  const char* status = result.Passed() ? "ok" : "not ok";
+
+  *stream << status << " " << *count << " - " <<
+             test_case_name << "." << test_info.name() << "\n";
+  *stream << "  ---\n";
+  *stream << "  duration_ms: " <<
+             FormatTimeInMillisAsSeconds(result.elapsed_time()) << "\n";
+
+  if (result.total_part_count() > 0) {
+    *stream << "  stack: |-\n";
+    for (int i = 0; i < result.total_part_count(); ++i) {
+      const TestPartResult& part = result.GetTestPartResult(i);
+      OutputTapComment(stream, part.message());
+    }
+  }
+  *stream << "  ...\n";
+  *count += 1;
+}
+
+void TapUnitTestResultPrinter::OutputTapComment(::std::ostream* stream,
+                                                const char* comment) {
+  const char* start = comment;
+  while (const char* end = strchr(start, '\n')) {
+    *stream << "    " << std::string(start, end) << "\n";
+    start = end + 1;
+  }
+  if (*start)
+    *stream << "    " << start << "\n";
+}
+
 // Formats the given time in milliseconds as seconds.
 std::string FormatTimeInMillisAsSeconds(TimeInMillis ms) {
   ::std::stringstream ss;
@@ -4365,6 +4486,9 @@ void UnitTestImpl::ConfigureXmlOutput() {
   if (output_format == "xml") {
     listeners()->SetDefaultXmlGenerator(new XmlUnitTestResultPrinter(
         UnitTestOptions::GetAbsolutePathToOutputFile().c_str()));
+  } else if (output_format == "tap") {
+    listeners()->SetDefaultXmlGenerator(new TapUnitTestResultPrinter(
+        UnitTestOptions::GetAbsolutePathToOutputFile().c_str()));
   } else if (output_format != "") {
     printf("WARNING: unrecognized output format \"%s\" ignored.\n",
            output_format.c_str());

deps/gtest/src/gtest_main.cc

@@ -32,7 +32,6 @@
 #include "gtest/gtest.h"
 
 GTEST_API_ int main(int argc, char **argv) {
-  printf("Running main() from gtest_main.cc\n");
   testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }

tools/test.py

@@ -256,11 +256,15 @@ def HasRun(self, output):
 
 class TapProgressIndicator(SimpleProgressIndicator):
 
-  def _printDiagnostic(self, messages):
-    for l in messages.splitlines():
-      logger.info('# ' + l)
+  def _printDiagnostic(self, traceback, severity):
+    logger.info('  severity: %s', severity)
+    logger.info('  stack: |-')
+
+    for l in traceback.splitlines():
+      logger.info('    ' + l)
 
   def Starting(self):
+    logger.info('TAP version 13')
     logger.info('1..%i' % len(self.cases))
     self._done = 0
 
@@ -269,6 +273,8 @@ def AboutToRun(self, case):
 
   def HasRun(self, output):
     self._done += 1
+    self.traceback = ''
+    self.severity = 'ok'
 
     # Print test name as (for example) "parallel/test-assert".  Tests that are
     # scraped from the addons documentation are all named test.js, making it
@@ -281,19 +287,23 @@ def HasRun(self, output):
 
     if output.UnexpectedOutput():
       status_line = 'not ok %i %s' % (self._done, command)
+      self.severity = 'fail'
+      self.traceback = output.output.stdout + output.output.stderr
+
       if FLAKY in output.test.outcomes and self.flaky_tests_mode == DONTCARE:
         status_line = status_line + ' # TODO : Fix flaky test'
+        self.severity = 'flaky'
+
       logger.info(status_line)
-      self._printDiagnostic("\n".join(output.diagnostic))
 
       if output.HasCrashed():
-        self._printDiagnostic(PrintCrashed(output.output.exit_code))
+        self.severity = 'crashed'
+        exit_code = output.output.exit_code 
+        self.traceback = "oh no!\nexit code: " + PrintCrashed(exit_code)
 
       if output.HasTimedOut():
-        self._printDiagnostic('TIMEOUT')
+        self.severity = 'fail'
 
-      self._printDiagnostic(output.output.stderr)
-      self._printDiagnostic(output.output.stdout)
     else:
       skip = skip_regex.search(output.output.stdout)
       if skip:
@@ -304,7 +314,11 @@ def HasRun(self, output):
         if FLAKY in output.test.outcomes:
           status_line = status_line + ' # TODO : Fix flaky test'
         logger.info(status_line)
-      self._printDiagnostic("\n".join(output.diagnostic))
+
+      if output.diagnostic:
+        self.severity = 'ok'
+        self.traceback = output.diagnostic
+
 
     duration = output.test.duration
 
@@ -313,7 +327,12 @@ def HasRun(self, output):
       (duration.seconds + duration.days * 24 * 3600) * 10**6) / 10**6
 
     logger.info('  ---')
-    logger.info('  duration_ms: %d.%d' % (total_seconds, duration.microseconds / 1000))
+    logger.info('  duration_ms: %d.%d' %
+      (total_seconds, duration.microseconds / 1000))
+    if self.severity is not 'ok' or self.traceback is not '':
+      if output.HasTimedOut():
+        self.traceback = 'timeout'
+      self._printDiagnostic(self.traceback, self.severity)
     logger.info('  ...')
 
   def Done(self):

vcbuild.bat

@@ -20,6 +20,7 @@ set noprojgen=
 set nobuild=
 set nosign=
 set nosnapshot=
+set cctest_args=
 set test_args=
 set package=
 set msi=
@@ -56,7 +57,7 @@ if /i "%1"=="noetw"         set noetw=1&goto arg-ok
 if /i "%1"=="noperfctr"     set noperfctr=1&goto arg-ok
 if /i "%1"=="licensertf"    set licensertf=1&goto arg-ok
 if /i "%1"=="test"          set test_args=%test_args% addons doctool sequential parallel message -J&set jslint=1&set build_addons=1&goto arg-ok
-if /i "%1"=="test-ci"       set test_args=%test_args% %test_ci_args% -p tap --logfile test.tap addons doctool message sequential parallel&set build_addons=1&goto arg-ok
+if /i "%1"=="test-ci"       set test_args=%test_args% %test_ci_args% -p tap --logfile test.tap addons doctool message sequential parallel&set cctest_args=%cctest_args% --gtest_output=tap:cctest.tap&set build_addons=1&goto arg-ok
 if /i "%1"=="test-addons"   set test_args=%test_args% addons&set build_addons=1&goto arg-ok
 if /i "%1"=="test-simple"   set test_args=%test_args% sequential parallel -J&goto arg-ok
 if /i "%1"=="test-message"  set test_args=%test_args% message&goto arg-ok
@@ -351,8 +352,8 @@ goto run-tests
 if "%test_args%"=="" goto jslint
 if "%config%"=="Debug" set test_args=--mode=debug %test_args%
 if "%config%"=="Release" set test_args=--mode=release %test_args%
-echo running 'cctest'
-"%config%\cctest"
+echo running 'cctest %cctest_args%'
+"%config%\cctest" %cctest_args%
 echo running 'python tools\test.py %test_args%'
 python tools\test.py %test_args%
 goto jslint