From f72f02faa9897e87215a304aca481351fb3b8a7e Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Tue, 9 Aug 2016 16:37:21 +0200 Subject: [PATCH 1/6] deps: make gtest output tap Teach gtest to produce TAP so we can integrate it better with our CI tooling. TAP is printed to stdout but it can also be written to file by passing the `--gtest_output=tap:filename.tap` switch to cctest. PR-URL: https://github.com/nodejs/node/pull/8034 Reviewed-By: James M Snell --- deps/gtest/src/gtest.cc | 124 ++++++++++++++++++++++++++++++++++- deps/gtest/src/gtest_main.cc | 1 - 2 files changed, 123 insertions(+), 2 deletions(-) diff --git a/deps/gtest/src/gtest.cc b/deps/gtest/src/gtest.cc index 7fd5f298dc04ed..970c0976cb5acf 100644 --- a/deps/gtest/src/gtest.cc +++ b/deps/gtest/src/gtest.cc @@ -3498,6 +3498,125 @@ std::string XmlUnitTestResultPrinter::RemoveInvalidXmlCharacters( // // +class TapUnitTestResultPrinter : public EmptyTestEventListener { + public: + TapUnitTestResultPrinter(); + explicit TapUnitTestResultPrinter(const char* output_file); + virtual void OnTestIterationEnd(const UnitTest& unit_test, int iteration); + + private: + static void PrintTapUnitTest(::std::ostream* stream, + const UnitTest& unit_test); + static void PrintTapTestCase(int* count, + ::std::ostream* stream, + const TestCase& test_case); + static void OutputTapTestInfo(int* count, + ::std::ostream* stream, + const char* test_case_name, + const TestInfo& test_info); + static void OutputTapComment(::std::ostream* stream, const char* comment); + + const std::string output_file_; + GTEST_DISALLOW_COPY_AND_ASSIGN_(TapUnitTestResultPrinter); +}; + +TapUnitTestResultPrinter::TapUnitTestResultPrinter() {} + +TapUnitTestResultPrinter::TapUnitTestResultPrinter(const char* output_file) + : output_file_(output_file) { + if (output_file_.c_str() == NULL || output_file_.empty()) { + fprintf(stderr, "TAP output file may not be null\n"); + fflush(stderr); + exit(EXIT_FAILURE); + } +} + +void TapUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test, + int /*iteration*/) { + FILE* tapout = stdout; + + if (!output_file_.empty()) { + FilePath output_file(output_file_); + FilePath output_dir(output_file.RemoveFileName()); + + tapout = NULL; + if (output_dir.CreateDirectoriesRecursively()) + tapout = posix::FOpen(output_file_.c_str(), "w"); + + if (tapout == NULL) { + fprintf(stderr, "Unable to open file \"%s\"\n", output_file_.c_str()); + fflush(stderr); + exit(EXIT_FAILURE); + } + } + + std::stringstream stream; + PrintTapUnitTest(&stream, unit_test); + fprintf(tapout, "%s", StringStreamToString(&stream).c_str()); + fflush(tapout); + + if (tapout != stdout) + fclose(tapout); +} + +void TapUnitTestResultPrinter::PrintTapUnitTest(std::ostream* stream, + const UnitTest& unit_test) { + *stream << "TAP version 13\n"; + *stream << "1.." << unit_test.reportable_test_count() << "\n"; + + int count = 1; + for (int i = 0; i < unit_test.total_test_case_count(); ++i) { + const TestCase& test_case = *unit_test.GetTestCase(i); + if (test_case.reportable_test_count() > 0) + PrintTapTestCase(&count, stream, test_case); + } + + *stream << "# failures: " << unit_test.failed_test_count() << "\n"; +} + +void TapUnitTestResultPrinter::PrintTapTestCase(int* count, + std::ostream* stream, + const TestCase& test_case) { + for (int i = 0; i < test_case.total_test_count(); ++i) { + const TestInfo& test_info = *test_case.GetTestInfo(i); + if (test_info.is_reportable()) + OutputTapTestInfo(count, stream, test_case.name(), test_info); + } +} + +void TapUnitTestResultPrinter::OutputTapTestInfo(int* count, + ::std::ostream* stream, + const char* test_case_name, + const TestInfo& test_info) { + const TestResult& result = *test_info.result(); + const char* status = result.Passed() ? "ok" : "not ok"; + + *stream << status << " " << *count << " - " << + test_case_name << "." << test_info.name() << "\n"; + *stream << " ---\n"; + *stream << " duration_ms: " << + FormatTimeInMillisAsSeconds(result.elapsed_time()) << "\n"; + *stream << " ...\n"; + + for (int i = 0; i < result.total_part_count(); ++i) { + const TestPartResult& part = result.GetTestPartResult(i); + OutputTapComment(stream, part.message()); + } + + *count += 1; +} + +void TapUnitTestResultPrinter::OutputTapComment(::std::ostream* stream, + const char* comment) { + const char* start = comment; + while (const char* end = strchr(start, '\n')) { + *stream << "# " << std::string(start, end) << "\n"; + start = end + 1; + } + if (*start) + *stream << "# " << start << "\n"; +} + // Formats the given time in milliseconds as seconds. std::string FormatTimeInMillisAsSeconds(TimeInMillis ms) { ::std::stringstream ss; @@ -4314,7 +4433,7 @@ UnitTestImpl::UnitTestImpl(UnitTest* parent) #endif // Will be overridden by the flag before first use. catch_exceptions_(false) { - listeners()->SetDefaultResultPrinter(new PrettyUnitTestResultPrinter); + listeners()->SetDefaultResultPrinter(new TapUnitTestResultPrinter); } UnitTestImpl::~UnitTestImpl() { @@ -4365,6 +4484,9 @@ void UnitTestImpl::ConfigureXmlOutput() { if (output_format == "xml") { listeners()->SetDefaultXmlGenerator(new XmlUnitTestResultPrinter( UnitTestOptions::GetAbsolutePathToOutputFile().c_str())); + } else if (output_format == "tap") { + listeners()->SetDefaultXmlGenerator(new TapUnitTestResultPrinter( + UnitTestOptions::GetAbsolutePathToOutputFile().c_str())); } else if (output_format != "") { printf("WARNING: unrecognized output format \"%s\" ignored.\n", output_format.c_str()); diff --git a/deps/gtest/src/gtest_main.cc b/deps/gtest/src/gtest_main.cc index f3028225523306..4cf03e59bac5df 100644 --- a/deps/gtest/src/gtest_main.cc +++ b/deps/gtest/src/gtest_main.cc @@ -32,7 +32,6 @@ #include "gtest/gtest.h" GTEST_API_ int main(int argc, char **argv) { - printf("Running main() from gtest_main.cc\n"); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } From 938e67a0b658ead48807612548b448a6e38ca6e9 Mon Sep 17 00:00:00 2001 From: Ben Noordhuis Date: Wed, 10 Aug 2016 11:47:46 +0200 Subject: [PATCH 2/6] build: run cctests as part of test-ci target Enable the cctests on the CI now that they know how to write TAP output. PR-URL: https://github.com/nodejs/node/pull/8034 Reviewed-By: James M Snell --- Makefile | 1 + vcbuild.bat | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index f42d50cbab9e91..82103da01155a5 100644 --- a/Makefile +++ b/Makefile @@ -192,6 +192,7 @@ test-ci-js: $(TEST_CI_ARGS) $(CI_JS_SUITES) test-ci: | build-addons + out/Release/cctest --gtest_output=tap:cctest.tap $(PYTHON) tools/test.py -p tap --logfile test.tap --mode=release --flaky-tests=$(FLAKY_TESTS) \ $(TEST_CI_ARGS) $(CI_NATIVE_SUITES) $(CI_JS_SUITES) diff --git a/vcbuild.bat b/vcbuild.bat index 77c0d01a14f486..519e7168ecf976 100644 --- a/vcbuild.bat +++ b/vcbuild.bat @@ -20,6 +20,7 @@ set noprojgen= set nobuild= set nosign= set nosnapshot= +set cctest_args= set test_args= set package= set msi= @@ -56,7 +57,7 @@ if /i "%1"=="noetw" set noetw=1&goto arg-ok if /i "%1"=="noperfctr" set noperfctr=1&goto arg-ok if /i "%1"=="licensertf" set licensertf=1&goto arg-ok if /i "%1"=="test" set test_args=%test_args% addons doctool sequential parallel message -J&set jslint=1&set build_addons=1&goto arg-ok -if /i "%1"=="test-ci" set test_args=%test_args% %test_ci_args% -p tap --logfile test.tap addons doctool message sequential parallel&set build_addons=1&goto arg-ok +if /i "%1"=="test-ci" set test_args=%test_args% %test_ci_args% -p tap --logfile test.tap addons doctool message sequential parallel&set cctest_args=%cctest_args% --gtest_output=tap:cctest.tap&set build_addons=1&goto arg-ok if /i "%1"=="test-addons" set test_args=%test_args% addons&set build_addons=1&goto arg-ok if /i "%1"=="test-simple" set test_args=%test_args% sequential parallel -J&goto arg-ok if /i "%1"=="test-message" set test_args=%test_args% message&goto arg-ok @@ -351,8 +352,8 @@ goto run-tests if "%test_args%"=="" goto jslint if "%config%"=="Debug" set test_args=--mode=debug %test_args% if "%config%"=="Release" set test_args=--mode=release %test_args% -echo running 'cctest' -"%config%\cctest" +echo running 'cctest %cctest_args%' +"%config%\cctest" %cctest_args% echo running 'python tools\test.py %test_args%' python tools\test.py %test_args% goto jslint From 7f819198b50a96937a663d8719a554e878c442b0 Mon Sep 17 00:00:00 2001 From: Brian White Date: Mon, 17 Oct 2016 11:12:54 -0400 Subject: [PATCH 3/6] deps: revert default gtest reporter change PR-URL: https://github.com/nodejs/node/pull/8948 Reviewed-By: Jeremiah Senkpiel Reviewed-By: Anna Henningsen Reviewed-By: James M Snell Reviewed-By: Sakthipriyan Vairamani Reviewed-By: Ben Noordhuis --- deps/gtest/src/gtest.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/gtest/src/gtest.cc b/deps/gtest/src/gtest.cc index 970c0976cb5acf..213740ca0c7943 100644 --- a/deps/gtest/src/gtest.cc +++ b/deps/gtest/src/gtest.cc @@ -4433,7 +4433,7 @@ UnitTestImpl::UnitTestImpl(UnitTest* parent) #endif // Will be overridden by the flag before first use. catch_exceptions_(false) { - listeners()->SetDefaultResultPrinter(new TapUnitTestResultPrinter); + listeners()->SetDefaultResultPrinter(new PrettyUnitTestResultPrinter); } UnitTestImpl::~UnitTestImpl() { From d2693b16bdb9250efa1f4957999f3fb33dacf6c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bergstr=C3=B6m?= Date: Wed, 12 Oct 2016 11:34:28 -0300 Subject: [PATCH 4/6] test: output tap13 instead of almost-tap Produce a tap13-compatible output which makes it simpler to parse. Output is still readable by the jenkins tap plugin. PR-URL: https://github.com/nodejs/node/pull/9262 Reviewed-By: Gibson Fahnestock Reviewed-By: Ben Noordhuis Reviewed-By: Myles Borins --- tools/test.py | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/tools/test.py b/tools/test.py index 97880fb3cd2667..ae18158c0c9329 100755 --- a/tools/test.py +++ b/tools/test.py @@ -256,11 +256,15 @@ def HasRun(self, output): class TapProgressIndicator(SimpleProgressIndicator): - def _printDiagnostic(self, messages): - for l in messages.splitlines(): - logger.info('# ' + l) + def _printDiagnostic(self, traceback, severity): + logger.info(' severity: %s', severity) + logger.info(' stack: |-') + + for l in traceback.splitlines(): + logger.info(' ' + l) def Starting(self): + logger.info('TAP version 13') logger.info('1..%i' % len(self.cases)) self._done = 0 @@ -269,6 +273,8 @@ def AboutToRun(self, case): def HasRun(self, output): self._done += 1 + self.traceback = '' + self.severity = 'ok' # Print test name as (for example) "parallel/test-assert". Tests that are # scraped from the addons documentation are all named test.js, making it @@ -281,19 +287,23 @@ def HasRun(self, output): if output.UnexpectedOutput(): status_line = 'not ok %i %s' % (self._done, command) + self.severity = 'fail' + self.traceback = output.output.stdout + output.output.stderr + if FLAKY in output.test.outcomes and self.flaky_tests_mode == DONTCARE: status_line = status_line + ' # TODO : Fix flaky test' + self.severity = 'flaky' + logger.info(status_line) - self._printDiagnostic("\n".join(output.diagnostic)) if output.HasCrashed(): - self._printDiagnostic(PrintCrashed(output.output.exit_code)) + self.severity = 'crashed' + exit_code = output.output.exit_code + self.traceback = "oh no!\nexit code: " + PrintCrashed(exit_code) if output.HasTimedOut(): - self._printDiagnostic('TIMEOUT') + self.severity = 'fail' - self._printDiagnostic(output.output.stderr) - self._printDiagnostic(output.output.stdout) else: skip = skip_regex.search(output.output.stdout) if skip: @@ -304,7 +314,11 @@ def HasRun(self, output): if FLAKY in output.test.outcomes: status_line = status_line + ' # TODO : Fix flaky test' logger.info(status_line) - self._printDiagnostic("\n".join(output.diagnostic)) + + if output.diagnostic: + self.severity = 'ok' + self.traceback = output.diagnostic + duration = output.test.duration @@ -313,7 +327,12 @@ def HasRun(self, output): (duration.seconds + duration.days * 24 * 3600) * 10**6) / 10**6 logger.info(' ---') - logger.info(' duration_ms: %d.%d' % (total_seconds, duration.microseconds / 1000)) + logger.info(' duration_ms: %d.%d' % + (total_seconds, duration.microseconds / 1000)) + if self.severity is not 'ok' or self.traceback is not '': + if output.HasTimedOut(): + self.traceback = 'timeout' + self._printDiagnostic(self.traceback, self.severity) logger.info(' ...') def Done(self): From a7d90df8032f0f363461edf9d5a05a030c796a32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bergstr=C3=B6m?= Date: Tue, 25 Oct 2016 12:23:48 -0300 Subject: [PATCH 5/6] gtest: output tap comments as yamlish This makes yaml-ish parsers happy. Note: gtest still seems to output the expected/result slightly different making the full traceback less informational. PR-URL: https://github.com/nodejs/node/pull/9262 Reviewed-By: Gibson Fahnestock Reviewed-By: Ben Noordhuis Reviewed-By: Myles Borins --- deps/gtest/src/gtest.cc | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/deps/gtest/src/gtest.cc b/deps/gtest/src/gtest.cc index 213740ca0c7943..87b67a2230fe55 100644 --- a/deps/gtest/src/gtest.cc +++ b/deps/gtest/src/gtest.cc @@ -3596,13 +3596,15 @@ void TapUnitTestResultPrinter::OutputTapTestInfo(int* count, *stream << " ---\n"; *stream << " duration_ms: " << FormatTimeInMillisAsSeconds(result.elapsed_time()) << "\n"; - *stream << " ...\n"; - for (int i = 0; i < result.total_part_count(); ++i) { - const TestPartResult& part = result.GetTestPartResult(i); - OutputTapComment(stream, part.message()); + if (result.total_part_count() > 0) { + *stream << " stack: |-\n"; + for (int i = 0; i < result.total_part_count(); ++i) { + const TestPartResult& part = result.GetTestPartResult(i); + OutputTapComment(stream, part.message()); + } } - + *stream << " ...\n"; *count += 1; } @@ -3610,11 +3612,11 @@ void TapUnitTestResultPrinter::OutputTapComment(::std::ostream* stream, const char* comment) { const char* start = comment; while (const char* end = strchr(start, '\n')) { - *stream << "# " << std::string(start, end) << "\n"; + *stream << " " << std::string(start, end) << "\n"; start = end + 1; } if (*start) - *stream << "# " << start << "\n"; + *stream << " " << start << "\n"; } // Formats the given time in milliseconds as seconds. From 0b8be2a11302907375f2b0d4c7604537a5d30142 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bergstr=C3=B6m?= Date: Tue, 25 Oct 2016 13:26:46 -0300 Subject: [PATCH 6/6] gitignore: ignore all tap files We now have multiple tap producers; just ignore all files with the `.tap` extension. PR-URL: https://github.com/nodejs/node/pull/9262 Reviewed-By: Gibson Fahnestock Reviewed-By: Ben Noordhuis Reviewed-By: Myles Borins --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 8416e3bfccb1b8..8a6cc149115708 100644 --- a/.gitignore +++ b/.gitignore @@ -85,7 +85,7 @@ deps/npm/node_modules/.bin/ # test artifacts tools/faketime icu_config.gypi -test.tap +*.tap # Xcode workspaces and project folders *.xcodeproj