diff --git a/.gitignore b/.gitignore index 8416e3bfccb1b8..8a6cc149115708 100644 --- a/.gitignore +++ b/.gitignore @@ -85,7 +85,7 @@ deps/npm/node_modules/.bin/ # test artifacts tools/faketime icu_config.gypi -test.tap +*.tap # Xcode workspaces and project folders *.xcodeproj diff --git a/Makefile b/Makefile index f42d50cbab9e91..82103da01155a5 100644 --- a/Makefile +++ b/Makefile @@ -192,6 +192,7 @@ test-ci-js: $(TEST_CI_ARGS) $(CI_JS_SUITES) test-ci: | build-addons + out/Release/cctest --gtest_output=tap:cctest.tap $(PYTHON) tools/test.py -p tap --logfile test.tap --mode=release --flaky-tests=$(FLAKY_TESTS) \ $(TEST_CI_ARGS) $(CI_NATIVE_SUITES) $(CI_JS_SUITES) diff --git a/deps/gtest/src/gtest.cc b/deps/gtest/src/gtest.cc index 7fd5f298dc04ed..87b67a2230fe55 100644 --- a/deps/gtest/src/gtest.cc +++ b/deps/gtest/src/gtest.cc @@ -3498,6 +3498,127 @@ std::string XmlUnitTestResultPrinter::RemoveInvalidXmlCharacters( // // +class TapUnitTestResultPrinter : public EmptyTestEventListener { + public: + TapUnitTestResultPrinter(); + explicit TapUnitTestResultPrinter(const char* output_file); + virtual void OnTestIterationEnd(const UnitTest& unit_test, int iteration); + + private: + static void PrintTapUnitTest(::std::ostream* stream, + const UnitTest& unit_test); + static void PrintTapTestCase(int* count, + ::std::ostream* stream, + const TestCase& test_case); + static void OutputTapTestInfo(int* count, + ::std::ostream* stream, + const char* test_case_name, + const TestInfo& test_info); + static void OutputTapComment(::std::ostream* stream, const char* comment); + + const std::string output_file_; + GTEST_DISALLOW_COPY_AND_ASSIGN_(TapUnitTestResultPrinter); +}; + +TapUnitTestResultPrinter::TapUnitTestResultPrinter() {} + +TapUnitTestResultPrinter::TapUnitTestResultPrinter(const char* output_file) + : output_file_(output_file) { + if (output_file_.c_str() == NULL || output_file_.empty()) { + fprintf(stderr, "TAP output file may not be null\n"); + fflush(stderr); + exit(EXIT_FAILURE); + } +} + +void TapUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test, + int /*iteration*/) { + FILE* tapout = stdout; + + if (!output_file_.empty()) { + FilePath output_file(output_file_); + FilePath output_dir(output_file.RemoveFileName()); + + tapout = NULL; + if (output_dir.CreateDirectoriesRecursively()) + tapout = posix::FOpen(output_file_.c_str(), "w"); + + if (tapout == NULL) { + fprintf(stderr, "Unable to open file \"%s\"\n", output_file_.c_str()); + fflush(stderr); + exit(EXIT_FAILURE); + } + } + + std::stringstream stream; + PrintTapUnitTest(&stream, unit_test); + fprintf(tapout, "%s", StringStreamToString(&stream).c_str()); + fflush(tapout); + + if (tapout != stdout) + fclose(tapout); +} + +void TapUnitTestResultPrinter::PrintTapUnitTest(std::ostream* stream, + const UnitTest& unit_test) { + *stream << "TAP version 13\n"; + *stream << "1.." << unit_test.reportable_test_count() << "\n"; + + int count = 1; + for (int i = 0; i < unit_test.total_test_case_count(); ++i) { + const TestCase& test_case = *unit_test.GetTestCase(i); + if (test_case.reportable_test_count() > 0) + PrintTapTestCase(&count, stream, test_case); + } + + *stream << "# failures: " << unit_test.failed_test_count() << "\n"; +} + +void TapUnitTestResultPrinter::PrintTapTestCase(int* count, + std::ostream* stream, + const TestCase& test_case) { + for (int i = 0; i < test_case.total_test_count(); ++i) { + const TestInfo& test_info = *test_case.GetTestInfo(i); + if (test_info.is_reportable()) + OutputTapTestInfo(count, stream, test_case.name(), test_info); + } +} + +void TapUnitTestResultPrinter::OutputTapTestInfo(int* count, + ::std::ostream* stream, + const char* test_case_name, + const TestInfo& test_info) { + const TestResult& result = *test_info.result(); + const char* status = result.Passed() ? "ok" : "not ok"; + + *stream << status << " " << *count << " - " << + test_case_name << "." << test_info.name() << "\n"; + *stream << " ---\n"; + *stream << " duration_ms: " << + FormatTimeInMillisAsSeconds(result.elapsed_time()) << "\n"; + + if (result.total_part_count() > 0) { + *stream << " stack: |-\n"; + for (int i = 0; i < result.total_part_count(); ++i) { + const TestPartResult& part = result.GetTestPartResult(i); + OutputTapComment(stream, part.message()); + } + } + *stream << " ...\n"; + *count += 1; +} + +void TapUnitTestResultPrinter::OutputTapComment(::std::ostream* stream, + const char* comment) { + const char* start = comment; + while (const char* end = strchr(start, '\n')) { + *stream << " " << std::string(start, end) << "\n"; + start = end + 1; + } + if (*start) + *stream << " " << start << "\n"; +} + // Formats the given time in milliseconds as seconds. std::string FormatTimeInMillisAsSeconds(TimeInMillis ms) { ::std::stringstream ss; @@ -4365,6 +4486,9 @@ void UnitTestImpl::ConfigureXmlOutput() { if (output_format == "xml") { listeners()->SetDefaultXmlGenerator(new XmlUnitTestResultPrinter( UnitTestOptions::GetAbsolutePathToOutputFile().c_str())); + } else if (output_format == "tap") { + listeners()->SetDefaultXmlGenerator(new TapUnitTestResultPrinter( + UnitTestOptions::GetAbsolutePathToOutputFile().c_str())); } else if (output_format != "") { printf("WARNING: unrecognized output format \"%s\" ignored.\n", output_format.c_str()); diff --git a/deps/gtest/src/gtest_main.cc b/deps/gtest/src/gtest_main.cc index f3028225523306..4cf03e59bac5df 100644 --- a/deps/gtest/src/gtest_main.cc +++ b/deps/gtest/src/gtest_main.cc @@ -32,7 +32,6 @@ #include "gtest/gtest.h" GTEST_API_ int main(int argc, char **argv) { - printf("Running main() from gtest_main.cc\n"); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/tools/test.py b/tools/test.py index 97880fb3cd2667..ae18158c0c9329 100755 --- a/tools/test.py +++ b/tools/test.py @@ -256,11 +256,15 @@ def HasRun(self, output): class TapProgressIndicator(SimpleProgressIndicator): - def _printDiagnostic(self, messages): - for l in messages.splitlines(): - logger.info('# ' + l) + def _printDiagnostic(self, traceback, severity): + logger.info(' severity: %s', severity) + logger.info(' stack: |-') + + for l in traceback.splitlines(): + logger.info(' ' + l) def Starting(self): + logger.info('TAP version 13') logger.info('1..%i' % len(self.cases)) self._done = 0 @@ -269,6 +273,8 @@ def AboutToRun(self, case): def HasRun(self, output): self._done += 1 + self.traceback = '' + self.severity = 'ok' # Print test name as (for example) "parallel/test-assert". Tests that are # scraped from the addons documentation are all named test.js, making it @@ -281,19 +287,23 @@ def HasRun(self, output): if output.UnexpectedOutput(): status_line = 'not ok %i %s' % (self._done, command) + self.severity = 'fail' + self.traceback = output.output.stdout + output.output.stderr + if FLAKY in output.test.outcomes and self.flaky_tests_mode == DONTCARE: status_line = status_line + ' # TODO : Fix flaky test' + self.severity = 'flaky' + logger.info(status_line) - self._printDiagnostic("\n".join(output.diagnostic)) if output.HasCrashed(): - self._printDiagnostic(PrintCrashed(output.output.exit_code)) + self.severity = 'crashed' + exit_code = output.output.exit_code + self.traceback = "oh no!\nexit code: " + PrintCrashed(exit_code) if output.HasTimedOut(): - self._printDiagnostic('TIMEOUT') + self.severity = 'fail' - self._printDiagnostic(output.output.stderr) - self._printDiagnostic(output.output.stdout) else: skip = skip_regex.search(output.output.stdout) if skip: @@ -304,7 +314,11 @@ def HasRun(self, output): if FLAKY in output.test.outcomes: status_line = status_line + ' # TODO : Fix flaky test' logger.info(status_line) - self._printDiagnostic("\n".join(output.diagnostic)) + + if output.diagnostic: + self.severity = 'ok' + self.traceback = output.diagnostic + duration = output.test.duration @@ -313,7 +327,12 @@ def HasRun(self, output): (duration.seconds + duration.days * 24 * 3600) * 10**6) / 10**6 logger.info(' ---') - logger.info(' duration_ms: %d.%d' % (total_seconds, duration.microseconds / 1000)) + logger.info(' duration_ms: %d.%d' % + (total_seconds, duration.microseconds / 1000)) + if self.severity is not 'ok' or self.traceback is not '': + if output.HasTimedOut(): + self.traceback = 'timeout' + self._printDiagnostic(self.traceback, self.severity) logger.info(' ...') def Done(self): diff --git a/vcbuild.bat b/vcbuild.bat index 77c0d01a14f486..519e7168ecf976 100644 --- a/vcbuild.bat +++ b/vcbuild.bat @@ -20,6 +20,7 @@ set noprojgen= set nobuild= set nosign= set nosnapshot= +set cctest_args= set test_args= set package= set msi= @@ -56,7 +57,7 @@ if /i "%1"=="noetw" set noetw=1&goto arg-ok if /i "%1"=="noperfctr" set noperfctr=1&goto arg-ok if /i "%1"=="licensertf" set licensertf=1&goto arg-ok if /i "%1"=="test" set test_args=%test_args% addons doctool sequential parallel message -J&set jslint=1&set build_addons=1&goto arg-ok -if /i "%1"=="test-ci" set test_args=%test_args% %test_ci_args% -p tap --logfile test.tap addons doctool message sequential parallel&set build_addons=1&goto arg-ok +if /i "%1"=="test-ci" set test_args=%test_args% %test_ci_args% -p tap --logfile test.tap addons doctool message sequential parallel&set cctest_args=%cctest_args% --gtest_output=tap:cctest.tap&set build_addons=1&goto arg-ok if /i "%1"=="test-addons" set test_args=%test_args% addons&set build_addons=1&goto arg-ok if /i "%1"=="test-simple" set test_args=%test_args% sequential parallel -J&goto arg-ok if /i "%1"=="test-message" set test_args=%test_args% message&goto arg-ok @@ -351,8 +352,8 @@ goto run-tests if "%test_args%"=="" goto jslint if "%config%"=="Debug" set test_args=--mode=debug %test_args% if "%config%"=="Release" set test_args=--mode=release %test_args% -echo running 'cctest' -"%config%\cctest" +echo running 'cctest %cctest_args%' +"%config%\cctest" %cctest_args% echo running 'python tools\test.py %test_args%' python tools\test.py %test_args% goto jslint