diff --git a/cmake/coveralls.cmake b/cmake/coveralls.cmake index ca1471cabb57c..c69dff2a509d2 100644 --- a/cmake/coveralls.cmake +++ b/cmake/coveralls.cmake @@ -1,10 +1,12 @@ # CMake script for code coverage. # If _COVERALLS_UPLOAD is ON, it will upload json files to overalls.io automatically. +# Param _GCOV_EXECUTABLE Gcov executable. # Param _COVERAGE_SRCS A list of coverage source files. +# Param _GIT_PR_ID Git pull request number. # Param _COVERALLS_UPLOAD Upload the result to coveralls. # Param _CMAKE_SCRIPT_PATH CMake script path. -function(code_coverage _COVERAGE_SRCS _COVERALLS_UPLOAD _CMAKE_SCRIPT_PATH) +function(code_coverage _GCOV_EXECUTABLE _COVERAGE_SRCS _GIT_PR_ID _COVERALLS_UPLOAD _CMAKE_SCRIPT_PATH) # clean previous gcov data. file(REMOVE_RECURSE ${PROJECT_BINARY_DIR}/*.gcda) @@ -23,28 +25,25 @@ function(code_coverage _COVERAGE_SRCS _COVERALLS_UPLOAD _CMAKE_SCRIPT_PATH) set(COVERAGE_SRCS "${COVERAGE_SRCS}*${SINGLE_SRC}") endforeach() - # query number of logical cores - cmake_host_system_information(RESULT core_size QUERY NUMBER_OF_LOGICAL_CORES) # coveralls json file. set(COVERALLS_FILE ${PROJECT_BINARY_DIR}/coveralls.json) add_custom_target(coveralls_generate - # Run regress tests. - COMMAND ${CMAKE_CTEST_COMMAND} - -j ${core_size} - --output-on-failure + # Run unit tests. + COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure # Generate Gcov and translate it into coveralls JSON. COMMAND ${CMAKE_COMMAND} -DCOVERAGE_SRCS="${COVERAGE_SRCS}" -DCOVERALLS_OUTPUT_FILE="${COVERALLS_FILE}" -DCOV_PATH="${PROJECT_BINARY_DIR}" -DPROJECT_ROOT="${PROJECT_SOURCE_DIR}" + -DGCOV_EXECUTABLE="${_GCOV_EXECUTABLE}" + -DGIT_PR_ID="${_GIT_PR_ID}" -P "${_CMAKE_SCRIPT_PATH}/coverallsGcovJsons.cmake" WORKING_DIRECTORY ${PROJECT_BINARY_DIR} COMMENT "Coveralls: generating coveralls output..." ) if (_COVERALLS_UPLOAD) - message("COVERALLS UPLOAD: ON") # Upload the JSON to coveralls. add_custom_target(coveralls_upload COMMAND ${CURL_EXECUTABLE} @@ -56,47 +55,56 @@ function(code_coverage _COVERAGE_SRCS _COVERALLS_UPLOAD _CMAKE_SCRIPT_PATH) add_custom_target(coveralls DEPENDS coveralls_upload) else() - message("COVERALLS UPLOAD: OFF") add_custom_target(coveralls DEPENDS coveralls_generate) endif() endfunction() if(WITH_COVERAGE) set(CMAKE_BUILD_TYPE "Debug") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage") + set(CODE_COVERAGE_FLAGS "-g -O0 -fprofile-arcs -ftest-coverage") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CODE_COVERAGE_FLAGS}") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CODE_COVERAGE_FLAGS}") - set(EXCLUDE_DIRS - "demo/" - "build/" - "tests/" - ".test_env/" - ) + file(GLOB_RECURSE PADDLE_SOURCES + ${PADDLE_SOURCE_DIR}/paddle/memory/*.cc + ${PADDLE_SOURCE_DIR}/paddle/memory/*.cu + ${PADDLE_SOURCE_DIR}/paddle/memory/*.h + ${PADDLE_SOURCE_DIR}/paddle/framework/*.h + ${PADDLE_SOURCE_DIR}/paddle/framework/*.cu + ${PADDLE_SOURCE_DIR}/paddle/framework/*.cc + ${PADDLE_SOURCE_DIR}/paddle/operators/*.h + ${PADDLE_SOURCE_DIR}/paddle/operators/*.cu + ${PADDLE_SOURCE_DIR}/paddle/operators/*.cc + ${PADDLE_SOURCE_DIR}/paddle/platform/*.h + ${PADDLE_SOURCE_DIR}/paddle/platform/*.cu + ${PADDLE_SOURCE_DIR}/paddle/platform/*.cc) - if(WITH_GPU) - file(GLOB_RECURSE PADDLE_SOURCES RELATIVE "${PROJECT_SOURCE_DIR}" "*.cpp" "*.cc" ".c" "*.cu") - else() - file(GLOB_RECURSE PADDLE_SOURCES RELATIVE "${PROJECT_SOURCE_DIR}" "*.cpp" "*.cc" "*.c") - endif() - - # exclude trivial files in PADDLE_SOURCES - foreach(EXCLUDE_DIR ${EXCLUDE_DIRS}) - foreach(TMP_PATH ${PADDLE_SOURCES}) - string(FIND ${TMP_PATH} ${EXCLUDE_DIR} EXCLUDE_DIR_FOUND) - if(NOT ${EXCLUDE_DIR_FOUND} EQUAL -1) - list(REMOVE_ITEM PADDLE_SOURCES ${TMP_PATH}) - endif() - endforeach(TMP_PATH) + # exclude trivial tests in PADDLE_SOURCES + foreach(TEST_SOURCE ${PADDLE_SOURCES}) + string(REGEX MATCH "[/A-Za-z0-9_]*test.[a-z]*" TEST_SOURCE ${TEST_SOURCE}) + if (TEST_SOURCE) + list(REMOVE_ITEM PADDLE_SOURCES ${TEST_SOURCE}) + endif() endforeach() - # convert to absolute path - set(PADDLE_SRCS "") - foreach(PADDLE_SRC ${PADDLE_SOURCES}) - set(PADDLE_SRCS "${PADDLE_SRCS};${PROJECT_SOURCE_DIR}/${PADDLE_SRC}") - endforeach() + message("-- Code Coverage: ${WITH_COVERAGE}") + message("-- Coveralls Upload: ${COVERALLS_UPLOAD}") + + get_filename_component(CXX_DIR ${CMAKE_CXX_COMPILER} DIRECTORY) + set(GCOV_EXECUTABLE ${CXX_DIR}/gcov) + if (NOT GCOV_EXECUTABLE) + message(FATAL_ERROR "gcov not found! Aborting...") + else() + message("-- Found gcov: ${GCOV_EXECUTABLE}") + endif() + # CI needs to pass pull request number into code coverage + # in order to bind coveralls with github. + set(GIT_PR_ID) code_coverage( - "${PADDLE_SRCS}" + "${GCOV_EXECUTABLE}" + "${PADDLE_SOURCES}" + "${GIT_PR_ID}" ${COVERALLS_UPLOAD} "${PROJECT_SOURCE_DIR}/cmake" ) diff --git a/cmake/coverallsGcovJsons.cmake b/cmake/coverallsGcovJsons.cmake index 4641184fcf527..97bef42bfc940 100644 --- a/cmake/coverallsGcovJsons.cmake +++ b/cmake/coverallsGcovJsons.cmake @@ -18,6 +18,7 @@ # SOFTWARE. # # Copyright (C) 2014 Joakim Söderberg +# Copyright (C) 2017 Gang Liao # # This is intended to be run by a custom target in a CMake project like this. # 0. Compile program with coverage support. @@ -40,14 +41,9 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) # CMake list format. string(REGEX REPLACE "\\*" ";" COVERAGE_SRCS ${COVERAGE_SRCS}) -find_program(GCOV_EXECUTABLE gcov) -if (NOT GCOV_EXECUTABLE) - message(FATAL_ERROR "gcov not found! Aborting...") -endif() find_package(Git) -# TODO: Add these git things to the coveralls json. if (GIT_FOUND) # Branch. execute_process( @@ -66,20 +62,64 @@ if (GIT_FOUND) ) endmacro() - git_log_format(an GIT_AUTHOR_EMAIL) + git_log_format(an GIT_AUTHOR_NAME) git_log_format(ae GIT_AUTHOR_EMAIL) git_log_format(cn GIT_COMMITTER_NAME) git_log_format(ce GIT_COMMITTER_EMAIL) git_log_format(B GIT_COMMIT_MESSAGE) + git_log_format(H GIT_COMMIT_HASH) + git_log_format(ai GIT_DATE_ISO_8601) message("Git exe: ${GIT_EXECUTABLE}") message("Git branch: ${GIT_BRANCH}") message("Git author: ${GIT_AUTHOR_NAME}") + message("Git author date: ${GIT_DATE_ISO_8601}") message("Git e-mail: ${GIT_AUTHOR_EMAIL}") message("Git commiter name: ${GIT_COMMITTER_NAME}") message("Git commiter e-mail: ${GIT_COMMITTER_EMAIL}") message("Git commit message: ${GIT_COMMIT_MESSAGE}") + message("Git commit hash: ${GIT_COMMIT_HASH}") + + # + # Store git commit infomation into coveralls json + # + # For example: + # "git": { + # "head": { + # "id": "b31f08d07ae564b08237e5a336e478b24ccc4a65", + # "author_name": "Nick Merwin", + # "author_email": "...", + # "committer_name": "Nick Merwin", + # "committer_email": "...", + # "message": "version bump" + # }, + # "branch": "master", + # "remotes": [ + # { + # "name": "origin", + # "url": "git@github.com:lemurheavy/coveralls-ruby.git" + # } + # ] + # }, + # + set(JSON_GIT_INFO + "{ + \"head\": { + \"author_name\": \"${GIT_AUTHOR_NAME}\", + \"author_email\": \"${GIT_AUTHOR_EMAIL}\", + \"committer_name\": \"${GIT_COMMITTER_NAME}\", + \"committer_email\": \"${GIT_COMMITTER_EMAIL}\", + \"message\": \"${GIT_COMMIT_MESSAGE}\" + }, + \"branch\": \"${GIT_BRANCH}\", + \"remotes\": [{ + \"name\": \"origin\", + \"url\": \"https://github.com/PaddlePaddle/Paddle.git\" + }] + }") + + message("${JSON_GIT_INFO}") endif() ############################# Macros ######################################### @@ -109,9 +149,7 @@ endmacro() ############################################################################## # Get the coverage data. -file(GLOB_RECURSE GCDA_FILES "${COV_PATH}" "*.gcda") -message("Process GCDA files:") -message("===============================") +file(GLOB_RECURSE GCDA_FILES "${COV_PATH}/*.gcda") # Get a list of all the object directories needed by gcov # (The directories the .gcda files and .o files are found in) @@ -134,13 +172,13 @@ foreach(GCDA ${GCDA_FILES}) # If -p is not specified then the file is named only "the_file.c.gcov" # execute_process( - COMMAND ${GCOV_EXECUTABLE} -p -o ${GCDA_DIR} ${GCDA} >/dev/null - WORKING_DIRECTORY ${GCDA_DIR} + COMMAND ${GCOV_EXECUTABLE} -p -o ${GCDA_DIR} ${GCDA} >/dev/null 2>/dev/null + WORKING_DIRECTORY ${COV_PATH} ) endforeach() # TODO: Make these be absolute path -file(GLOB_RECURSE ALL_GCOV_FILES "${COV_PATH}" "*.gcov") +file(GLOB ALL_GCOV_FILES ${COV_PATH}/*.gcov) # Get only the filenames to use for filtering. #set(COVERAGE_SRCS_NAMES "") @@ -189,7 +227,7 @@ foreach (GCOV_FILE ${ALL_GCOV_FILES}) list(FIND COVERAGE_SRCS ${GCOV_SRC_PATH} WAS_FOUND) if (NOT WAS_FOUND EQUAL -1) - message("YES: ${GCOV_FILE}") + # message("YES: ${GCOV_FILE}") list(APPEND GCOV_FILES ${GCOV_FILE}) # We remove it from the list, so we don't bother searching for it again. @@ -197,21 +235,34 @@ foreach (GCOV_FILE ${ALL_GCOV_FILES}) # have coverage data generated from them (no lines are covered). list(REMOVE_ITEM COVERAGE_SRCS_REMAINING ${GCOV_SRC_PATH}) else() - message("NO: ${GCOV_FILE}") + # message("NO: ${GCOV_FILE}") endif() endforeach() -# TODO: Enable setting these -set(JSON_SERVICE_NAME "travis-ci") -set(JSON_SERVICE_JOB_ID $ENV{TRAVIS_JOB_ID}) - -set(JSON_TEMPLATE -"{ - \"service_name\": \"\@JSON_SERVICE_NAME\@\", - \"service_job_id\": \"\@JSON_SERVICE_JOB_ID\@\", - \"source_files\": \@JSON_GCOV_FILES\@ -}" -) +if(ON_TRAVIS) + set(JSON_SERVICE_NAME "travis-ci") + set(JSON_SERVICE_JOB_ID $ENV{TRAVIS_JOB_ID}) + set(JSON_TEMPLATE + "{ + \"service_name\": \"\@JSON_SERVICE_NAME\@\", + \"service_job_id\": \"\@JSON_SERVICE_JOB_ID\@\", + \"git\": ${JSON_GIT_INFO}, + \"run_at\": \"\@GIT_DATE_ISO_8601\@\", + \"source_files\": \@JSON_GCOV_FILES\@ + }") +else(ON_TRAVIS) + set(JSON_SERVICE_NAME "teamcity") + set(JSON_REPO_TOKEN "JSUOs6TF6fD2i30OJ5o2S55V8XWv6euen") + set(JSON_TEMPLATE + "{ + \"repo_token\": \"\@JSON_REPO_TOKEN\@\", + \"git\": ${JSON_GIT_INFO}, + \"run_at\": \"\@GIT_DATE_ISO_8601\@\", + \"service_pull_request\": \"${GIT_PR_ID}\", + \"service_name\": \"\@JSON_SERVICE_NAME\@\", + \"source_files\": \@JSON_GCOV_FILES\@ + }") +endif(ON_TRAVIS) set(SRC_FILE_TEMPLATE "{ @@ -235,7 +286,7 @@ foreach (GCOV_FILE ${GCOV_FILES}) # The new coveralls API doesn't need the entire source (Yay!) # However, still keeping that part for now. Will cleanup in the future. file(MD5 "${GCOV_SRC_PATH}" GCOV_CONTENTS_MD5) - message("MD5: ${GCOV_SRC_PATH} = ${GCOV_CONTENTS_MD5}") + # message("MD5: ${GCOV_SRC_PATH} = ${GCOV_CONTENTS_MD5}") # Loads the gcov file as a list of lines. # (We first open the file and replace all occurences of [] with _ @@ -324,6 +375,8 @@ foreach (GCOV_FILE ${GCOV_FILES}) # Translate the hitcount into valid JSON values. if (${HITCOUNT} STREQUAL "#####") set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}0, ") + elseif (${HITCOUNT} STREQUAL "=====") + set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}0, ") elseif (${HITCOUNT} STREQUAL "-") set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}null, ") else() @@ -341,7 +394,7 @@ foreach (GCOV_FILE ${GCOV_FILES}) math(EXPR GCOV_LINE_COUNT "${GCOV_LINE_COUNT}+1") endforeach() - message("${GCOV_LINE_COUNT} of ${LINE_COUNT} lines read!") + # message("${GCOV_LINE_COUNT} of ${LINE_COUNT} lines read!") # Advanced way of removing the trailing comma in the JSON array. # "[1, 2, 3, " -> "[1, 2, 3" @@ -351,7 +404,7 @@ foreach (GCOV_FILE ${GCOV_FILES}) set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}]") # Generate the final JSON for this file. - message("Generate JSON for file: ${GCOV_SRC_REL_PATH}...") + # message("Generate JSON for file: ${GCOV_SRC_REL_PATH}...") string(CONFIGURE ${SRC_FILE_TEMPLATE} FILE_JSON) set(JSON_GCOV_FILES "${JSON_GCOV_FILES}${FILE_JSON}, ") diff --git a/go/pserver/.gitignore b/go/pserver/.gitignore new file mode 100644 index 0000000000000..bb5a5b2a1385a --- /dev/null +++ b/go/pserver/.gitignore @@ -0,0 +1 @@ +optimizer.go diff --git a/go/pserver/CMakeLists.txt b/go/pserver/CMakeLists.txt index 4fe0a8cb021e8..8bd4146ace23b 100644 --- a/go/pserver/CMakeLists.txt +++ b/go/pserver/CMakeLists.txt @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # + +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/optimizer.go.in ${CMAKE_CURRENT_SOURCE_DIR}/optimizer.go) + if(WITH_TESTING) go_test(pserver_test DEPS paddle_go_optimizer) endif() diff --git a/go/pserver/client/c/CMakeLists.txt b/go/pserver/client/c/CMakeLists.txt index a932791c7cb00..583e96c5a1329 100644 --- a/go/pserver/client/c/CMakeLists.txt +++ b/go/pserver/client/c/CMakeLists.txt @@ -13,7 +13,7 @@ # limitations under the License. # cc_library(paddle_go_optimizer DEPS paddle_optimizer paddle_proto glog gflags protobuf) -target_link_libraries(paddle_go_optimizer stdc++ m) +target_link_libraries(paddle_go_optimizer stdc++ m ${CODE_COVERAGE_FLAGS}) # Copy library to the required place. # See: go/pserver/optimizer.go: diff --git a/go/pserver/optimizer.go b/go/pserver/optimizer.go.in similarity index 96% rename from go/pserver/optimizer.go rename to go/pserver/optimizer.go.in index ae7359073494b..cd14eae2b592c 100644 --- a/go/pserver/optimizer.go +++ b/go/pserver/optimizer.go.in @@ -15,7 +15,7 @@ package pserver // #cgo CFLAGS: -I ../../ -// #cgo LDFLAGS: ${SRCDIR}/client/c/libpaddle_go_optimizer.a -lstdc++ -lm +// #cgo LDFLAGS: ${CMAKE_CURRENT_SOURCE_DIR}/client/c/libpaddle_go_optimizer.a -lstdc++ -lm ${CODE_COVERAGE_FLAGS} // #include "paddle/optimizer/optimizer.h" // #include // #include diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 2941662f349ba..1e583db1c1cb4 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -39,6 +39,8 @@ Configuring cmake in /paddle/build ... -DCUDNN_ROOT=/usr/ -DWITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} -DWITH_TESTING=${WITH_TESTING:-OFF} + -DWITH_COVERAGE=${WITH_COVERAGE:-OFF} + -DCOVERALLS_UPLOAD=${COVERALLS_UPLOAD:-OFF} -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ======================================== EOF @@ -58,6 +60,8 @@ cmake .. \ -DCUDNN_ROOT=/usr/ \ -DWITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} \ -DWITH_TESTING=${WITH_TESTING:-OFF} \ + -DWITH_COVERAGE=${WITH_COVERAGE:-OFF} \ + -DCOVERALLS_UPLOAD=${COVERALLS_UPLOAD:-OFF} \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON cat <