Enable Code Coverage for Paddle refactor code

cmake/coveralls.cmake

@@ -1,10 +1,12 @@
 # CMake script for code coverage.
 # If _COVERALLS_UPLOAD is ON, it will upload json files to overalls.io automatically.
 
+# Param _GCOV_EXECUTABLE        Gcov executable.
 # Param _COVERAGE_SRCS          A list of coverage source files.
+# Param _GIT_PR_ID              Git pull request number.
 # Param _COVERALLS_UPLOAD       Upload the result to coveralls.
 # Param _CMAKE_SCRIPT_PATH      CMake script path.
-function(code_coverage _COVERAGE_SRCS _COVERALLS_UPLOAD _CMAKE_SCRIPT_PATH)
+function(code_coverage _GCOV_EXECUTABLE _COVERAGE_SRCS _GIT_PR_ID _COVERALLS_UPLOAD _CMAKE_SCRIPT_PATH)
     # clean previous gcov data.
     file(REMOVE_RECURSE ${PROJECT_BINARY_DIR}/*.gcda)
 
@@ -23,28 +25,25 @@ function(code_coverage _COVERAGE_SRCS _COVERALLS_UPLOAD _CMAKE_SCRIPT_PATH)
         set(COVERAGE_SRCS "${COVERAGE_SRCS}*${SINGLE_SRC}")
     endforeach()
 
-    # query number of logical cores
-    cmake_host_system_information(RESULT core_size QUERY NUMBER_OF_LOGICAL_CORES)
     # coveralls json file.
     set(COVERALLS_FILE ${PROJECT_BINARY_DIR}/coveralls.json)
     add_custom_target(coveralls_generate
-        # Run regress tests.
-        COMMAND ${CMAKE_CTEST_COMMAND}
-                -j ${core_size}
-                --output-on-failure
+        # Run unit tests.
+        COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure
         # Generate Gcov and translate it into coveralls JSON.
         COMMAND ${CMAKE_COMMAND}
                 -DCOVERAGE_SRCS="${COVERAGE_SRCS}"
                 -DCOVERALLS_OUTPUT_FILE="${COVERALLS_FILE}"
                 -DCOV_PATH="${PROJECT_BINARY_DIR}"
                 -DPROJECT_ROOT="${PROJECT_SOURCE_DIR}"
+                -DGCOV_EXECUTABLE="${_GCOV_EXECUTABLE}"
+                -DGIT_PR_ID="${_GIT_PR_ID}"
                 -P "${_CMAKE_SCRIPT_PATH}/coverallsGcovJsons.cmake"
         WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
         COMMENT "Coveralls: generating coveralls output..."
     )
 
     if (_COVERALLS_UPLOAD)
-        message("COVERALLS UPLOAD: ON")
         # Upload the JSON to coveralls.
         add_custom_target(coveralls_upload
             COMMAND ${CURL_EXECUTABLE}
@@ -56,47 +55,56 @@ function(code_coverage _COVERAGE_SRCS _COVERALLS_UPLOAD _CMAKE_SCRIPT_PATH)
 
         add_custom_target(coveralls DEPENDS coveralls_upload)
     else()
-        message("COVERALLS UPLOAD: OFF")
         add_custom_target(coveralls DEPENDS coveralls_generate)
     endif()
 endfunction()
 
 if(WITH_COVERAGE)
     set(CMAKE_BUILD_TYPE "Debug")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage")
+    set(CODE_COVERAGE_FLAGS "-g -O0 -fprofile-arcs -ftest-coverage")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CODE_COVERAGE_FLAGS}")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CODE_COVERAGE_FLAGS}")
 
-    set(EXCLUDE_DIRS
-        "demo/"
-        "build/"
-        "tests/"
-        ".test_env/"
-    )
+    file(GLOB_RECURSE PADDLE_SOURCES
+        ${PADDLE_SOURCE_DIR}/paddle/memory/*.cc
+        ${PADDLE_SOURCE_DIR}/paddle/memory/*.cu
+        ${PADDLE_SOURCE_DIR}/paddle/memory/*.h
+        ${PADDLE_SOURCE_DIR}/paddle/framework/*.h
+        ${PADDLE_SOURCE_DIR}/paddle/framework/*.cu
+        ${PADDLE_SOURCE_DIR}/paddle/framework/*.cc
+        ${PADDLE_SOURCE_DIR}/paddle/operators/*.h
+        ${PADDLE_SOURCE_DIR}/paddle/operators/*.cu
+        ${PADDLE_SOURCE_DIR}/paddle/operators/*.cc
+        ${PADDLE_SOURCE_DIR}/paddle/platform/*.h
+        ${PADDLE_SOURCE_DIR}/paddle/platform/*.cu
+        ${PADDLE_SOURCE_DIR}/paddle/platform/*.cc)
 
-    if(WITH_GPU)
-        file(GLOB_RECURSE PADDLE_SOURCES RELATIVE "${PROJECT_SOURCE_DIR}" "*.cpp" "*.cc" ".c" "*.cu")
-    else()
-        file(GLOB_RECURSE PADDLE_SOURCES RELATIVE "${PROJECT_SOURCE_DIR}" "*.cpp" "*.cc" "*.c")
-    endif()
-
-    # exclude trivial files in PADDLE_SOURCES
-    foreach(EXCLUDE_DIR ${EXCLUDE_DIRS})
-        foreach(TMP_PATH ${PADDLE_SOURCES})
-            string(FIND ${TMP_PATH} ${EXCLUDE_DIR} EXCLUDE_DIR_FOUND)
-            if(NOT ${EXCLUDE_DIR_FOUND} EQUAL -1)
-                list(REMOVE_ITEM PADDLE_SOURCES ${TMP_PATH})
-            endif()
-        endforeach(TMP_PATH)
+    # exclude trivial tests in PADDLE_SOURCES
+    foreach(TEST_SOURCE ${PADDLE_SOURCES}) 
+        string(REGEX MATCH "[/A-Za-z0-9_]*test.[a-z]*" TEST_SOURCE ${TEST_SOURCE})
+        if (TEST_SOURCE)
+            list(REMOVE_ITEM PADDLE_SOURCES ${TEST_SOURCE}) 
+        endif()
     endforeach()
 
-    # convert to absolute path
-    set(PADDLE_SRCS "")
-    foreach(PADDLE_SRC ${PADDLE_SOURCES})
-        set(PADDLE_SRCS "${PADDLE_SRCS};${PROJECT_SOURCE_DIR}/${PADDLE_SRC}")
-    endforeach()
+    message("-- Code Coverage: ${WITH_COVERAGE}")
+    message("-- Coveralls Upload: ${COVERALLS_UPLOAD}")
+
+    get_filename_component(CXX_DIR ${CMAKE_CXX_COMPILER} DIRECTORY)
+    set(GCOV_EXECUTABLE ${CXX_DIR}/gcov)
+    if (NOT GCOV_EXECUTABLE)
+        message(FATAL_ERROR "gcov not found! Aborting...")
+    else()
+        message("-- Found gcov: ${GCOV_EXECUTABLE}")
+    endif()
 
+    # CI needs to pass pull request number into code coverage
+    # in order to bind coveralls with github.
+    set(GIT_PR_ID)
     code_coverage(
-        "${PADDLE_SRCS}"
+        "${GCOV_EXECUTABLE}"
+        "${PADDLE_SOURCES}"
+        "${GIT_PR_ID}"
         ${COVERALLS_UPLOAD}
         "${PROJECT_SOURCE_DIR}/cmake"
     )

cmake/coverallsGcovJsons.cmake

@@ -18,6 +18,7 @@
 # SOFTWARE.
 #
 # Copyright (C) 2014 Joakim Söderberg <joakim.soderberg@gmail.com>
+# Copyright (C) 2017 Gang Liao <liaogang@baidu.com>
 #
 # This is intended to be run by a custom target in a CMake project like this.
 # 0. Compile program with coverage support.
@@ -40,14 +41,9 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 # CMake list format.
 string(REGEX REPLACE "\\*" ";" COVERAGE_SRCS ${COVERAGE_SRCS})
 
-find_program(GCOV_EXECUTABLE gcov)
-if (NOT GCOV_EXECUTABLE)
-	message(FATAL_ERROR "gcov not found! Aborting...")
-endif()
 
 find_package(Git)
 
-# TODO: Add these git things to the coveralls json.
 if (GIT_FOUND)
 	# Branch.
 	execute_process(
@@ -66,20 +62,64 @@ if (GIT_FOUND)
 		)
 	endmacro()
 
-	git_log_format(an GIT_AUTHOR_EMAIL)
+	git_log_format(an GIT_AUTHOR_NAME)
 	git_log_format(ae GIT_AUTHOR_EMAIL)
 	git_log_format(cn GIT_COMMITTER_NAME)
 	git_log_format(ce GIT_COMMITTER_EMAIL)
 	git_log_format(B GIT_COMMIT_MESSAGE)
+	git_log_format(H GIT_COMMIT_HASH)
+	git_log_format(ai GIT_DATE_ISO_8601)
 
 	message("Git exe: ${GIT_EXECUTABLE}")
 	message("Git branch: ${GIT_BRANCH}")
 	message("Git author: ${GIT_AUTHOR_NAME}")
+	message("Git author date: ${GIT_DATE_ISO_8601}")
 	message("Git e-mail: ${GIT_AUTHOR_EMAIL}")
 	message("Git commiter name: ${GIT_COMMITTER_NAME}")
 	message("Git commiter e-mail: ${GIT_COMMITTER_EMAIL}")
 	message("Git commit message: ${GIT_COMMIT_MESSAGE}")
+	message("Git commit hash: ${GIT_COMMIT_HASH}")
+
+	#
+	# Store git commit infomation into coveralls json
+	#
+	# For example:
+	#	"git": {
+	#		"head": {
+	#		  "id": "b31f08d07ae564b08237e5a336e478b24ccc4a65",
+	#		  "author_name": "Nick Merwin",
+	#		  "author_email": "...",
+	#		  "committer_name": "Nick Merwin",
+	#		  "committer_email": "...",
+	#		  "message": "version bump"
+	#		},
+	#		"branch": "master",
+	#		"remotes": [
+	#		  {
+	#			"name": "origin",
+	#			"url": "git@github.com:lemurheavy/coveralls-ruby.git"
+	#		  }
+	#		]
+	#	  },
+	#
 
+	set(JSON_GIT_INFO
+	"{
+	  \"head\": {
+	    \"author_name\": \"${GIT_AUTHOR_NAME}\",
+	    \"author_email\": \"${GIT_AUTHOR_EMAIL}\",
+	    \"committer_name\": \"${GIT_COMMITTER_NAME}\",
+	    \"committer_email\": \"${GIT_COMMITTER_EMAIL}\",
+	    \"message\": \"${GIT_COMMIT_MESSAGE}\"
+	  },
+	  \"branch\": \"${GIT_BRANCH}\",
+	  \"remotes\": [{
+	    \"name\": \"origin\",
+	    \"url\": \"https://github.com/PaddlePaddle/Paddle.git\"
+	  }]
+	}")
+
+	message("${JSON_GIT_INFO}")
 endif()
 
 ############################# Macros #########################################
@@ -109,9 +149,7 @@ endmacro()
 ##############################################################################
 
 # Get the coverage data.
-file(GLOB_RECURSE GCDA_FILES "${COV_PATH}" "*.gcda")
-message("Process GCDA files:")
-message("===============================")
+file(GLOB_RECURSE GCDA_FILES "${COV_PATH}/*.gcda")
 
 # Get a list of all the object directories needed by gcov
 # (The directories the .gcda files and .o files are found in)
@@ -134,13 +172,13 @@ foreach(GCDA ${GCDA_FILES})
 	# If -p is not specified then the file is named only "the_file.c.gcov"
 	#
 	execute_process(
-		COMMAND ${GCOV_EXECUTABLE} -p -o ${GCDA_DIR} ${GCDA} >/dev/null
-		WORKING_DIRECTORY ${GCDA_DIR}
+		COMMAND ${GCOV_EXECUTABLE} -p -o ${GCDA_DIR} ${GCDA} >/dev/null 2>/dev/null
+		WORKING_DIRECTORY ${COV_PATH}
 	)
 endforeach()
 
 # TODO: Make these be absolute path
-file(GLOB_RECURSE ALL_GCOV_FILES "${COV_PATH}" "*.gcov")
+file(GLOB ALL_GCOV_FILES ${COV_PATH}/*.gcov)
 
 # Get only the filenames to use for filtering.
 #set(COVERAGE_SRCS_NAMES "")
@@ -189,29 +227,42 @@ foreach (GCOV_FILE ${ALL_GCOV_FILES})
 	list(FIND COVERAGE_SRCS ${GCOV_SRC_PATH} WAS_FOUND)
 
 	if (NOT WAS_FOUND EQUAL -1)
-		message("YES: ${GCOV_FILE}")
+		# message("YES: ${GCOV_FILE}")
 		list(APPEND GCOV_FILES ${GCOV_FILE})
 
 		# We remove it from the list, so we don't bother searching for it again.
 		# Also files left in COVERAGE_SRCS_REMAINING after this loop ends should
 		# have coverage data generated from them (no lines are covered).
 		list(REMOVE_ITEM COVERAGE_SRCS_REMAINING ${GCOV_SRC_PATH})
 	else()
-		message("NO:  ${GCOV_FILE}")
+		# message("NO:  ${GCOV_FILE}")
 	endif()
 endforeach()
 
-# TODO: Enable setting these
-set(JSON_SERVICE_NAME "travis-ci")
-set(JSON_SERVICE_JOB_ID $ENV{TRAVIS_JOB_ID})
-
-set(JSON_TEMPLATE
-"{
-  \"service_name\": \"\@JSON_SERVICE_NAME\@\",
-  \"service_job_id\": \"\@JSON_SERVICE_JOB_ID\@\",
-  \"source_files\": \@JSON_GCOV_FILES\@
-}"
-)
+if(ON_TRAVIS)
+    set(JSON_SERVICE_NAME "travis-ci")
+    set(JSON_SERVICE_JOB_ID $ENV{TRAVIS_JOB_ID})
+	set(JSON_TEMPLATE
+	"{
+	  \"service_name\": \"\@JSON_SERVICE_NAME\@\",
+	  \"service_job_id\": \"\@JSON_SERVICE_JOB_ID\@\",
+	  \"git\": ${JSON_GIT_INFO},
+      \"run_at\": \"\@GIT_DATE_ISO_8601\@\",
+	  \"source_files\": \@JSON_GCOV_FILES\@
+	}")
+else(ON_TRAVIS)
+	set(JSON_SERVICE_NAME "teamcity")
+    set(JSON_REPO_TOKEN "JSUOs6TF6fD2i30OJ5o2S55V8XWv6euen")
+	set(JSON_TEMPLATE
+	"{
+	  \"repo_token\": \"\@JSON_REPO_TOKEN\@\",
+	  \"git\": ${JSON_GIT_INFO},
+	  \"run_at\": \"\@GIT_DATE_ISO_8601\@\",
+	  \"service_pull_request\": \"${GIT_PR_ID}\",
+	  \"service_name\": \"\@JSON_SERVICE_NAME\@\",
+	  \"source_files\": \@JSON_GCOV_FILES\@
+	}")
+endif(ON_TRAVIS)
 
 set(SRC_FILE_TEMPLATE
 "{
@@ -235,7 +286,7 @@ foreach (GCOV_FILE ${GCOV_FILES})
 	# The new coveralls API doesn't need the entire source (Yay!)
 	# However, still keeping that part for now. Will cleanup in the future.
 	file(MD5 "${GCOV_SRC_PATH}" GCOV_CONTENTS_MD5)
-	message("MD5: ${GCOV_SRC_PATH} = ${GCOV_CONTENTS_MD5}")
+	# message("MD5: ${GCOV_SRC_PATH} = ${GCOV_CONTENTS_MD5}")
 
 	# Loads the gcov file as a list of lines.
 	# (We first open the file and replace all occurences of [] with _
@@ -324,6 +375,8 @@ foreach (GCOV_FILE ${GCOV_FILES})
 					# Translate the hitcount into valid JSON values.
 					if (${HITCOUNT} STREQUAL "#####")
 						set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}0, ")
+					elseif (${HITCOUNT} STREQUAL "=====")
+						set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}0, ")
 					elseif (${HITCOUNT} STREQUAL "-")
 						set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}null, ")
 					else()
@@ -341,7 +394,7 @@ foreach (GCOV_FILE ${GCOV_FILES})
 		math(EXPR GCOV_LINE_COUNT "${GCOV_LINE_COUNT}+1")
 	endforeach()
 
-	message("${GCOV_LINE_COUNT} of ${LINE_COUNT} lines read!")
+	# message("${GCOV_LINE_COUNT} of ${LINE_COUNT} lines read!")
 
 	# Advanced way of removing the trailing comma in the JSON array.
 	# "[1, 2, 3, " -> "[1, 2, 3"
@@ -351,7 +404,7 @@ foreach (GCOV_FILE ${GCOV_FILES})
 	set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}]")
 
 	# Generate the final JSON for this file.
-	message("Generate JSON for file: ${GCOV_SRC_REL_PATH}...")
+	# message("Generate JSON for file: ${GCOV_SRC_REL_PATH}...")
 	string(CONFIGURE ${SRC_FILE_TEMPLATE} FILE_JSON)
 
 	set(JSON_GCOV_FILES "${JSON_GCOV_FILES}${FILE_JSON}, ")

go/pserver/.gitignore

@@ -0,0 +1 @@
+optimizer.go

go/pserver/CMakeLists.txt

@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/optimizer.go.in ${CMAKE_CURRENT_SOURCE_DIR}/optimizer.go)
+
 if(WITH_TESTING)
   go_test(pserver_test DEPS paddle_go_optimizer)
 endif()

go/pserver/client/c/CMakeLists.txt

@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 cc_library(paddle_go_optimizer DEPS paddle_optimizer paddle_proto glog gflags protobuf)
-target_link_libraries(paddle_go_optimizer stdc++ m)
+target_link_libraries(paddle_go_optimizer stdc++ m ${CODE_COVERAGE_FLAGS})
 
 # Copy library to the required place.
 # See: go/pserver/optimizer.go:

go/pserver/optimizer.go → go/pserver/optimizer.go.in

@@ -15,7 +15,7 @@
 package pserver
 
 // #cgo CFLAGS: -I ../../
-// #cgo LDFLAGS: ${SRCDIR}/client/c/libpaddle_go_optimizer.a -lstdc++ -lm
+// #cgo LDFLAGS: ${CMAKE_CURRENT_SOURCE_DIR}/client/c/libpaddle_go_optimizer.a -lstdc++ -lm ${CODE_COVERAGE_FLAGS} 
 // #include "paddle/optimizer/optimizer.h"
 // #include <stdlib.h>
 // #include <string.h>