Skip to content

Commit

Permalink
Fix JUnit failure reporting (#10121)
Browse files Browse the repository at this point in the history 
* Fix spacing

* Add try..finally everywhere

* trigger ci

* Fix pytest invocations

* Remove junit collection where no files exist

Co-authored-by: driazati <driazati@users.noreply.github.com>
  • Loading branch information
@driazati
driazati and driazati committed Feb 16, 2022
1 parent 64e94ab commit 646d155
Show file tree
Hide file tree
Showing 5 changed files with 141 additions and 114 deletions.
221 changes: 124 additions & 97 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -256,20 +256,20 @@ def cpp_unittest(image) {
}

stage('Build') {
parallel 'BUILD: GPU': {
if (!skip_ci) {
node('GPUBUILD') {
ws(per_exec_ws('tvm/build-gpu')) {
init_git()
sh "${docker_run} ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh"
make(ci_gpu, 'build', '-j2')
pack_lib('gpu', tvm_multilib)
// compiler test
sh "${docker_run} ${ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh"
make(ci_gpu, 'build2', '-j2')
}
parallel 'BUILD: GPU': {
if (!skip_ci) {
node('GPUBUILD') {
ws(per_exec_ws('tvm/build-gpu')) {
init_git()
sh "${docker_run} ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh"
make(ci_gpu, 'build', '-j2')
pack_lib('gpu', tvm_multilib)
// compiler test
sh "${docker_run} ${ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh"
make(ci_gpu, 'build2', '-j2')
}
}
}
},
'BUILD: CPU': {
if (!skip_ci && is_docs_only_build != 1) {
Expand Down Expand Up @@ -360,17 +360,20 @@ stage('Build') {
script: "${docker_run} ${ci_qemu} ./tests/scripts/task_config_build_qemu.sh",
label: 'Create QEMU cmake config',
)
make(ci_qemu, 'build', '-j2')
timeout(time: max_time, unit: 'MINUTES') {
ci_setup(ci_qemu)
sh (
script: "${docker_run} ${ci_qemu} ./tests/scripts/task_python_microtvm.sh",
label: 'Run microTVM tests',
)
sh (
script: "${docker_run} ${ci_qemu} ./tests/scripts/task_demo_microtvm.sh",
label: 'Run microTVM demos',
)
try {
make(ci_qemu, 'build', '-j2')
timeout(time: max_time, unit: 'MINUTES') {
ci_setup(ci_qemu)
sh (
script: "${docker_run} ${ci_qemu} ./tests/scripts/task_python_microtvm.sh",
label: 'Run microTVM tests',
)
sh (
script: "${docker_run} ${ci_qemu} ./tests/scripts/task_demo_microtvm.sh",
label: 'Run microTVM demos',
)
}
} finally {
junit 'build/pytest-results/*.xml'
}
}
Expand All @@ -382,10 +385,11 @@ stage('Build') {
}

stage('Test') {
parallel 'unittest: GPU': {
if (!skip_ci && is_docs_only_build != 1) {
node('TensorCore') {
ws(per_exec_ws('tvm/ut-python-gpu')) {
parallel 'unittest: GPU': {
if (!skip_ci && is_docs_only_build != 1) {
node('TensorCore') {
ws(per_exec_ws('tvm/ut-python-gpu')) {
try {
init_git()
unpack_lib('gpu', tvm_multilib)
timeout(time: max_time, unit: 'MINUTES') {
Expand All @@ -402,18 +406,21 @@ stage('Test') {
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_integration_gpuonly.sh",
label: 'Run Python GPU integration tests',
)
junit 'build/pytest-results/*.xml'
}
} finally {
junit 'build/pytest-results/*.xml'
}
}
} else {
Utils.markStageSkippedForConditional('unittest: GPU')
}
},
'integration: CPU': {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU') {
ws(per_exec_ws('tvm/ut-python-cpu')) {
} else {
Utils.markStageSkippedForConditional('unittest: GPU')
}
},
'integration: CPU': {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU') {
ws(per_exec_ws('tvm/ut-python-cpu')) {
try {
init_git()
unpack_lib('cpu', tvm_multilib_tsim)
timeout(time: max_time, unit: 'MINUTES') {
Expand All @@ -422,18 +429,21 @@ stage('Test') {
script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh",
label: 'Run CPU integration tests',
)
junit 'build/pytest-results/*.xml'
}
} finally {
junit 'build/pytest-results/*.xml'
}
}
} else {
Utils.markStageSkippedForConditional('integration: CPU')
}
},
'unittest: CPU': {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU') {
ws(per_exec_ws("tvm/ut-python-cpu")) {
} else {
Utils.markStageSkippedForConditional('integration: CPU')
}
},
'unittest: CPU': {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU') {
ws(per_exec_ws("tvm/ut-python-cpu")) {
try {
init_git()
unpack_lib('cpu', tvm_multilib_tsim)
timeout(time: max_time, unit: 'MINUTES') {
Expand All @@ -444,18 +454,21 @@ stage('Test') {
script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_vta_tsim.sh",
label: "Run VTA tests in TSIM",
)
junit "build/pytest-results/*.xml"
}
} finally {
junit 'build/pytest-results/*.xml'
}
}
} else {
Utils.markStageSkippedForConditional('unittest: CPU')
}
},
'python3: i386': {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU') {
ws(per_exec_ws('tvm/ut-python-i386')) {
} else {
Utils.markStageSkippedForConditional('unittest: CPU')
}
},
'python3: i386': {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU') {
ws(per_exec_ws('tvm/ut-python-i386')) {
try {
init_git()
unpack_lib('i386', tvm_multilib)
timeout(time: max_time, unit: 'MINUTES') {
Expand All @@ -466,18 +479,21 @@ stage('Test') {
label: 'Run i386 integration tests',
)
fsim_test(ci_i386)
junit 'build/pytest-results/*.xml'
}
} finally {
junit 'build/pytest-results/*.xml'
}
}
} else {
Utils.markStageSkippedForConditional('python3: i386')
}
},
'python3: arm': {
if (!skip_ci && is_docs_only_build != 1) {
node('ARM') {
ws(per_exec_ws('tvm/ut-python-arm')) {
} else {
Utils.markStageSkippedForConditional('python3: i386')
}
},
'python3: arm': {
if (!skip_ci && is_docs_only_build != 1) {
node('ARM') {
ws(per_exec_ws('tvm/ut-python-arm')) {
try {
init_git()
unpack_lib('arm', tvm_multilib)
timeout(time: max_time, unit: 'MINUTES') {
Expand All @@ -487,47 +503,55 @@ stage('Test') {
script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_arm_compute_library.sh",
label: 'Run test_arm_compute_lib test',
)
junit 'build/pytest-results/*.xml'
// sh "${docker_run} ${ci_arm} ./tests/scripts/task_python_integration.sh"
}
} finally {
junit 'build/pytest-results/*.xml'
}
}
} else {
Utils.markStageSkippedForConditional('python3: arm')
}
},
} else {
Utils.markStageSkippedForConditional('python3: arm')
}
},
'topi: GPU': {
if (!skip_ci && is_docs_only_build != 1) {
node('GPU') {
ws(per_exec_ws('tvm/topi-python-gpu')) {
init_git()
unpack_lib('gpu', tvm_multilib)
timeout(time: max_time, unit: 'MINUTES') {
ci_setup(ci_gpu)
sh (
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh",
label: 'Run TOPI tests',
)
junit 'build/pytest-results/*.xml'
if (!skip_ci && is_docs_only_build != 1) {
node('GPU') {
ws(per_exec_ws('tvm/topi-python-gpu')) {
try {
init_git()
unpack_lib('gpu', tvm_multilib)
timeout(time: max_time, unit: 'MINUTES') {
ci_setup(ci_gpu)
sh (
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh",
label: 'Run TOPI tests',
)
}
} finally {
junit 'build/pytest-results/*.xml'
}
}
}
}
} else {
Utils.markStageSkippedForConditional('topi: GPU')
}
}
},
'frontend: GPU': {
if (!skip_ci && is_docs_only_build != 1) {
node('GPU') {
ws(per_exec_ws('tvm/frontend-python-gpu')) {
init_git()
unpack_lib('gpu', tvm_multilib)
timeout(time: max_time, unit: 'MINUTES') {
ci_setup(ci_gpu)
sh (
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh",
label: 'Run Python frontend tests',
)
try {
init_git()
unpack_lib('gpu', tvm_multilib)
timeout(time: max_time, unit: 'MINUTES') {
ci_setup(ci_gpu)
sh (
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh",
label: 'Run Python frontend tests',
)
}
} finally {
junit 'build/pytest-results/*.xml'
}
}
Expand All @@ -540,14 +564,17 @@ stage('Test') {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU') {
ws(per_exec_ws('tvm/frontend-python-cpu')) {
init_git()
unpack_lib('cpu', tvm_multilib)
timeout(time: max_time, unit: 'MINUTES') {
ci_setup(ci_cpu)
sh (
script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_frontend_cpu.sh",
label: 'Run Python frontend tests',
)
try {
init_git()
unpack_lib('cpu', tvm_multilib)
timeout(time: max_time, unit: 'MINUTES') {
ci_setup(ci_cpu)
sh (
script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_frontend_cpu.sh",
label: 'Run Python frontend tests',
)
}
} finally {
junit 'build/pytest-results/*.xml'
}
}
Expand Down Expand Up @@ -595,13 +622,13 @@ stage('Build packages') {
*/

stage('Deploy') {
node('doc') {
ws(per_exec_ws('tvm/deploy-docs')) {
if (env.BRANCH_NAME == 'main') {
node('doc') {
ws(per_exec_ws('tvm/deploy-docs')) {
if (env.BRANCH_NAME == 'main') {
unpack_lib('docs', 'docs.tgz')
sh 'cp docs.tgz /var/docs/docs.tgz'
sh 'tar xf docs.tgz -C /var/docs'
}
}
}
}
}
4 changes: 3 additions & 1 deletion tests/scripts/task_python_frontend.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,11 @@ run_pytest cython python-frontend-coreml tests/python/frontend/coreml
echo "Running relay Tensorflow frontend test..."
# Note: Tensorflow tests often have memory issues, so invoke each one separately
TENSORFLOW_TESTS=$(./tests/scripts/pytest_ids.py --folder tests/python/frontend/tensorflow)
i=0
for node_id in $TENSORFLOW_TESTS; do
echo "$node_id"
run_pytest cython python-frontend-tensorflow "$node_id"
run_pytest cython "python-frontend-tensorflow-$i" "$node_id"
i=$((i+1))
done

echo "Running relay caffe2 frontend test..."
Expand Down
13 changes: 6 additions & 7 deletions tests/scripts/task_python_integration.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,7 @@
# specific language governing permissions and limitations
# under the License.

set -e
set -u
set -euxo pipefail

source tests/scripts/setup-pytest-env.sh
export PYTHONPATH=${PYTHONPATH}:${TVM_PATH}/apps/extension/python
Expand All @@ -44,22 +43,22 @@ rm -rf lib
make
cd ../..

run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-extensions apps/extension/tests
run_pytest cython ${TVM_INTEGRATION_TESTSUITE_NAME}-extensions apps/extension/tests
run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-extensions-0 apps/extension/tests
run_pytest cython ${TVM_INTEGRATION_TESTSUITE_NAME}-extensions-1 apps/extension/tests

# Test dso plugin
cd apps/dso_plugin_module
rm -rf lib
make
cd ../..
run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-dso_plugin_module apps/dso_plugin_module
run_pytest cython ${TVM_INTEGRATION_TESTSUITE_NAME}-dso_plugin_module apps/dso_plugin_module
run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-dso_plugin_module-0 apps/dso_plugin_module
run_pytest cython ${TVM_INTEGRATION_TESTSUITE_NAME}-dso_plugin_module-1 apps/dso_plugin_module

# Do not enable TensorFlow op
# TVM_FFI=cython sh prepare_and_test_tfop_module.sh
# TVM_FFI=ctypes sh prepare_and_test_tfop_module.sh

run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME} tests/python/integration
run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-integration tests/python/integration

# Ignoring Arm(R) Ethos(TM)-U NPU tests in the collective to run to run them in parallel in the next step.
run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-contrib tests/python/contrib --ignore=tests/python/contrib/test_ethosu
Expand Down
Loading

0 comments on commit 646d155

Please sign in to comment.