diff --git a/Jenkinsfile b/Jenkinsfile
index b0e5f77612f5..34ad79c6b43f 100755
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -256,20 +256,20 @@ def cpp_unittest(image) {
 }
 
 stage('Build') {
-    parallel 'BUILD: GPU': {
-      if (!skip_ci) {
-        node('GPUBUILD') {
-          ws(per_exec_ws('tvm/build-gpu')) {
-            init_git()
-            sh "${docker_run} ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh"
-            make(ci_gpu, 'build', '-j2')
-            pack_lib('gpu', tvm_multilib)
-            // compiler test
-            sh "${docker_run} ${ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh"
-            make(ci_gpu, 'build2', '-j2')
-          }
+  parallel 'BUILD: GPU': {
+    if (!skip_ci) {
+      node('GPUBUILD') {
+        ws(per_exec_ws('tvm/build-gpu')) {
+          init_git()
+          sh "${docker_run} ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh"
+          make(ci_gpu, 'build', '-j2')
+          pack_lib('gpu', tvm_multilib)
+          // compiler test
+          sh "${docker_run} ${ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh"
+          make(ci_gpu, 'build2', '-j2')
         }
       }
+    }
   },
   'BUILD: CPU': {
     if (!skip_ci && is_docs_only_build != 1) {
@@ -360,17 +360,20 @@ stage('Build') {
             script: "${docker_run} ${ci_qemu} ./tests/scripts/task_config_build_qemu.sh",
             label: 'Create QEMU cmake config',
           )
-          make(ci_qemu, 'build', '-j2')
-          timeout(time: max_time, unit: 'MINUTES') {
-            ci_setup(ci_qemu)
-            sh (
-              script: "${docker_run} ${ci_qemu} ./tests/scripts/task_python_microtvm.sh",
-              label: 'Run microTVM tests',
-            )
-            sh (
-              script: "${docker_run} ${ci_qemu} ./tests/scripts/task_demo_microtvm.sh",
-              label: 'Run microTVM demos',
-            )
+          try {
+            make(ci_qemu, 'build', '-j2')
+            timeout(time: max_time, unit: 'MINUTES') {
+              ci_setup(ci_qemu)
+              sh (
+                script: "${docker_run} ${ci_qemu} ./tests/scripts/task_python_microtvm.sh",
+                label: 'Run microTVM tests',
+              )
+              sh (
+                script: "${docker_run} ${ci_qemu} ./tests/scripts/task_demo_microtvm.sh",
+                label: 'Run microTVM demos',
+              )
+            }
+          } finally {
             junit 'build/pytest-results/*.xml'
           }
         }
@@ -382,10 +385,11 @@ stage('Build') {
 }
 
 stage('Test') {
-    parallel 'unittest: GPU': {
-      if (!skip_ci && is_docs_only_build != 1) {
-        node('TensorCore') {
-          ws(per_exec_ws('tvm/ut-python-gpu')) {
+  parallel 'unittest: GPU': {
+    if (!skip_ci && is_docs_only_build != 1) {
+      node('TensorCore') {
+        ws(per_exec_ws('tvm/ut-python-gpu')) {
+          try {
             init_git()
             unpack_lib('gpu', tvm_multilib)
             timeout(time: max_time, unit: 'MINUTES') {
@@ -402,18 +406,21 @@ stage('Test') {
                 script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_integration_gpuonly.sh",
                 label: 'Run Python GPU integration tests',
               )
-              junit 'build/pytest-results/*.xml'
             }
+          } finally {
+            junit 'build/pytest-results/*.xml'
           }
         }
-      } else {
-        Utils.markStageSkippedForConditional('unittest: GPU')
       }
-    },
-    'integration: CPU': {
-      if (!skip_ci && is_docs_only_build != 1) {
-        node('CPU') {
-          ws(per_exec_ws('tvm/ut-python-cpu')) {
+    } else {
+      Utils.markStageSkippedForConditional('unittest: GPU')
+    }
+  },
+  'integration: CPU': {
+    if (!skip_ci && is_docs_only_build != 1) {
+      node('CPU') {
+        ws(per_exec_ws('tvm/ut-python-cpu')) {
+          try {
             init_git()
             unpack_lib('cpu', tvm_multilib_tsim)
             timeout(time: max_time, unit: 'MINUTES') {
@@ -422,18 +429,21 @@ stage('Test') {
                 script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh",
                 label: 'Run CPU integration tests',
               )
-              junit 'build/pytest-results/*.xml'
             }
+          } finally {
+            junit 'build/pytest-results/*.xml'
           }
         }
-      } else {
-        Utils.markStageSkippedForConditional('integration: CPU')
       }
-    },
-    'unittest: CPU': {
-      if (!skip_ci && is_docs_only_build != 1) {
-        node('CPU') {
-          ws(per_exec_ws("tvm/ut-python-cpu")) {
+    } else {
+      Utils.markStageSkippedForConditional('integration: CPU')
+    }
+  },
+  'unittest: CPU': {
+    if (!skip_ci && is_docs_only_build != 1) {
+      node('CPU') {
+        ws(per_exec_ws("tvm/ut-python-cpu")) {
+          try {
             init_git()
             unpack_lib('cpu', tvm_multilib_tsim)
             timeout(time: max_time, unit: 'MINUTES') {
@@ -444,18 +454,21 @@ stage('Test') {
                 script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_vta_tsim.sh",
                 label: "Run VTA tests in TSIM",
               )
-              junit "build/pytest-results/*.xml"
             }
+          } finally {
+            junit 'build/pytest-results/*.xml'
           }
         }
-      } else {
-        Utils.markStageSkippedForConditional('unittest: CPU')
       }
-    },
-    'python3: i386': {
-      if (!skip_ci && is_docs_only_build != 1) {
-        node('CPU') {
-          ws(per_exec_ws('tvm/ut-python-i386')) {
+    } else {
+      Utils.markStageSkippedForConditional('unittest: CPU')
+    }
+  },
+  'python3: i386': {
+    if (!skip_ci && is_docs_only_build != 1) {
+      node('CPU') {
+        ws(per_exec_ws('tvm/ut-python-i386')) {
+          try {
             init_git()
             unpack_lib('i386', tvm_multilib)
             timeout(time: max_time, unit: 'MINUTES') {
@@ -466,18 +479,21 @@ stage('Test') {
                 label: 'Run i386 integration tests',
               )
               fsim_test(ci_i386)
-              junit 'build/pytest-results/*.xml'
             }
+          } finally {
+            junit 'build/pytest-results/*.xml'
           }
         }
-     } else {
-        Utils.markStageSkippedForConditional('python3: i386')
       }
-    },
-    'python3: arm': {
-      if (!skip_ci && is_docs_only_build != 1) {
-        node('ARM') {
-          ws(per_exec_ws('tvm/ut-python-arm')) {
+    } else {
+      Utils.markStageSkippedForConditional('python3: i386')
+    }
+  },
+  'python3: arm': {
+    if (!skip_ci && is_docs_only_build != 1) {
+      node('ARM') {
+        ws(per_exec_ws('tvm/ut-python-arm')) {
+          try {
             init_git()
             unpack_lib('arm', tvm_multilib)
             timeout(time: max_time, unit: 'MINUTES') {
@@ -487,47 +503,55 @@ stage('Test') {
                 script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_arm_compute_library.sh",
                 label: 'Run test_arm_compute_lib test',
               )
-              junit 'build/pytest-results/*.xml'
             // sh "${docker_run} ${ci_arm} ./tests/scripts/task_python_integration.sh"
             }
+          } finally {
+            junit 'build/pytest-results/*.xml'
           }
         }
-      } else {
-         Utils.markStageSkippedForConditional('python3: arm')
       }
-    },
+    } else {
+      Utils.markStageSkippedForConditional('python3: arm')
+    }
+  },
   'topi: GPU': {
-  if (!skip_ci && is_docs_only_build != 1) {
-    node('GPU') {
-      ws(per_exec_ws('tvm/topi-python-gpu')) {
-        init_git()
-        unpack_lib('gpu', tvm_multilib)
-        timeout(time: max_time, unit: 'MINUTES') {
-          ci_setup(ci_gpu)
-          sh (
-            script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh",
-            label: 'Run TOPI tests',
-          )
-          junit 'build/pytest-results/*.xml'
+    if (!skip_ci && is_docs_only_build != 1) {
+      node('GPU') {
+        ws(per_exec_ws('tvm/topi-python-gpu')) {
+          try {
+            init_git()
+            unpack_lib('gpu', tvm_multilib)
+            timeout(time: max_time, unit: 'MINUTES') {
+              ci_setup(ci_gpu)
+              sh (
+                script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh",
+                label: 'Run TOPI tests',
+              )
+            }
+          } finally {
+            junit 'build/pytest-results/*.xml'
+          }
         }
       }
-    }
     } else {
       Utils.markStageSkippedForConditional('topi: GPU')
-  }
+    }
   },
   'frontend: GPU': {
     if (!skip_ci && is_docs_only_build != 1) {
       node('GPU') {
         ws(per_exec_ws('tvm/frontend-python-gpu')) {
-          init_git()
-          unpack_lib('gpu', tvm_multilib)
-          timeout(time: max_time, unit: 'MINUTES') {
-            ci_setup(ci_gpu)
-            sh (
-              script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh",
-              label: 'Run Python frontend tests',
-            )
+          try {
+            init_git()
+            unpack_lib('gpu', tvm_multilib)
+            timeout(time: max_time, unit: 'MINUTES') {
+              ci_setup(ci_gpu)
+              sh (
+                script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh",
+                label: 'Run Python frontend tests',
+              )
+            }
+          } finally {
             junit 'build/pytest-results/*.xml'
           }
         }
@@ -540,14 +564,17 @@ stage('Test') {
     if (!skip_ci && is_docs_only_build != 1) {
       node('CPU') {
         ws(per_exec_ws('tvm/frontend-python-cpu')) {
-          init_git()
-          unpack_lib('cpu', tvm_multilib)
-          timeout(time: max_time, unit: 'MINUTES') {
-            ci_setup(ci_cpu)
-            sh (
-              script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_frontend_cpu.sh",
-              label: 'Run Python frontend tests',
-            )
+          try {
+            init_git()
+            unpack_lib('cpu', tvm_multilib)
+            timeout(time: max_time, unit: 'MINUTES') {
+              ci_setup(ci_cpu)
+              sh (
+                script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_frontend_cpu.sh",
+                label: 'Run Python frontend tests',
+              )
+            }
+          } finally {
             junit 'build/pytest-results/*.xml'
           }
         }
@@ -595,13 +622,13 @@ stage('Build packages') {
 */
 
 stage('Deploy') {
-    node('doc') {
-      ws(per_exec_ws('tvm/deploy-docs')) {
-        if (env.BRANCH_NAME == 'main') {
+  node('doc') {
+    ws(per_exec_ws('tvm/deploy-docs')) {
+      if (env.BRANCH_NAME == 'main') {
         unpack_lib('docs', 'docs.tgz')
         sh 'cp docs.tgz /var/docs/docs.tgz'
         sh 'tar xf docs.tgz -C /var/docs'
-        }
       }
     }
+  }
 }
diff --git a/tests/scripts/task_python_frontend.sh b/tests/scripts/task_python_frontend.sh
index 6a46f8845f4e..6c0cd8df67f5 100755
--- a/tests/scripts/task_python_frontend.sh
+++ b/tests/scripts/task_python_frontend.sh
@@ -42,9 +42,11 @@ run_pytest cython python-frontend-coreml tests/python/frontend/coreml
 echo "Running relay Tensorflow frontend test..."
 # Note: Tensorflow tests often have memory issues, so invoke each one separately
 TENSORFLOW_TESTS=$(./tests/scripts/pytest_ids.py --folder tests/python/frontend/tensorflow)
+i=0
 for node_id in $TENSORFLOW_TESTS; do
     echo "$node_id"
-    run_pytest cython python-frontend-tensorflow "$node_id"
+    run_pytest cython "python-frontend-tensorflow-$i" "$node_id"
+    i=$((i+1))
 done
 
 echo "Running relay caffe2 frontend test..."
diff --git a/tests/scripts/task_python_integration.sh b/tests/scripts/task_python_integration.sh
index 4992bfa678de..3bd063e2b5d5 100755
--- a/tests/scripts/task_python_integration.sh
+++ b/tests/scripts/task_python_integration.sh
@@ -16,8 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -e
-set -u
+set -euxo pipefail
 
 source tests/scripts/setup-pytest-env.sh
 export PYTHONPATH=${PYTHONPATH}:${TVM_PATH}/apps/extension/python
@@ -44,22 +43,22 @@ rm -rf lib
 make
 cd ../..
 
-run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-extensions apps/extension/tests
-run_pytest cython ${TVM_INTEGRATION_TESTSUITE_NAME}-extensions apps/extension/tests
+run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-extensions-0 apps/extension/tests
+run_pytest cython ${TVM_INTEGRATION_TESTSUITE_NAME}-extensions-1 apps/extension/tests
 
 # Test dso plugin
 cd apps/dso_plugin_module
 rm -rf lib
 make
 cd ../..
-run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-dso_plugin_module apps/dso_plugin_module
-run_pytest cython ${TVM_INTEGRATION_TESTSUITE_NAME}-dso_plugin_module apps/dso_plugin_module
+run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-dso_plugin_module-0 apps/dso_plugin_module
+run_pytest cython ${TVM_INTEGRATION_TESTSUITE_NAME}-dso_plugin_module-1 apps/dso_plugin_module
 
 # Do not enable TensorFlow op
 # TVM_FFI=cython sh prepare_and_test_tfop_module.sh
 # TVM_FFI=ctypes sh prepare_and_test_tfop_module.sh
 
-run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME} tests/python/integration
+run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-integration tests/python/integration
 
 # Ignoring Arm(R) Ethos(TM)-U NPU tests in the collective to run to run them in parallel in the next step.
 run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-contrib tests/python/contrib --ignore=tests/python/contrib/test_ethosu
diff --git a/tests/scripts/task_python_unittest.sh b/tests/scripts/task_python_unittest.sh
index 54a36f6dcfd4..7ff433cb9db1 100755
--- a/tests/scripts/task_python_unittest.sh
+++ b/tests/scripts/task_python_unittest.sh
@@ -16,8 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -e
-set -u
+set -euxo pipefail
 
 source tests/scripts/setup-pytest-env.sh
 
@@ -31,9 +30,9 @@ if [ -z "${TVM_UNITTEST_TESTSUITE_NAME:-}" ]; then
 fi
 
 # First run minimal test on both ctypes and cython.
-run_pytest ctypes ${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test tests/python/all-platform-minimal-test
-run_pytest cython ${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test tests/python/all-platform-minimal-test
+run_pytest ctypes ${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test-0 tests/python/all-platform-minimal-test
+run_pytest cython ${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test-1 tests/python/all-platform-minimal-test
 
 # Then run all unittests on both ctypes and cython.
-run_pytest ctypes ${TVM_UNITTEST_TESTSUITE_NAME} tests/python/unittest
-run_pytest cython ${TVM_UNITTEST_TESTSUITE_NAME} tests/python/unittest
+run_pytest ctypes ${TVM_UNITTEST_TESTSUITE_NAME}-0 tests/python/unittest
+run_pytest cython ${TVM_UNITTEST_TESTSUITE_NAME}-1 tests/python/unittest
diff --git a/tests/scripts/task_python_unittest_gpuonly.sh b/tests/scripts/task_python_unittest_gpuonly.sh
index 54dd085f1817..85fa293c31af 100755
--- a/tests/scripts/task_python_unittest_gpuonly.sh
+++ b/tests/scripts/task_python_unittest_gpuonly.sh
@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -euo pipefail
+set -euxo pipefail
 
 export PYTEST_ADDOPTS="-m gpu ${PYTEST_ADDOPTS:-}"
 
@@ -33,5 +33,5 @@ export TVM_UNITTEST_TESTSUITE_NAME=python-unittest-vulkan
 
 source tests/scripts/setup-pytest-env.sh
 
-run_pytest ctypes ${TVM_UNITTEST_TESTSUITE_NAME} tests/python/unittest/test_target_codegen_vulkan.py
-run_pytest cython ${TVM_UNITTEST_TESTSUITE_NAME} tests/python/unittest/test_target_codegen_vulkan.py
+run_pytest ctypes ${TVM_UNITTEST_TESTSUITE_NAME}-0 tests/python/unittest/test_target_codegen_vulkan.py
+run_pytest cython ${TVM_UNITTEST_TESTSUITE_NAME}-1 tests/python/unittest/test_target_codegen_vulkan.py