Enable usage of sanitizers in tests (NVIDIA#3377)

* Enable usage of sanitizers in tests

Signed-off-by: Janusz Lisiecki <jlisiecki@nvidia.com>

docker/Dockerfile.deps

@@ -8,7 +8,7 @@ FROM ${BUILDER_EXTRA_DEPS} as extra_deps
 FROM ${FROM_IMAGE_NAME}
 
 # Install yum Dependencies
-RUN yum install -y wget nasm doxygen graphviz gettext xz openssl autogen libasan4
+RUN yum install -y wget nasm doxygen graphviz gettext xz openssl autogen
 ENV ACLOCAL_PATH=/usr/share/aclocal/:/usr/local/share/aclocal
 
 # add devtoolset-7 to paths
@@ -17,7 +17,7 @@ ENV PATH=/opt/rh/devtoolset-7/root/usr/bin:$PATH \
 
 # CUDA 10 doesn't support gcc > 7 so install 7 and remove other
 RUN yum remove -y devtoolset* && \
-    yum install -y devtoolset-7
+    yum install -y devtoolset-7 devtoolset-7-libasan-devel devtoolset-7-liblsan-devel devtoolset-7-libtsan-devel devtoolset-7-libubsan-devel
 
 # Don't want the short-unicode version for Python 2.7
 RUN rm -f /opt/python/cp27-cp27m

qa/leak.sup

@@ -0,0 +1,10 @@
+# TODO when all test passes in CI we can set suppress false-positives
+leak:paddle
+leak:llvm
+#leak:PyCode_NewWithPosOnlyArgs
+#leak:PyLong_FromLong
+#leak:_PyLong_New
+#leak:PyBytes_FromString
+#leak:PyCapsule_New
+#leak:av_realloc_f
+#leak:memalign

qa/test_template.sh

@@ -45,13 +45,54 @@ epilog=${epilog-:}
 # get the number of elements in `prolog` array
 numer_of_prolog_elms=${#prolog[@]}
 
+# turn on sanitizers
+enable_sanitizer() {
+    # supress leaks that are false positive or not related to DALI
+    export LSAN_OPTIONS=suppressions=$topdir/qa/leak.sup
+    export ASAN_OPTIONS=symbolize=1:protect_shadow_gap=0:log_path=sanitizer.log:start_deactivated=true:allocator_may_return_null=1::detect_leaks=1
+    export ASAN_SYMBOLIZER_PATH=$(which llvm-symbolizer)
+    # avoid python false positives
+    export PYTHONMALLOC=malloc
+}
+
+# turn off sanitizer to avoid breaking any non-related system built-ins
+disable_sanitizer() {
+    export ASAN_OPTIONS=start_deactivated=true:detect_leaks=0
+    unset ASAN_SYMBOLIZER_PATH
+    unset PYTHONMALLOC
+}
+
 # Wrap the test_body in a subshell, where we can safely execute it with `set -e`
 # and turn it off in current shell to intercept the error code
+# when sanitizers are on, do set +e to run all the test no matter what the result is
+# and collect as much of sanitizers output as possible
 test_body_wrapper() {(
-    set -e
+    if [ -n "$DALI_ENABLE_SANITIZERS" ]; then
+        set +e
+        enable_sanitizer
+    else
+        set -e
+    fi
+
     test_body
+
+    if [ -n "$DALI_ENABLE_SANITIZERS" ]; then
+        disable_sanitizer
+    fi
 )}
 
+process_sanitizers_logs() {
+    find $topdir -iname "sanitizer.log.*" -print0 | xargs -0 -I file cat file > $topdir/sanitizer.log
+    if [ -e $topdir/sanitizer.log ]; then
+        cat $topdir/sanitizer.log
+        grep -q ERROR $topdir/sanitizer.log || true
+        # ToDo - enable when the suppression file is completed
+        # grep -q ERROR $topdir/sanitizer.log && exit 1 || true
+    fi
+    # rm so the consequitive test won't reread the same logs over and over
+    find $topdir -iname "sanitizer.log.*" -delete
+}
+
 # get extra index url for given packages
 extra_indices=$($topdir/qa/setup_packages.py -u $pip_packages --cuda ${CUDA_VERSION} -e)
 extra_indices_string=""
@@ -93,7 +134,8 @@ do
         test_body_wrapper
         RV=$?
         set -e
-        if [ $RV -gt 0 ] ; then
+        # if sanitizers are enabled run test until the end so we have as much data as possible
+        if [ $RV -gt 0 ] && [ -z "$DALI_ENABLE_SANITIZERS" ]; then
             mkdir -p $topdir/core_artifacts
             cp core* $topdir/core_artifacts || true
             exit ${RV}
@@ -106,3 +148,7 @@ do
         ${epilog[variant]}
     done
 done
+
+if [ -n "$DALI_ENABLE_SANITIZERS" ]; then
+    process_sanitizers_logs
+fi

tools/test_bundled_libs.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -43,8 +43,10 @@ def main():
                         "librt.so.1",
                         "libstdc++.so.6",
                         "libgcc_s.so.1",
-                        "libz.so.1",
-                        "liblzma.so.5"
+                        "libasan.so",
+                        "liblsan.so",
+                        "libubsan.so",
+                        "libtsan.so"
                     ]}
 
     bundled_libs = argv[1:]