From 799236dace4f8d7ac9d62d64e06d3cbcee8084c9 Mon Sep 17 00:00:00 2001 From: "Gromadzki, Tomasz" Date: Wed, 10 May 2023 12:56:31 +0200 Subject: [PATCH 01/56] common: enable manual trigger of some builds Signed-off-by: Gromadzki, Tomasz --- .github/workflows/bandit.yml | 2 +- .github/workflows/codeql.yml | 2 +- .github/workflows/gha.yml | 2 +- .github/workflows/ras_linux.yml | 2 +- .github/workflows/unittest.yml | 2 +- .github/workflows/unittest_long.yml | 1 + 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml index 570a3008645..bd80dca0d85 100644 --- a/.github/workflows/bandit.yml +++ b/.github/workflows/bandit.yml @@ -1,7 +1,7 @@ # Run the Bandit scanner for finding vulnerabilities in the python code. name: bandit -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] env: GITHUB_REPO: pmem/pmdk diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 5eac157c2e6..1624dd270ab 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -3,7 +3,7 @@ # This workflow does not use docker containers. name: CodeQL -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] env: GITHUB_REPO: pmem/pmdk diff --git a/.github/workflows/gha.yml b/.github/workflows/gha.yml index 3eed7399709..58b5c4c8cc4 100644 --- a/.github/workflows/gha.yml +++ b/.github/workflows/gha.yml @@ -2,7 +2,7 @@ # checks basic builds with various compilers and executes all sets of tests. name: PMDK -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] env: GITHUB_REPO: pmem/pmdk diff --git a/.github/workflows/ras_linux.yml b/.github/workflows/ras_linux.yml index 852e7b1b0a6..20102347daa 100644 --- a/.github/workflows/ras_linux.yml +++ b/.github/workflows/ras_linux.yml @@ -24,7 +24,7 @@ # utils/gha-runners/run-ras-linux.yml name: RAS_Linux -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] jobs: linux: diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 645ca915435..f81f92f5c4d 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -3,7 +3,7 @@ # This workflow is run on 'self-hosted' runners. name: PMDK_runners -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] jobs: linux: diff --git a/.github/workflows/unittest_long.yml b/.github/workflows/unittest_long.yml index 455cfe46b79..fc25542ee04 100644 --- a/.github/workflows/unittest_long.yml +++ b/.github/workflows/unittest_long.yml @@ -4,6 +4,7 @@ name: Long_run_tests on: + workflow_dispatch: schedule: - cron: '5 0 * * 6' # At 00:05 on Saturday. From 952bdf4411e485d10382fb76c9b37fdc46a0714c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Tue, 9 May 2023 13:52:18 +0200 Subject: [PATCH 02/56] common: fix prepare-for-build.sh to properly pass extra arguments --- utils/docker/prepare-for-build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/docker/prepare-for-build.sh b/utils/docker/prepare-for-build.sh index c74e65cb628..332605cd89d 100755 --- a/utils/docker/prepare-for-build.sh +++ b/utils/docker/prepare-for-build.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2016-2022, Intel Corporation +# Copyright 2016-2023, Intel Corporation # # prepare-for-build.sh - is called inside a Docker container; prepares @@ -21,5 +21,5 @@ fi # should be preserved KEEP_TEST_CONFIG=${KEEP_TEST_CONFIG:-0} if [[ "$KEEP_TEST_CONFIG" == 0 ]]; then - ./configure-tests.sh + ./configure-tests.sh $@ fi From 52b139644c39b390633c17ea24251c4692feeac6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Tue, 9 May 2023 13:55:24 +0200 Subject: [PATCH 03/56] common: extend run-build-package.sh with extra checks Make use of extra tests for packages we have up our sleeve. Since we're running this test on Fedora, we'd have to also extend list of installed packages, for these new checks to work. Make the script a little more verbose, in the process. --- utils/docker/images/Dockerfile.fedora-37 | 9 +++++ utils/docker/run-build-package.sh | 42 +++++++++++++++++++----- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/utils/docker/images/Dockerfile.fedora-37 b/utils/docker/images/Dockerfile.fedora-37 index cef6e817458..47f6e92d56e 100644 --- a/utils/docker/images/Dockerfile.fedora-37 +++ b/utils/docker/images/Dockerfile.fedora-37 @@ -55,6 +55,7 @@ ENV TESTS_DEPS "\ libunwind-devel \ ndctl \ openssh-server \ + python3 \ strace" # packaging dependencies @@ -70,6 +71,7 @@ ENV MISC_DEPS "\ lbzip2 \ man \ python3-flake8 \ + python3-pip \ rsync \ shadow-utils \ sudo \ @@ -77,6 +79,11 @@ ENV MISC_DEPS "\ which \ xmlto" +# python scripts dependencies +ENV PYTHON_DEPS "\ + distro \ + xmlrunner" + # Copy install valgrind script COPY install-valgrind.sh install-valgrind.sh @@ -92,6 +99,8 @@ RUN dnf update -y && dnf install -y \ && ./install-valgrind.sh fedora \ && dnf clean all +RUN pip3 install $PYTHON_DEPS + # Add and switch user ENV USER pmdkuser ENV USERPASS pmdkpass diff --git a/utils/docker/run-build-package.sh b/utils/docker/run-build-package.sh index 4d8eda44342..034ae57b866 100755 --- a/utils/docker/run-build-package.sh +++ b/utils/docker/run-build-package.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2016-2021, Intel Corporation +# Copyright 2016-2023, Intel Corporation # # run-build-package.sh - is called inside a Docker container; prepares @@ -17,27 +17,37 @@ git config user.email "test@package.com" git config user.name "test package" git tag -a 1.4.99 -m "1.4" HEAD~1 || true -# Build all and run tests -cd $WORKDIR +echo "## Build package (and run basic tests)" +pushd $WORKDIR export PCHECK_OPTS="-j2 BLACKLIST_FILE=${BLACKLIST_FILE}" make -j$(nproc) $PACKAGE_MANAGER -# Install packages +echo "## Build PMDK once more (clobber from packaging process cleared out some required files)" +make -j$(nproc) + +echo "## Test built packages" +python3 $SCRIPTSDIR/packages/test-build-packages.py -r $(pwd) + +echo "## Install packages" if [[ "$PACKAGE_MANAGER" == "dpkg" ]]; then - cd $PACKAGE_MANAGER + pushd $PACKAGE_MANAGER echo $USERPASS | sudo -S dpkg --install *.deb else RPM_ARCH=$(uname -m) - cd $PACKAGE_MANAGER/$RPM_ARCH + pushd $PACKAGE_MANAGER/$RPM_ARCH echo $USERPASS | sudo -S rpm --install *.rpm fi +popd + +echo "## Test installed packages" +python3 $SCRIPTSDIR/packages/test-packages-installation.py -r $(pwd) -# Compile and run standalone test -cd $WORKDIR/utils/docker/test_package +echo "## Compile and run standalone test" +pushd $SCRIPTSDIR/test_package make -j$(nproc) LIBPMEMOBJ_MIN_VERSION=1.4 ./test_package testfile1 -# Use pmreorder installed in the system +echo "## Use pmreorder installed in the system" pmreorder_version="$(pmreorder -v)" pmreorder_pattern="pmreorder\.py .+$" (echo "$pmreorder_version" | grep -Ev "$pmreorder_pattern") && echo "pmreorder version failed" && exit 1 @@ -45,3 +55,17 @@ pmreorder_pattern="pmreorder\.py .+$" touch testfile2 touch logfile1 pmreorder -p testfile2 -l logfile1 +popd + +echo "## Run tests (against PMDK installed in the system)" +pushd $WORKDIR/src/test +make -j$(nproc) clobber +make -j$(nproc) + +# Prepare test config once more. Now, with path to PMDK set in the OS +# (rather than in the git tree) - for testing packages installed in the system. +$SCRIPTSDIR/configure-tests.sh PKG +./RUNTESTS -t check +popd + +popd From 69cc71bf0bf37f96474ea1208e0caea732c51109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Tue, 9 May 2023 14:00:38 +0200 Subject: [PATCH 04/56] common: remove redundant, not used script --- utils/docker/packages/cleanup-packages.py | 112 ---------------------- 1 file changed, 112 deletions(-) delete mode 100644 utils/docker/packages/cleanup-packages.py diff --git a/utils/docker/packages/cleanup-packages.py b/utils/docker/packages/cleanup-packages.py deleted file mode 100644 index 8a2f8c2ffcc..00000000000 --- a/utils/docker/packages/cleanup-packages.py +++ /dev/null @@ -1,112 +0,0 @@ -#!usr/bin/env python3 -# -# SPDX-License-Identifier: BSD-3-Clause -# Copyright 2020-2023, Intel Corporation - -""" -This module includes functions which clean up environment system after -installation of rpm packages from PMDK library. -""" - -from os import listdir, path, linesep -import distro -from subprocess import check_output -from argparse import ArgumentParser -import re -import json -from pathlib import Path - -PMDK_VERSION = '' -SYSTEM_ARCHITECTURE = '' - - -def get_package_version_and_system_architecture(pmdk_path): - """ - Returns packages version and system architecture from names of directories - from rpm directory. - """ - version = '' - architecture = '' - os_distro=distro.id() - - # first check if there is a json file with version and architecture - pkg_version_file_path = Path(pmdk_path).joinpath("pkgVersion.json") - if pkg_version_file_path.exists() and pkg_version_file_path.is_file(): - with open(pkg_version_file_path) as pkg_version_file: - version_from_file = {} - try: - version_from_file = json.load(pkg_version_file) - version = version_from_file.get('PMDK_VERSION', '') - architecture = version_from_file.get('SYSTEM_ARCHITECTURE', '') - except: - pass - - # if cannot read values from json file, read them from rpms: - if version == '' or architecture == '': - if os_distro != 'ubuntu': - pkg_directory = path.join(pmdk_path, 'rpm') - for elem in listdir(pkg_directory): - if '.src.rpm' in elem: - version = re.search(r'[\s]*pmdk-([\S]+).src.rpm', elem).group(1) - else: - architecture = elem - - else: - pkg_directory = path.join(pmdk_path, 'dpkg') - for elem in listdir(pkg_directory): - if '.changes' in elem: - # looks for the version number of dpkg package in dpkg package name - version = re.search(r'pmdk_*(.+)_(.+).changes', elem).group(1) - architecture = re.search(r'pmdk_*(.+)_(.+).changes', elem).group(2) - - return version, architecture - - -def uninstall_rpm_packages(pmdk_path): - """ - Removes binaries installed from packages from PMDK library. - """ - try: - rpm_to_uninstall = check_output( - 'rpm -qa | grep ' + PMDK_VERSION, cwd=pmdk_path, shell=True) - pkg_to_uninstall = rpm_to_uninstall.decode( - 'UTF-8').replace(linesep, ' ') - except: - pkg_to_uninstall = '' - - if pkg_to_uninstall: - check_output('rpm -e ' + pkg_to_uninstall, cwd=pmdk_path, shell=True) - - -def uninstall_dpkg_packages(pmdk_path): - """ - Removes binaries installed from packages from PMDK library. - """ - try: - dpkg_to_uninstall = check_output( - 'dpkg-query --show | grep ' + PMDK_VERSION + ''' | awk '{print $1}' ''', cwd=pmdk_path, shell=True) - pkg_to_uninstall = dpkg_to_uninstall.decode( - 'UTF-8').replace(linesep, ' ') - except: - pkg_to_uninstall = '' - - if pkg_to_uninstall: - check_output('dpkg -r ' + pkg_to_uninstall, cwd=pmdk_path, shell=True) - - -if __name__ == '__main__': - parser = ArgumentParser( - description='''Clean up system environment after installation rpm - packages from PMDK library''') - parser.add_argument("-r", "--pmdk-path", required=True, - help="the PMDK library root path.") - - args = parser.parse_args() - PMDK_VERSION, SYSTEM_ARCHITECTURE =\ - get_package_version_and_system_architecture(args.pmdk_path) - - os_distro=distro.id() - if os_distro != 'ubuntu': - uninstall_rpm_packages(args.pmdk_path) - else: - uninstall_dpkg_packages(args.pmdk_path) From 37693b689c7413eb386ae93833b9e1b182a134f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Mon, 8 May 2023 16:54:04 +0200 Subject: [PATCH 05/56] common: add param to package testing script to skip daxio --- utils/docker/packages/test-build-packages.py | 18 +++++++++++++++--- utils/docker/run-build-package.sh | 3 ++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/utils/docker/packages/test-build-packages.py b/utils/docker/packages/test-build-packages.py index 82247028e2e..5d96c772009 100644 --- a/utils/docker/packages/test-build-packages.py +++ b/utils/docker/packages/test-build-packages.py @@ -10,8 +10,12 @@ -if all required packages are built, -if built packages are consistent with names of libraries read from .so files and other elements (tools and "PMDK"). + Required arguments: -r the PMDK library root path. + +Optional arguments: +--skip-daxio to be set if daxio was not built (e.g., if NDCTL_ENABLE was set to 'n') """ from os import listdir, path, linesep @@ -148,10 +152,12 @@ def find_missing_packages(packages_path, pmdk_path, pmdk_debuginfo_package_name) 'pmempool': PACKAGES_INFO(basic=True, devel=False, debug=False, debuginfo=True, debug_debuginfo=False), 'pmreorder': PACKAGES_INFO(basic=True, devel=False, debug=False, - debuginfo=False, debug_debuginfo=False), - 'daxio': PACKAGES_INFO(basic=True, devel=False, debug=False, - debuginfo=True, debug_debuginfo=False) + debuginfo=False, debug_debuginfo=False) } + if not skip_daxio: + tools['daxio'] = PACKAGES_INFO(basic=True, devel=False, debug=False, + debuginfo=True, debug_debuginfo=False) + tools_packages = get_names_of_packages(tools) missing_tools_packages = [ elem for elem in tools_packages if elem not in built_packages] @@ -237,11 +243,17 @@ def test_completeness_of_name_of_libraries_and_tools(self): if __name__ == '__main__': path_argument = '-r' + daxio_build_argument = "--skip-daxio" + skip_daxio = False if '-h' in sys.argv or '--help' in sys.argv: print(__doc__) unittest.main() elif path_argument in sys.argv: pmdk_path = parse_argument(path_argument) + if daxio_build_argument in sys.argv: + skip_daxio = True + index = sys.argv.index(daxio_build_argument) + sys.argv.pop(index) if pmdk_path: PMDK_VERSION, SYSTEM_ARCHITECTURE =\ get_package_version_and_system_architecture(pmdk_path) diff --git a/utils/docker/run-build-package.sh b/utils/docker/run-build-package.sh index 034ae57b866..a06b8c85c49 100755 --- a/utils/docker/run-build-package.sh +++ b/utils/docker/run-build-package.sh @@ -26,7 +26,8 @@ echo "## Build PMDK once more (clobber from packaging process cleared out some r make -j$(nproc) echo "## Test built packages" -python3 $SCRIPTSDIR/packages/test-build-packages.py -r $(pwd) +[ "$NDCTL_ENABLE" == "n" ] && extra_params="--skip-daxio" || extra_params="" +python3 $SCRIPTSDIR/packages/test-build-packages.py -r $(pwd) ${extra_params} echo "## Install packages" if [[ "$PACKAGE_MANAGER" == "dpkg" ]]; then From c58670b14007480c7be960b0eadba514def12385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Tue, 9 May 2023 14:14:16 +0200 Subject: [PATCH 06/56] common: remove (now redundant) second script to test packages 'build-and-test-pmdk-packages.sh' script was using the extra checks for packages, but now they are all part of 'run-build-package.sh'. Packages GHA workflow now use the old 'MAKE_PKG' command. --- .github/workflows/pkg_linux.yml | 2 +- utils/docker/build-CI.sh | 1 - utils/docker/build-and-test-pmdk-packages.sh | 67 ------- utils/docker/packages/install-packages.py | 188 ------------------- 4 files changed, 1 insertion(+), 257 deletions(-) delete mode 100755 utils/docker/build-and-test-pmdk-packages.sh delete mode 100644 utils/docker/packages/install-packages.py diff --git a/.github/workflows/pkg_linux.yml b/.github/workflows/pkg_linux.yml index ee2a84d7d64..230dfe5c1ad 100644 --- a/.github/workflows/pkg_linux.yml +++ b/.github/workflows/pkg_linux.yml @@ -23,7 +23,7 @@ jobs: PMDK_CXX: g++ VALGRIND: 1 SRC_CHECKERS: 0 - CHECK_PKGS: 1 + MAKE_PKG: 1 steps: - name: Clone the git repo diff --git a/utils/docker/build-CI.sh b/utils/docker/build-CI.sh index 4498d3dc574..ce2313536a4 100755 --- a/utils/docker/build-CI.sh +++ b/utils/docker/build-CI.sh @@ -38,7 +38,6 @@ if [[ $MAKE_PKG -eq 1 ]] ; then command="./run-build-package.sh"; fi if [[ $COVERAGE -eq 1 ]] ; then command="./run-coverage.sh"; fi if [[ $BANDIT -eq 1 ]] ; then command="./run-bandit.sh"; fi if [[ "$COVERITY" -eq 1 ]]; then command="./run-coverity.sh"; fi -if [[ $CHECK_PKGS -eq 1 ]] ; then command="./build-and-test-pmdk-packages.sh"; fi if [ -n "$DNS_SERVER" ]; then DNS_SETTING=" --dns=$DNS_SERVER "; fi if [[ -f $CI_FILE_SKIP_BUILD_PKG_CHECK ]]; then BUILD_PACKAGE_CHECK=n; else BUILD_PACKAGE_CHECK=y; fi diff --git a/utils/docker/build-and-test-pmdk-packages.sh b/utils/docker/build-and-test-pmdk-packages.sh deleted file mode 100755 index 224f54bc627..00000000000 --- a/utils/docker/build-and-test-pmdk-packages.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: BSD-3-Clause -# Copyright 2023, Intel Corporation - -# build-and-test-pmdk-packages.sh - is called inside a Docker container; Script for building and testing pmdk packages. - -set -eo pipefail - -# Prepare build environment -# Adding 'PKG' parameter is required for PKG tests. -./prepare-for-build.sh PKG - -# build_pmdk_package -- build pmdk package depending on the Linux distribution. -function build_pmdk_package { - cd $WORKDIR - echo "********** make package **********" - date - if [ $OS = opensuse/leap ] || [ $OS = rockylinux/rockylinux ]; then - make rpm EXPERIMENTAL=y BUILD_PACKAGE_CHECK=n PMEM2_INSTALL=y - elif [ $OS = ubuntu ]; then - make dpkg EXPERIMENTAL=y BUILD_PACKAGE_CHECK=n PMEM2_INSTALL=y - fi -} - -# Build pmdk to create src/nondebug directory used by python scripts in tests. -function build_pmdk { - cd $WORKDIR - echo "********** make pmdk **********" - make EXTRA_CFLAGS=-DUSE_VALGRIND -} - -# test_built_packages -- test if the packages are built correctly. -function test_built_packages { - echo "********** test built packages **********" - python3 $SCRIPTSDIR/packages/test-build-packages.py -r $(pwd) -} - -# install_packages -- install built packages. -function install_packages { - echo "********** install packages **********" - sudo python3 $SCRIPTSDIR/packages/install-packages.py -r $(pwd) -} - -# test_installed_packages -- test if the packages were installed correctly. -function test_installed_packages { - echo "********** test installed packages **********" - python3 $SCRIPTSDIR/packages/test-packages-installation.py -r $(pwd) -} - -function build_unittests { - echo "********** build unittests **********" - cd $WORKDIR/src/test && make clobber - make test -} - -function run_check_unittests { - echo "********** run tests **********" - cd $WORKDIR/src/test && ./RUNTESTS -t check -o 60m -} - -build_pmdk_package -build_pmdk -test_built_packages -install_packages -test_installed_packages -build_unittests -run_check_unittests diff --git a/utils/docker/packages/install-packages.py b/utils/docker/packages/install-packages.py deleted file mode 100644 index b79b37cd2c3..00000000000 --- a/utils/docker/packages/install-packages.py +++ /dev/null @@ -1,188 +0,0 @@ -#!usr/bin/env python3 -# -# SPDX-License-Identifier: BSD-3-Clause -# Copyright 2020-2023, Intel Corporation - -""" -This module includes functions which install packages from PMDK library. -""" - -from os import listdir, path, linesep -from subprocess import check_output -from argparse import ArgumentParser -import distro -import re -import json - -PMDK_VERSION = '' -SYSTEM_ARCHITECTURE = '' - -def get_package_version_and_system_architecture(pmdk_path): - """ - Returns packages version and system architecture from names of directories - from packages directory. - """ - os_distro=distro.id() - if os_distro != 'ubuntu': - pkg_directory = path.join(pmdk_path, 'rpm') - else: - pkg_directory = path.join(pmdk_path, 'dpkg') - - version = '' - architecture = '' - for elem in listdir(pkg_directory): - if os_distro != 'ubuntu': - if '.src.rpm' in elem: - # looks for the version number of package in package name - version = re.search(r'[\s]*pmdk-([\S]+).src.rpm', elem).group(1) - else: - architecture = elem - - elif os_distro == 'ubuntu': - if '.changes' in elem: - # looks for the version number of packages in package name - version = re.search(r'pmdk_*(.+)_(.+).changes', elem).group(1) - architecture = re.search(r'pmdk_*(.+)_(.+).changes', elem).group(2) - return version, architecture - - -def save_pkg_version(pkg_version_file_path): - values_to_save = { - 'PMDK_VERSION': PMDK_VERSION, - 'SYSTEM_ARCHITECTURE': SYSTEM_ARCHITECTURE - } - with open(pkg_version_file_path, 'w') as file: - json.dump(values_to_save, file, indent=2) - - -def get_built_rpm_packages(pmdk_path): - """ - Returns built pkg packages from pkg directory. - """ - path_to_rpm_files = path.join(pmdk_path, 'rpm', SYSTEM_ARCHITECTURE) - packages = listdir(path_to_rpm_files) - return packages - - -def get_built_dpkg_packages(pmdk_path): - """ - Returns built pkg packages from pkg directory. - """ - path_to_dpkg_files = path.join(pmdk_path, 'dpkg') - packages ='' - for elem in listdir(path_to_dpkg_files): - if '.deb' in elem: - packages += elem + ' ' - - return packages - - -def install_rpm_packages(pmdk_path): - """ - Install packages from PMDK library. - """ - packages = ' '.join(get_built_rpm_packages(pmdk_path)) - path_to_rpm_files = path.join(pmdk_path, 'rpm', SYSTEM_ARCHITECTURE) - check_output('rpm -i --nodeps ' + packages, - cwd=path_to_rpm_files, shell=True) - - -def install_dpkg_packages(pmdk_path): - """ - Install packages from PMDK library. - """ - packages = get_built_dpkg_packages(pmdk_path) - path_to_dpkg_files = path.join(pmdk_path, 'dpkg') - check_output('dpkg -i ' + packages, - cwd=path_to_dpkg_files, shell=True) - - -def get_names_of_rpm_content(): - """ - Returns names of elements, for which are installed from packages from PMDK - library. - """ - packages_path = path.join(args.pmdk_path, 'rpm', SYSTEM_ARCHITECTURE) - installed_packages = check_output( - 'ls | grep ' + PMDK_VERSION, cwd=packages_path, shell=True) - delimiter = '-' - - installed_packages = installed_packages.decode( - 'UTF-8').split(linesep) - - libraries_names = [item.split( - '-')[0] for item in installed_packages if item.split('-')[0]] - return set(libraries_names) - - -def get_names_of_dpkg_content(): - """ - Returns names of elements, for which are installed from packages from PMDK - library. - """ - packages_path = path.join(args.pmdk_path, 'dpkg') - installed_packages = check_output( - 'ls | grep ' + PMDK_VERSION + ' | grep .deb', cwd=packages_path, shell=True) - - installed_packages = installed_packages.decode( - 'UTF-8').split(linesep) - - libraries_names = [item.split( - '_')[0] for item in installed_packages if item.split('-')[0]] - return set(libraries_names) - - -def get_installed_packages(so_path, split_param, packages_path): - """ - Returns names of packages from PMDK library, which are installed. - """ - libraries = get_names_of_pkg_content_func() - installed_packages = [] - - for library in libraries: - if library == "pmempool" and check_output( - 'find /usr/bin/ -name ' + library, cwd=args.pmdk_path, - shell=True): - installed_packages.append(library) - elif library == "libpmemobj++" and check_output( - 'find /usr/include/' + library + ' -name *.hpp', - cwd=args.pmdk_path, shell=True): - installed_packages.append(library) - elif library == "pmdk": - pass - elif check_output('find ' + so_path + ' -name ' + library + '.so', - cwd=args.pmdk_path, shell=True): - installed_packages.append(library) - return installed_packages - - -if __name__ == '__main__': - parser = ArgumentParser( - description='Install packages from PMDK library') - parser.add_argument("-r", "--pmdk-path", required=True, - help="the PMDK library root path.") - - args = parser.parse_args() - os_distro=distro.id() - PMDK_VERSION, SYSTEM_ARCHITECTURE =\ - get_package_version_and_system_architecture(args.pmdk_path) - save_pkg_version(args.pmdk_path + "/pkgVersion.json") - if os_distro != 'ubuntu': - so_path = '/usr/lib64/' - split_param = '-' - packages_path = path.join(args.pmdk_path, 'rpm', SYSTEM_ARCHITECTURE) - install_cmd = 'rpm -i --nodeps ' - install_func = install_rpm_packages - get_names_of_pkg_content_func = get_names_of_rpm_content - else: - so_path = '/lib/x86_64-linux-gnu/' - split_param = '_' - packages_path = path.join(args.pmdk_path, 'dpkg') - install_cmd = 'dpkg -i ' - install_func = install_dpkg_packages - get_names_of_pkg_content_func = get_names_of_dpkg_content - - if not get_installed_packages(so_path, packages_path, get_names_of_pkg_content_func): - install_func(args.pmdk_path) - else: - print("PMDK library is still installed") From 36670d5a9e3f8576ea72c2ed65c0ad1c44ec0180 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Tue, 9 May 2023 14:24:29 +0200 Subject: [PATCH 07/56] common: move all MAKE_PKG checks to nightly build update RockyLinux 8 image to work with new MAKE_PKG scripts. Don't execute rpm checks on Fedora - it's duplicated now by RockyLinux --- .github/workflows/gha.yml | 9 ++--- .github/workflows/nightly.yml | 7 ++-- .github/workflows/pkg_linux.yml | 39 --------------------- utils/docker/images/Dockerfile.rockylinux-8 | 8 +++++ 4 files changed, 16 insertions(+), 47 deletions(-) delete mode 100644 .github/workflows/pkg_linux.yml diff --git a/.github/workflows/gha.yml b/.github/workflows/gha.yml index 3eed7399709..3cf419c0066 100644 --- a/.github/workflows/gha.yml +++ b/.github/workflows/gha.yml @@ -26,16 +26,13 @@ jobs: SRC_CHECKERS: 0 strategy: matrix: - CONFIG: ["N=1 OS=ubuntu OS_VER=22.04 FAULT_INJECTION=1 TEST_BUILD=debug", + CONFIG: ["N=1 OS=ubuntu OS_VER=22.04 FAULT_INJECTION=1 TEST_BUILD=debug PUSH_IMAGE=1", "N=2 OS=ubuntu OS_VER=22.04 FAULT_INJECTION=1 TEST_BUILD=nondebug UBSAN=1", "N=3 OS=ubuntu OS_VER=22.04 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=debug SRC_CHECKERS=1", "N=4 OS=ubuntu OS_VER=22.04 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=nondebug", - "N=5 OS=fedora OS_VER=37 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=debug", + "N=5 OS=fedora OS_VER=37 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=debug PUSH_IMAGE=1", "N=6 OS=fedora OS_VER=37 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=nondebug AUTO_DOC_UPDATE=1", - "N=7 OS=fedora OS_VER=37 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 PUSH_IMAGE=1", - "N=8 OS=ubuntu OS_VER=22.04 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 NDCTL_ENABLE=n PUSH_IMAGE=1", - "N=9 OS=ubuntu OS_VER=22.04 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 NDCTL_ENABLE=n PMDK_CC=clang PMDK_CXX=clang++", - "N=10 OS=ubuntu OS_VER=22.04 COVERAGE=1 FAULT_INJECTION=1 TEST_BUILD=debug" + "N=7 OS=ubuntu OS_VER=22.04 COVERAGE=1 FAULT_INJECTION=1 TEST_BUILD=debug" ] steps: - name: Clone the git repo diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 95e1cf6f7fd..c6ee3369a38 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -30,10 +30,13 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - CONFIG: ["OS=debian OS_VER=11 FAULT_INJECTION=1 TEST_BUILD=debug", + CONFIG: ["OS=ubuntu OS_VER=22.04 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 NDCTL_ENABLE=n", + "OS=ubuntu OS_VER=22.04 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 NDCTL_ENABLE=n PMDK_CC=clang PMDK_CXX=clang++", + "OS=debian OS_VER=11 FAULT_INJECTION=1 TEST_BUILD=debug", "OS=debian OS_VER=11 FAULT_INJECTION=1 TEST_BUILD=nondebug PUSH_IMAGE=1", "OS=opensuse-leap OS_VER=15 FAULT_INJECTION=1 TEST_BUILD=debug", - "OS=opensuse-leap OS_VER=15 FAULT_INJECTION=1 TEST_BUILD=nondebug PUSH_IMAGE=1", + "OS=opensuse-leap OS_VER=15 FAULT_INJECTION=1 TEST_BUILD=nondebug", + "OS=opensuse-leap OS_VER=15 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 PUSH_IMAGE=1", "OS=rockylinux OS_VER=8 TEST_BUILD=debug", "OS=rockylinux OS_VER=8 TEST_BUILD=nondebug", "OS=rockylinux OS_VER=8 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 PUSH_IMAGE=1", diff --git a/.github/workflows/pkg_linux.yml b/.github/workflows/pkg_linux.yml deleted file mode 100644 index 230dfe5c1ad..00000000000 --- a/.github/workflows/pkg_linux.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Run extra checks for making package. -name: PKG_linux - -on: [push, pull_request] - -jobs: - linux: - name: PKG_Linux - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - CONFIG: ["OS=ubuntu OS_VER=22.04", - "OS=rockylinux OS_VER=9", - "OS=opensuse-leap OS_VER=15" - ] - env: - GITHUB_REPO: pmem/pmdk - DOCKER_REPO: ghcr.io/pmem/pmdk - HOST_WORKDIR: /home/runner/work/pmdk/pmdk - WORKDIR: utils/docker - PMDK_CC: gcc - PMDK_CXX: g++ - VALGRIND: 1 - SRC_CHECKERS: 0 - MAKE_PKG: 1 - - steps: - - name: Clone the git repo - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Pull or rebuild the image - run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh - - # This step will create docker containers, build pmdk packages and test it. - - name: Build the CI and test packages - run: cd $WORKDIR && ${{ matrix.CONFIG }} ./build-CI.sh diff --git a/utils/docker/images/Dockerfile.rockylinux-8 b/utils/docker/images/Dockerfile.rockylinux-8 index 03a2daab5f8..b562f6a13ef 100644 --- a/utils/docker/images/Dockerfile.rockylinux-8 +++ b/utils/docker/images/Dockerfile.rockylinux-8 @@ -71,6 +71,7 @@ ENV MISC_DEPS "\ man \ python3 \ python3-flake8 \ + python3-pip \ rsync \ shadow-utils \ sudo \ @@ -78,6 +79,11 @@ ENV MISC_DEPS "\ which \ xmlto" +# python scripts dependencies +ENV PYTHON_DEPS "\ + distro \ + xmlrunner" + # Copy install valgrind script COPY install-valgrind.sh install-valgrind.sh @@ -96,6 +102,8 @@ RUN dnf update -y && dnf install -y epel-release \ && ./install-valgrind.sh rockylinux \ && dnf clean all +RUN pip3 install $PYTHON_DEPS + # Add ans switch user ENV USER pmdkuser ENV USERPASS pmdkpass From 61fb7b526463d5ac07414d4e756666fee7ca4227 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Tue, 9 May 2023 14:29:57 +0200 Subject: [PATCH 08/56] common: move packages' tests into utils/docker/test_packages dir --- utils/docker/run-build-package.sh | 4 ++-- utils/docker/test_package/README | 7 +++++-- .../test-built-packages.py} | 0 .../test-packages-installation.py | 0 4 files changed, 7 insertions(+), 4 deletions(-) rename utils/docker/{packages/test-build-packages.py => test_package/test-built-packages.py} (100%) rename utils/docker/{packages => test_package}/test-packages-installation.py (100%) diff --git a/utils/docker/run-build-package.sh b/utils/docker/run-build-package.sh index a06b8c85c49..c211441a524 100755 --- a/utils/docker/run-build-package.sh +++ b/utils/docker/run-build-package.sh @@ -27,7 +27,7 @@ make -j$(nproc) echo "## Test built packages" [ "$NDCTL_ENABLE" == "n" ] && extra_params="--skip-daxio" || extra_params="" -python3 $SCRIPTSDIR/packages/test-build-packages.py -r $(pwd) ${extra_params} +python3 $SCRIPTSDIR/test_package/test-built-packages.py -r $(pwd) ${extra_params} echo "## Install packages" if [[ "$PACKAGE_MANAGER" == "dpkg" ]]; then @@ -41,7 +41,7 @@ fi popd echo "## Test installed packages" -python3 $SCRIPTSDIR/packages/test-packages-installation.py -r $(pwd) +python3 $SCRIPTSDIR/test_package/test-packages-installation.py -r $(pwd) echo "## Compile and run standalone test" pushd $SCRIPTSDIR/test_package diff --git a/utils/docker/test_package/README b/utils/docker/test_package/README index de81a6a5b8e..e4015df0f78 100644 --- a/utils/docker/test_package/README +++ b/utils/docker/test_package/README @@ -2,5 +2,8 @@ Persistent Memory Development Kit This is utils/docker/test_package/README. -This directory contains simple application which uses libpmemobj. -It can be used to test whether libpmemobj was installed properly. +This directory contains: +- Simple application which uses libpmemobj. + It can be used to test whether libpmemobj was installed properly. +- Test python modules with various checks for PMDK packages. + Each module contains a detailed description of its contents. diff --git a/utils/docker/packages/test-build-packages.py b/utils/docker/test_package/test-built-packages.py similarity index 100% rename from utils/docker/packages/test-build-packages.py rename to utils/docker/test_package/test-built-packages.py diff --git a/utils/docker/packages/test-packages-installation.py b/utils/docker/test_package/test-packages-installation.py similarity index 100% rename from utils/docker/packages/test-packages-installation.py rename to utils/docker/test_package/test-packages-installation.py From 321a12a017da34b2838bd2a35c365a7b372e8af5 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 16 May 2023 11:00:46 +0200 Subject: [PATCH 09/56] test: disable test due to #5641 Signed-off-by: Tomasz Gromadzki --- src/test/ex_libpmem2/TESTS.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/test/ex_libpmem2/TESTS.py b/src/test/ex_libpmem2/TESTS.py index 5183cf91cef..9f4bd08ab03 100755 --- a/src/test/ex_libpmem2/TESTS.py +++ b/src/test/ex_libpmem2/TESTS.py @@ -4,6 +4,7 @@ # import futils import testframework as t +from testframework import granularity as g @t.require_build(['debug', 'release']) @@ -73,7 +74,10 @@ def run(self, ctx): ctx.exec(example_path, *args, stdout_file='out4.log') -class TEST5(EX_LIBPMEM2): +# XXX Disable the test execution under pmemcheck with g.PAGE until fixed. +# https://github.com/pmem/pmdk/issues/5641 +# additionall test TEST501 has been added to cover non-pmemcheck configs. +class EX_LIBPMEM2_TEST5(EX_LIBPMEM2): def run(self, ctx): example_path = futils.get_example_path(ctx, 'pmem2', 'unsafe_shutdown') @@ -82,6 +86,19 @@ def run(self, ctx): ctx.exec(example_path, "read", file_path, stdout_file='out5.log') +@g.require_granularity(g.CACHELINE, g.BYTE) # to be removed when fixed +@t.require_valgrind_enabled('pmemcheck') # to be removed when fixed +class TEST5(EX_LIBPMEM2_TEST5): + + pass + + +@t.require_valgrind_disabled('pmemcheck') +class TEST501(EX_LIBPMEM2_TEST5): # to be removed when fixed + + pass + + @t.windows_exclude # This test case would require two VALGRIND_SET_CLEAN() calls # to be added to the "src/examples/libpmem2/ringbuf/ringbuf.c" From b41c2a1e2c1b2e58a02345f469591245777caf47 Mon Sep 17 00:00:00 2001 From: Oksana Salyk Date: Tue, 16 May 2023 13:55:26 +0200 Subject: [PATCH 10/56] test: disable pmem2_badblock tests due to #5636 Temporary solution until a platform supporting the tested functionality is ready --- src/test/pmem2_badblock/TESTS.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/test/pmem2_badblock/TESTS.py b/src/test/pmem2_badblock/TESTS.py index d42f70aac08..5b57c1c475e 100755 --- a/src/test/pmem2_badblock/TESTS.py +++ b/src/test/pmem2_badblock/TESTS.py @@ -1,6 +1,6 @@ #!../env.py # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2021, Intel Corporation +# Copyright 2021-2023, Intel Corporation # import os @@ -69,7 +69,8 @@ def run_test(self, ctx, filepath): bbTool.clear_all(filepath) -class TEST0(PMEM2_BADBLOCK_COUNT): +# XXX - https://github.com/pmem/pmdk/issues/5636 +class DISABLE_TEST0(PMEM2_BADBLOCK_COUNT): """ compares the number of bad blocks returned by pmem2 and ndctl on fsdax """ @@ -80,8 +81,9 @@ def run(self, ctx): self.run_test(ctx, filepath) +# XXX - https://github.com/pmem/pmdk/issues/5636 @t.require_devdax(t.DevDax('devdax1')) -class TEST1(PMEM2_BADBLOCK_COUNT): +class DISABLE_TEST1(PMEM2_BADBLOCK_COUNT): """ compares the number of bad blocks returned by pmem2 and ndctl on devdax """ @@ -92,7 +94,8 @@ def run(self, ctx): self.run_test(ctx, ddpath) -class TEST2(PMEM2_BADBLOCK): +# XXX - https://github.com/pmem/pmdk/issues/5636 +class DISABLE_TEST2(PMEM2_BADBLOCK): """test mcsafe read operation with encountered badblock""" test_case = "test_pmem2_src_mcsafe_badblock_read" @@ -101,7 +104,8 @@ def run(self, ctx): self.run_test(ctx, filepath) -class TEST3(PMEM2_BADBLOCK): +# XXX - https://github.com/pmem/pmdk/issues/5636 +class DISABLE_TEST3(PMEM2_BADBLOCK): """test mcsafe write operation with encountered badblock""" test_case = "test_pmem2_src_mcsafe_badblock_write" @@ -110,8 +114,9 @@ def run(self, ctx): self.run_test(ctx, filepath) +# XXX - https://github.com/pmem/pmdk/issues/5636 @t.require_devdax(t.DevDax('devdax1')) -class TEST4(PMEM2_BADBLOCK): +class DISABLE_TEST4(PMEM2_BADBLOCK): """test mcsafe read operation with encountered badblock on devdax""" test_case = "test_pmem2_src_mcsafe_badblock_read" @@ -120,8 +125,9 @@ def run(self, ctx): self.run_test(ctx, ddpath) +# XXX - https://github.com/pmem/pmdk/issues/5636 @t.require_devdax(t.DevDax('devdax1')) -class TEST5(PMEM2_BADBLOCK): +class DISABLE_TEST5(PMEM2_BADBLOCK): """test mcsafe write operation with encountered badblock on devdax""" test_case = "test_pmem2_src_mcsafe_badblock_write" From 046e4584550db0a5fe31851830762e0ccb496d6e Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 17 May 2023 13:37:28 +0200 Subject: [PATCH 11/56] test: disable test until #5599 fixed Signed-off-by: Tomasz Gromadzki --- src/test/pmem2_map_prot/TESTS.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/pmem2_map_prot/TESTS.py b/src/test/pmem2_map_prot/TESTS.py index 0e945eaa5ef..fd84f515356 100644 --- a/src/test/pmem2_map_prot/TESTS.py +++ b/src/test/pmem2_map_prot/TESTS.py @@ -1,6 +1,6 @@ #!../env.py # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2020, Intel Corporation +# Copyright 2020-2023, Intel Corporation # import testframework as t @@ -72,6 +72,8 @@ class TEST6(PMEM2_MAP_PROT): @t.require_architectures('x86_64') +# XXX disable the test until https://github.com/pmem/pmdk/issues/5599 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST7(PMEM2_MAP_PROT): """ READ|EXEC protection on file opened in read|write|exec mode; test runs From 3249d23130e24479262711133ad439a8261bb5ce Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 17 May 2023 15:32:01 +0200 Subject: [PATCH 12/56] test: print out build system version Signed-off-by: Tomasz Gromadzki --- .github/workflows/gha.yml | 3 ++ .github/workflows/nightly.yml | 3 ++ utils/docker/get-system-info.sh | 47 ++++++++++++++++++++++++++++ utils/gha-runners/get-system-info.sh | 4 +++ 4 files changed, 57 insertions(+) create mode 100755 utils/docker/get-system-info.sh diff --git a/.github/workflows/gha.yml b/.github/workflows/gha.yml index 0eb1d748ded..d23684130dc 100644 --- a/.github/workflows/gha.yml +++ b/.github/workflows/gha.yml @@ -40,6 +40,9 @@ jobs: with: fetch-depth: 50 + - name: Get system information + run: ./$WORKDIR/get-system-info.sh + - name: Pull or rebuild the image run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index c6ee3369a38..42205d5bd9c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -50,6 +50,9 @@ jobs: with: fetch-depth: 50 + - name: Get system information + run: ./$WORKDIR/get-system-info.sh + - name: Pull or rebuild the image run: cd $WORKDIR && ${{ matrix.CONFIG }} ./pull-or-rebuild-image.sh rebuild diff --git a/utils/docker/get-system-info.sh b/utils/docker/get-system-info.sh new file mode 100755 index 00000000000..01bdad34e91 --- /dev/null +++ b/utils/docker/get-system-info.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2023, Intel Corporation + +# +# get-system-info.sh - Script for printing system info +# + +function system_info { + echo "********** system_info **********" + cat /etc/os-release | grep -oP "PRETTY_NAME=\K.*" + uname -r + echo "libndctl: $(pkg-config --modversion libndctl || echo 'libndctl not found')" + echo "valgrind: $(pkg-config --modversion valgrind || echo 'valgrind not found')" + echo "*************** installed-packages ***************" + # Instructions below will return some minor errors, as they are dependent on the Linux distribution. + zypper se --installed-only 2>/dev/null || true + apt list --installed 2>/dev/null || true + yum list installed 2>/dev/null || true + echo "**********/proc/cmdline**********" + cat /proc/cmdline + echo "**********/proc/modules**********" + cat /proc/modules + echo "**********/proc/cpuinfo**********" + cat /proc/cpuinfo + echo "**********/proc/meminfo**********" + cat /proc/meminfo + echo "**********/proc/swaps**********" + cat /proc/swaps + echo "**********/proc/version**********" + cat /proc/version + echo "**********check-updates**********" + # Instructions below will return some minor errors, as they are dependent on the Linux distribution. + zypper list-updates 2>/dev/null || true + apt-get update 2>/dev/null || true + apt upgrade --dry-run 2>/dev/null || true + dnf check-update 2>/dev/null || true + echo "**********list-enviroment**********" + env + echo "******list-build-system-versions*******" + gcc --version + clang --version + make --version +} + +# Call the function above to print system info. +system_info diff --git a/utils/gha-runners/get-system-info.sh b/utils/gha-runners/get-system-info.sh index ad061d1ce08..185d89c1c9f 100755 --- a/utils/gha-runners/get-system-info.sh +++ b/utils/gha-runners/get-system-info.sh @@ -51,6 +51,10 @@ function system_info { ls -la /sys/bus/nd/devices/ndbus*/region*/pfn*/resource ls -la /sys/bus/nd/devices/ndbus*/region*/namespace*/resource ls -la /sys/bus/nd/devices/region*/deep_flush + echo "******list-build-system-versions*******" + gcc --version + clang --version + make --version } # Call the function above to print system info. From b5d69f4fba0322c9136d622c54ce4a1a7b0486ea Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 17 May 2023 17:36:26 +0200 Subject: [PATCH 13/56] common: move Fedora builds to nightly builds Fedora builds fails regularyly but randomly. Builds have been moved to nightly build to avoid CI/GHA being blcked with many false positives. Signed-off-by: Tomasz Gromadzki --- .github/workflows/gha.yml | 9 ++++----- .github/workflows/nightly.yml | 7 +++++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/gha.yml b/.github/workflows/gha.yml index 0eb1d748ded..8abe3e4c02f 100644 --- a/.github/workflows/gha.yml +++ b/.github/workflows/gha.yml @@ -26,13 +26,12 @@ jobs: SRC_CHECKERS: 0 strategy: matrix: - CONFIG: ["N=1 OS=ubuntu OS_VER=22.04 FAULT_INJECTION=1 TEST_BUILD=debug PUSH_IMAGE=1", + CONFIG: [ + "N=1 OS=ubuntu OS_VER=22.04 FAULT_INJECTION=1 TEST_BUILD=debug PUSH_IMAGE=1", "N=2 OS=ubuntu OS_VER=22.04 FAULT_INJECTION=1 TEST_BUILD=nondebug UBSAN=1", "N=3 OS=ubuntu OS_VER=22.04 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=debug SRC_CHECKERS=1", - "N=4 OS=ubuntu OS_VER=22.04 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=nondebug", - "N=5 OS=fedora OS_VER=37 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=debug PUSH_IMAGE=1", - "N=6 OS=fedora OS_VER=37 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=nondebug AUTO_DOC_UPDATE=1", - "N=7 OS=ubuntu OS_VER=22.04 COVERAGE=1 FAULT_INJECTION=1 TEST_BUILD=debug" + "N=4 OS=ubuntu OS_VER=22.04 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=nondebug AUTO_DOC_UPDATE=1", + "N=5 OS=ubuntu OS_VER=22.04 COVERAGE=1 FAULT_INJECTION=1 TEST_BUILD=debug", ] steps: - name: Clone the git repo diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index c6ee3369a38..3f476309fa2 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -30,7 +30,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - CONFIG: ["OS=ubuntu OS_VER=22.04 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 NDCTL_ENABLE=n", + CONFIG: [ + "OS=ubuntu OS_VER=22.04 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 NDCTL_ENABLE=n", "OS=ubuntu OS_VER=22.04 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 NDCTL_ENABLE=n PMDK_CC=clang PMDK_CXX=clang++", "OS=debian OS_VER=11 FAULT_INJECTION=1 TEST_BUILD=debug", "OS=debian OS_VER=11 FAULT_INJECTION=1 TEST_BUILD=nondebug PUSH_IMAGE=1", @@ -42,7 +43,9 @@ jobs: "OS=rockylinux OS_VER=8 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 PUSH_IMAGE=1", "OS=rockylinux OS_VER=9 TEST_BUILD=debug", "OS=rockylinux OS_VER=9 TEST_BUILD=nondebug", - "OS=rockylinux OS_VER=9 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 PUSH_IMAGE=1" + "OS=rockylinux OS_VER=9 MAKE_PKG=1 EXPERIMENTAL=y VALGRIND=0 PUSH_IMAGE=1", + "OS=fedora OS_VER=37 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=debug PUSH_IMAGE=1", + "OS=fedora OS_VER=37 PMDK_CC=clang PMDK_CXX=clang++ TEST_BUILD=nondebug", ] steps: - name: Clone the git repo From 252b4f1a9a1bf844a869c0707f87987a91b6f87d Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 17 May 2023 23:17:22 +0200 Subject: [PATCH 14/56] common: avoid error message when tools are not available --- utils/docker/get-system-info.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/docker/get-system-info.sh b/utils/docker/get-system-info.sh index 01bdad34e91..648a98f7081 100755 --- a/utils/docker/get-system-info.sh +++ b/utils/docker/get-system-info.sh @@ -38,9 +38,9 @@ function system_info { echo "**********list-enviroment**********" env echo "******list-build-system-versions*******" - gcc --version - clang --version - make --version + gcc --version 2>/dev/null || true + clang --version 2>/dev/null || true + make --version2>/dev/null || true } # Call the function above to print system info. From 3aca5be0ac8ce74a81785e42657b6c51c8825293 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 17 May 2023 23:22:37 +0200 Subject: [PATCH 15/56] common: avoid error message when tools are not available --- utils/docker/get-system-info.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/docker/get-system-info.sh b/utils/docker/get-system-info.sh index 648a98f7081..dde7230d213 100755 --- a/utils/docker/get-system-info.sh +++ b/utils/docker/get-system-info.sh @@ -40,7 +40,7 @@ function system_info { echo "******list-build-system-versions*******" gcc --version 2>/dev/null || true clang --version 2>/dev/null || true - make --version2>/dev/null || true + make --version 2>/dev/null || true } # Call the function above to print system info. From 4f29fd87438540714b78781e9b35d4ce38db5104 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 17 May 2023 23:46:40 +0200 Subject: [PATCH 16/56] common: avoid error message when tools are not available --- utils/gha-runners/get-system-info.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/gha-runners/get-system-info.sh b/utils/gha-runners/get-system-info.sh index 185d89c1c9f..9a41e1c6427 100755 --- a/utils/gha-runners/get-system-info.sh +++ b/utils/gha-runners/get-system-info.sh @@ -52,9 +52,9 @@ function system_info { ls -la /sys/bus/nd/devices/ndbus*/region*/namespace*/resource ls -la /sys/bus/nd/devices/region*/deep_flush echo "******list-build-system-versions*******" - gcc --version - clang --version - make --version + gcc --version 2>/dev/null || true + clang --version 2>/dev/null || true + make --version 2>/dev/null || true } # Call the function above to print system info. From 174336da16773340f59ba41b8e075dc565fcb615 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 16 May 2023 11:41:14 +0200 Subject: [PATCH 17/56] test: enable Python tests (short+medium) to be run on PMem Signed-off-by: Tomasz Gromadzki --- .github/workflows/unittest.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index f81f92f5c4d..fb91049a060 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -30,5 +30,8 @@ jobs: - name: Create testconfig files run: ./$WORKDIR/create-testconfig.sh - - name: Run tests + - name: Run tests (Bash) run: make check + + - name: Run tests (Python) + run: cd src/test/ && ./RUNTESTS.py -t check From ff6a07d98fa42ceaceeaf7f52374e0e5f873392e Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 18 May 2023 15:29:04 +0200 Subject: [PATCH 18/56] test: disable avx512f tests under Valgrind Disable avx512f tests until Valgrind until Valigrind will provide support for avx512f instruction. See: #5640 Signed-off-by: Tomasz Gromadzki --- src/test/pmem2_mem_ext/TESTS.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/test/pmem2_mem_ext/TESTS.py b/src/test/pmem2_mem_ext/TESTS.py index a750c61a0cd..4f316c23483 100755 --- a/src/test/pmem2_mem_ext/TESTS.py +++ b/src/test/pmem2_mem_ext/TESTS.py @@ -1,6 +1,6 @@ #!../env.py # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2020-2022, Intel Corporation +# Copyright 2020-2023, Intel Corporation # import testframework as t @@ -115,6 +115,13 @@ def setup(self, ctx): ret = tools.Tools(ctx.env, ctx.build).cpufd() self.check_arch(ctx.variant(), ret.returncode) +# All tests with variant VARIANT_AVX512F are disabled under Valgrind +# until the issue https://github.com/pmem/pmdk/issues/5640 is fixed. + if ctx.valgrind is not None: + if ctx.valgrind.tool.name != "NONE": + if ctx.variant() == VARIANT_AVX512F: + raise futils.Skip("AVX512F unavailable under Valigrind") + def check_arch(self, variant, available_arch): if variant == VARIANT_MOVDIR64B: if available_arch < MOVDIR64B: From a9c535f91bc6dc453a85c8849e0f83ded57e11c0 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 18 May 2023 16:23:53 +0200 Subject: [PATCH 19/56] test: fix test for win32 platform Signed-off-by: Tomasz Gromadzki --- src/test/pmem2_mem_ext/TESTS.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/test/pmem2_mem_ext/TESTS.py b/src/test/pmem2_mem_ext/TESTS.py index 4f316c23483..08fa241a18e 100755 --- a/src/test/pmem2_mem_ext/TESTS.py +++ b/src/test/pmem2_mem_ext/TESTS.py @@ -117,10 +117,11 @@ def setup(self, ctx): # All tests with variant VARIANT_AVX512F are disabled under Valgrind # until the issue https://github.com/pmem/pmdk/issues/5640 is fixed. - if ctx.valgrind is not None: - if ctx.valgrind.tool.name != "NONE": - if ctx.variant() == VARIANT_AVX512F: - raise futils.Skip("AVX512F unavailable under Valigrind") + if not sys.platform.startswith('win32'): + if ctx.valgrind is not None: + if ctx.valgrind.tool.name != "NONE": + if ctx.variant() == VARIANT_AVX512F: + raise futils.Skip("AVX512F unavailable under Valigrind") def check_arch(self, variant, available_arch): if variant == VARIANT_MOVDIR64B: From a890dda80f3cac9bfd143558dcade480a306826d Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 18 May 2023 16:52:43 +0200 Subject: [PATCH 20/56] test: speedup tests on PMem Signed-off-by: Tomasz Gromadzki --- .github/workflows/unittest.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index fb91049a060..70e63bd858c 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -14,6 +14,8 @@ jobs: fail-fast: false matrix: os: [[self-hosted, rhel],[self-hosted, opensuse]] + config: ['make check', + 'make pycheck'] env: WORKDIR: utils/gha-runners @@ -30,8 +32,5 @@ jobs: - name: Create testconfig files run: ./$WORKDIR/create-testconfig.sh - - name: Run tests (Bash) - run: make check - - - name: Run tests (Python) - run: cd src/test/ && ./RUNTESTS.py -t check + - name: Run tests + run: ${{ matrix.config }} From 966206f79037539b063e3ddda79d86a740fbdaef Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 18 May 2023 20:36:36 +0200 Subject: [PATCH 21/56] test: make build as fast as possible Execute build in a paralel way, but execute test one by one. Signed-off-by: Tomasz Gromadzki --- .github/workflows/unittest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 70e63bd858c..33f98e08333 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -14,8 +14,8 @@ jobs: fail-fast: false matrix: os: [[self-hosted, rhel],[self-hosted, opensuse]] - config: ['make check', - 'make pycheck'] + config: ['make -j$(nproc); make -j$(nproc) test; make check', + 'make -j$(nproc); make -j$(nproc) test; make pycheck'] env: WORKDIR: utils/gha-runners From f1ffe190ea85f0f7f491fa17b8c56bf2e6dd1d6a Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 18 May 2023 16:46:29 +0200 Subject: [PATCH 22/56] test: fix cstyle issue Signed-off-by: Tomasz Gromadzki --- src/test/pmem2_mem_ext/TESTS.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/test/pmem2_mem_ext/TESTS.py b/src/test/pmem2_mem_ext/TESTS.py index 08fa241a18e..3d0c55e1570 100755 --- a/src/test/pmem2_mem_ext/TESTS.py +++ b/src/test/pmem2_mem_ext/TESTS.py @@ -115,13 +115,13 @@ def setup(self, ctx): ret = tools.Tools(ctx.env, ctx.build).cpufd() self.check_arch(ctx.variant(), ret.returncode) -# All tests with variant VARIANT_AVX512F are disabled under Valgrind -# until the issue https://github.com/pmem/pmdk/issues/5640 is fixed. - if not sys.platform.startswith('win32'): - if ctx.valgrind is not None: - if ctx.valgrind.tool.name != "NONE": - if ctx.variant() == VARIANT_AVX512F: - raise futils.Skip("AVX512F unavailable under Valigrind") + # XXX all tests with VARIANT_AVX512F are disabled under Valgrind + # until the issue https://github.com/pmem/pmdk/issues/5640 is fixed. + # "win32" `if`` is related to unknown `is not None` by Windows Python + if not sys.platform.startswith('win32') and ctx.valgrind is not None: + if ctx.valgrind.tool.name != "NONE": + if ctx.variant() == VARIANT_AVX512F: + raise futils.Skip("AVX512F unavailable under Valigrind") def check_arch(self, variant, available_arch): if variant == VARIANT_MOVDIR64B: From 90ef249294d3a01aeb68d6b2e8cc12bb27e4183c Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 19 May 2023 09:39:47 +0200 Subject: [PATCH 23/56] test: disable pmem2_vm_reservation TEST11,32 until #5592 fixed Disable pmem2_vm_reservation TEST11 and TEST32 for Valgrind until #5592 is fixed. Signed-off-by: Tomasz Gromadzki --- src/test/pmem2_vm_reservation/TESTS.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/test/pmem2_vm_reservation/TESTS.py b/src/test/pmem2_vm_reservation/TESTS.py index f95befefdea..2bc4757bae1 100755 --- a/src/test/pmem2_vm_reservation/TESTS.py +++ b/src/test/pmem2_vm_reservation/TESTS.py @@ -1,6 +1,6 @@ #!../env.py # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2020-2021, Intel Corporation +# Copyright 2020-2023, Intel Corporation # import os @@ -140,6 +140,9 @@ class TEST10(PMEM2_VM_RESERVATION): test_case = "test_vm_reserv_new_region_occupied_map_below" +# XXX disable the test for 'pmemcheck', 'drd', 'helgrind', 'memcheck' +# until https://github.com/pmem/pmdk/issues/5592 is fixed. +@t.require_valgrind_disabled('pmemcheck', 'drd', 'helgrind', 'memcheck') class TEST11(PMEM2_VM_RESERVATION_DEVDAX): """ DevDax create a reservation in the region overlapping lower half of @@ -293,6 +296,9 @@ class TEST31(PMEM2_VM_RESERVATION_ASYNC): ops_per_thread = 1000 +# XXX disable the test for 'pmemcheck', 'drd', 'helgrind', 'memcheck' +# until https://github.com/pmem/pmdk/issues/5592 is fixed. +@t.require_valgrind_disabled('pmemcheck', 'drd', 'helgrind', 'memcheck') class TEST32(PMEM2_VM_RESERVATION_ASYNC_DEVDAX): """ DevDax map and unmap asynchronously multiple times to the whole From dab04b83895ad1710e4b92a92196663045a3cb54 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Fri, 19 May 2023 09:52:00 +0200 Subject: [PATCH 24/56] test: disable pmem2_map_from_existing TEST[0-2] until #5594 is fixed Disable tests pmem2_map_from_existing TEST[0-2] for memcheck until #5594 is fixed. Signed-off-by: Tomasz Gromadzki --- src/test/pmem2_map_from_existing/TESTS.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/test/pmem2_map_from_existing/TESTS.py b/src/test/pmem2_map_from_existing/TESTS.py index 9d8e50dd41b..28b20975a45 100644 --- a/src/test/pmem2_map_from_existing/TESTS.py +++ b/src/test/pmem2_map_from_existing/TESTS.py @@ -1,6 +1,6 @@ #!../env.py # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2020-2021, Intel Corporation +# Copyright 2020-2023, Intel Corporation # @@ -17,16 +17,25 @@ def run(self, ctx): ctx.exec('pmem2_map_from_existing', self.test_case, filepath) +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5594 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST0(Pmem2_from_existing): """try to create two the same mappings""" test_case = "test_two_same_mappings" +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5594 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST1(Pmem2_from_existing): """try to map which overlap bottom part of existing mapping""" test_case = "test_mapping_overlap_bottom" +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5594 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST2(Pmem2_from_existing): """try to map which overlap upper part of existing mapping""" test_case = "test_mapping_overlap_upper" From 7955520f36fbdf62ea3f28cbd36201339486f2e4 Mon Sep 17 00:00:00 2001 From: Oksana Salyk Date: Fri, 19 May 2023 13:29:27 +0200 Subject: [PATCH 25/56] test: disable obj_sync TEST0,2,5,6 disable obj_sync TEST0,2,5,6 until #5644 and #5646 are fixed --- src/test/obj_sync/TEST0 | 5 ++++- src/test/obj_sync/TEST2 | 10 +++++++--- src/test/obj_sync/TEST5 | 10 +++++++--- src/test/obj_sync/TEST6 | 10 +++++++--- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/test/obj_sync/TEST0 b/src/test/obj_sync/TEST0 index 824a32eae8f..570b86a6b8f 100755 --- a/src/test/obj_sync/TEST0 +++ b/src/test/obj_sync/TEST0 @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2015-2019, Intel Corporation +# Copyright 2015-2023, Intel Corporation # # src/test/obj_sync/TEST0 -- unit test for PMEM-resident locks @@ -12,6 +12,9 @@ require_test_type medium require_fs_type none require_build_type debug nondebug +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5643 is fixed. +configure_valgrind pmemcheck force-disable setup diff --git a/src/test/obj_sync/TEST2 b/src/test/obj_sync/TEST2 index cda7a4dc13f..130e6991ab6 100755 --- a/src/test/obj_sync/TEST2 +++ b/src/test/obj_sync/TEST2 @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2015-2019, Intel Corporation +# Copyright 2015-2023, Intel Corporation # # src/test/obj_sync/TEST2 -- unit test for PMEM-resident locks @@ -12,8 +12,12 @@ require_test_type medium require_fs_type none require_build_type debug nondebug -require_valgrind 3.10 -configure_valgrind helgrind force-enable +# XXX disable the test for 'drd' and 'helgrind' +# until https://github.com/pmem/pmdk/issues/5646 is fixed. +#require_valgrind 3.10 +#configure_valgrind helgrind force-enable +configure_valgrind drd force-disable +configure_valgrind helgrind force-disable setup diff --git a/src/test/obj_sync/TEST5 b/src/test/obj_sync/TEST5 index fc13b065c16..53a7bbe2fe7 100755 --- a/src/test/obj_sync/TEST5 +++ b/src/test/obj_sync/TEST5 @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2015-2019, Intel Corporation +# Copyright 2015-2023, Intel Corporation # # src/test/obj_sync/TEST5 -- unit test for PMEM-resident locks @@ -12,8 +12,12 @@ require_test_type medium require_fs_type none require_build_type debug nondebug -require_valgrind 3.10 -configure_valgrind drd force-enable +# XXX disable the test for 'drd' and 'helgrind' +# until https://github.com/pmem/pmdk/issues/5646 is fixed. +#require_valgrind 3.10 +#configure_valgrind drd force-enable +configure_valgrind drd force-disable +configure_valgrind helgrind force-disable setup expect_normal_exit ./obj_sync$EXESUFFIX c 10 5 diff --git a/src/test/obj_sync/TEST6 b/src/test/obj_sync/TEST6 index 8c11d2ad23f..bf720bb57c6 100755 --- a/src/test/obj_sync/TEST6 +++ b/src/test/obj_sync/TEST6 @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2015-2019, Intel Corporation +# Copyright 2015-2023, Intel Corporation # # src/test/obj_sync/TEST6 -- unit test for PMEM-resident locks @@ -12,8 +12,12 @@ require_test_type medium require_fs_type none require_build_type debug nondebug -require_valgrind 3.10 -configure_valgrind helgrind force-enable +# XXX disable the test for 'drd' and 'helgrind' +# until https://github.com/pmem/pmdk/issues/5646 is fixed. +#require_valgrind 3.10 +#configure_valgrind helgrind force-enable +configure_valgrind drd force-disable +configure_valgrind helgrind force-disable setup expect_normal_exit ./obj_sync$EXESUFFIX c 10 5 From 6d203527d0ac802ac50ce2140a2fd563b0986b95 Mon Sep 17 00:00:00 2001 From: Oksana Salyk Date: Fri, 19 May 2023 14:15:26 +0200 Subject: [PATCH 26/56] common: remove install-libndctl.sh script --- utils/docker/images/install-libndctl.sh | 60 ------------------------- 1 file changed, 60 deletions(-) delete mode 100755 utils/docker/images/install-libndctl.sh diff --git a/utils/docker/images/install-libndctl.sh b/utils/docker/images/install-libndctl.sh deleted file mode 100755 index fc8cee3fb88..00000000000 --- a/utils/docker/images/install-libndctl.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: BSD-3-Clause -# Copyright 2017-2019, Intel Corporation - -# -# install-libndctl.sh - installs libndctl -# - -set -e - -OS=$2 - -echo "==== clone ndctl repo ====" -git clone https://github.com/pmem/ndctl.git -cd ndctl -git checkout $1 - -if [ "$OS" = "fedora" ]; then - -echo "==== setup rpmbuild tree ====" -rpmdev-setuptree - -RPMDIR=$HOME/rpmbuild/ -VERSION=$(./git-version) -SPEC=./rhel/ndctl.spec - -echo "==== create source tarball =====" -git archive --format=tar --prefix="ndctl-${VERSION}/" HEAD | gzip > "$RPMDIR/SOURCES/ndctl-${VERSION}.tar.gz" - -echo "==== build ndctl ====" -./autogen.sh -./configure --disable-docs -make -j$(nproc) - -echo "==== build ndctl packages ====" -rpmbuild -ba $SPEC - -echo "==== install ndctl packages ====" -RPM_ARCH=$(uname -m) -rpm -i $RPMDIR/RPMS/$RPM_ARCH/*.rpm - -echo "==== cleanup ====" -rm -rf $RPMDIR - -else - -echo "==== build ndctl ====" -./autogen.sh -./configure --disable-docs -make -j$(nproc) - -echo "==== install ndctl ====" -make -j$(nproc) install - -echo "==== cleanup ====" - -fi - -cd .. -rm -rf ndctl From a2b42e7ff53efee99d81187308da6b4566745ad4 Mon Sep 17 00:00:00 2001 From: Oksana Salyk Date: Fri, 19 May 2023 13:07:05 +0200 Subject: [PATCH 27/56] test: disable ex_libpmem2 TEST6 until #5638 and #5593 are fixed --- src/test/ex_libpmem2/TESTS.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/ex_libpmem2/TESTS.py b/src/test/ex_libpmem2/TESTS.py index 9f4bd08ab03..4d32d04aba8 100755 --- a/src/test/ex_libpmem2/TESTS.py +++ b/src/test/ex_libpmem2/TESTS.py @@ -100,6 +100,12 @@ class TEST501(EX_LIBPMEM2_TEST5): # to be removed when fixed @t.windows_exclude +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5638 is fixed. +@t.require_valgrind_disabled('memcheck') +# XXX disable the test for `drd' +# until https://github.com/pmem/pmdk/issues/5593 is fixed. +@t.require_valgrind_disabled('drd') # This test case would require two VALGRIND_SET_CLEAN() calls # to be added to the "src/examples/libpmem2/ringbuf/ringbuf.c" # example (see https://github.com/pmem/pmdk/pull/5604) From a4b3c7fab4b7af4f0d90e54aad6816cf0422c594 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 22 May 2023 06:57:26 +0200 Subject: [PATCH 28/56] test:@t.req...val..._disable() used only once in ex_libpmem2/TEST6 @t.require_valgrind_disabled() can only be used one time as a class decorator. This fix improves the fix #5664 Signed-off-by: Tomasz Gromadzki --- src/test/ex_libpmem2/TESTS.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/test/ex_libpmem2/TESTS.py b/src/test/ex_libpmem2/TESTS.py index 4d32d04aba8..133cd83bc07 100755 --- a/src/test/ex_libpmem2/TESTS.py +++ b/src/test/ex_libpmem2/TESTS.py @@ -102,17 +102,19 @@ class TEST501(EX_LIBPMEM2_TEST5): # to be removed when fixed @t.windows_exclude # XXX disable the test for `memcheck' # until https://github.com/pmem/pmdk/issues/5638 is fixed. -@t.require_valgrind_disabled('memcheck') +# @t.require_valgrind_disabled('memcheck') # XXX disable the test for `drd' # until https://github.com/pmem/pmdk/issues/5593 is fixed. -@t.require_valgrind_disabled('drd') +# @t.require_valgrind_disabled('drd') # This test case would require two VALGRIND_SET_CLEAN() calls # to be added to the "src/examples/libpmem2/ringbuf/ringbuf.c" # example (see https://github.com/pmem/pmdk/pull/5604) # in order to pass under pmemcheck, but examples # do not use valgrind macros on purpose (to avoid unnecessary # complication), so this test case just should not be run under pmemcheck. -@t.require_valgrind_disabled('pmemcheck') +# @t.require_valgrind_disabled('pmemcheck') +# XXX _disabled() can be used only once. +@t.require_valgrind_disabled('memcheck', 'drd', 'pmemcheck') class TEST6(EX_LIBPMEM2): def run(self, ctx): From 72c2b5e7e3bcbace87b6bfbc8930b236bd7cd891 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 22 May 2023 07:23:09 +0200 Subject: [PATCH 29/56] test: disable ex_libpmem2/TEST6 for 'helgrind' until #5638 is fixed Disable ex_libpmem2/TEST6 for 'helgrind' until the issue #5638 is fixed. Signed-off-by: Tomasz Gromadzki --- src/test/ex_libpmem2/TESTS.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/ex_libpmem2/TESTS.py b/src/test/ex_libpmem2/TESTS.py index 133cd83bc07..bea56ee9758 100755 --- a/src/test/ex_libpmem2/TESTS.py +++ b/src/test/ex_libpmem2/TESTS.py @@ -100,9 +100,9 @@ class TEST501(EX_LIBPMEM2_TEST5): # to be removed when fixed @t.windows_exclude -# XXX disable the test for `memcheck' +# XXX disable the test for `memcheck' and 'helgrind' # until https://github.com/pmem/pmdk/issues/5638 is fixed. -# @t.require_valgrind_disabled('memcheck') +# @t.require_valgrind_disabled('memcheck', 'helgrind') # XXX disable the test for `drd' # until https://github.com/pmem/pmdk/issues/5593 is fixed. # @t.require_valgrind_disabled('drd') @@ -114,7 +114,7 @@ class TEST501(EX_LIBPMEM2_TEST5): # to be removed when fixed # complication), so this test case just should not be run under pmemcheck. # @t.require_valgrind_disabled('pmemcheck') # XXX _disabled() can be used only once. -@t.require_valgrind_disabled('memcheck', 'drd', 'pmemcheck') +@t.require_valgrind_disabled('memcheck', 'drd', 'pmemcheck', 'helgrind') class TEST6(EX_LIBPMEM2): def run(self, ctx): From 7f317c7a37f1218402cb0307f31f622259b72db0 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 22 May 2023 09:03:06 +0200 Subject: [PATCH 30/56] test: disable pmem2_integration/TEST[39,41] w/ memcheck Disable pmem2_integration/TEST[39,41] with memcheck until https://github.com/pmem/pmdk/issues/5637 is fixed. Signed-off-by: Tomasz Gromadzki --- src/test/pmem2_integration/TESTS.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/test/pmem2_integration/TESTS.py b/src/test/pmem2_integration/TESTS.py index 2b6b3500a21..c438cfc459a 100755 --- a/src/test/pmem2_integration/TESTS.py +++ b/src/test/pmem2_integration/TESTS.py @@ -1,6 +1,6 @@ #!../env.py # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2019-2021, Intel Corporation +# Copyright 2019-2023, Intel Corporation # @@ -289,6 +289,9 @@ class TEST38(PMEM2_INTEGRATION): test_case = "test_unaligned_persist" +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5637 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST39(PMEM2_INTEGRATION): """compare normal map vs map_from_existing""" test_case = "test_map_from_existing" @@ -300,6 +303,9 @@ class TES40(PMEM2_INTEGRATION): @t.windows_exclude +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5637 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST41(PMEM2_INTEGRATION_DEV_DAXES): """compare normal map vs map_from_existing on devdax""" test_case = "test_map_from_existing" From cdd634036a492b4bdcd33be20d1fc92afee93bad Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 22 May 2023 09:52:36 +0200 Subject: [PATCH 31/56] test: disable pmem2_future/TEST[0-3]: w/ pmemcheck until #5596 is fixed Signed-off-by: Tomasz Gromadzki --- src/test/pmem2_future/TESTS.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/test/pmem2_future/TESTS.py b/src/test/pmem2_future/TESTS.py index 60b6a735878..237b8c3b9b7 100755 --- a/src/test/pmem2_future/TESTS.py +++ b/src/test/pmem2_future/TESTS.py @@ -1,6 +1,6 @@ #!../env.py # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2022, Intel Corporation +# Copyright 2022-2023, Intel Corporation # import sys import testframework as t @@ -72,21 +72,33 @@ def run(self, ctx): F"Got {len(matches)} instead.") +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5596 is fixed. +@t.require_valgrind_disabled('pmemcheck') class TEST0(PMEM2_FUTURE): size = 64 test_case = 'test_pmem2_future_mover' +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5596 is fixed. +@t.require_valgrind_disabled('pmemcheck') class TEST1(PMEM2_FUTURE): size = 4096 test_case = 'test_pmem2_future_mover' +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5596 is fixed. +@t.require_valgrind_disabled('pmemcheck') class TEST2(PMEM2_FUTURE): size = 64 test_case = 'test_pmem2_future_vdm' +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5596 is fixed. +@t.require_valgrind_disabled('pmemcheck') class TEST3(PMEM2_FUTURE): size = 4096 test_case = 'test_pmem2_future_vdm' From df91120340ad9e10e94768912bf61fc23239b826 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 22 May 2023 09:18:21 +0200 Subject: [PATCH 32/56] test: disable pmem2_map tests with memcheck until #5600 is fixed Disable pmem2_mappmem2_map /TEST[0,1,4,5,11,12,13,14,15,16,24,25,26,27,29,30] with memcheck until https://github.com/pmem/pmdk/issues/5600 is fixed. Signed-off-by: Tomasz Gromadzki --- src/test/pmem2_map/TESTS.py | 50 ++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/src/test/pmem2_map/TESTS.py b/src/test/pmem2_map/TESTS.py index 23427017781..9fd6a4b7d25 100755 --- a/src/test/pmem2_map/TESTS.py +++ b/src/test/pmem2_map/TESTS.py @@ -1,6 +1,6 @@ #!../env.py # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2019-2021, Intel Corporation +# Copyright 2019-2023, Intel Corporation # import os @@ -43,12 +43,18 @@ def run(self, ctx): ctx.exec('pmem2_map', self.test_case, dd.path) +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST0(PMEM2_MAP): """map a O_RDWR file""" test_case = "test_map_rdrw_file" with_size = False +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST1(PMEM2_MAP_DEVDAX): """DevDax map a O_RDWR file""" test_case = "test_map_rdrw_file" @@ -67,11 +73,17 @@ class TEST3(PMEM2_MAP_DEVDAX): with_size = False +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST4(PMEM2_MAP): """map valid memory ranges""" test_case = "test_map_valid_ranges" +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST5(PMEM2_MAP_DEVDAX): """DevDax map valid memory ranges""" test_case = "test_map_valid_ranges" @@ -102,21 +114,33 @@ class TEST10(PMEM2_MAP): test_case = "test_map_invalid_fd" +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST11(PMEM2_MAP): """unmap valid pmem2 mapping""" test_case = "test_unmap_valid" +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST12(PMEM2_MAP_DEVDAX): """DevDax unmap valid pmem2 mapping""" test_case = "test_unmap_valid" +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST13(PMEM2_MAP): """unmap a pmem2 mapping with an invalid length""" test_case = "test_unmap_zero_length" +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST14(PMEM2_MAP_DEVDAX): """DevDax unmap a pmem2 mapping with an invalid length""" test_case = "test_unmap_zero_length" @@ -124,11 +148,17 @@ class TEST14(PMEM2_MAP_DEVDAX): # UnmapViewOfFile does not care about the address alignment @t.windows_exclude +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST15(PMEM2_MAP): """unmap a pmem2 mapping with an unaligned address""" test_case = "test_unmap_unaligned_addr" +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST16(PMEM2_MAP_DEVDAX): """DevDax unmap a pmem2 mapping with an unaligned address""" test_case = "test_unmap_unaligned_addr" @@ -177,6 +207,9 @@ class TEST23(PMEM2_MAP): filesize = 0 +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST24(PMEM2_MAP): """ map a file with PMEM2_SHARED sharing, changes in the mapping are visible @@ -186,6 +219,9 @@ class TEST24(PMEM2_MAP): with_size = False +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST25(PMEM2_MAP): """ map a file with PMEM2_PRIVATE sharing, changes in the mapping are not @@ -195,6 +231,9 @@ class TEST25(PMEM2_MAP): with_size = False +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST26(PMEM2_MAP): """ map a file with PMEM2_PRIVATE sharing, changes in the mapping are not @@ -204,6 +243,9 @@ class TEST26(PMEM2_MAP): with_size = False +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST27(PMEM2_MAP): """ map O_RDONLY file with PMEM2_PRIVATE sharing @@ -220,6 +262,9 @@ class TEST28(PMEM2_MAP_DEVDAX): @t.linux_only @t.require_architectures('x86_64') +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST29(PMEM2_MAP): """map alignment test for huge pages""" test_case = "test_map_huge_alignment" @@ -228,6 +273,9 @@ class TEST29(PMEM2_MAP): @t.linux_only @t.require_architectures('x86_64') +# XXX disable the test for `memcheck' +# until https://github.com/pmem/pmdk/issues/5600 is fixed. +@t.require_valgrind_disabled('memcheck') class TEST30(PMEM2_MAP): """map alignment test for small pages""" test_case = "test_map_huge_alignment" From 62643d85394fb7d975d90f8d26ccda0d54630193 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 22 May 2023 12:31:54 +0200 Subject: [PATCH 33/56] test: disable pmem2_mover/TEST[0-5, 9] w/ pmemcheck until #5595 is fixed Signed-off-by: Tomasz Gromadzki --- src/test/pmem2_mover/TESTS.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/test/pmem2_mover/TESTS.py b/src/test/pmem2_mover/TESTS.py index 90c358453e0..4b22ffc0ef2 100644 --- a/src/test/pmem2_mover/TESTS.py +++ b/src/test/pmem2_mover/TESTS.py @@ -1,6 +1,6 @@ #!../env.py # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2022, Intel Corporation +# Copyright 2022-2023, Intel Corporation # @@ -39,31 +39,49 @@ def run(self, ctx): ctx.exec('pmem2_mover', self.test_case, self.filepath, self.thread_num) +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5595 is fixed. +@t.require_valgrind_disabled('pmemcheck') class TEST0(PMEM2_MOVER): """verify pmem2 mover memcpy functionality""" test_case = "test_mover_memcpy_basic" +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5595 is fixed. +@t.require_valgrind_disabled('pmemcheck') class TEST1(PMEM2_MOVER): """verify pmem2 mover memmove functionality""" test_case = "test_mover_memmove_basic" +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5595 is fixed. +@t.require_valgrind_disabled('pmemcheck') class TEST2(PMEM2_MOVER): """verify pmem2 mover memset functionality""" test_case = "test_mover_memset_basic" +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5595 is fixed. +@t.require_valgrind_disabled('pmemcheck') class TEST3(PMEM2_MOVER_MT): """verify pmem2 mover multi-threaded memcpy functionality""" test_case = "test_mover_memcpy_multithreaded" +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5595 is fixed. +@t.require_valgrind_disabled('pmemcheck') class TEST4(PMEM2_MOVER_MT): """verify pmem2 mover multi-threaded memmove functionality""" test_case = "test_mover_memmove_multithreaded" +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5595 is fixed. +@t.require_valgrind_disabled('pmemcheck') class TEST5(PMEM2_MOVER_MT): """verify pmem2 mover multi-threaded memset functionality""" test_case = "test_mover_memset_multithreaded" @@ -90,6 +108,9 @@ class TEST8(PMEM2_MOVER_MT): test_case = "test_mover_memset_multithreaded" +# XXX disable the test for 'pmemcheck' +# until https://github.com/pmem/pmdk/issues/5595 is fixed. +@t.require_valgrind_disabled('pmemcheck') class TEST9(PMEM2_MOVER): """verify pmem2 mover functionality""" test_case = "test_miniasync_mover" From 377f875e2abffaea3716cf4e2985a29f05a6470f Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 22 May 2023 06:42:14 +0200 Subject: [PATCH 34/56] test: disable ex_libpmem2/TEST5 with memcheck Disable ex_libpmem2/TEST5 with memcheck until issue #5635 is fixed. Signed-off-by: Tomasz Gromadzki --- src/test/ex_libpmem2/TESTS.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/test/ex_libpmem2/TESTS.py b/src/test/ex_libpmem2/TESTS.py index 4d32d04aba8..7b9656b0192 100755 --- a/src/test/ex_libpmem2/TESTS.py +++ b/src/test/ex_libpmem2/TESTS.py @@ -74,8 +74,8 @@ def run(self, ctx): ctx.exec(example_path, *args, stdout_file='out4.log') -# XXX Disable the test execution under pmemcheck with g.PAGE until fixed. -# https://github.com/pmem/pmdk/issues/5641 +# XXX Disable the test execution under pmemcheck with g.PAGE until the issue +# https://github.com/pmem/pmdk/issues/5641 is fixed. # additionall test TEST501 has been added to cover non-pmemcheck configs. class EX_LIBPMEM2_TEST5(EX_LIBPMEM2): @@ -93,7 +93,12 @@ class TEST5(EX_LIBPMEM2_TEST5): pass -@t.require_valgrind_disabled('pmemcheck') +# XXX Disable the test execution with 'memcheck' until the issue: +# https://github.com/pmem/pmdk/issues/5635 is fixed. +# additionall test TEST501 has been added to cover non-pmemcheck configs. +# @t.require_valgrind_disabled('memcheck') # to be removed when fixed +# @t.require_valgrind_disabled('pmemcheck') +@t.require_valgrind_disabled('pmemcheck', 'memcheck') class TEST501(EX_LIBPMEM2_TEST5): # to be removed when fixed pass From 2c0fb2698271c3b3802c5c91a938fcbd724ce48b Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 22 May 2023 15:12:44 +0200 Subject: [PATCH 35/56] common: install hub on Ubuntu to build proper documentation Fixes: #5674 Signed-off-by: Tomasz Gromadzki --- utils/docker/images/Dockerfile.ubuntu-22.04 | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/docker/images/Dockerfile.ubuntu-22.04 b/utils/docker/images/Dockerfile.ubuntu-22.04 index b60bd18766b..fc5c0632e7e 100644 --- a/utils/docker/images/Dockerfile.ubuntu-22.04 +++ b/utils/docker/images/Dockerfile.ubuntu-22.04 @@ -69,6 +69,7 @@ ENV COVERITY_DEPS ruby gcc g++ wget # miscellaneous dependencies (mostly for CI) ENV MISC_DEPS "clang \ + hub \ clang-format \ flake8 \ python3-pip \ From 448eb22e5bd7d3758e9ae06f1f2edd0df07dede4 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 22 May 2023 16:57:37 +0200 Subject: [PATCH 36/56] common: install rsync on Ubuntu to build proper documentation Signed-off-by: Tomasz Gromadzki --- utils/docker/images/Dockerfile.ubuntu-22.04 | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/docker/images/Dockerfile.ubuntu-22.04 b/utils/docker/images/Dockerfile.ubuntu-22.04 index fc5c0632e7e..cbf2b8949d0 100644 --- a/utils/docker/images/Dockerfile.ubuntu-22.04 +++ b/utils/docker/images/Dockerfile.ubuntu-22.04 @@ -70,6 +70,7 @@ ENV COVERITY_DEPS ruby gcc g++ wget # miscellaneous dependencies (mostly for CI) ENV MISC_DEPS "clang \ hub \ + rsync \ clang-format \ flake8 \ python3-pip \ From ceab4f15a7f512e38581b07d716a669c5e3314b3 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Mon, 22 May 2023 22:03:56 +0200 Subject: [PATCH 37/56] test: display execution time of each test Signed-off-by: Tomasz Gromadzki --- utils/gha-runners/create-testconfig.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/gha-runners/create-testconfig.sh b/utils/gha-runners/create-testconfig.sh index ad0ae5783f9..ed72a96951c 100755 --- a/utils/gha-runners/create-testconfig.sh +++ b/utils/gha-runners/create-testconfig.sh @@ -21,6 +21,7 @@ DEVICE_DAX_PATH=($(ndctl list -X | jq -r '.[].daxregion.devices[].chardev' | awk KEEP_GOING=y TEST_TIMEOUT=120m ENABLE_SUDO_TESTS=y +TM=1 EOL # Create config file for py tests. From 6418a2d3a01d660e685895e9972c9e01f569c0a1 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 23 May 2023 06:28:04 +0200 Subject: [PATCH 38/56] test: add missing persist in obj_tx_alloc_mt/TEST1 Fixes: #5660 Signed-off-by: Tomasz Gromadzki --- src/test/obj_tx_alloc_mt/obj_tx_alloc_mt.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/obj_tx_alloc_mt/obj_tx_alloc_mt.c b/src/test/obj_tx_alloc_mt/obj_tx_alloc_mt.c index 9199a9ef418..ff95f7bcd1d 100644 --- a/src/test/obj_tx_alloc_mt/obj_tx_alloc_mt.c +++ b/src/test/obj_tx_alloc_mt/obj_tx_alloc_mt.c @@ -183,6 +183,7 @@ main(int argc, char *argv[]) util_cond_init(&a->cond); } } + pmemobj_persist(pop, r, sizeof(struct root)); /* * Reduce the number of lanes to a value smaller than the number of From c7272214346b3b806d554278b13811f93bce9ac0 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 23 May 2023 06:04:00 +0200 Subject: [PATCH 39/56] test: fix pmem2_mover/TEST1 with memcheck for #5678 Fixes: #5678. Signed-off-by: Tomasz Gromadzki --- src/test/pmem2_mover/pmem2_mover.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/pmem2_mover/pmem2_mover.c b/src/test/pmem2_mover/pmem2_mover.c index 45d9deb6d1d..50a94b6c0fc 100644 --- a/src/test/pmem2_mover/pmem2_mover.c +++ b/src/test/pmem2_mover/pmem2_mover.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: BSD-3-Clause -/* Copyright 2019-2022, Intel Corporation */ +/* Copyright 2019-2023, Intel Corporation */ /* * pmem2_mover.c -- pmem2 mover tests @@ -113,6 +113,7 @@ test_mover_memmove_basic(const struct test_case *tc, int argc, char *argv[]) if (memcmp(data + string_size / 2, buffer_data, string_size)) UT_FATAL("data should be equal"); + free(buffer_data); pmem2_map_delete(&map); pmem2_config_delete(&cfg); pmem2_source_delete(&src); From 6f533b16d13db75371c51b6577ad83413793745e Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 23 May 2023 12:53:32 +0200 Subject: [PATCH 40/56] test: (fix) remove obsolete data structure Signed-off-by: Tomasz Gromadzki --- src/test/obj_tx_alloc_mt/obj_tx_alloc_mt.c | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/test/obj_tx_alloc_mt/obj_tx_alloc_mt.c b/src/test/obj_tx_alloc_mt/obj_tx_alloc_mt.c index ff95f7bcd1d..0a0da795658 100644 --- a/src/test/obj_tx_alloc_mt/obj_tx_alloc_mt.c +++ b/src/test/obj_tx_alloc_mt/obj_tx_alloc_mt.c @@ -23,15 +23,8 @@ static unsigned Threads; static unsigned Ops_per_thread; static unsigned Tx_per_thread; -struct action { - struct pobj_action pact; - os_mutex_t lock; - os_cond_t cond; -}; - struct root { uint64_t offs[MAX_THREADS][MAX_OPS_PER_THREAD]; - struct action actions[MAX_THREADS][MAX_OPS_PER_THREAD]; }; struct worker_args { @@ -177,11 +170,6 @@ main(int argc, char *argv[]) args[i].pop = pop; args[i].r = r; args[i].idx = i; - for (unsigned j = 0; j < Ops_per_thread; ++j) { - struct action *a = &r->actions[i][j]; - util_mutex_init(&a->lock); - util_cond_init(&a->cond); - } } pmemobj_persist(pop, r, sizeof(struct root)); From 5bed7f510650c4c53446f88d8f423de5b38a552d Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 23 May 2023 13:08:29 +0200 Subject: [PATCH 41/56] test: remove support for obsolete pmemobj_root Signed-off-by: Tomasz Gromadzki --- src/test/obj_tx_alloc_mt/obj_tx_alloc_mt.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/test/obj_tx_alloc_mt/obj_tx_alloc_mt.c b/src/test/obj_tx_alloc_mt/obj_tx_alloc_mt.c index 0a0da795658..fb34e4f5c0b 100644 --- a/src/test/obj_tx_alloc_mt/obj_tx_alloc_mt.c +++ b/src/test/obj_tx_alloc_mt/obj_tx_alloc_mt.c @@ -23,13 +23,8 @@ static unsigned Threads; static unsigned Ops_per_thread; static unsigned Tx_per_thread; -struct root { - uint64_t offs[MAX_THREADS][MAX_OPS_PER_THREAD]; -}; - struct worker_args { PMEMobjpool *pop; - struct root *r; unsigned idx; }; @@ -160,18 +155,12 @@ main(int argc, char *argv[]) UT_FATAL("!pmemobj_open"); } - PMEMoid oid = pmemobj_root(pop, sizeof(struct root)); - struct root *r = pmemobj_direct(oid); - UT_ASSERTne(r, NULL); - struct worker_args args[MAX_THREADS]; for (unsigned i = 0; i < Threads; ++i) { args[i].pop = pop; - args[i].r = r; args[i].idx = i; } - pmemobj_persist(pop, r, sizeof(struct root)); /* * Reduce the number of lanes to a value smaller than the number of From 1cfb23d758b824e7b6958ce3618ee296a3b6878b Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 23 May 2023 08:45:11 +0200 Subject: [PATCH 42/56] pmem2: remove data_mover memory leak Automatically created data_mover is deleted during the map delete process. Fixes: #5637 Signed-off-by: Tomasz Gromadzki --- src/libpmem2/map_posix.c | 7 ++++--- src/libpmem2/mover.c | 7 +++++-- src/test/pmem2_integration/TESTS.py | 6 ------ 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/libpmem2/map_posix.c b/src/libpmem2/map_posix.c index e914d5251d0..073fe88ec0e 100644 --- a/src/libpmem2/map_posix.c +++ b/src/libpmem2/map_posix.c @@ -621,10 +621,11 @@ pmem2_map_delete(struct pmem2_map **map_ptr) if (ret) goto err_register_map; } - - if (!map->custom_vdm) - mover_delete(map->vdm); } + + if (!map->custom_vdm) + mover_delete(map->vdm); + Free(map); *map_ptr = NULL; diff --git a/src/libpmem2/mover.c b/src/libpmem2/mover.c index 25f5b12b490..e2dbadc1ea5 100644 --- a/src/libpmem2/mover.c +++ b/src/libpmem2/mover.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: BSD-3-Clause -/* Copyright 2021-2022, Intel Corporation */ +/* Copyright 2021-2023, Intel Corporation */ /* * mover.c -- default pmem2 data mover @@ -12,6 +12,7 @@ #include "out.h" #include "pmem2_utils.h" #include "util.h" +#include "alloc.h" #include #include #include @@ -194,6 +195,7 @@ mover_new(struct pmem2_map *map, struct vdm **vdm) goto membuf_failed; } + LOG(3, "dms %p", dms); return 0; membuf_failed: @@ -207,8 +209,9 @@ mover_new(struct pmem2_map *map, struct vdm **vdm) void mover_delete(struct vdm *dms) { + LOG(3, "dms %p", dms); membuf_delete(((struct data_mover *)dms)->membuf); - free((struct data_mover *)dms); + Free((struct data_mover *)dms); } /* diff --git a/src/test/pmem2_integration/TESTS.py b/src/test/pmem2_integration/TESTS.py index c438cfc459a..527fe027156 100755 --- a/src/test/pmem2_integration/TESTS.py +++ b/src/test/pmem2_integration/TESTS.py @@ -289,9 +289,6 @@ class TEST38(PMEM2_INTEGRATION): test_case = "test_unaligned_persist" -# XXX disable the test for `memcheck' -# until https://github.com/pmem/pmdk/issues/5637 is fixed. -@t.require_valgrind_disabled('memcheck') class TEST39(PMEM2_INTEGRATION): """compare normal map vs map_from_existing""" test_case = "test_map_from_existing" @@ -303,9 +300,6 @@ class TES40(PMEM2_INTEGRATION): @t.windows_exclude -# XXX disable the test for `memcheck' -# until https://github.com/pmem/pmdk/issues/5637 is fixed. -@t.require_valgrind_disabled('memcheck') class TEST41(PMEM2_INTEGRATION_DEV_DAXES): """compare normal map vs map_from_existing on devdax""" test_case = "test_map_from_existing" From a614348755dd56dfbbef2f01c043705c4b07b2fa Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 23 May 2023 15:09:49 +0200 Subject: [PATCH 43/56] test: fix pmem2_map_from_existing/TEST[0-2] w/ memcheck Fixes: #5594 Signed-off-by: Tomasz Gromadzki --- src/test/pmem2_map_from_existing/TESTS.py | 9 --------- .../pmem2_map_from_existing/pmem2_map_from_existing.c | 5 ++++- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/test/pmem2_map_from_existing/TESTS.py b/src/test/pmem2_map_from_existing/TESTS.py index 28b20975a45..ab8044bbca2 100644 --- a/src/test/pmem2_map_from_existing/TESTS.py +++ b/src/test/pmem2_map_from_existing/TESTS.py @@ -17,25 +17,16 @@ def run(self, ctx): ctx.exec('pmem2_map_from_existing', self.test_case, filepath) -# XXX disable the test for `memcheck' -# until https://github.com/pmem/pmdk/issues/5594 is fixed. -@t.require_valgrind_disabled('memcheck') class TEST0(Pmem2_from_existing): """try to create two the same mappings""" test_case = "test_two_same_mappings" -# XXX disable the test for `memcheck' -# until https://github.com/pmem/pmdk/issues/5594 is fixed. -@t.require_valgrind_disabled('memcheck') class TEST1(Pmem2_from_existing): """try to map which overlap bottom part of existing mapping""" test_case = "test_mapping_overlap_bottom" -# XXX disable the test for `memcheck' -# until https://github.com/pmem/pmdk/issues/5594 is fixed. -@t.require_valgrind_disabled('memcheck') class TEST2(Pmem2_from_existing): """try to map which overlap upper part of existing mapping""" test_case = "test_mapping_overlap_upper" diff --git a/src/test/pmem2_map_from_existing/pmem2_map_from_existing.c b/src/test/pmem2_map_from_existing/pmem2_map_from_existing.c index 3868f1da83e..7cb4371bc59 100644 --- a/src/test/pmem2_map_from_existing/pmem2_map_from_existing.c +++ b/src/test/pmem2_map_from_existing/pmem2_map_from_existing.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: BSD-3-Clause -/* Copyright 2020-2021, Intel Corporation */ +/* Copyright 2020-2023, Intel Corporation */ /* * pmem2_map_from_existing.c -- pmem2_map_from_existing unittests @@ -37,6 +37,7 @@ test_two_same_mappings(const struct test_case *tc, int argc, char *argv[]) UT_ASSERTeq(map2, NULL); pmem2_map_delete(&map1); + pmem2_source_delete(&src); CLOSE(fd); return 1; } @@ -69,6 +70,7 @@ test_mapping_overlap_bottom(const struct test_case *tc, int argc, char *argv[]) UT_ASSERTeq(map2, NULL); pmem2_map_delete(&map1); + pmem2_source_delete(&src); CLOSE(fd); return 1; } @@ -101,6 +103,7 @@ test_mapping_overlap_upper(const struct test_case *tc, int argc, char *argv[]) UT_ASSERTeq(map2, NULL); pmem2_map_delete(&map1); + pmem2_source_delete(&src); CLOSE(fd); return 1; } From 01670d31a51eac8b78148d31592b3aadada2f731 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 24 May 2023 09:04:31 +0200 Subject: [PATCH 44/56] common: print elapsed time for failed tests Signed-off-by: Tomasz Gromadzki --- src/test/RUNTESTS.py | 13 +++++++++---- src/test/unittest/basetest.py | 4 +++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/test/RUNTESTS.py b/src/test/RUNTESTS.py index 1bc50e489ab..46742bae003 100755 --- a/src/test/RUNTESTS.py +++ b/src/test/RUNTESTS.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2019-2022, Intel Corporation +# Copyright 2019-2023, Intel Corporation """Main script for unit tests execution. @@ -132,9 +132,14 @@ def run_tests(self): return ret def _test_failed(self, tc, ctx, fail): - self.msg.print('{}: {}FAILED{}\t({}/{})' + """Print message specific for failed test""" + if self.config.tm: + tm = '\t[{:06.3F} s]'.format(tc.elapsed) + else: + tm = '' + self.msg.print('{}: {}FAILED{}\t({}/{}) {}' .format(tc, futils.Color.RED, - futils.Color.END, tc.test_type, ctx)) + futils.Color.END, tc.test_type, ctx, tm)) self.msg.print(fail) if not self.config.keep_going: @@ -143,7 +148,7 @@ def _test_failed(self, tc, ctx, fail): def _test_passed(self, tc): """Print message specific for passed test""" if self.config.tm: - tm = '\t\t\t[{:06.3F} s]'.format(tc.elapsed) + tm = '\t[{:06.3F} s]'.format(tc.elapsed) else: tm = '' diff --git a/src/test/unittest/basetest.py b/src/test/unittest/basetest.py index c4f9cc8d6a6..eda966ef0f0 100644 --- a/src/test/unittest/basetest.py +++ b/src/test/unittest/basetest.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2019-2020, Intel Corporation +# Copyright 2019-2023, Intel Corporation """Base tests class and its functionalities. @@ -179,6 +179,7 @@ def _execute(self, c): self.check(c) except futils.Fail: + self.elapsed = (datetime.now() - start_time).total_seconds() self._on_fail() raise @@ -191,6 +192,7 @@ def _execute(self, c): msg = '{}: {}TIMEOUT{}\t({})'.format(self, futils.Color.RED, futils.Color.END, self.ctx) + self.elapsed = (datetime.now() - start_time).total_seconds() raise futils.Fail(msg) else: From 39001909f9c6923adf72217de1e42c218c9f6ff2 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 24 May 2023 06:38:33 +0200 Subject: [PATCH 45/56] common: Increasing the parallelism of test execution Signed-off-by: Tomasz Gromadzki --- .github/workflows/unittest_valgrind_py.yml | 8 +++----- .github/workflows/unittest_valgrind_sh.yml | 8 +++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/workflows/unittest_valgrind_py.yml b/.github/workflows/unittest_valgrind_py.yml index b9827681665..ea07c446163 100644 --- a/.github/workflows/unittest_valgrind_py.yml +++ b/.github/workflows/unittest_valgrind_py.yml @@ -18,10 +18,8 @@ jobs: fail-fast: false matrix: os: [[self-hosted, rhel],[self-hosted, opensuse]] - config: ['RUNTESTS.py --force-enable memcheck', - 'RUNTESTS.py --force-enable pmemcheck', - 'RUNTESTS.py --force-enable drd', - 'RUNTESTS.py --force-enable helgrind'] + build: ['debug', 'release', 'static_debug', 'static_release'] + config: ['memcheck', 'pmemcheck', 'drd', 'helgrind'] env: WORKDIR: utils/gha-runners @@ -39,4 +37,4 @@ jobs: run: ./$WORKDIR/create-testconfig.sh - name: Run tests - run: cd src/test/ && ./${{ matrix.config }} + run: cd src/test/ && ./RUNTESTS.py --force-enable ${{ matrix.config }} -b ${{ matrix.build }} diff --git a/.github/workflows/unittest_valgrind_sh.yml b/.github/workflows/unittest_valgrind_sh.yml index 2f8c590656d..bdab874129b 100644 --- a/.github/workflows/unittest_valgrind_sh.yml +++ b/.github/workflows/unittest_valgrind_sh.yml @@ -18,10 +18,8 @@ jobs: fail-fast: false matrix: os: [[self-hosted, rhel],[self-hosted, opensuse]] - config: ['RUNTESTS -m force-enable', - 'RUNTESTS -p force-enable', - 'RUNTESTS -d force-enable', - 'RUNTESTS -e force-enable'] + build: ['debug', 'nondebug', 'static-debug', 'static-nondebug'] + config: ['-m', '-p', '-d', '-e'] env: WORKDIR: utils/gha-runners @@ -39,4 +37,4 @@ jobs: run: ./$WORKDIR/create-testconfig.sh - name: Run tests - run: cd src/test/ && ./${{ matrix.config }} + run: cd src/test/ && ./$RUNTESTS -b ${{ matrix.build }} ${{ matrix.config }} force-enable From 4022b87467ea0df07ed9297f139f73f1782eb6b3 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 24 May 2023 09:29:24 +0200 Subject: [PATCH 46/56] common: run 'check' tests in parallel Signed-off-by: Tomasz Gromadzki --- .github/workflows/unittest.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 33f98e08333..dfc4c3ea861 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -14,8 +14,14 @@ jobs: fail-fast: false matrix: os: [[self-hosted, rhel],[self-hosted, opensuse]] - config: ['make -j$(nproc); make -j$(nproc) test; make check', - 'make -j$(nproc); make -j$(nproc) test; make pycheck'] + config: ['check TEST_BUILD=debug', + 'check TEST_BUILD=nondebug', + 'check TEST_BUILD=static-debug', + 'check TEST_BUILD=static-nondebug', + 'pycheck TEST_BUILD=debug', + 'pycheck TEST_BUILD=release', + 'pycheck TEST_BUILD=static_debug', + 'pycheck TEST_BUILD=static_release'] env: WORKDIR: utils/gha-runners @@ -33,4 +39,4 @@ jobs: run: ./$WORKDIR/create-testconfig.sh - name: Run tests - run: ${{ matrix.config }} + run: make ${{ matrix.config }} From 8989c65f1612cd6c7bb70351b49179ad7541289c Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 24 May 2023 09:58:00 +0200 Subject: [PATCH 47/56] common: run 'long' tests in parallel Signed-off-by: Tomasz Gromadzki --- .github/workflows/unittest_long.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unittest_long.yml b/.github/workflows/unittest_long.yml index fc25542ee04..503259d4a7f 100644 --- a/.github/workflows/unittest_long.yml +++ b/.github/workflows/unittest_long.yml @@ -18,8 +18,15 @@ jobs: fail-fast: false matrix: os: [[self-hosted, rhel],[self-hosted, opensuse]] - config: ['RUNTESTS -t long', - 'RUNTESTS.py -t long'] + config: ['RUNTESTS -t long -b debug', + 'RUNTESTS -t long -b nondebug', + 'RUNTESTS -t long -b static-debug', + 'RUNTESTS -t long -b static-nondebug', + 'RUNTESTS.py -t long -b debug', + 'RUNTESTS.py -t long -b release', + 'RUNTESTS.py -t long -b static_debug', + 'RUNTESTS.py -t long -b static_release'] + env: WORKDIR: utils/gha-runners From 050e71fdbb2eabe59f8c82eb67f50b70e97dc0b6 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 24 May 2023 11:18:22 +0200 Subject: [PATCH 48/56] common: use as many cores for build as possible Signed-off-by: Tomasz Gromadzki --- utils/gha-runners/build-pmdk.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utils/gha-runners/build-pmdk.sh b/utils/gha-runners/build-pmdk.sh index 9d66b16e58d..467acbff43a 100755 --- a/utils/gha-runners/build-pmdk.sh +++ b/utils/gha-runners/build-pmdk.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2022, Intel Corporation +# Copyright 2022-2023, Intel Corporation # # build-pmdk.sh - Script for building pmdk project @@ -16,10 +16,10 @@ set -eo pipefail # function build_pmdk { echo "********** make pmdk **********" - cd ${PMDK_PATH} && make clean - cd ${PMDK_PATH} && make EXTRA_CFLAGS=-DUSE_VALGRIND + cd ${PMDK_PATH} && make -j$(nproc) clean + cd ${PMDK_PATH} && make -j$(nproc) EXTRA_CFLAGS=-DUSE_VALGRIND echo "********** make pmdk test **********" - cd ${PMDK_PATH}/ && make test + cd ${PMDK_PATH}/ && make -j$(nproc) test } build_pmdk From f9b0210cc7347c6971dcec9c394e9d30b78b0d7d Mon Sep 17 00:00:00 2001 From: Oksana Salyk Date: Wed, 24 May 2023 12:11:17 +0200 Subject: [PATCH 49/56] test: fix memory leak in pmem2_mover test --- src/test/pmem2_mover/pmem2_mover.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/pmem2_mover/pmem2_mover.c b/src/test/pmem2_mover/pmem2_mover.c index 50a94b6c0fc..3e953551054 100644 --- a/src/test/pmem2_mover/pmem2_mover.c +++ b/src/test/pmem2_mover/pmem2_mover.c @@ -163,6 +163,7 @@ test_mover_memset_basic(const struct test_case *tc, int argc, char *argv[]) if (memcmp(data, memset_result, array_size)) UT_FATAL("data should be equal"); + free(memset_result); pmem2_map_delete(&map); pmem2_config_delete(&cfg); pmem2_source_delete(&src); From 541ad0824fb528c23fe0fb42f09a56f87f8da37d Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 24 May 2023 13:24:46 +0200 Subject: [PATCH 50/56] test: disable pmem2_mover/TEST[6-8] (pmemcheck/byte) Disable the test for 'pmemcheck' and 'BYTE' granularity until https://github.com/pmem/pmdk/issues/5686 is fixed. Signed-off-by: Tomasz Gromadzki --- src/test/pmem2_mover/TESTS.py | 45 ++++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/src/test/pmem2_mover/TESTS.py b/src/test/pmem2_mover/TESTS.py index 4b22ffc0ef2..4a372d2668a 100644 --- a/src/test/pmem2_mover/TESTS.py +++ b/src/test/pmem2_mover/TESTS.py @@ -87,27 +87,66 @@ class TEST5(PMEM2_MOVER_MT): test_case = "test_mover_memset_multithreaded" -class TEST6(PMEM2_MOVER_MT): +# XXX disable the test for 'pmemcheck' and 'BYTE' granurality +# until https://github.com/pmem/pmdk/issues/5686 is fixed. +class PMEM2_MOVER_MT_TEST6(PMEM2_MOVER_MT): test_type = t.Long thread_num = 16 """verify pmem2 mover multi-threaded memcpy functionality (Long)""" test_case = "test_mover_memcpy_multithreaded" -class TEST7(PMEM2_MOVER_MT): +@g.require_granularity(g.PAGE, g.CACHELINE) +class TEST6(PMEM2_MOVER_MT_TEST6): + pass + + +@g.require_granularity(g.BYTE) +@t.require_valgrind_disabled('pmemcheck') +class TEST61(PMEM2_MOVER_MT_TEST6): + pass + + +# XXX disable the test for 'pmemcheck' and 'BYTE' granurality +# until https://github.com/pmem/pmdk/issues/5686 is fixed. +class PMEM2_MOVER_MT_TEST7(PMEM2_MOVER_MT): test_type = t.Long thread_num = 16 """verify pmem2 mover multi-threaded memmove functionality (Long)""" test_case = "test_mover_memmove_multithreaded" -class TEST8(PMEM2_MOVER_MT): +@g.require_granularity(g.PAGE, g.CACHELINE) +class TEST7(PMEM2_MOVER_MT_TEST7): + pass + + +@g.require_granularity(g.BYTE) +@t.require_valgrind_disabled('pmemcheck') +class TEST71(PMEM2_MOVER_MT_TEST7): + pass + + +# XXX disable the test for 'pmemcheck' and 'BYTE' granurality +# until https://github.com/pmem/pmdk/issues/5686 is fixed. +class PMEM2_MOVER_MT_TEST8(PMEM2_MOVER_MT): test_type = t.Long thread_num = 16 """verify pmem2 mover multi-threaded memset functionality (Long)""" test_case = "test_mover_memset_multithreaded" +@g.require_granularity(g.PAGE, g.CACHELINE) +class TEST8(PMEM2_MOVER_MT_TEST8): + pass + + +@g.require_granularity(g.BYTE) +@t.require_valgrind_disabled('pmemcheck') +class TEST81(PMEM2_MOVER_MT_TEST8): + pass + + # XXX disable the test for 'pmemcheck' # until https://github.com/pmem/pmdk/issues/5595 is fixed. @t.require_valgrind_disabled('pmemcheck') From e6c38fac36a91c93d64f7fb7186d55a721f7e9f0 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 24 May 2023 13:53:37 +0200 Subject: [PATCH 51/56] test: disable pmem2_vm_reservation/TEST22 w/ Valgrind Disable the test for 'pmemcheck', 'drd', 'helgrind', 'memcheck' until https://github.com/pmem/pmdk/issues/5592 is fixed. Signed-off-by: Tomasz Gromadzki --- src/test/pmem2_vm_reservation/TESTS.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/pmem2_vm_reservation/TESTS.py b/src/test/pmem2_vm_reservation/TESTS.py index 2bc4757bae1..e5cdfa85638 100755 --- a/src/test/pmem2_vm_reservation/TESTS.py +++ b/src/test/pmem2_vm_reservation/TESTS.py @@ -215,6 +215,9 @@ class TEST21(PMEM2_VM_RESERVATION): test_case = "test_vm_reserv_map_unmap_multiple_files" +# XXX disable the test for 'pmemcheck', 'drd', 'helgrind', 'memcheck' +# until https://github.com/pmem/pmdk/issues/5592 is fixed. +@t.require_valgrind_disabled('pmemcheck', 'drd', 'helgrind', 'memcheck') class TEST22(PMEM2_VM_RESERVATION_DEVDAX): """ DevDax map multiple files to a vm reservation, then From 5a363c93b2d90014edf85692fd7eb17b1b511f71 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 24 May 2023 14:07:44 +0200 Subject: [PATCH 52/56] test: disable obj_defrag_advanced/TEST5 until #5690 is fixed Disable the test for 'drd' until https://github.com/pmem/pmdk/issues/5690 is fixed. Signed-off-by: Tomasz Gromadzki --- src/test/obj_defrag_advanced/TESTS.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/test/obj_defrag_advanced/TESTS.py b/src/test/obj_defrag_advanced/TESTS.py index e4447bcd390..a33200c44dd 100755 --- a/src/test/obj_defrag_advanced/TESTS.py +++ b/src/test/obj_defrag_advanced/TESTS.py @@ -1,6 +1,6 @@ #!../env.py # SPDX-License-Identifier: BSD-3-Clause -# Copyright 2020, Intel Corporation +# Copyright 2020-2023, Intel Corporation # import testframework as t @@ -103,8 +103,12 @@ class TEST4(ObjDefragAdvancedMt): ncycles = 25 +# XXX disable the test for 'drd' +# until https://github.com/pmem/pmdk/issues/5690 is fixed. +# previousely the test has been disabled for other Valgrind options # This test last too long under helgrind/memcheck/pmemcheck -@t.require_valgrind_disabled('helgrind', 'memcheck', 'pmemcheck') +# @t.require_valgrind_disabled('helgrind', 'memcheck', 'pmemcheck') +@t.require_valgrind_disabled('helgrind', 'memcheck', 'pmemcheck', 'drd') class TEST5(ObjDefragAdvancedMt): max_nodes = 256 From 0b4b3f865970a024edb1b7d2f008d4ed75232360 Mon Sep 17 00:00:00 2001 From: Oksana Salyk Date: Thu, 25 May 2023 11:39:24 +0200 Subject: [PATCH 53/56] common: 1.13.1-rc1 release --- ChangeLog | 28 ++++++++++++++++++++++++++++ GIT_VERSION | 1 - VERSION | 1 + 3 files changed, 29 insertions(+), 1 deletion(-) delete mode 100644 GIT_VERSION create mode 100644 VERSION diff --git a/ChangeLog b/ChangeLog index f2156164b4b..1221a1cacd6 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,31 @@ +Thu May 25 2023 Oksana Sałyk + + * Version 1.13.1-rc1 + + This release : + - Identifies and temporarily disables tests permanently or sporadically + failing in some configurations: + - ex_libpmem2 (#5593, #5635, #5638, #5641) + - obj_defrag_advanced (#5690) + - obj_sync (#5643, #5646) + - pmem2_badblock (#5652, #5636) + - pmem2_future (#5596) + - pmem2_map (#5600) + - pmem2_map_prot (#5599) + - pmem2_mem_ext (#5640) + - pmem2_mover (partially fixed) (#5595, #5686) + - pmem2_vm_reservation (#5592) + + - Fixes failing tests: + - obj_tx_alloc_mt (#5680) + - pmem2_integration (#5637) + - pmem2_map_from_existing (#5594) + + Other changes and notable bug fixes: + - test: update permissions of DAX devices after clearing bad blocks (#5611) + - pmem2: fix valgrind instrumentation in pmem2_map_from_exisiting (#5605) + - test: disable avx512f tests (pmem2_mem_ext) under Valgrind (#5658) + Wed Apr 26 2023 Łukasz Stolarczuk * Version 1.13.0 diff --git a/GIT_VERSION b/GIT_VERSION deleted file mode 100644 index 6002f22fa25..00000000000 --- a/GIT_VERSION +++ /dev/null @@ -1 +0,0 @@ -$Format:%h$ \ No newline at end of file diff --git a/VERSION b/VERSION new file mode 100644 index 00000000000..eec4d766c7c --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +1.13.1-rc1 From b2b4e48f121e528315ba5889df0ab9c5001f9546 Mon Sep 17 00:00:00 2001 From: Oksana Salyk Date: Thu, 25 May 2023 11:46:38 +0200 Subject: [PATCH 54/56] common: git versions --- GIT_VERSION | 1 + VERSION | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 GIT_VERSION delete mode 100644 VERSION diff --git a/GIT_VERSION b/GIT_VERSION new file mode 100644 index 00000000000..6002f22fa25 --- /dev/null +++ b/GIT_VERSION @@ -0,0 +1 @@ +$Format:%h$ \ No newline at end of file diff --git a/VERSION b/VERSION deleted file mode 100644 index eec4d766c7c..00000000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -1.13.1-rc1 From 9f36cff856c7c822f41b59a9eb51fd0c205c98c9 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 25 May 2023 13:07:36 +0200 Subject: [PATCH 55/56] test: shortening the overall execution of valgrind tests Reorder test execution to place long tests sequence (drd/pmemcheck) at the beginning to complete all tests execution in a shorter time. Signed-off-by: Tomasz Gromadzki --- .github/workflows/unittest_valgrind_py.yml | 5 +++-- .github/workflows/unittest_valgrind_sh.yml | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/unittest_valgrind_py.yml b/.github/workflows/unittest_valgrind_py.yml index ea07c446163..b9a6bca86c1 100644 --- a/.github/workflows/unittest_valgrind_py.yml +++ b/.github/workflows/unittest_valgrind_py.yml @@ -17,9 +17,10 @@ jobs: strategy: fail-fast: false matrix: - os: [[self-hosted, rhel],[self-hosted, opensuse]] + config: ['drd', 'pmemcheck', 'memcheck', 'helgrind'] build: ['debug', 'release', 'static_debug', 'static_release'] - config: ['memcheck', 'pmemcheck', 'drd', 'helgrind'] + os: [[self-hosted, rhel],[self-hosted, opensuse]] + env: WORKDIR: utils/gha-runners diff --git a/.github/workflows/unittest_valgrind_sh.yml b/.github/workflows/unittest_valgrind_sh.yml index bdab874129b..4670fae2286 100644 --- a/.github/workflows/unittest_valgrind_sh.yml +++ b/.github/workflows/unittest_valgrind_sh.yml @@ -17,9 +17,10 @@ jobs: strategy: fail-fast: false matrix: - os: [[self-hosted, rhel],[self-hosted, opensuse]] + config: ['-d', '-p', '-m', '-e'] build: ['debug', 'nondebug', 'static-debug', 'static-nondebug'] - config: ['-m', '-p', '-d', '-e'] + os: [[self-hosted, rhel],[self-hosted, opensuse]] + env: WORKDIR: utils/gha-runners From 1aa215ce2851e902d179c4ae41e28cd6982fd6fd Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Thu, 25 May 2023 18:27:20 +0200 Subject: [PATCH 56/56] test: disable pmem2_mover/TEST[6-8] for drd Disable the test until #5694 is fixed. Signed-off-by: Tomasz Gromadzki --- src/test/pmem2_mover/TESTS.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/test/pmem2_mover/TESTS.py b/src/test/pmem2_mover/TESTS.py index 4a372d2668a..b7bd292f2b6 100644 --- a/src/test/pmem2_mover/TESTS.py +++ b/src/test/pmem2_mover/TESTS.py @@ -96,13 +96,18 @@ class PMEM2_MOVER_MT_TEST6(PMEM2_MOVER_MT): test_case = "test_mover_memcpy_multithreaded" +# XXX disable the test for 'drd`` +# until https://github.com/pmem/pmdk/issues/5694 is fixed. +@t.require_valgrind_disabled('drd') @g.require_granularity(g.PAGE, g.CACHELINE) class TEST6(PMEM2_MOVER_MT_TEST6): pass +# XXX disable the test for 'drd`` +# until https://github.com/pmem/pmdk/issues/5694 is fixed. +@t.require_valgrind_disabled('pmemcheck', 'drd') @g.require_granularity(g.BYTE) -@t.require_valgrind_disabled('pmemcheck') class TEST61(PMEM2_MOVER_MT_TEST6): pass @@ -116,13 +121,18 @@ class PMEM2_MOVER_MT_TEST7(PMEM2_MOVER_MT): test_case = "test_mover_memmove_multithreaded" +# XXX disable the test for 'drd`` +# until https://github.com/pmem/pmdk/issues/5694 is fixed. +@t.require_valgrind_disabled('drd') @g.require_granularity(g.PAGE, g.CACHELINE) class TEST7(PMEM2_MOVER_MT_TEST7): pass +# XXX disable the test for 'drd`` +# until https://github.com/pmem/pmdk/issues/5694 is fixed. +@t.require_valgrind_disabled('pmemcheck', 'drd') @g.require_granularity(g.BYTE) -@t.require_valgrind_disabled('pmemcheck') class TEST71(PMEM2_MOVER_MT_TEST7): pass @@ -136,13 +146,18 @@ class PMEM2_MOVER_MT_TEST8(PMEM2_MOVER_MT): test_case = "test_mover_memset_multithreaded" +# XXX disable the test for 'drd`` +# until https://github.com/pmem/pmdk/issues/5694 is fixed. +@t.require_valgrind_disabled('drd') @g.require_granularity(g.PAGE, g.CACHELINE) class TEST8(PMEM2_MOVER_MT_TEST8): pass +# XXX disable the test for 'drd`` +# until https://github.com/pmem/pmdk/issues/5694 is fixed. +@t.require_valgrind_disabled('pmemcheck', 'drd') @g.require_granularity(g.BYTE) -@t.require_valgrind_disabled('pmemcheck') class TEST81(PMEM2_MOVER_MT_TEST8): pass