From d517dc76440fc3f70a990de9a15a8ba33895535d Mon Sep 17 00:00:00 2001 From: Mariusz Zaborski Date: Thu, 13 Jun 2024 07:37:57 -0400 Subject: [PATCH 1/6] [tests,ci] Allow to build only tests Signed-off-by: Mariusz Zaborski --- .ci/lib/stage-build-test-only.jenkinsfile | 23 +++++++++++++ .ci/linux-tests-only.jenkinsfile | 14 ++++++++ libos/meson.build | 6 ++-- libos/test/regression/meson.build | 2 +- meson.build | 40 +++++++++++++++-------- meson_options.txt | 2 ++ pal/meson.build | 10 +++--- python/graminelibos/__init__.py | 3 +- 8 files changed, 78 insertions(+), 22 deletions(-) create mode 100644 .ci/lib/stage-build-test-only.jenkinsfile create mode 100644 .ci/linux-tests-only.jenkinsfile diff --git a/.ci/lib/stage-build-test-only.jenkinsfile b/.ci/lib/stage-build-test-only.jenkinsfile new file mode 100644 index 0000000000..3c6770df1f --- /dev/null +++ b/.ci/lib/stage-build-test-only.jenkinsfile @@ -0,0 +1,23 @@ +stage('build') { + try { + sh ''' + meson setup build/ \ + --werror \ + --prefix="$PREFIX" \ + --buildtype="$BUILDTYPE" \ + -Dskeleton=disabled \ + -Ddirect=disabled \ + -Dsgx=disabled \ + -Dtests=enabled \ + $MESON_OPTIONS + ninja -vC build/ + ''' + } finally { + archiveArtifacts ''' + build/meson-logs/**/*, + ''' + } + + sh 'rm -rf build' + sh 'git clean -Xf subprojects' +} diff --git a/.ci/linux-tests-only.jenkinsfile b/.ci/linux-tests-only.jenkinsfile new file mode 100644 index 0000000000..ec6299bbb0 --- /dev/null +++ b/.ci/linux-tests-only.jenkinsfile @@ -0,0 +1,14 @@ +node('nonsgx_slave && aesni') { + checkout scm + + load '.ci/lib/config-docker.jenkinsfile' + docker.build( + "local:${env.BUILD_TAG}", + '-f .ci/ubuntu20.04.dockerfile .' + ).inside("${env.DOCKER_ARGS_COMMON} --security-opt seccomp=${env.WORKSPACE}/scripts/docker_seccomp_mar_2021.json") { + load '.ci/lib/config.jenkinsfile' + load '.ci/lib/config-clang.jenkinsfile' + + load '.ci/lib/stage-build-test-only.jenkinsfile' + } +} diff --git a/libos/meson.build b/libos/meson.build index f99fa39edc..9365f244d3 100644 --- a/libos/meson.build +++ b/libos/meson.build @@ -17,8 +17,10 @@ if host_machine.cpu_family() == 'x86_64' cflags_libos += ['-mfxsr', '-mxsave'] endif -subdir('include') -subdir('src') +if src_build + subdir('include') + subdir('src') +endif if enable_tests subdir('test') diff --git a/libos/test/regression/meson.build b/libos/test/regression/meson.build index c1270607e5..44c33a7a55 100644 --- a/libos/test/regression/meson.build +++ b/libos/test/regression/meson.build @@ -192,7 +192,7 @@ if host_machine.cpu_family() == 'x86_64' } endif -if sgx +if sgx or force_sgx_tests tests += { 'attestation': { # for `sgx_arch.h` diff --git a/meson.build b/meson.build index 329272b844..8e7f5db233 100644 --- a/meson.build +++ b/meson.build @@ -50,6 +50,9 @@ vtune = get_option('vtune') == 'enabled' enable_libgomp = get_option('libgomp') == 'enabled' enable_tests = get_option('tests') == 'enabled' +force_sgx_tests = get_option('force_sgx_tests') == 'enabled' + +src_build = skeleton or direct or sgx cc = meson.get_compiler('c') host_has_glibc = cc.get_define('__GLIBC__', prefix: '#include ') != '' @@ -230,19 +233,22 @@ endif # # Dependencies # +if src_build + tomlc99_proj = subproject('tomlc99-208203af46bdbdb29ba199660ed78d09c220b6c5') + tomlc99_dep = tomlc99_proj.get_variable('tomlc99_dep') + tomlc99_src = tomlc99_proj.get_variable('tomlc99_src') -tomlc99_proj = subproject('tomlc99-208203af46bdbdb29ba199660ed78d09c220b6c5') -tomlc99_dep = tomlc99_proj.get_variable('tomlc99_dep') -tomlc99_src = tomlc99_proj.get_variable('tomlc99_src') - -uthash_dep = subproject('uthash-2.1.0').get_variable('uthash_dep') + uthash_dep = subproject('uthash-2.1.0').get_variable('uthash_dep') +endif mbedtls_proj = subproject('mbedtls-3.6.0') mbedtls_static_dep = mbedtls_proj.get_variable('mbedtls_static_dep') mbedtls_pal_dep = mbedtls_proj.get_variable('mbedtls_pal_dep') -curl_proj = subproject('curl-8.8.0') -cjson_proj = subproject('cJSON-1.7.12') +if src_build + curl_proj = subproject('curl-8.8.0') + cjson_proj = subproject('cJSON-1.7.12') +endif if sgx # XXX: do not call subproject() from under "if sgx" conditional, because it @@ -288,16 +294,22 @@ endif # The compilation # -subdir('common') -subdir('pal') +if src_build + subdir('common') + subdir('pal') +endif subdir('libos') subdir('python') -subdir('tools') +if src_build + subdir('tools') +endif -if get_option('libc') == 'glibc' - subproject('glibc-2.39-1') -elif get_option('libc') == 'musl' - subproject('musl-1.2.4') +if src_build + if get_option('libc') == 'glibc' + subproject('glibc-2.39-1') + elif get_option('libc') == 'musl' + subproject('musl-1.2.4') + endif endif if enable_libgomp diff --git a/meson_options.txt b/meson_options.txt index e29929f6d7..4787eb5085 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -11,6 +11,8 @@ option('libc', type: 'combo', choices: ['none', 'glibc', 'musl'], option('tests', type: 'combo', choices: ['disabled', 'enabled'], description: 'Build test binaries') +option('force_sgx_tests', type: 'combo', choices: ['disabled', 'enabled'], + description: 'Build sgx tests even if sgx is not enabled') option('dcap', type: 'combo', choices: ['disabled', 'enabled'], description: 'Build additional utilities linked against DCAP library') diff --git a/pal/meson.build b/pal/meson.build index 8c5799ac8f..254d21dee7 100644 --- a/pal/meson.build +++ b/pal/meson.build @@ -1,8 +1,10 @@ -subdir('include') -subdir('src') +if src_build + subdir('include') + subdir('src') -if enable_tests - subdir('regression') + if enable_tests + subdir('regression') + endif endif if debug diff --git a/python/graminelibos/__init__.py b/python/graminelibos/__init__.py index a96d57e768..5c19e7ddfc 100644 --- a/python/graminelibos/__init__.py +++ b/python/graminelibos/__init__.py @@ -6,7 +6,8 @@ _CONFIG_PKGLIBDIR = '@PKGLIBDIR@' _CONFIG_LIBDIR = '@LIBDIR@' _CONFIG_SYSLIBDIR = '@SYSLIBDIR@' -_CONFIG_SGX_ENABLED = '@SGX_ENABLED@' == '1' +_CONFIG_SGX_ENABLED = _os.path.exists('@PKGLIBDIR@/sgx/libpal.so') + if __version__.startswith('@') and not _os.getenv('GRAMINE_IMPORT_FOR_SPHINX_ANYWAY') == '1': raise RuntimeError( From dcd70a50f3e0b2c8743998a3053ed03766149590 Mon Sep 17 00:00:00 2001 From: Mariusz Zaborski Date: Tue, 18 Jun 2024 12:38:01 -0400 Subject: [PATCH 2/6] squash! [tests,ci] Allow to build only tests !TODO: Use this commit msg: [tests,CI] Allow to build only tests Signed-off-by: Mariusz Zaborski --- libos/meson.build | 4 ++-- meson.build | 32 ++++++++++++++++---------------- pal/meson.build | 10 +++++----- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/libos/meson.build b/libos/meson.build index 9365f244d3..cb2f4f2be8 100644 --- a/libos/meson.build +++ b/libos/meson.build @@ -18,8 +18,8 @@ if host_machine.cpu_family() == 'x86_64' endif if src_build - subdir('include') - subdir('src') + subdir('include') + subdir('src') endif if enable_tests diff --git a/meson.build b/meson.build index 8e7f5db233..c1d98e08d2 100644 --- a/meson.build +++ b/meson.build @@ -23,9 +23,9 @@ project( # We can't change to c23 for any supported versions yet (requires at # least gcc 14). if meson.get_compiler('c').has_argument('-std=c23') - add_project_arguments('-std=c23', language: 'c') + add_project_arguments('-std=c23', language: 'c') elif meson.get_compiler('c').has_argument('-std=c2x') - add_project_arguments('-std=c2x', language: 'c') + add_project_arguments('-std=c2x', language: 'c') endif # we need this subdir() early, because we need scripts defined there for setting up global vars @@ -234,11 +234,11 @@ endif # Dependencies # if src_build - tomlc99_proj = subproject('tomlc99-208203af46bdbdb29ba199660ed78d09c220b6c5') - tomlc99_dep = tomlc99_proj.get_variable('tomlc99_dep') - tomlc99_src = tomlc99_proj.get_variable('tomlc99_src') + tomlc99_proj = subproject('tomlc99-208203af46bdbdb29ba199660ed78d09c220b6c5') + tomlc99_dep = tomlc99_proj.get_variable('tomlc99_dep') + tomlc99_src = tomlc99_proj.get_variable('tomlc99_src') - uthash_dep = subproject('uthash-2.1.0').get_variable('uthash_dep') + uthash_dep = subproject('uthash-2.1.0').get_variable('uthash_dep') endif mbedtls_proj = subproject('mbedtls-3.6.0') @@ -246,8 +246,8 @@ mbedtls_static_dep = mbedtls_proj.get_variable('mbedtls_static_dep') mbedtls_pal_dep = mbedtls_proj.get_variable('mbedtls_pal_dep') if src_build - curl_proj = subproject('curl-8.8.0') - cjson_proj = subproject('cJSON-1.7.12') + curl_proj = subproject('curl-8.8.0') + cjson_proj = subproject('cJSON-1.7.12') endif if sgx @@ -295,21 +295,21 @@ endif # if src_build - subdir('common') - subdir('pal') + subdir('common') + subdir('pal') endif subdir('libos') subdir('python') if src_build - subdir('tools') + subdir('tools') endif if src_build - if get_option('libc') == 'glibc' - subproject('glibc-2.39-1') - elif get_option('libc') == 'musl' - subproject('musl-1.2.4') - endif + if get_option('libc') == 'glibc' + subproject('glibc-2.39-1') + elif get_option('libc') == 'musl' + subproject('musl-1.2.4') + endif endif if enable_libgomp diff --git a/pal/meson.build b/pal/meson.build index 254d21dee7..6004a752fc 100644 --- a/pal/meson.build +++ b/pal/meson.build @@ -1,10 +1,10 @@ if src_build - subdir('include') - subdir('src') + subdir('include') + subdir('src') - if enable_tests - subdir('regression') - endif + if enable_tests + subdir('regression') + endif endif if debug From 6205cfae7439c39c71dc3725b80d64518e06a481 Mon Sep 17 00:00:00 2001 From: Mariusz Zaborski Date: Wed, 19 Jun 2024 07:36:36 -0400 Subject: [PATCH 3/6] fixup! [tests,ci] Allow to build only tests Signed-off-by: Mariusz Zaborski --- subprojects/packagefiles/mbedtls/meson.build | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/subprojects/packagefiles/mbedtls/meson.build b/subprojects/packagefiles/mbedtls/meson.build index bc084767ef..66daf12612 100644 --- a/subprojects/packagefiles/mbedtls/meson.build +++ b/subprojects/packagefiles/mbedtls/meson.build @@ -61,7 +61,8 @@ pkgconfig.generate( if get_option('libc') == 'glibc' and host_has_glibc foreach output : mbedtls_libs_output meson.add_install_script('/bin/sh', '-c', - ('ln -sf ../../../@0@ ' + + ('mkdir -p "$MESON_INSTALL_DESTDIR_PREFIX"/@1@/gramine/runtime/@2@/i && ' + + 'ln -sf ../../../@0@ ' + '"$MESON_INSTALL_DESTDIR_PREFIX"/@1@/gramine/runtime/@2@/').format( output, get_option('libdir'), get_option('libc'))) endforeach From 9dfb2fa8b8d6e5c55d3c01132a8026cda4f709d5 Mon Sep 17 00:00:00 2001 From: Mariusz Zaborski Date: Mon, 24 Jun 2024 07:19:16 -0400 Subject: [PATCH 4/6] squash! [tests,ci] Allow to build only tests !TODO: Use this commit msg: [tests,CI] Allow to build only tests This commit allows the test suite to run on a Gramine version that was installed on the system by other means (e.g., via a package or Docker). Signed-off-by: Mariusz Zaborski --- .ci/linux-tests-only.jenkinsfile | 2 +- libos/test/regression/meson.build | 24 +++++++++++------------- meson.build | 2 +- meson_options.txt | 2 -- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/.ci/linux-tests-only.jenkinsfile b/.ci/linux-tests-only.jenkinsfile index ec6299bbb0..422c11c083 100644 --- a/.ci/linux-tests-only.jenkinsfile +++ b/.ci/linux-tests-only.jenkinsfile @@ -1,4 +1,4 @@ -node('nonsgx_slave && aesni') { +node('nonsgx_slave') { checkout scm load '.ci/lib/config-docker.jenkinsfile' diff --git a/libos/test/regression/meson.build b/libos/test/regression/meson.build index 44c33a7a55..43bf7be896 100644 --- a/libos/test/regression/meson.build +++ b/libos/test/regression/meson.build @@ -192,20 +192,18 @@ if host_machine.cpu_family() == 'x86_64' } endif -if sgx or force_sgx_tests - tests += { - 'attestation': { - # for `sgx_arch.h` - 'include_directories': include_directories('../../../pal/src/host/linux-sgx'), - 'source': [ - 'attestation.c', - '../../../pal/src/host/linux-sgx/enclave_api.S', - ], +tests += { + 'attestation': { + # for `sgx_arch.h` + 'include_directories': include_directories('../../../pal/src/host/linux-sgx'), + 'source': [ + 'attestation.c', + '../../../pal/src/host/linux-sgx/enclave_api.S', + ], - 'dependencies': mbedtls_static_dep, - }, - } -endif + 'dependencies': mbedtls_static_dep, + }, +} install_dir = pkglibdir / 'tests' / 'libos' / 'regression' diff --git a/meson.build b/meson.build index c1d98e08d2..a87290cd44 100644 --- a/meson.build +++ b/meson.build @@ -50,7 +50,6 @@ vtune = get_option('vtune') == 'enabled' enable_libgomp = get_option('libgomp') == 'enabled' enable_tests = get_option('tests') == 'enabled' -force_sgx_tests = get_option('force_sgx_tests') == 'enabled' src_build = skeleton or direct or sgx @@ -241,6 +240,7 @@ if src_build uthash_dep = subproject('uthash-2.1.0').get_variable('uthash_dep') endif +# Build Mbed TLS because some tests, such as attestation tests, require it. mbedtls_proj = subproject('mbedtls-3.6.0') mbedtls_static_dep = mbedtls_proj.get_variable('mbedtls_static_dep') mbedtls_pal_dep = mbedtls_proj.get_variable('mbedtls_pal_dep') diff --git a/meson_options.txt b/meson_options.txt index 4787eb5085..e29929f6d7 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -11,8 +11,6 @@ option('libc', type: 'combo', choices: ['none', 'glibc', 'musl'], option('tests', type: 'combo', choices: ['disabled', 'enabled'], description: 'Build test binaries') -option('force_sgx_tests', type: 'combo', choices: ['disabled', 'enabled'], - description: 'Build sgx tests even if sgx is not enabled') option('dcap', type: 'combo', choices: ['disabled', 'enabled'], description: 'Build additional utilities linked against DCAP library') From 0f1b5ed68c63045f469a082386688fc747fc1068 Mon Sep 17 00:00:00 2001 From: Mariusz Zaborski Date: Tue, 2 Jul 2024 09:43:35 -0400 Subject: [PATCH 5/6] fixup! [tests,ci] Allow to build only tests Signed-off-by: Mariusz Zaborski --- .ci/linux-tests-only.jenkinsfile | 1 + meson.build | 3 ++- subprojects/packagefiles/mbedtls/meson.build | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.ci/linux-tests-only.jenkinsfile b/.ci/linux-tests-only.jenkinsfile index 422c11c083..231d59f9e3 100644 --- a/.ci/linux-tests-only.jenkinsfile +++ b/.ci/linux-tests-only.jenkinsfile @@ -10,5 +10,6 @@ node('nonsgx_slave') { load '.ci/lib/config-clang.jenkinsfile' load '.ci/lib/stage-build-test-only.jenkinsfile' + # XXX: we should also run test } } diff --git a/meson.build b/meson.build index a87290cd44..b26dc60245 100644 --- a/meson.build +++ b/meson.build @@ -240,7 +240,8 @@ if src_build uthash_dep = subproject('uthash-2.1.0').get_variable('uthash_dep') endif -# Build Mbed TLS because some tests, such as attestation tests, require it. +# Build Mbed TLS even if !src_build because some tests (attestation tests) +# require it. mbedtls_proj = subproject('mbedtls-3.6.0') mbedtls_static_dep = mbedtls_proj.get_variable('mbedtls_static_dep') mbedtls_pal_dep = mbedtls_proj.get_variable('mbedtls_pal_dep') diff --git a/subprojects/packagefiles/mbedtls/meson.build b/subprojects/packagefiles/mbedtls/meson.build index 66daf12612..d362560993 100644 --- a/subprojects/packagefiles/mbedtls/meson.build +++ b/subprojects/packagefiles/mbedtls/meson.build @@ -61,7 +61,7 @@ pkgconfig.generate( if get_option('libc') == 'glibc' and host_has_glibc foreach output : mbedtls_libs_output meson.add_install_script('/bin/sh', '-c', - ('mkdir -p "$MESON_INSTALL_DESTDIR_PREFIX"/@1@/gramine/runtime/@2@/i && ' + + ('mkdir -p "$MESON_INSTALL_DESTDIR_PREFIX"/@1@/gramine/runtime/@2@ && ' + 'ln -sf ../../../@0@ ' + '"$MESON_INSTALL_DESTDIR_PREFIX"/@1@/gramine/runtime/@2@/').format( output, get_option('libdir'), get_option('libc'))) From 7828b305f9d72dbc3cc391e0ae7d47d6f92179ba Mon Sep 17 00:00:00 2001 From: Mariusz Zaborski Date: Wed, 3 Jul 2024 13:51:52 -0400 Subject: [PATCH 6/6] fixup! [tests,ci] Allow to build only tests Signed-off-by: Mariusz Zaborski --- subprojects/packagefiles/mbedtls/meson.build | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/subprojects/packagefiles/mbedtls/meson.build b/subprojects/packagefiles/mbedtls/meson.build index d362560993..104a3260a3 100644 --- a/subprojects/packagefiles/mbedtls/meson.build +++ b/subprojects/packagefiles/mbedtls/meson.build @@ -58,11 +58,10 @@ pkgconfig.generate( ], ) -if get_option('libc') == 'glibc' and host_has_glibc +if get_option('libc') == 'glibc' and host_has_glibc and src_build foreach output : mbedtls_libs_output meson.add_install_script('/bin/sh', '-c', - ('mkdir -p "$MESON_INSTALL_DESTDIR_PREFIX"/@1@/gramine/runtime/@2@ && ' + - 'ln -sf ../../../@0@ ' + + ('ln -sf ../../../@0@ ' + '"$MESON_INSTALL_DESTDIR_PREFIX"/@1@/gramine/runtime/@2@/').format( output, get_option('libdir'), get_option('libc'))) endforeach