Enable more tests for wasm32-wasi builds

.github/workflows/ci.yml

@@ -371,13 +371,20 @@ jobs:
     runs-on: ${{matrix.os}}
 
     steps:
+    - name: Set environment
+      run: |
+        echo "builddir=${{github.workspace}}/build.wasm" >> ${GITHUB_ENV}
+        echo "TOYWASM_NATIVE=${{github.workspace}}/build.native/toywasm" >> ${GITHUB_ENV}
+
     - name: Install dependencies (ubuntu)
       if: startsWith(matrix.os, 'ubuntu-')
-      run: sudo apt-get update && sudo apt-get install -y pax ninja-build
+      run: sudo apt-get update && sudo apt-get install -y wabt pax virtualenv ninja-build
 
     - name: Install dependencies (macOS)
       if: startsWith(matrix.os, 'macos-')
       run: |
+        brew install wabt
+        brew install virtualenv
         brew install ninja
 
     - uses: actions/checkout@v3
@@ -390,6 +397,24 @@ jobs:
         EXTRA_CMAKE_OPTIONS: -G Ninja -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/dist -DTOYWASM_TARBALL_SUFFIX=-${{matrix.name}} ${{matrix.extra}}
       run: ./wasm-on-wasm.sh
 
+    - name: toywasm --version
+      env:
+        TOYWASM_WASM: ${{env.builddir}}/toywasm
+      working-directory: ${{env.builddir}}
+      run: |
+        ${{github.workspace}}/test/toywasm-on-toywasm.py --version
+
+    - name: Test
+      working-directory: ${{env.builddir}}
+      run: |
+        ctest -V -LE slow
+
+    - name: Test (slow)
+      if: false
+      working-directory: ${{env.builddir}}
+      run: |
+        ctest -V -L slow
+
     - name: Install
       run: |
         cmake --build build.wasm --target install

CMakeLists.txt

@@ -25,6 +25,21 @@ if(TOYWASM_ENABLE_WASI)
 add_subdirectory(libwasi)
 endif()
 
+if(CMAKE_C_COMPILER_TARGET MATCHES "wasm")
+set(TOYWASM_CLI "${CMAKE_CURRENT_SOURCE_DIR}/test/toywasm-on-toywasm.py")
+if(NOT DEFINED ENV{TOYWASM_NATIVE})
+list(APPEND TEST_ENV "TOYWASM_NATIVE=${CMAKE_CURRENT_SOURCE_DIR}/build.native/toywasm")
+endif()
+if(NOT DEFINED ENV{TOYWASM_WASM})
+list(APPEND TEST_ENV "TOYWASM_WASM=${CMAKE_BINARY_DIR}/toywasm")
+endif()
+else()
+set(TOYWASM_CLI "${CMAKE_BINARY_DIR}/toywasm")
+endif()
+
+# for wasi-testsuite-adapter.py
+list(APPEND TEST_ENV "TEST_RUNTIME_EXE=${TOYWASM_CLI}")
+
 # cli
 
 set(cli_sources
@@ -37,7 +52,7 @@ target_link_libraries(toywasm-cli toywasm-lib-core $<$<BOOL:${TOYWASM_ENABLE_WAS
 set_target_properties(toywasm-cli PROPERTIES OUTPUT_NAME toywasm)
 
 add_test(NAME toywasm-cli-simple-module COMMAND
-	toywasm-cli --load spectest.wasm --invoke "print_i32 123"
+	${TOYWASM_CLI} --load=spectest.wasm "--invoke=print_i32 123"
 )
 set_tests_properties(toywasm-cli-simple-module PROPERTIES ENVIRONMENT "${TEST_ENV}")
 
@@ -48,7 +63,7 @@ if(NOT TOYWASM_ENABLE_WASM_MULTI_MEMORY)
 add_test(NAME toywasm-cli-wasm3-spec-test
 	# Note: arbitrary limits for stack overflow tests in call.wast.
 	# (--max-frames and --max-stack-cells)
-	COMMAND ./test/run-wasm3-spec-test-opam-2.0.0.sh --exec "${CMAKE_BINARY_DIR}/toywasm --max-frames 2000 --max-stack-cells 10000 --repl --repl-prompt wasm3" --timeout 60 --spectest ${CMAKE_BINARY_DIR}/spectest.wasm
+	COMMAND ./test/run-wasm3-spec-test-opam-2.0.0.sh --exec "${TOYWASM_CLI} --max-frames=2000 --max-stack-cells=10000 --repl --repl-prompt=wasm3" --timeout 60 --spectest ${CMAKE_BINARY_DIR}/spectest.wasm
 	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 )
 set_tests_properties(toywasm-cli-wasm3-spec-test PROPERTIES ENVIRONMENT "${TEST_ENV}")
@@ -57,7 +72,7 @@ endif()
 
 if(TOYWASM_ENABLE_WASM_SIMD)
 add_test(NAME toywasm-cli-wasm3-spec-test-simd
-	COMMAND ./test/run-wasm3-spec-test-simd.sh --exec "${CMAKE_BINARY_DIR}/toywasm --repl --repl-prompt wasm3" --timeout 60 --spectest ${CMAKE_BINARY_DIR}/spectest.wasm
+	COMMAND ./test/run-wasm3-spec-test-simd.sh --exec "${TOYWASM_CLI} --repl --repl-prompt=wasm3" --timeout 60 --spectest ${CMAKE_BINARY_DIR}/spectest.wasm
 	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 )
 set_tests_properties(toywasm-cli-wasm3-spec-test-simd PROPERTIES ENVIRONMENT "${TEST_ENV}")
@@ -66,7 +81,7 @@ endif()
 
 if(TOYWASM_ENABLE_WASM_EXTENDED_CONST)
 add_test(NAME toywasm-cli-wasm3-spec-test-extended-const
-	COMMAND ./test/run-wasm3-spec-test-extended-const.sh --exec "${CMAKE_BINARY_DIR}/toywasm --repl --repl-prompt wasm3" --timeout 60 --spectest ${CMAKE_BINARY_DIR}/spectest.wasm
+	COMMAND ./test/run-wasm3-spec-test-extended-const.sh --exec "${TOYWASM_CLI} --repl --repl-prompt=wasm3" --timeout 60 --spectest ${CMAKE_BINARY_DIR}/spectest.wasm
 	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 )
 set_tests_properties(toywasm-cli-wasm3-spec-test-extended-const PROPERTIES ENVIRONMENT "${TEST_ENV}")
@@ -75,7 +90,7 @@ endif()
 
 if(TOYWASM_ENABLE_WASM_MULTI_MEMORY)
 add_test(NAME toywasm-cli-wasm3-spec-test-multi-memory
-	COMMAND ./test/run-wasm3-spec-test-multi-memory.sh --exec "${CMAKE_BINARY_DIR}/toywasm --repl --repl-prompt wasm3" --timeout 60 --spectest ${CMAKE_BINARY_DIR}/spectest.wasm
+	COMMAND ./test/run-wasm3-spec-test-multi-memory.sh --exec "${TOYWASM_CLI} --repl --repl-prompt=wasm3" --timeout 60 --spectest ${CMAKE_BINARY_DIR}/spectest.wasm
 	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 )
 set_tests_properties(toywasm-cli-wasm3-spec-test-multi-memory PROPERTIES ENVIRONMENT "${TEST_ENV}")
@@ -84,7 +99,7 @@ endif()
 
 if(TOYWASM_ENABLE_WASM_TAILCALL)
 add_test(NAME toywasm-cli-wasm3-spec-test-tailcall
-	COMMAND ./test/run-wasm3-spec-test-tailcall.sh --exec "${CMAKE_BINARY_DIR}/toywasm --repl --repl-prompt wasm3" --timeout 60 --spectest ${CMAKE_BINARY_DIR}/spectest.wasm
+	COMMAND ./test/run-wasm3-spec-test-tailcall.sh --exec "${TOYWASM_CLI} --repl --repl-prompt=wasm3" --timeout 60 --spectest ${CMAKE_BINARY_DIR}/spectest.wasm
 	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 )
 set_tests_properties(toywasm-cli-wasm3-spec-test-tailcall PROPERTIES ENVIRONMENT "${TEST_ENV}")
@@ -93,7 +108,7 @@ endif()
 
 if(TOYWASM_ENABLE_WASM_THREADS)
 add_test(NAME toywasm-cli-wasm3-spec-test-threads
-	COMMAND ./test/run-wasm3-spec-test-threads.sh --exec "${CMAKE_BINARY_DIR}/toywasm --repl --repl-prompt wasm3" --timeout 60 --spectest ${CMAKE_BINARY_DIR}/spectest.wasm
+	COMMAND ./test/run-wasm3-spec-test-threads.sh --exec "${TOYWASM_CLI} --repl --repl-prompt=wasm3" --timeout 60 --spectest ${CMAKE_BINARY_DIR}/spectest.wasm
 	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 )
 set_tests_properties(toywasm-cli-wasm3-spec-test-threads PROPERTIES ENVIRONMENT "${TEST_ENV}")
@@ -105,24 +120,24 @@ add_test(NAME toywasm-cli-wasi-testsuite
 	COMMAND ./test/run-wasi-testsuite.sh
 	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 )
-set_tests_properties(toywasm-cli-wasi-testsuite PROPERTIES ENVIRONMENT "${TEST_ENV};TOYWASM=${CMAKE_BINARY_DIR}/toywasm;$<$<BOOL:${TOYWASM_ENABLE_WASI_THREADS}>:TESTS=proposals/wasi-threads/>")
+set_tests_properties(toywasm-cli-wasi-testsuite PROPERTIES ENVIRONMENT "${TEST_ENV};TOYWASM=${TOYWASM_CLI};$<$<BOOL:${TOYWASM_ENABLE_WASI_THREADS}>:TESTS=proposals/wasi-threads/>")
 set_tests_properties(toywasm-cli-wasi-testsuite PROPERTIES LABELS "wasi-testsuite")
 
 add_test(NAME toywasm-cli-wasmtime-wasi-tests
-	COMMAND ./test/run-wasmtime-wasi-tests.sh "${CMAKE_BINARY_DIR}/toywasm --wasi --wasi-dir ."
+	COMMAND ./test/run-wasmtime-wasi-tests.sh "${TOYWASM_CLI} --wasi --wasi-dir=."
 	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 )
 set_tests_properties(toywasm-cli-wasmtime-wasi-tests PROPERTIES ENVIRONMENT "${TEST_ENV}")
 set_tests_properties(toywasm-cli-wasmtime-wasi-tests PROPERTIES LABELS "wasmtime-wasi-tests")
 
 add_test(NAME toywasm-cli-spidermonkey
-	COMMAND ./test/run-spidermonkey.sh ${CMAKE_BINARY_DIR}/toywasm --wasi
+	COMMAND ./test/run-spidermonkey.sh ${TOYWASM_CLI} --wasi
 	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 )
 set_tests_properties(toywasm-cli-spidermonkey PROPERTIES ENVIRONMENT "${TEST_ENV}")
 
 add_test(NAME toywasm-cli-ffmpeg
-	COMMAND ./test/run-ffmpeg.sh ${CMAKE_BINARY_DIR}/toywasm --wasi --wasi-dir .video --
+	COMMAND ./test/run-ffmpeg.sh ${TOYWASM_CLI} --wasi --wasi-dir=.video --
 	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 )
 set_tests_properties(toywasm-cli-ffmpeg PROPERTIES ENVIRONMENT "${TEST_ENV}")
@@ -133,7 +148,7 @@ if(NOT TOYWASM_ENABLE_WASM_MULTI_MEMORY)
 add_test(NAME toywasm-cli-wasm3-spec-test-disable-optimizations
 	# Note: arbitrary limits for stack overflow tests in call.wast.
 	# (--max-frames and --max-stack-cells)
-	COMMAND ./test/run-wasm3-spec-test-opam-2.0.0.sh --exec "${CMAKE_BINARY_DIR}/toywasm --disable-jump-table --disable-resulttype-cellidx --disable-localtype-cellidx --max-frames 2000 --max-stack-cells 10000 --repl --repl-prompt wasm3" --timeout 60 --spectest ${CMAKE_BINARY_DIR}/spectest.wasm
+	COMMAND ./test/run-wasm3-spec-test-opam-2.0.0.sh --exec "${TOYWASM_CLI} --disable-jump-table --disable-resulttype-cellidx --disable-localtype-cellidx --max-frames=2000 --max-stack-cells=10000 --repl --repl-prompt=wasm3" --timeout 60 --spectest ${CMAKE_BINARY_DIR}/spectest.wasm
 	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 )
 set_tests_properties(toywasm-cli-wasm3-spec-test-disable-optimizations PROPERTIES ENVIRONMENT "${TEST_ENV}")
@@ -142,7 +157,7 @@ endif()
 
 if(TOYWASM_ENABLE_WASI)
 add_test(NAME toywasm-cli-wasm3-wasi-test
-	COMMAND ./test/run-wasm3-wasi-test.sh --exec "${CMAKE_BINARY_DIR}/toywasm --wasi --wasi-dir ." --separate-args --timeout 1200
+	COMMAND ./test/run-wasm3-wasi-test.sh --exec "${TOYWASM_CLI} --wasi --wasi-dir=." --separate-args --timeout 1200
 	WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 )
 set_tests_properties(toywasm-cli-wasm3-wasi-test PROPERTIES ENVIRONMENT "${TEST_ENV}")
@@ -171,6 +186,7 @@ endif()
 # unit test
 
 if(BUILD_TESTING)
+if(TOYWASM_BUILD_UNITTEST)
 find_package(cmocka REQUIRED)
 
 # compat with cmocka <1.1.6
@@ -196,6 +212,7 @@ target_link_libraries(toywasm-test toywasm-lib-core m ${CMOCKA_LIBRARY})
 add_test(NAME toywasm-test COMMAND toywasm-test)
 set_tests_properties(toywasm-test PROPERTIES ENVIRONMENT "${TEST_ENV}")
 target_link_libraries(toywasm-test cmocka::cmocka)
+endif() # TOYWASM_BUILD_UNITTEST
 endif()
 
 # XXX Is there a way to create the file list from install() commands?

README.md

@@ -84,13 +84,21 @@ It includes
 * macOS/amd64
 * Ubuntu/amd64
 
-#### With qemu (less coverage because of slowness)
+#### With qemu
 
 * Ubuntu/arm64
 * Ubuntu/armhf (Note: 32-bit)
+
+#### With qemu (less coverage because of slowness)
+
 * Ubuntu/s390x (Note: big endian)
+* Ubuntu/risc-v
+
+(I haven't investigated why they are slower than others.
+It might be related to the fact that our build disables LTO
+for them for toolchain issues.)
 
-#### with even less coverage
+#### With toywasm (less coverage because of slowness)
 
 * wasm32-wasi
 

build-wasm32-wasi.sh

@@ -50,7 +50,7 @@ cmake \
 -B ${BUILD_DIR} \
 -DWASI_SDK_PREFIX=${WASI_SDK_DIR} \
 -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} \
--DBUILD_TESTING=OFF \
+-DTOYWASM_BUILD_UNITTEST=OFF \
 -DTOYWASM_USE_TAILCALL=${USE_TAILCALL:-OFF} \
 ${EXTRA_CMAKE_OPTIONS} \
 .

cmake/ToywasmConfig.cmake

@@ -26,6 +26,8 @@ option(TOYWASM_ENABLE_WASM_THREADS "Enable WASM threads proposal" OFF)
 option(TOYWASM_ENABLE_WASI "Enable WASI snapshow preview1" ON)
 option(TOYWASM_ENABLE_WASI_THREADS "Enable wasi-threads proposal" OFF)
 
+option(TOYWASM_BUILD_UNITTEST "Build toywasm-test" ON)
+
 if(NOT DEFINED USE_LSAN)
 set(USE_LSAN ON)
 endif()

test/run-wasmtime-wasi-tests.sh

@@ -36,7 +36,7 @@ if [ "${WASM_ON_WASM:-0}" -ne 0 ]; then
 else
     run_wasi_test()
     {
-        ${EXE} --wasi-dir ${TMP} ${w} ${TMP};
+        ${EXE} --wasi-dir=${TMP} -- ${w} ${TMP};
     }
     if ${EXE} --version | grep -F "sizeof(void *) = 4"; then
         SKIPLIST="${SKIPLIST} ${THIS_DIR}/wasmtime-wasi-tests-skip-32bit.txt"

test/toywasm-on-toywasm.py

@@ -7,7 +7,7 @@
 import sys
 import subprocess
 
-executable = os.getenv("TOYWASM", "./build.native/toywasm")
+executable = os.getenv("TOYWASM_NATIVE", "./build.native/toywasm")
 executable_wasm = os.getenv("TOYWASM_WASM", "./build.wasm/toywasm")
 
 parser = argparse.ArgumentParser(allow_abbrev=False)
@@ -16,6 +16,7 @@
 args, unknown = parser.parse_known_args()
 
 sys_prefix = "@TOYWASM@"
+debug = False
 
 
 def translate_path(cat, name):
@@ -38,13 +39,40 @@ def translate_path(cat, name):
     g, h = x.split("::", maxsplit=2)
     options.append(f"--wasi-dir={h}")
 
-# assume that the first argument which doesn't
-# start with "--" is a wasm module.
+# assume that the first argument which doesn't start with "--" is
+# a wasm module.
+# also, translate --load argument.
+# note: this logic recognizes only a subset of toywasm cli options.
+# you can use "--invoke=foo" instead of "--invoke foo" to avoid
+# misinterpretations.
 user_args = sys.argv[1:]
+if debug:
+    print(f"Original: {user_args}")
+load_arg = False
+options_done = False
 for i in range(0, len(user_args)):
-    if not user_args[i].startswith("--"):
+    if load_arg:
+        user_args[i] = translate_path(f"load-wasm-{i}", user_args[i])
+        load_arg = False
+        continue
+    if not options_done:
+        if user_args[i] == "--":
+            options_done = True
+            continue
+        if user_args[i] == "--load":
+            load_arg = True
+            continue
+        if user_args[i].startswith("--load="):
+            _, wasm = user_args[i].split("=", 1)
+            translated = translate_path(f"load-wasm-{i}", wasm)
+            user_args[i] = f"--load={translated}"
+            continue
+    if options_done or not user_args[i].startswith("--"):
         user_args[i] = translate_path("user-wasm", user_args[i])
         break
+if debug:
+    print(f"Translated: {user_args}")
+    print(f"Host toywasm options: {options}")
 
 cmd = (
     shlex.split(executable) + ["--wasi"] + options + ["--", executable_wasm] + user_args

test/wasi-testsuite-adapter.py

@@ -30,9 +30,9 @@
 
 options = []
 for x in args.env:
-    options.extend(["--wasi-env", x])
+    options.extend([f"--wasi-env={x}"])
 for x in args.dir:
-    options.extend(["--wasi-dir", x])
+    options.extend([f"--wasi-dir={x}"])
 result = subprocess.run(
     shlex.split(executable) + ["--wasi"] + options + ["--", args.test_file] + args.arg
 )