Skip to content

Commit

Permalink
Flattens std lib test cases (#1884)
Browse files Browse the repository at this point in the history 
Signed-off-by: Takeshi Yoneda <t.y.mathetake@gmail.com>
  • Loading branch information
@mathetake
mathetake committed Dec 20, 2023
1 parent 823d573 commit 8c71d4d
Show file tree
Hide file tree
Showing 2 changed files with 147 additions and 129 deletions.
7 changes: 3 additions & 4 deletions .github/workflows/integration.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,8 +151,7 @@ jobs:
if: ${{ matrix.compiler != 'optimizing' }}
run: |
cd ${{ env.STDLIB_TESTS }}
go test -bench='./zig' -benchtime=1x
go test -bench='BenchmarkZig' -benchtime=1x
build_tinygo_test_binary:
name: Build TinyGo test binary
Expand Down Expand Up @@ -269,7 +268,7 @@ jobs:
if: ${{ matrix.compiler != 'optimizing' }}
run: |
cd ${{ env.STDLIB_TESTS }}
go test -bench='./tinygo' -benchtime=1x
go test -bench='BenchmarkTinyGo' -benchtime=1x
wasi-testsuite:
name: wasi-testsuite
Expand Down Expand Up @@ -441,4 +440,4 @@ jobs:
if: ${{ matrix.compiler != 'optimizing' }}
run: |
cd ${{ env.STDLIB_TESTS }}
go test -bench='./wasip1' -benchtime=1x
go test -bench='BenchmarkWasip1' -benchtime=1x
269 changes: 144 additions & 125 deletions internal/integration_test/stdlibs/bench_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,143 +16,162 @@ import (
"github.com/tetratelabs/wazero/sys"
)

func BenchmarkStdlibs(b *testing.B) {
type testConfig struct {
name string
config wazero.RuntimeConfig
func BenchmarkZig(b *testing.B) {
if runtime.GOARCH == "arm64" {
b.Run("optimizing", func(b *testing.B) {
c := opt.NewRuntimeConfigOptimizingCompiler()
runtBench(b, context.Background(), c, zigTestCase)
})
}
configs := []testConfig{
{name: "baseline", config: wazero.NewRuntimeConfigCompiler()},
b.Run("baseline", func(b *testing.B) {
c := wazero.NewRuntimeConfigCompiler()
runtBench(b, context.Background(), c, zigTestCase)
})
}

func BenchmarkTinyGo(b *testing.B) {
if runtime.GOARCH == "arm64" {
b.Run("optimizing", func(b *testing.B) {
c := opt.NewRuntimeConfigOptimizingCompiler()
runtBench(b, context.Background(), c, tinyGoTestCase)
})
}
b.Run("baseline", func(b *testing.B) {
c := wazero.NewRuntimeConfigCompiler()
runtBench(b, context.Background(), c, tinyGoTestCase)
})
}

func BenchmarkWasip1(b *testing.B) {
if runtime.GOARCH == "arm64" {
configs = append(configs, testConfig{name: "optimizing", config: opt.NewRuntimeConfigOptimizingCompiler()})
b.Run("optimizing", func(b *testing.B) {
c := opt.NewRuntimeConfigOptimizingCompiler()
runtBench(b, context.Background(), c, wasip1TestCase)
})
}
b.Run("baseline", func(b *testing.B) {
c := wazero.NewRuntimeConfigCompiler()
runtBench(b, context.Background(), c, wasip1TestCase)
})
}

cwd, _ := os.Getwd()
defer os.Chdir(cwd) //nolint
ctx := context.Background()

testCases := []struct {
name, dir string
readTestCase func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error)
}{
{
name: "zig",
dir: "testdata/zig/",
readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) {
bin, err := os.ReadFile(fpath)
modCfg := defaultModuleConfig().
WithFSConfig(wazero.NewFSConfig().WithDirMount(".", "/")).
WithArgs("test.wasm")

return bin, modCfg, err
},
type testCase struct {
name, dir string
readTestCase func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error)
}

var (
zigTestCase = testCase{
name: "zig",
dir: "testdata/zig/",
readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) {
bin, err := os.ReadFile(fpath)
modCfg := defaultModuleConfig().
WithFSConfig(wazero.NewFSConfig().WithDirMount(".", "/")).
WithArgs("test.wasm")

return bin, modCfg, err
},
{
name: "tinygo",
dir: "testdata/tinygo/",
readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) {
if !strings.HasSuffix(fname, ".test") {
return nil, nil, nil
}
bin, err := os.ReadFile(fpath)
fsconfig := wazero.NewFSConfig().
WithDirMount(".", "/").
WithDirMount(os.TempDir(), "/tmp")
modCfg := defaultModuleConfig().
WithFSConfig(fsconfig).
WithArgs(fname, "-test.v")

return bin, modCfg, err
},
}
tinyGoTestCase = testCase{
name: "tinygo",
dir: "testdata/tinygo/",
readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) {
if !strings.HasSuffix(fname, ".test") {
return nil, nil, nil
}
bin, err := os.ReadFile(fpath)
fsconfig := wazero.NewFSConfig().
WithDirMount(".", "/").
WithDirMount(os.TempDir(), "/tmp")
modCfg := defaultModuleConfig().
WithFSConfig(fsconfig).
WithArgs(fname, "-test.v")

return bin, modCfg, err
},
{
name: "wasip1",
dir: "testdata/go/",
readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) {
if !strings.HasSuffix(fname, ".test") {
return nil, nil, nil
}
bin, err := os.ReadFile(fpath)
if err != nil {
return nil, nil, err
}
fsuffixstripped := strings.ReplaceAll(fname, ".test", "")
inferredpath := strings.ReplaceAll(fsuffixstripped, "_", "/")
testdir := filepath.Join(runtime.GOROOT(), inferredpath)
err = os.Chdir(testdir)

sysroot := filepath.VolumeName(testdir) + string(os.PathSeparator)
normalizedTestdir := normalizeOsPath(testdir)

modCfg := defaultModuleConfig().
WithFSConfig(
wazero.NewFSConfig().
WithDirMount(sysroot, "/").
WithDirMount(os.TempDir(), "/tmp")).
WithEnv("PWD", normalizedTestdir)

args := []string{fname, "-test.short", "-test.v"}

// Skip tests that are fragile on Windows.
if runtime.GOOS == "windows" {
modCfg = modCfg.
WithEnv("GOROOT", normalizeOsPath(runtime.GOROOT()))

args = append(args,
"-test.skip=TestRenameCaseDifference/dir|"+
"TestDirFSPathsValid|TestDirFS|TestDevNullFile|"+
"TestOpenError|TestSymlinkWithTrailingSlash")
}
modCfg = modCfg.WithArgs(args...)

return bin, modCfg, err
},
}
wasip1TestCase = testCase{
name: "wasip1",
dir: "testdata/go/",
readTestCase: func(fpath string, fname string) ([]byte, wazero.ModuleConfig, error) {
if !strings.HasSuffix(fname, ".test") {
return nil, nil, nil
}
bin, err := os.ReadFile(fpath)
if err != nil {
return nil, nil, err
}
fsuffixstripped := strings.ReplaceAll(fname, ".test", "")
inferredpath := strings.ReplaceAll(fsuffixstripped, "_", "/")
testdir := filepath.Join(runtime.GOROOT(), inferredpath)
err = os.Chdir(testdir)

sysroot := filepath.VolumeName(testdir) + string(os.PathSeparator)
normalizedTestdir := normalizeOsPath(testdir)

modCfg := defaultModuleConfig().
WithFSConfig(
wazero.NewFSConfig().
WithDirMount(sysroot, "/").
WithDirMount(os.TempDir(), "/tmp")).
WithEnv("PWD", normalizedTestdir)

args := []string{fname, "-test.short", "-test.v"}

// Skip tests that are fragile on Windows.
if runtime.GOOS == "windows" {
modCfg = modCfg.
WithEnv("GOROOT", normalizeOsPath(runtime.GOROOT()))

args = append(args,
"-test.skip=TestRenameCaseDifference/dir|"+
"TestDirFSPathsValid|TestDirFS|TestDevNullFile|"+
"TestOpenError|TestSymlinkWithTrailingSlash")
}
modCfg = modCfg.WithArgs(args...)

return bin, modCfg, err
},
}
)

for _, tc := range testCases {
b.Run(tc.name, func(b *testing.B) {
files, err := os.ReadDir(tc.dir)
require.NoError(b, err)
for _, f := range files {
fname := f.Name()
// Ensure we are on root dir.
err = os.Chdir(cwd)
func runtBench(b *testing.B, ctx context.Context, rc wazero.RuntimeConfig, tc testCase) {
cwd, _ := os.Getwd()
files, err := os.ReadDir(tc.dir)
require.NoError(b, err)
for _, f := range files {
fname := f.Name()
// Ensure we are on root dir.
err = os.Chdir(cwd)
require.NoError(b, err)

fpath := filepath.Join(cwd, tc.dir, fname)
bin, modCfg, err := tc.readTestCase(fpath, fname)
require.NoError(b, err)
if bin == nil {
continue
}
b.Run(fname, func(b *testing.B) {
r := wazero.NewRuntimeWithConfig(ctx, rc)
wasi_snapshot_preview1.MustInstantiate(ctx, r)
b.Cleanup(func() { r.Close(ctx) })

var cm wazero.CompiledModule
b.Run("Compile", func(b *testing.B) {
b.ResetTimer()
var err error
cm, err = r.CompileModule(ctx, bin)
require.NoError(b, err)
})

fpath := filepath.Join(cwd, tc.dir, fname)
bin, modCfg, err := tc.readTestCase(fpath, fname)
require.NoError(b, err)
if bin == nil {
// skip
continue
}

b.Run(fname, func(b *testing.B) {
for _, cfg := range configs {
r := wazero.NewRuntimeWithConfig(ctx, cfg.config)
wasi_snapshot_preview1.MustInstantiate(ctx, r)
b.Cleanup(func() { r.Close(ctx) })

m, err := r.CompileModule(ctx, bin)
require.NoError(b, err)

b.Run(cfg.name, func(b *testing.B) {
b.Run("Compile", func(b *testing.B) {
_, err := r.CompileModule(ctx, bin)
require.NoError(b, err)
})
im, err := r.InstantiateModule(ctx, m, modCfg)
require.NoError(b, err)
b.Run("Run", func(b *testing.B) {
_, err := im.ExportedFunction("_start").Call(ctx)
requireZeroExitCode(b, err)
})
})
}
})
}
im, err := r.InstantiateModule(ctx, cm, modCfg)
require.NoError(b, err)
b.Run("Run", func(b *testing.B) {
b.ResetTimer()
_, err := im.ExportedFunction("_start").Call(ctx)
requireZeroExitCode(b, err)
})
})
}
}
Expand Down

0 comments on commit 8c71d4d

Please sign in to comment.