From 6d79b86219154eab9fdca81b3488fd4357c92013 Mon Sep 17 00:00:00 2001 From: Chao Chen Date: Thu, 15 Jun 2023 15:30:34 -0700 Subject: [PATCH] Enable failpoint by default in integration tests Signed-off-by: Chao Chen --- .github/workflows/tests-arm64.yaml | 3 +++ .github/workflows/tests.yaml | 3 +++ bill-of-materials.json | 9 ++++++++ go.mod | 1 + go.sum | 2 ++ server/etcdserver/raft.go | 6 ++++-- tests/framework/integration/testing.go | 24 ++++++++++++++++++++++ tests/go.mod | 1 + tests/go.sum | 2 ++ tests/integration/clientv3/cluster_test.go | 1 + 10 files changed, 50 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests-arm64.yaml b/.github/workflows/tests-arm64.yaml index 637fed942b5..f92e7648fc9 100644 --- a/.github/workflows/tests-arm64.yaml +++ b/.github/workflows/tests-arm64.yaml @@ -40,12 +40,15 @@ jobs: export JUNIT_REPORT_DIR=$(realpath ${TARGET}) case "${TARGET}" in linux-arm64-integration-1-cpu) + make gofail-enable GOOS=linux GOARCH=arm64 CPU=1 make test-integration ;; linux-arm64-integration-2-cpu) + make gofail-enable GOOS=linux GOARCH=arm64 CPU=2 make test-integration ;; linux-arm64-integration-4-cpu) + make gofail-enable GOOS=linux GOARCH=arm64 CPU=4 make test-integration ;; linux-arm64-unit-4-cpu-race) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 6c99d24e7a5..ca20d2d76ba 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -31,12 +31,15 @@ jobs: export JUNIT_REPORT_DIR=$(realpath ${TARGET}) case "${TARGET}" in linux-amd64-integration-1-cpu) + make gofail-enable GOOS=linux GOARCH=amd64 CPU=1 make test-integration ;; linux-amd64-integration-2-cpu) + make gofail-enable GOOS=linux GOARCH=amd64 CPU=2 make test-integration ;; linux-amd64-integration-4-cpu) + make gofail-enable GOOS=linux GOARCH=amd64 CPU=4 make test-integration ;; linux-amd64-unit-4-cpu-race) diff --git a/bill-of-materials.json b/bill-of-materials.json index db45756dc76..b5cf5d624f0 100644 --- a/bill-of-materials.json +++ b/bill-of-materials.json @@ -521,6 +521,15 @@ } ] }, + { + "project": "go.etcd.io/gofail/runtime", + "licenses": [ + { + "type": "Apache License 2.0", + "confidence": 1 + } + ] + }, { "project": "go.etcd.io/raft/v3", "licenses": [ diff --git a/go.mod b/go.mod index 1655299ce3d..73694a1c10f 100644 --- a/go.mod +++ b/go.mod @@ -79,6 +79,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect + go.etcd.io/gofail v0.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 // indirect go.opentelemetry.io/otel v1.14.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2 // indirect diff --git a/go.sum b/go.sum index 9dba68e2e56..d29fa83bc86 100644 --- a/go.sum +++ b/go.sum @@ -275,6 +275,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.etcd.io/gofail v0.1.0 h1:XItAMIhOojXFQMgrxjnd2EIIHun/d5qL0Pf7FzVTkFg= +go.etcd.io/gofail v0.1.0/go.mod h1:VZBCXYGZhHAinaBiiqYvuDynvahNsAyLFwB3kEHKz1M= go.etcd.io/raft/v3 v3.0.0-20221201111702-eaa6808e1f7a h1:Znv2XJyAf/fsJsFNt9toO8uyXwwHQ44wxqsvdSxipj4= go.etcd.io/raft/v3 v3.0.0-20221201111702-eaa6808e1f7a/go.mod h1:eMshmuwXLWZrjHXN8ZgYrOMQRSbHqi5M84DEZWhG+o4= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= diff --git a/server/etcdserver/raft.go b/server/etcdserver/raft.go index 6e50b417f6a..e2f26117d98 100644 --- a/server/etcdserver/raft.go +++ b/server/etcdserver/raft.go @@ -23,12 +23,13 @@ import ( "go.uber.org/zap" + "go.etcd.io/raft/v3" + "go.etcd.io/raft/v3/raftpb" + "go.etcd.io/etcd/client/pkg/v3/logutil" "go.etcd.io/etcd/pkg/v3/contention" "go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp" serverstorage "go.etcd.io/etcd/server/v3/storage" - "go.etcd.io/raft/v3" - "go.etcd.io/raft/v3/raftpb" ) const ( @@ -307,6 +308,7 @@ func (r *raftNode) start(rh *raftReadyHandler) { notifyc <- struct{}{} } + // gofail: var raftBeforeAdvance struct{} r.Advance() case <-r.stopped: return diff --git a/tests/framework/integration/testing.go b/tests/framework/integration/testing.go index 7a4613a8ac2..b0580dcd670 100644 --- a/tests/framework/integration/testing.go +++ b/tests/framework/integration/testing.go @@ -19,10 +19,13 @@ import ( "testing" grpc_logsettable "github.com/grpc-ecosystem/go-grpc-middleware/logging/settable" + "github.com/stretchr/testify/require" "go.uber.org/zap/zapcore" "go.uber.org/zap/zapgrpc" "go.uber.org/zap/zaptest" + gofail "go.etcd.io/gofail/runtime" + "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/client/pkg/v3/verify" clientv3 "go.etcd.io/etcd/client/v3" @@ -39,6 +42,12 @@ func init() { type testOptions struct { goLeakDetection bool skipInShort bool + failpoint *failpoint +} + +type failpoint struct { + name string + payload string } func newTestOptions(opts ...TestOption) *testOptions { @@ -60,6 +69,11 @@ func WithoutSkipInShort() TestOption { return func(opt *testOptions) { opt.skipInShort = false } } +// WithFailpoint registers a go fail point +func WithFailpoint(name, payload string) TestOption { + return func(opt *testOptions) { opt.failpoint = &failpoint{name: name, payload: payload} } +} + // BeforeTestExternal initializes test context and is targeted for external APIs. // In general the `integration` package is not targeted to be used outside of // etcd project, but till the dedicated package is developed, this is @@ -84,6 +98,16 @@ func BeforeTest(t testutil.TB, opts ...TestOption) { testutil.RegisterLeakDetection(t) } + if options.failpoint != nil && len(options.failpoint.name) != 0 { + if len(gofail.List()) == 0 { + t.Skip("please run 'make gofail-enable' before running the test") + } + require.NoError(t, gofail.Enable(options.failpoint.name, options.failpoint.payload)) + t.Cleanup(func() { + require.NoError(t, gofail.Disable(options.failpoint.name)) + }) + } + previousWD, err := os.Getwd() if err != nil { t.Fatal(err) diff --git a/tests/go.mod b/tests/go.mod index 7893fc6288e..ad98e6b9ef4 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -32,6 +32,7 @@ require ( go.etcd.io/etcd/etcdutl/v3 v3.6.0-alpha.0 go.etcd.io/etcd/pkg/v3 v3.6.0-alpha.0 go.etcd.io/etcd/server/v3 v3.6.0-alpha.0 + go.etcd.io/gofail v0.1.0 go.etcd.io/raft/v3 v3.0.0-20221201111702-eaa6808e1f7a go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 go.opentelemetry.io/otel v1.14.0 diff --git a/tests/go.sum b/tests/go.sum index 00cdbd7b226..f9393b46522 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -260,6 +260,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.etcd.io/gofail v0.1.0 h1:XItAMIhOojXFQMgrxjnd2EIIHun/d5qL0Pf7FzVTkFg= +go.etcd.io/gofail v0.1.0/go.mod h1:VZBCXYGZhHAinaBiiqYvuDynvahNsAyLFwB3kEHKz1M= go.etcd.io/raft/v3 v3.0.0-20221201111702-eaa6808e1f7a h1:Znv2XJyAf/fsJsFNt9toO8uyXwwHQ44wxqsvdSxipj4= go.etcd.io/raft/v3 v3.0.0-20221201111702-eaa6808e1f7a/go.mod h1:eMshmuwXLWZrjHXN8ZgYrOMQRSbHqi5M84DEZWhG+o4= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= diff --git a/tests/integration/clientv3/cluster_test.go b/tests/integration/clientv3/cluster_test.go index 822855f1365..6ac1e20c69e 100644 --- a/tests/integration/clientv3/cluster_test.go +++ b/tests/integration/clientv3/cluster_test.go @@ -294,6 +294,7 @@ func TestMemberPromote(t *testing.T) { // TestMemberPromoteMemberNotLearner ensures that promoting a voting member fails. func TestMemberPromoteMemberNotLearner(t *testing.T) { + // TODO enable this test with integration2.WithFailpoint("raftBeforeAdvance", `sleep(100)`) after PR 15708 is merged integration2.BeforeTest(t) clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 3})