Enable failpoint by default in integration tests

Signed-off-by: Chao Chen <chaochn@amazon.com>

.github/workflows/tests-arm64.yaml

@@ -40,12 +40,15 @@ jobs:
           export JUNIT_REPORT_DIR=$(realpath ${TARGET})
           case "${TARGET}" in
             linux-arm64-integration-1-cpu)
+              make gofail-enable
               GOOS=linux GOARCH=arm64 CPU=1 make test-integration
               ;;
             linux-arm64-integration-2-cpu)
+              make gofail-enable
               GOOS=linux GOARCH=arm64 CPU=2 make test-integration
               ;;
             linux-arm64-integration-4-cpu)
+              make gofail-enable
               GOOS=linux GOARCH=arm64 CPU=4 make test-integration
               ;;
             linux-arm64-unit-4-cpu-race)

.github/workflows/tests.yaml

@@ -31,12 +31,15 @@ jobs:
           export JUNIT_REPORT_DIR=$(realpath ${TARGET})
           case "${TARGET}" in
             linux-amd64-integration-1-cpu)
+              make gofail-enable
               GOOS=linux GOARCH=amd64 CPU=1 make test-integration
               ;;
             linux-amd64-integration-2-cpu)
+              make gofail-enable
               GOOS=linux GOARCH=amd64 CPU=2 make test-integration
               ;;
             linux-amd64-integration-4-cpu)
+              make gofail-enable
               GOOS=linux GOARCH=amd64 CPU=4 make test-integration
               ;;
             linux-amd64-unit-4-cpu-race)

bill-of-materials.json

@@ -521,6 +521,15 @@
 			}
 		]
 	},
+	{
+		"project": "go.etcd.io/gofail/runtime",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
+	},
 	{
 		"project": "go.etcd.io/raft/v3",
 		"licenses": [

go.mod

@@ -79,6 +79,7 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
 	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
+	go.etcd.io/gofail v0.1.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 // indirect
 	go.opentelemetry.io/otel v1.14.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2 // indirect

go.sum

@@ -275,6 +275,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
 go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
+go.etcd.io/gofail v0.1.0 h1:XItAMIhOojXFQMgrxjnd2EIIHun/d5qL0Pf7FzVTkFg=
+go.etcd.io/gofail v0.1.0/go.mod h1:VZBCXYGZhHAinaBiiqYvuDynvahNsAyLFwB3kEHKz1M=
 go.etcd.io/raft/v3 v3.0.0-20221201111702-eaa6808e1f7a h1:Znv2XJyAf/fsJsFNt9toO8uyXwwHQ44wxqsvdSxipj4=
 go.etcd.io/raft/v3 v3.0.0-20221201111702-eaa6808e1f7a/go.mod h1:eMshmuwXLWZrjHXN8ZgYrOMQRSbHqi5M84DEZWhG+o4=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=

server/etcdserver/raft.go

@@ -23,12 +23,13 @@ import (
 
 	"go.uber.org/zap"
 
+	"go.etcd.io/raft/v3"
+	"go.etcd.io/raft/v3/raftpb"
+
 	"go.etcd.io/etcd/client/pkg/v3/logutil"
 	"go.etcd.io/etcd/pkg/v3/contention"
 	"go.etcd.io/etcd/server/v3/etcdserver/api/rafthttp"
 	serverstorage "go.etcd.io/etcd/server/v3/storage"
-	"go.etcd.io/raft/v3"
-	"go.etcd.io/raft/v3/raftpb"
 )
 
 const (
@@ -307,6 +308,7 @@ func (r *raftNode) start(rh *raftReadyHandler) {
 					notifyc <- struct{}{}
 				}
 
+				// gofail: var raftBeforeAdvance struct{}
 				r.Advance()
 			case <-r.stopped:
 				return

tests/framework/integration/testing.go

@@ -19,10 +19,13 @@ import (
 	"testing"
 
 	grpc_logsettable "github.com/grpc-ecosystem/go-grpc-middleware/logging/settable"
+	"github.com/stretchr/testify/require"
 	"go.uber.org/zap/zapcore"
 	"go.uber.org/zap/zapgrpc"
 	"go.uber.org/zap/zaptest"
 
+	gofail "go.etcd.io/gofail/runtime"
+
 	"go.etcd.io/etcd/client/pkg/v3/testutil"
 	"go.etcd.io/etcd/client/pkg/v3/verify"
 	clientv3 "go.etcd.io/etcd/client/v3"
@@ -39,6 +42,12 @@ func init() {
 type testOptions struct {
 	goLeakDetection bool
 	skipInShort     bool
+	failpoint       *failpoint
+}
+
+type failpoint struct {
+	name    string
+	payload string
 }
 
 func newTestOptions(opts ...TestOption) *testOptions {
@@ -60,6 +69,11 @@ func WithoutSkipInShort() TestOption {
 	return func(opt *testOptions) { opt.skipInShort = false }
 }
 
+// WithFailpoint registers a go fail point
+func WithFailpoint(name, payload string) TestOption {
+	return func(opt *testOptions) { opt.failpoint = &failpoint{name: name, payload: payload} }
+}
+
 // BeforeTestExternal initializes test context and is targeted for external APIs.
 // In general the `integration` package is not targeted to be used outside of
 // etcd project, but till the dedicated package is developed, this is
@@ -84,6 +98,16 @@ func BeforeTest(t testutil.TB, opts ...TestOption) {
 		testutil.RegisterLeakDetection(t)
 	}
 
+	if options.failpoint != nil && len(options.failpoint.name) != 0 {
+		if len(gofail.List()) == 0 {
+			t.Skip("please run 'make gofail-enable' before running the test")
+		}
+		require.NoError(t, gofail.Enable(options.failpoint.name, options.failpoint.payload))
+		t.Cleanup(func() {
+			require.NoError(t, gofail.Disable(options.failpoint.name))
+		})
+	}
+
 	previousWD, err := os.Getwd()
 	if err != nil {
 		t.Fatal(err)

tests/go.mod

@@ -32,6 +32,7 @@ require (
 	go.etcd.io/etcd/etcdutl/v3 v3.6.0-alpha.0
 	go.etcd.io/etcd/pkg/v3 v3.6.0-alpha.0
 	go.etcd.io/etcd/server/v3 v3.6.0-alpha.0
+	go.etcd.io/gofail v0.1.0
 	go.etcd.io/raft/v3 v3.0.0-20221201111702-eaa6808e1f7a
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0
 	go.opentelemetry.io/otel v1.14.0

tests/go.sum

@@ -260,6 +260,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
 go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
+go.etcd.io/gofail v0.1.0 h1:XItAMIhOojXFQMgrxjnd2EIIHun/d5qL0Pf7FzVTkFg=
+go.etcd.io/gofail v0.1.0/go.mod h1:VZBCXYGZhHAinaBiiqYvuDynvahNsAyLFwB3kEHKz1M=
 go.etcd.io/raft/v3 v3.0.0-20221201111702-eaa6808e1f7a h1:Znv2XJyAf/fsJsFNt9toO8uyXwwHQ44wxqsvdSxipj4=
 go.etcd.io/raft/v3 v3.0.0-20221201111702-eaa6808e1f7a/go.mod h1:eMshmuwXLWZrjHXN8ZgYrOMQRSbHqi5M84DEZWhG+o4=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=

tests/integration/clientv3/cluster_test.go

@@ -294,6 +294,7 @@ func TestMemberPromote(t *testing.T) {
 
 // TestMemberPromoteMemberNotLearner ensures that promoting a voting member fails.
 func TestMemberPromoteMemberNotLearner(t *testing.T) {
+	// TODO enable this test with integration2.WithFailpoint("raftBeforeAdvance", `sleep(100)`) after PR 15708 is merged
 	integration2.BeforeTest(t)
 
 	clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 3})