diff --git a/clientv3/integration/kv_test.go b/clientv3/integration/kv_test.go index ec2db2d66284..ddf284aa1859 100644 --- a/clientv3/integration/kv_test.go +++ b/clientv3/integration/kv_test.go @@ -708,6 +708,7 @@ func TestKVGetRetry(t *testing.T) { time.Sleep(100 * time.Millisecond) clus.Members[fIdx].Restart(t) + clus.Members[fIdx].WaitOK(t) select { case <-time.After(5 * time.Second): @@ -792,7 +793,7 @@ func TestKVGetStoppedServerAndClose(t *testing.T) { // this Get fails and triggers an asynchronous connection retry _, err := cli.Get(ctx, "abc") cancel() - if err != nil && err != context.DeadlineExceeded { + if err != nil && !isServerUnavailable(err) { t.Fatal(err) } } @@ -821,7 +822,7 @@ func TestKVPutStoppedServerAndClose(t *testing.T) { // this Put fails and triggers an asynchronous connection retry _, err = cli.Put(ctx, "abc", "123") cancel() - if err != nil && err != context.DeadlineExceeded { + if err != nil && !isServerUnavailable(err) { t.Fatal(err) } } @@ -906,7 +907,7 @@ func TestKVLargeRequests(t *testing.T) { maxCallSendBytesClient: 10 * 1024 * 1024, maxCallRecvBytesClient: 0, valueSize: 10 * 1024 * 1024, - expectError: grpc.Errorf(codes.ResourceExhausted, "grpc: trying to send message larger than max "), + expectError: grpc.Errorf(codes.ResourceExhausted, "trying to send message larger than max "), }, { maxRequestBytesServer: 10 * 1024 * 1024, @@ -920,7 +921,7 @@ func TestKVLargeRequests(t *testing.T) { maxCallSendBytesClient: 10 * 1024 * 1024, maxCallRecvBytesClient: 0, valueSize: 10*1024*1024 + 5, - expectError: grpc.Errorf(codes.ResourceExhausted, "grpc: trying to send message larger than max "), + expectError: grpc.Errorf(codes.ResourceExhausted, "trying to send message larger than max "), }, } for i, test := range tests { @@ -940,7 +941,7 @@ func TestKVLargeRequests(t *testing.T) { t.Errorf("#%d: expected %v, got %v", i, test.expectError, err) } } else if err != nil && !strings.HasPrefix(err.Error(), test.expectError.Error()) { - t.Errorf("#%d: expected %v, got %v", i, test.expectError, err) + t.Errorf("#%d: expected error starting with '%s', got '%s'", i, test.expectError.Error(), err.Error()) } // put request went through, now expects large response back diff --git a/clientv3/integration/server_shutdown_test.go b/clientv3/integration/server_shutdown_test.go index de5c708563fc..335f3491744b 100644 --- a/clientv3/integration/server_shutdown_test.go +++ b/clientv3/integration/server_shutdown_test.go @@ -104,7 +104,7 @@ func TestBalancerUnderServerShutdownWatch(t *testing.T) { if err == nil { break } - if err == context.DeadlineExceeded || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout || err == rpctypes.ErrTimeoutDueToLeaderFail { + if isClientTimeout(err) || isServerCtxTimeout(err) || err == rpctypes.ErrTimeout || err == rpctypes.ErrTimeoutDueToLeaderFail { continue } t.Fatal(err) @@ -366,6 +366,35 @@ func isServerCtxTimeout(err error) bool { return code == codes.DeadlineExceeded && strings.Contains(err.Error(), "context deadline exceeded") } +// In grpc v1.11.3+ dial timeouts can error out with transport.ErrConnClosing. Previously dial timeouts +// would always error out with context.DeadlineExceeded. +func isClientTimeout(err error) bool { + if err == nil { + return false + } + if err == context.DeadlineExceeded { + return true + } + ev, ok := status.FromError(err) + if !ok { + return false + } + code := ev.Code() + return code == codes.DeadlineExceeded || ev.Message() == transport.ErrConnClosing.Desc +} + +func isServerUnavailable(err error) bool { + if err == nil { + return false + } + ev, ok := status.FromError(err) + if !ok { + return false + } + code := ev.Code() + return code == codes.Unavailable +} + func isCanceled(err error) bool { if err == nil { return false