Merge remote-tracking branch 'oss/master' into f-nomad

* oss/master:
  Defer reader.Close that is used to determine sha256
  changelog++
  Avoid unseal failure if plugin backends fail to setup during postUnseal (#3686)
  Add logic for using Auth.Period when handling auth login/renew requests (#3677)
  plugins/database: use context with plugins that use database/sql package (#3691)
  changelog++
  Fix plaintext backup in transit (#3692)
  Database gRPC plugins (#3666)

CHANGELOG.md

@@ -33,6 +33,8 @@ FEATURES:
    operation that can export a given key, including all key versions and
    configuration, as well as a restore operation allowing import into another
    Vault.
+ * **gRPC Database Plugins**: Database plugins now use gRPC for transport,
+   allowing them to be written in other languages.
 
 IMPROVEMENTS:
 
@@ -46,6 +48,10 @@ IMPROVEMENTS:
    during database configuration. This establishes a session-wide [write
    concern](https://docs.mongodb.com/manual/reference/write-concern/) for the
    lifecycle of the mount [GH-3646]
+ * mfa/okta: Filter a given email address as a login filter, allowing operation
+   when login email and account email are different
+ * plugins: Make Vault more resilient when unsealing when plugins are
+   unavailable [GH-3686]
  * secret/pki: `allowed_domains` and `key_usage` can now be specified
    as a comma-separated string or an array of strings [GH-3642]
  * secret/ssh: Allow 4096-bit keys to be used in dynamic key method [GH-3593]
@@ -58,8 +64,12 @@ BUG FIXES:
  * auth/cert: Return `allowed_names` on role read [GH-3654]
  * auth/ldap: Fix incorrect control information being sent [GH-3402] [GH-3496]
    [GH-3625] [GH-3656]
+ * core: Fix seal status reporting when using an autoseal
+ * core: Add creation path to wrap info for a control group token
  * core: Fix potential panic that could occur using plugins when a node
    transitioned from active to standby [GH-3638]
+ * core: Fix memory ballooning when a connection would connect to the cluster
+   port and then go away -- redux! [GH-3680]
  * core: Replace recursive token revocation logic with depth-first logic, which
    can avoid hitting stack depth limits in extreme cases [GH-2348]
  * core/pkcs11 (enterprise): Fix panic when PKCS#11 library is not readable

Makefile

@@ -84,6 +84,7 @@ proto:
 	protoc -I helper/forwarding -I vault -I ../../.. helper/forwarding/types.proto --go_out=plugins=grpc:helper/forwarding
 	protoc -I physical physical/types.proto --go_out=plugins=grpc:physical
 	protoc -I helper/identity -I ../../.. helper/identity/types.proto --go_out=plugins=grpc:helper/identity
+	protoc  builtin/logical/database/dbplugin/*.proto --go_out=plugins=grpc:.
 	sed -i -e 's/Idp/IDP/' -e 's/Url/URL/' -e 's/Id/ID/' -e 's/EntityId/EntityID/' -e 's/Api/API/' -e 's/Qr/QR/' -e 's/protobuf:"/sentinel:"" protobuf:"/' helper/identity/types.pb.go helper/storagepacker/types.pb.go
 	sed -i -e 's/Iv/IV/' -e 's/Hmac/HMAC/' physical/types.pb.go
 

builtin/credential/app-id/backend_test.go

@@ -141,7 +141,7 @@ func testAccStepMapUserIdCidr(t *testing.T, cidr string) logicaltest.TestStep {
 func testAccLogin(t *testing.T, display string) logicaltest.TestStep {
 	checkTTL := func(resp *logical.Response) error {
 		if resp.Auth.LeaseOptions.TTL.String() != "768h0m0s" {
-			return fmt.Errorf("invalid TTL")
+			return fmt.Errorf("invalid TTL: got %s", resp.Auth.LeaseOptions.TTL)
 		}
 		return nil
 	}
@@ -165,7 +165,7 @@ func testAccLogin(t *testing.T, display string) logicaltest.TestStep {
 func testAccLoginAppIDInPath(t *testing.T, display string) logicaltest.TestStep {
 	checkTTL := func(resp *logical.Response) error {
 		if resp.Auth.LeaseOptions.TTL.String() != "768h0m0s" {
-			return fmt.Errorf("invalid TTL")
+			return fmt.Errorf("invalid TTL: got %s", resp.Auth.LeaseOptions.TTL)
 		}
 		return nil
 	}

builtin/credential/approle/path_login.go

@@ -3,7 +3,6 @@ package approle
 import (
 	"fmt"
 	"strings"
-	"time"
 
 	"github.com/hashicorp/vault/logical"
 	"github.com/hashicorp/vault/logical/framework"
@@ -68,20 +67,13 @@ func (b *backend) pathLoginUpdate(req *logical.Request, data *framework.FieldDat
 		Policies: role.Policies,
 		LeaseOptions: logical.LeaseOptions{
 			Renewable: true,
+			TTL:       role.TokenTTL,
 		},
 		Alias: &logical.Alias{
 			Name: role.RoleID,
 		},
 	}
 
-	// If 'Period' is set, use the value of 'Period' as the TTL.
-	// Otherwise, set the normal TokenTTL.
-	if role.Period > time.Duration(0) {
-		auth.TTL = role.Period
-	} else {
-		auth.TTL = role.TokenTTL
-	}
-
 	return &logical.Response{
 		Auth: auth,
 	}, nil
@@ -107,16 +99,12 @@ func (b *backend) pathLoginRenew(req *logical.Request, data *framework.FieldData
 		return nil, fmt.Errorf("role %s does not exist during renewal", roleName)
 	}
 
-	// If 'Period' is set on the Role, the token should never expire.
-	// Replenish the TTL with 'Period's value.
-	if role.Period > time.Duration(0) {
-		// If 'Period' was updated after the token was issued,
-		// token will bear the updated 'Period' value as its TTL.
-		req.Auth.TTL = role.Period
-		return &logical.Response{Auth: req.Auth}, nil
-	} else {
-		return framework.LeaseExtend(role.TokenTTL, role.TokenMaxTTL, b.System())(req, data)
+	resp, err := framework.LeaseExtend(role.TokenTTL, role.TokenMaxTTL, b.System())(req, data)
+	if err != nil {
+		return nil, err
 	}
+	resp.Auth.Period = role.Period
+	return resp, nil
 }
 
 const pathLoginHelpSys = "Issue a token based on the credentials supplied"

builtin/logical/database/backend.go

@@ -1,6 +1,7 @@
 package database
 
 import (
+	"context"
 	"fmt"
 	"net/rpc"
 	"strings"
@@ -87,7 +88,7 @@ func (b *databaseBackend) getDBObj(name string) (dbplugin.Database, bool) {
 // This function creates a new db object from the stored configuration and
 // caches it in the connections map. The caller of this function needs to hold
 // the backend's write lock
-func (b *databaseBackend) createDBObj(s logical.Storage, name string) (dbplugin.Database, error) {
+func (b *databaseBackend) createDBObj(ctx context.Context, s logical.Storage, name string) (dbplugin.Database, error) {
 	db, ok := b.connections[name]
 	if ok {
 		return db, nil
@@ -103,7 +104,7 @@ func (b *databaseBackend) createDBObj(s logical.Storage, name string) (dbplugin.
 		return nil, err
 	}
 
-	err = db.Initialize(config.ConnectionDetails, true)
+	err = db.Initialize(ctx, config.ConnectionDetails, true)
 	if err != nil {
 		return nil, err
 	}
@@ -170,7 +171,8 @@ func (b *databaseBackend) clearConnection(name string) {
 
 func (b *databaseBackend) closeIfShutdown(name string, err error) {
 	// Plugin has shutdown, close it so next call can reconnect.
-	if err == rpc.ErrShutdown {
+	switch err {
+	case rpc.ErrShutdown, dbplugin.ErrPluginShutdown:
 		b.Lock()
 		b.clearConnection(name)
 		b.Unlock()

builtin/logical/database/backend_test.go

@@ -488,9 +488,11 @@ func TestBackend_roleCrud(t *testing.T) {
 		RevocationStatements: defaultRevocationSQL,
 	}
 
-	var actual dbplugin.Statements
-	if err := mapstructure.Decode(resp.Data, &actual); err != nil {
-		t.Fatal(err)
+	actual := dbplugin.Statements{
+		CreationStatements:   resp.Data["creation_statements"].(string),
+		RevocationStatements: resp.Data["revocation_statements"].(string),
+		RollbackStatements:   resp.Data["rollback_statements"].(string),
+		RenewStatements:      resp.Data["renew_statements"].(string),
 	}
 
 	if !reflect.DeepEqual(expected, actual) {

builtin/logical/database/dbplugin/client.go

@@ -1,10 +1,8 @@
 package dbplugin
 
 import (
-	"fmt"
-	"net/rpc"
+	"errors"
 	"sync"
-	"time"
 
 	"github.com/hashicorp/go-plugin"
 	"github.com/hashicorp/vault/helper/pluginutil"
@@ -17,11 +15,11 @@ type DatabasePluginClient struct {
 	client *plugin.Client
 	sync.Mutex
 
-	*databasePluginRPCClient
+	Database
 }
 
 func (dc *DatabasePluginClient) Close() error {
-	err := dc.databasePluginRPCClient.Close()
+	err := dc.Database.Close()
 	dc.client.Kill()
 
 	return err
@@ -55,79 +53,20 @@ func newPluginClient(sys pluginutil.RunnerUtil, pluginRunner *pluginutil.PluginR
 
 	// We should have a database type now. This feels like a normal interface
 	// implementation but is in fact over an RPC connection.
-	databaseRPC := raw.(*databasePluginRPCClient)
+	var db Database
+	switch raw.(type) {
+	case *gRPCClient:
+		db = raw.(*gRPCClient)
+	case *databasePluginRPCClient:
+		logger.Warn("database: plugin is using deprecated net RPC transport, recompile plugin to upgrade to gRPC", "plugin", pluginRunner.Name)
+		db = raw.(*databasePluginRPCClient)
+	default:
+		return nil, errors.New("unsupported client type")
+	}
 
 	// Wrap RPC implimentation in DatabasePluginClient
 	return &DatabasePluginClient{
-		client:                  client,
-		databasePluginRPCClient: databaseRPC,
+		client:   client,
+		Database: db,
 	}, nil
 }
-
-// ---- RPC client domain ----
-
-// databasePluginRPCClient implements Database and is used on the client to
-// make RPC calls to a plugin.
-type databasePluginRPCClient struct {
-	client *rpc.Client
-}
-
-func (dr *databasePluginRPCClient) Type() (string, error) {
-	var dbType string
-	err := dr.client.Call("Plugin.Type", struct{}{}, &dbType)
-
-	return fmt.Sprintf("plugin-%s", dbType), err
-}
-
-func (dr *databasePluginRPCClient) CreateUser(statements Statements, usernameConfig UsernameConfig, expiration time.Time) (username string, password string, err error) {
-	req := CreateUserRequest{
-		Statements:     statements,
-		UsernameConfig: usernameConfig,
-		Expiration:     expiration,
-	}
-
-	var resp CreateUserResponse
-	err = dr.client.Call("Plugin.CreateUser", req, &resp)
-
-	return resp.Username, resp.Password, err
-}
-
-func (dr *databasePluginRPCClient) RenewUser(statements Statements, username string, expiration time.Time) error {
-	req := RenewUserRequest{
-		Statements: statements,
-		Username:   username,
-		Expiration: expiration,
-	}
-
-	err := dr.client.Call("Plugin.RenewUser", req, &struct{}{})
-
-	return err
-}
-
-func (dr *databasePluginRPCClient) RevokeUser(statements Statements, username string) error {
-	req := RevokeUserRequest{
-		Statements: statements,
-		Username:   username,
-	}
-
-	err := dr.client.Call("Plugin.RevokeUser", req, &struct{}{})
-
-	return err
-}
-
-func (dr *databasePluginRPCClient) Initialize(conf map[string]interface{}, verifyConnection bool) error {
-	req := InitializeRequest{
-		Config:           conf,
-		VerifyConnection: verifyConnection,
-	}
-
-	err := dr.client.Call("Plugin.Initialize", req, &struct{}{})
-
-	return err
-}
-
-func (dr *databasePluginRPCClient) Close() error {
-	err := dr.client.Call("Plugin.Close", struct{}{}, &struct{}{})
-
-	return err
-}