From 981f70883a25c5b8929ab21543204322478f32a9 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Sun, 30 Jun 2019 07:59:02 -0400
Subject: [PATCH 01/53] upgrade aws roles

---
 builtin/credential/aws/path_role.go | 43 +++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index 992d2e62b809..6fe17c0badbb 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -320,6 +320,49 @@ func (b *backend) setRole(ctx context.Context, s logical.Storage, roleName strin
 	return nil
 }
 
+// updateUpgradableRoleEntries upgrades and persists all of the role entries
+// that are in need of being upgraded.
+func (b *backend) updateUpgradableRoleEntries(ctx context.Context, s logical.Storage) error {
+
+	// Upgrade only if we are either: (1) a local mount, or (2) are _not_ a
+	// performance replicated standby cluster.
+	if b.System().LocalMount() || !b.System().ReplicationState().HasState(consts.ReplicationPerformanceSecondary|consts.ReplicationPerformanceStandby) {
+
+		// TODO come up with a way to record the fact that this has already
+		// been run and that therefore we don't need to do it every time
+
+		// Read all the role names.  We don't need to grab the mutex here for
+		// listing. The reason is that if we're in the process of creating a
+		// new mount, it will already happen on the active node (since it's a
+		// write) and will properly use the latest role structure.
+		roleNames, err := s.List(ctx, "role/")
+		if err != nil {
+			return err
+		}
+
+		// Upgrade the roles as necessary.
+		for _, roleName := range roleNames {
+			err := b.updateUpgradableRoleEntry(ctx, s, roleName)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// updateUpgradableRoleEntry uses the write lock to call roleInternal(), which
+// will do an upgrade and save it if need be.
+func (b *backend) updateUpgradableRoleEntry(ctx context.Context, s logical.Storage, roleName string) error {
+
+	b.roleMutex.Lock()
+	defer b.roleMutex.Unlock()
+
+	_, err := b.roleInternal(ctx, s, roleName)
+	return err
+}
+
 // If needed, updates the role entry and returns a bool indicating if it was updated
 // (and thus needs to be persisted)
 func (b *backend) upgradeRole(ctx context.Context, s logical.Storage, roleEntry *awsRoleEntry) (bool, error) {

From 1ead34871bae9694c83590e0c0fa2cc8686da244 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Sun, 30 Jun 2019 08:02:17 -0400
Subject: [PATCH 02/53] test upgrade aws roles

---
 builtin/credential/aws/path_role_test.go | 106 +++++++++++++++++++++++
 1 file changed, 106 insertions(+)

diff --git a/builtin/credential/aws/path_role_test.go b/builtin/credential/aws/path_role_test.go
index a593d459685f..a56bce3048fb 100644
--- a/builtin/credential/aws/path_role_test.go
+++ b/builtin/credential/aws/path_role_test.go
@@ -801,6 +801,112 @@ func TestRoleEntryUpgradeV(t *testing.T) {
 	}
 }
 
+func TestUpdateUpgradableRoleEntries(t *testing.T) {
+
+	config := logical.TestBackendConfig()
+	storage := &logical.InmemStorage{}
+	config.StorageView = storage
+	b, err := Backend(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ctx := context.Background()
+	err = b.Setup(ctx, config)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// create some role entries, some of which will need to be upgraded
+	type testData struct {
+		name  string
+		entry *awsRoleEntry
+	}
+
+	before := []testData{
+		{
+			name: "role1",
+			entry: &awsRoleEntry{
+				BoundIamRoleARNs:            []string{"arn:aws:iam::000000000001:role/my_role_prefix"},
+				BoundIamInstanceProfileARNs: []string{"arn:aws:iam::000000000001:instance-profile/my_profile-prefix"},
+				Version:                     1,
+			},
+		},
+		{
+			name: "role2",
+			entry: &awsRoleEntry{
+				BoundIamRoleARNs:            []string{"arn:aws:iam::000000000002:role/my_role_prefix"},
+				BoundIamInstanceProfileARNs: []string{"arn:aws:iam::000000000002:instance-profile/my_profile-prefix"},
+				Version:                     2,
+			},
+		},
+		{
+			name: "role3",
+			entry: &awsRoleEntry{
+				BoundIamRoleARNs:            []string{"arn:aws:iam::000000000003:role/my_role_prefix"},
+				BoundIamInstanceProfileARNs: []string{"arn:aws:iam::000000000003:instance-profile/my_profile-prefix"},
+				Version:                     currentRoleStorageVersion,
+			},
+		},
+	}
+
+	// put the entries in storage
+	for _, role := range before {
+		err = b.setRole(ctx, storage, role.name, role.entry)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// upgrade all the entries
+	err = b.updateUpgradableRoleEntries(ctx, storage)
+
+	// read the entries from storage
+	after := make([]testData, 0)
+	names, err := storage.List(ctx, "role/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, name := range names {
+		entry, err := b.role(ctx, storage, name)
+		if err != nil {
+			t.Fatal(err)
+		}
+		after = append(after, testData{name: name, entry: entry})
+	}
+
+	// make sure each entry is at the current version
+	expected := []testData{
+		{
+			name: "role1",
+			entry: &awsRoleEntry{
+				BoundIamRoleARNs:            []string{"arn:aws:iam::000000000001:role/my_role_prefix"},
+				BoundIamInstanceProfileARNs: []string{"arn:aws:iam::000000000001:instance-profile/my_profile-prefix"},
+				Version:                     currentRoleStorageVersion,
+			},
+		},
+		{
+			name: "role2",
+			entry: &awsRoleEntry{
+				BoundIamRoleARNs:            []string{"arn:aws:iam::000000000002:role/my_role_prefix"},
+				BoundIamInstanceProfileARNs: []string{"arn:aws:iam::000000000002:instance-profile/my_profile-prefix"},
+				Version:                     currentRoleStorageVersion,
+			},
+		},
+		{
+			name: "role3",
+			entry: &awsRoleEntry{
+				BoundIamRoleARNs:            []string{"arn:aws:iam::000000000003:role/my_role_prefix"},
+				BoundIamInstanceProfileARNs: []string{"arn:aws:iam::000000000003:instance-profile/my_profile-prefix"},
+				Version:                     currentRoleStorageVersion,
+			},
+		},
+	}
+	if diff := deep.Equal(expected, after); diff != nil {
+		t.Fatal(diff)
+	}
+}
+
 func resolveArnToFakeUniqueId(ctx context.Context, s logical.Storage, arn string) (string, error) {
 	return "FakeUniqueId1", nil
 }

From cccb56e66b1066afb835e70328c66d9e628e37d1 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Sun, 30 Jun 2019 10:47:33 -0400
Subject: [PATCH 03/53] Initialize aws credential backend at mount time

---
 builtin/credential/aws/backend.go   | 10 +++-
 builtin/credential/aws/path_role.go | 93 +++++++++++++++++++++++------
 vault/auth.go                       | 18 ++++++
 3 files changed, 101 insertions(+), 20 deletions(-)

diff --git a/builtin/credential/aws/backend.go b/builtin/credential/aws/backend.go
index 0209e6773c05..ccf6dae410a8 100644
--- a/builtin/credential/aws/backend.go
+++ b/builtin/credential/aws/backend.go
@@ -34,8 +34,14 @@ type backend struct {
 	// Lock to make changes to any of the backend's configuration endpoints.
 	configMutex sync.RWMutex
 
+	// Guards the initialize function
+	initializeCASGuard *uint32
+
+	// check whether initialize has already been called
+	initialized bool
+
 	// Lock to make changes to role entries
-	roleMutex sync.RWMutex
+	roleMutex sync.Mutex
 
 	// Lock to make changes to the blacklist entries
 	blacklistMutex sync.RWMutex
@@ -91,6 +97,7 @@ func Backend(conf *logical.BackendConfig) (*backend, error) {
 		EC2ClientsMap:         make(map[string]map[string]*ec2.EC2),
 		IAMClientsMap:         make(map[string]map[string]*iam.IAM),
 		iamUserIdToArnCache:   cache.New(7*24*time.Hour, 24*time.Hour),
+		initializeCASGuard:    new(uint32),
 		tidyBlacklistCASGuard: new(uint32),
 		tidyWhitelistCASGuard: new(uint32),
 		roleCache:             cache.New(cache.NoExpiration, cache.NoExpiration),
@@ -115,6 +122,7 @@ func Backend(conf *logical.BackendConfig) (*backend, error) {
 		},
 		Paths: []*framework.Path{
 			pathLogin(b),
+			pathInitialize(b),
 			pathListRole(b),
 			pathListRoles(b),
 			pathRole(b),
diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index 6fe17c0badbb..a7c23824189f 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -4,7 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net/http"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/hashicorp/errwrap"
@@ -191,6 +193,15 @@ auth_type is ec2.`,
 	}
 }
 
+func pathInitialize(b *backend) *framework.Path {
+	return &framework.Path{
+		Pattern: "initialize$",
+		Callbacks: map[logical.Operation]framework.OperationFunc{
+			logical.CreateOperation: b.pathRoleInitialize,
+		},
+	}
+}
+
 func pathListRole(b *backend) *framework.Path {
 	return &framework.Path{
 		Pattern: "role/?",
@@ -320,32 +331,76 @@ func (b *backend) setRole(ctx context.Context, s logical.Storage, roleName strin
 	return nil
 }
 
-// updateUpgradableRoleEntries upgrades and persists all of the role entries
-// that are in need of being upgraded.
-func (b *backend) updateUpgradableRoleEntries(ctx context.Context, s logical.Storage) error {
+// pathRoleInitialize is used to initialize the AWS roles
+func (b *backend) pathRoleInitialize(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
 
-	// Upgrade only if we are either: (1) a local mount, or (2) are _not_ a
-	// performance replicated standby cluster.
+	// Initialize only if we are either:
+	//   (1) A local mount.
+	//   (2) Are _not_ a replicated standby cluster.
 	if b.System().LocalMount() || !b.System().ReplicationState().HasState(consts.ReplicationPerformanceSecondary|consts.ReplicationPerformanceStandby) {
 
-		// TODO come up with a way to record the fact that this has already
-		// been run and that therefore we don't need to do it every time
+		// TODO we need to figure out a way to persist the fact that this has
+		// already been run in the past for a given currentRoleStorageVersion,
+		// so that we only ever run it once even if the vault server is
+		// restarted.
 
-		// Read all the role names.  We don't need to grab the mutex here for
-		// listing. The reason is that if we're in the process of creating a
-		// new mount, it will already happen on the active node (since it's a
-		// write) and will properly use the latest role structure.
-		roleNames, err := s.List(ctx, "role/")
-		if err != nil {
-			return err
+		// grab the guard
+		if !atomic.CompareAndSwapUint32(b.initializeCASGuard, 0, 1) {
+			resp := &logical.Response{}
+			resp.AddWarning("Initialize operation already in progress.")
+			return resp, nil
 		}
 
-		// Upgrade the roles as necessary.
-		for _, roleName := range roleNames {
-			err := b.updateUpgradableRoleEntry(ctx, s, roleName)
-			if err != nil {
-				return err
+		// check if this method has already been called
+		if b.initialized {
+			atomic.StoreUint32(b.initializeCASGuard, 0)
+			return logical.ErrorResponse("already initialized"), nil
+		}
+		b.initialized = true
+
+		// kick off the role upgrader
+		go func() {
+			defer atomic.StoreUint32(b.initializeCASGuard, 0)
+
+			// Don't cancel when the original client request goes away
+			ctx = context.Background()
+
+			logger := b.Logger().Named("initialize")
+			logger.Info("starting initialization")
+			err := b.updateUpgradableRoleEntries(ctx, req.Storage)
+			if err == nil {
+				logger.Info("initialization succeeded")
+			} else {
+				logger.Error("error running initialization", "error", err)
 			}
+		}()
+
+		resp := &logical.Response{}
+		resp.AddWarning("Initialization operation successfully started. Any information from the operation will be printed to Vault's server logs.")
+		return logical.RespondWithStatusCode(resp, req, http.StatusAccepted)
+	}
+
+	return nil, nil
+}
+
+// updateUpgradableRoleEntries upgrades and persists all of the role entries
+// that are in need of being upgraded.
+func (b *backend) updateUpgradableRoleEntries(ctx context.Context, s logical.Storage) error {
+
+	// Read all the role names.  We don't need to grab the mutex here for
+	// listing. The reason is that if we're in the process of creating a new
+	// mount, it will already happen on the active node (since it's a write)
+	// and will properly use the latest role structure.
+	roleNames, err := s.List(ctx, "role/")
+	if err != nil {
+		return err
+	}
+
+	// Upgrade the roles as necessary.
+	for _, roleName := range roleNames {
+		err := b.updateUpgradableRoleEntry(ctx, s, roleName)
+		if err != nil {
+			return err
 		}
 	}
 
diff --git a/vault/auth.go b/vault/auth.go
index a0a10dedc042..dad0de3a6945 100644
--- a/vault/auth.go
+++ b/vault/auth.go
@@ -185,6 +185,24 @@ func (c *Core) enableCredentialInternal(ctx context.Context, entry *MountEntry,
 		return err
 	}
 
+	// TODO Its obviously hackish to check specifically for the AWS auth
+	// builtin here.  However, we only want to make the initialization request
+	// on backends that expect to be initialized, and right now AWS auth is the
+	// only one.  What is the best solution that doesn't require e.g. adding a
+	// method to logical.Backend?  Maybe we can somehow detect whether an
+	// "auth/foo/initialize" path exists, before calling?
+	if entry.Table == "auth" && entry.Path == "aws/" {
+		req := &logical.Request{
+			Operation: logical.CreateOperation,
+			Path:      "auth/aws/initialize",
+		}
+
+		_, err = c.router.Route(ctx, req)
+		if err != nil {
+			return err
+		}
+	}
+
 	if c.logger.IsInfo() {
 		c.logger.Info("enabled credential backend", "path", entry.Path, "type", entry.Type)
 	}

From ce1dbe03962d51a3d5ff5ccc94187996523c80c7 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Sun, 30 Jun 2019 11:32:24 -0400
Subject: [PATCH 04/53] add a TODO

---
 vault/auth.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/vault/auth.go b/vault/auth.go
index dad0de3a6945..8c7b26224440 100644
--- a/vault/auth.go
+++ b/vault/auth.go
@@ -703,6 +703,9 @@ func (c *Core) setupCredentials(ctx context.Context) error {
 		return c.persistAuth(ctx, c.auth, nil)
 	}
 
+	// TODO do we need to check for AWS auth builtin backed, and initialize it?
+	// If so, is this the right place to do that?
+
 	return nil
 }
 

From 593048fd14f3bb9bb71af6ba387d5bb45e1e1165 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Sun, 30 Jun 2019 12:02:45 -0400
Subject: [PATCH 05/53] create end-to-end test for builtin/credential/aws

---
 builtin/credential/aws/backend_test.go | 32 ++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/builtin/credential/aws/backend_test.go b/builtin/credential/aws/backend_test.go
index b8b947350b31..9ee459249133 100644
--- a/builtin/credential/aws/backend_test.go
+++ b/builtin/credential/aws/backend_test.go
@@ -14,10 +14,15 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/sts"
+	hclog "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/vault/api"
 	logicaltest "github.com/hashicorp/vault/helper/testhelpers/logical"
+	vaulthttp "github.com/hashicorp/vault/http"
 	"github.com/hashicorp/vault/sdk/framework"
+	"github.com/hashicorp/vault/sdk/helper/logging"
 	"github.com/hashicorp/vault/sdk/helper/policyutil"
 	"github.com/hashicorp/vault/sdk/logical"
+	"github.com/hashicorp/vault/vault"
 )
 
 const testVaultHeaderValue = "VaultAcceptanceTesting"
@@ -1813,3 +1818,30 @@ func generateRenewRequest(s logical.Storage, auth *logical.Auth) *logical.Reques
 
 	return renewReq
 }
+
+func TestBackend_Initialize(t *testing.T) {
+
+	logger := logging.NewVaultLogger(hclog.Trace)
+	coreConfig := &vault.CoreConfig{
+		Logger: logger,
+		CredentialBackends: map[string]logical.Factory{
+			"aws": Factory,
+		},
+	}
+	cluster := vault.NewTestCluster(t, coreConfig, &vault.TestClusterOptions{
+		NumCores:    1,
+		HandlerFunc: vaulthttp.Handler,
+	})
+	cluster.Start()
+	defer cluster.Cleanup()
+
+	vault.TestWaitActive(t, cluster.Cores[0].Core)
+	client := cluster.Cores[0].Client
+
+	// Setup Vault
+	if err := client.Sys().EnableAuthWithOptions("aws", &api.EnableAuthOptions{
+		Type: "aws",
+	}); err != nil {
+		t.Fatal(err)
+	}
+}

From 7e900b691c92ba6de4f90eb7533de582569b09e1 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Sun, 30 Jun 2019 12:46:49 -0400
Subject: [PATCH 06/53] fix bug in initializer

---
 builtin/credential/aws/path_role.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index a7c23824189f..175e67667953 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -358,6 +358,8 @@ func (b *backend) pathRoleInitialize(ctx context.Context, req *logical.Request,
 		}
 		b.initialized = true
 
+		s := req.Storage
+
 		// kick off the role upgrader
 		go func() {
 			defer atomic.StoreUint32(b.initializeCASGuard, 0)
@@ -367,7 +369,7 @@ func (b *backend) pathRoleInitialize(ctx context.Context, req *logical.Request,
 
 			logger := b.Logger().Named("initialize")
 			logger.Info("starting initialization")
-			err := b.updateUpgradableRoleEntries(ctx, req.Storage)
+			err := b.updateUpgradableRoleEntries(ctx, s)
 			if err == nil {
 				logger.Info("initialization succeeded")
 			} else {

From add4b062aff124f5e802010a6c896116bbfb8e95 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Sun, 30 Jun 2019 12:55:05 -0400
Subject: [PATCH 07/53] improve comments

---
 builtin/credential/aws/backend_test.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/builtin/credential/aws/backend_test.go b/builtin/credential/aws/backend_test.go
index 9ee459249133..12bc114bf247 100644
--- a/builtin/credential/aws/backend_test.go
+++ b/builtin/credential/aws/backend_test.go
@@ -1821,6 +1821,10 @@ func generateRenewRequest(s logical.Storage, auth *logical.Auth) *logical.Reques
 
 func TestBackend_Initialize(t *testing.T) {
 
+	// TODO this test is not very good. it just outputs some log entries showing
+	// that initialization ran.  lets find a way to verify that the initializer
+	// actually did something, or at least somehow trap the fact that it ran.
+
 	logger := logging.NewVaultLogger(hclog.Trace)
 	coreConfig := &vault.CoreConfig{
 		Logger: logger,

From 24a10f5928dc24ac360c26bd29fd8a592435ab6a Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Sun, 30 Jun 2019 14:22:53 -0400
Subject: [PATCH 08/53] add Initialize() to logical.Backend

---
 sdk/framework/backend.go          | 16 ++++++++++++++++
 sdk/logical/logical.go            |  4 ++++
 sdk/plugin/grpc_backend_client.go |  4 ++++
 sdk/plugin/middleware.go          |  9 +++++++++
 vault/router_testing.go           |  2 +-
 vault/testing.go                  |  3 +--
 6 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/sdk/framework/backend.go b/sdk/framework/backend.go
index 721d23f3e776..607bb0b94a0a 100644
--- a/sdk/framework/backend.go
+++ b/sdk/framework/backend.go
@@ -49,6 +49,10 @@ type Backend struct {
 	// and ease specifying callbacks for revocation, renewal, etc.
 	Secrets []*Secret
 
+	// InitializeFunc is the callback, which if set, will be invoked via
+	// Initialize() just after a plugin has been mounted.
+	InitializeFunc InitializeFunc
+
 	// PeriodicFunc is the callback, which if set, will be invoked when the
 	// periodic timer of RollbackManager ticks. This can be used by
 	// backends to do anything it wishes to do periodically.
@@ -108,6 +112,18 @@ type CleanupFunc func(context.Context)
 // InvalidateFunc is the callback for backend key invalidation.
 type InvalidateFunc func(context.Context, string)
 
+// InitializeFunc is the callback, which if set, will be invoked via
+// Initialize() just after a plugin has been mounted.
+type InitializeFunc func(context.Context, *logical.Request) error
+
+// Initialize is the logical.Backend implementation.
+func (b *Backend) Initialize(ctx context.Context, req *logical.Request) error {
+	if b.InitializeFunc != nil {
+		return b.InitializeFunc(ctx, req)
+	}
+	return nil
+}
+
 // HandleExistenceCheck is the logical.Backend implementation.
 func (b *Backend) HandleExistenceCheck(ctx context.Context, req *logical.Request) (checkFound bool, exists bool, err error) {
 	b.once.Do(b.init)
diff --git a/sdk/logical/logical.go b/sdk/logical/logical.go
index cc3f6ae47533..b468af728524 100644
--- a/sdk/logical/logical.go
+++ b/sdk/logical/logical.go
@@ -38,6 +38,10 @@ func (b BackendType) String() string {
 // allows for a "procfs" like interaction, as internal state can be exposed by
 // acting like a logical backend and being mounted.
 type Backend interface {
+
+	// Initialize is used to initialize a plugin after it has been mounted.
+	Initialize(context.Context, *Request) error
+
 	// HandleRequest is used to handle a request and generate a response.
 	// The backends must check the operation type and handle appropriately.
 	HandleRequest(context.Context, *Request) (*Response, error)
diff --git a/sdk/plugin/grpc_backend_client.go b/sdk/plugin/grpc_backend_client.go
index 169c1e23a6f7..eb923d84d104 100644
--- a/sdk/plugin/grpc_backend_client.go
+++ b/sdk/plugin/grpc_backend_client.go
@@ -44,6 +44,10 @@ type backendGRPCPluginClient struct {
 	doneCtx    context.Context
 }
 
+func (b *backendGRPCPluginClient) Initialize(ctx context.Context, req *logical.Request) error {
+	panic("TODO")
+}
+
 func (b *backendGRPCPluginClient) HandleRequest(ctx context.Context, req *logical.Request) (*logical.Response, error) {
 	if b.metadataMode {
 		return nil, ErrClientInMetadataMode
diff --git a/sdk/plugin/middleware.go b/sdk/plugin/middleware.go
index cd92a42bd05b..b7388d7bce73 100644
--- a/sdk/plugin/middleware.go
+++ b/sdk/plugin/middleware.go
@@ -19,6 +19,15 @@ type backendTracingMiddleware struct {
 // Validate the backendTracingMiddle object satisfies the backend interface
 var _ logical.Backend = &backendTracingMiddleware{}
 
+func (b *backendTracingMiddleware) Initialize(ctx context.Context, req *logical.Request) (err error) {
+	defer func(then time.Time) {
+		b.logger.Trace("initialize", "path", req.Path, "status", "finished", "err", err, "took", time.Since(then))
+	}(time.Now())
+
+	b.logger.Trace("initialize", "path", req.Path, "status", "started")
+	return b.next.Initialize(ctx, req)
+}
+
 func (b *backendTracingMiddleware) HandleRequest(ctx context.Context, req *logical.Request) (resp *logical.Response, err error) {
 	defer func(then time.Time) {
 		b.logger.Trace("handle request", "path", req.Path, "status", "finished", "err", err, "took", time.Since(then))
diff --git a/vault/router_testing.go b/vault/router_testing.go
index bc287806eda1..bea608a3190b 100644
--- a/vault/router_testing.go
+++ b/vault/router_testing.go
@@ -103,7 +103,7 @@ func (n *NoopBackend) Logger() log.Logger {
 	return log.NewNullLogger()
 }
 
-func (n *NoopBackend) Initialize(ctx context.Context) error {
+func (n *NoopBackend) Initialize(ctx context.Context, req *logical.Request) error {
 	return nil
 }
 
diff --git a/vault/testing.go b/vault/testing.go
index c8fa642b672c..1c45ec050110 100644
--- a/vault/testing.go
+++ b/vault/testing.go
@@ -694,8 +694,7 @@ func (n *rawHTTP) Cleanup(ctx context.Context) {
 	// noop
 }
 
-func (n *rawHTTP) Initialize(ctx context.Context) error {
-	// noop
+func (n *rawHTTP) Initialize(ctx context.Context, req *logical.Request) error {
 	return nil
 }
 

From 6abe0eea4f7eb57b949967d2b38626768bca9859 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Sun, 30 Jun 2019 14:54:41 -0400
Subject: [PATCH 09/53] use Initialize() in Core.enableCredentialInternal()

---
 builtin/credential/aws/backend.go   |  8 ++---
 builtin/credential/aws/path_role.go | 35 +++++++------------
 builtin/plugin/backend.go           | 52 ++++++++++++++++++++++++++++-
 vault/auth.go                       | 23 ++++---------
 vault/router.go                     | 14 ++++++--
 5 files changed, 85 insertions(+), 47 deletions(-)

diff --git a/builtin/credential/aws/backend.go b/builtin/credential/aws/backend.go
index ccf6dae410a8..90e39cb0086b 100644
--- a/builtin/credential/aws/backend.go
+++ b/builtin/credential/aws/backend.go
@@ -38,7 +38,7 @@ type backend struct {
 	initializeCASGuard *uint32
 
 	// check whether initialize has already been called
-	initialized bool
+	isInitialized bool
 
 	// Lock to make changes to role entries
 	roleMutex sync.Mutex
@@ -122,7 +122,6 @@ func Backend(conf *logical.BackendConfig) (*backend, error) {
 		},
 		Paths: []*framework.Path{
 			pathLogin(b),
-			pathInitialize(b),
 			pathListRole(b),
 			pathListRoles(b),
 			pathRole(b),
@@ -142,8 +141,9 @@ func Backend(conf *logical.BackendConfig) (*backend, error) {
 			pathIdentityWhitelist(b),
 			pathTidyIdentityWhitelist(b),
 		},
-		Invalidate:  b.invalidate,
-		BackendType: logical.TypeCredential,
+		Invalidate:     b.invalidate,
+		InitializeFunc: b.initialize,
+		BackendType:    logical.TypeCredential,
 	}
 
 	return b, nil
diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index 175e67667953..cf1481e0b7a6 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"net/http"
 	"strings"
 	"sync/atomic"
 	"time"
@@ -193,15 +192,6 @@ auth_type is ec2.`,
 	}
 }
 
-func pathInitialize(b *backend) *framework.Path {
-	return &framework.Path{
-		Pattern: "initialize$",
-		Callbacks: map[logical.Operation]framework.OperationFunc{
-			logical.CreateOperation: b.pathRoleInitialize,
-		},
-	}
-}
-
 func pathListRole(b *backend) *framework.Path {
 	return &framework.Path{
 		Pattern: "role/?",
@@ -331,14 +321,16 @@ func (b *backend) setRole(ctx context.Context, s logical.Storage, roleName strin
 	return nil
 }
 
-// pathRoleInitialize is used to initialize the AWS roles
-func (b *backend) pathRoleInitialize(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
+// initialize is used to initialize the AWS roles
+func (b *backend) initialize(ctx context.Context, req *logical.Request) error {
 
 	// Initialize only if we are either:
 	//   (1) A local mount.
 	//   (2) Are _not_ a replicated standby cluster.
 	if b.System().LocalMount() || !b.System().ReplicationState().HasState(consts.ReplicationPerformanceSecondary|consts.ReplicationPerformanceStandby) {
 
+		logger := b.Logger().Named("initialize")
+
 		// TODO we need to figure out a way to persist the fact that this has
 		// already been run in the past for a given currentRoleStorageVersion,
 		// so that we only ever run it once even if the vault server is
@@ -346,17 +338,17 @@ func (b *backend) pathRoleInitialize(ctx context.Context, req *logical.Request,
 
 		// grab the guard
 		if !atomic.CompareAndSwapUint32(b.initializeCASGuard, 0, 1) {
-			resp := &logical.Response{}
-			resp.AddWarning("Initialize operation already in progress.")
-			return resp, nil
+			logger.Error("initializational already in progress")
+			return nil
 		}
 
 		// check if this method has already been called
-		if b.initialized {
+		if b.isInitialized {
 			atomic.StoreUint32(b.initializeCASGuard, 0)
-			return logical.ErrorResponse("already initialized"), nil
+			logger.Error("already initialized")
+			return nil
 		}
-		b.initialized = true
+		b.isInitialized = true
 
 		s := req.Storage
 
@@ -367,7 +359,6 @@ func (b *backend) pathRoleInitialize(ctx context.Context, req *logical.Request,
 			// Don't cancel when the original client request goes away
 			ctx = context.Background()
 
-			logger := b.Logger().Named("initialize")
 			logger.Info("starting initialization")
 			err := b.updateUpgradableRoleEntries(ctx, s)
 			if err == nil {
@@ -377,12 +368,10 @@ func (b *backend) pathRoleInitialize(ctx context.Context, req *logical.Request,
 			}
 		}()
 
-		resp := &logical.Response{}
-		resp.AddWarning("Initialization operation successfully started. Any information from the operation will be printed to Vault's server logs.")
-		return logical.RespondWithStatusCode(resp, req, http.StatusAccepted)
+		return nil
 	}
 
-	return nil, nil
+	return nil
 }
 
 // updateUpgradableRoleEntries upgrades and persists all of the role entries
diff --git a/builtin/plugin/backend.go b/builtin/plugin/backend.go
index c29842af628d..0f9932df4979 100644
--- a/builtin/plugin/backend.go
+++ b/builtin/plugin/backend.go
@@ -188,7 +188,7 @@ func (b *PluginBackend) HandleRequest(ctx context.Context, req *logical.Request)
 	return resp, err
 }
 
-// HandleExistenceCheck is a thin wrapper implementation of HandleRequest that includes automatic plugin reload.
+// HandleExistenceCheck is a thin wrapper implementation of HandleExistenceCheck that includes automatic plugin reload.
 func (b *PluginBackend) HandleExistenceCheck(ctx context.Context, req *logical.Request) (bool, bool, error) {
 	b.RLock()
 	canary := b.canary
@@ -237,3 +237,53 @@ func (b *PluginBackend) HandleExistenceCheck(ctx context.Context, req *logical.R
 	}
 	return checkFound, exists, err
 }
+
+// Initialize is a thin wrapper implementation of Initialize that includes automatic plugin reload.
+func (b *PluginBackend) Initialize(ctx context.Context, req *logical.Request) error {
+	b.RLock()
+	canary := b.canary
+
+	// Lazy-load backend
+	if !b.loaded {
+		// Upgrade lock
+		b.RUnlock()
+		b.Lock()
+		// Check once more after lock swap
+		if !b.loaded {
+			err := b.startBackend(ctx)
+			if err != nil {
+				b.Unlock()
+				return err
+			}
+		}
+		b.Unlock()
+		b.RLock()
+	}
+
+	err := b.Backend.Initialize(ctx, req)
+	b.RUnlock()
+	if err != nil &&
+		(err.Error() == rpc.ErrShutdown.Error() || err == bplugin.ErrPluginShutdown) {
+		// Reload plugin if it's an rpc.ErrShutdown
+		b.Lock()
+		if b.canary == canary {
+			err := b.reloadBackend(ctx)
+			if err != nil {
+				b.Unlock()
+				return err
+			}
+			b.canary, err = uuid.GenerateUUID()
+			if err != nil {
+				b.Unlock()
+				return err
+			}
+		}
+		b.Unlock()
+
+		// Try request once more
+		b.RLock()
+		defer b.RUnlock()
+		return b.Backend.Initialize(ctx, req)
+	}
+	return err
+}
diff --git a/vault/auth.go b/vault/auth.go
index 8c7b26224440..9f9f73a19bc4 100644
--- a/vault/auth.go
+++ b/vault/auth.go
@@ -185,22 +185,13 @@ func (c *Core) enableCredentialInternal(ctx context.Context, entry *MountEntry,
 		return err
 	}
 
-	// TODO Its obviously hackish to check specifically for the AWS auth
-	// builtin here.  However, we only want to make the initialization request
-	// on backends that expect to be initialized, and right now AWS auth is the
-	// only one.  What is the best solution that doesn't require e.g. adding a
-	// method to logical.Backend?  Maybe we can somehow detect whether an
-	// "auth/foo/initialize" path exists, before calling?
-	if entry.Table == "auth" && entry.Path == "aws/" {
-		req := &logical.Request{
-			Operation: logical.CreateOperation,
-			Path:      "auth/aws/initialize",
-		}
-
-		_, err = c.router.Route(ctx, req)
-		if err != nil {
-			return err
-		}
+	// Initialize
+	req := &logical.Request{
+		Path: entry.Table + "/" + entry.Path,
+	}
+	err = c.router.RouteInitialize(ctx, req)
+	if err != nil {
+		return err
 	}
 
 	if c.logger.IsInfo() {
diff --git a/vault/router.go b/vault/router.go
index 1eccb3ff93f3..f001906e2238 100644
--- a/vault/router.go
+++ b/vault/router.go
@@ -462,17 +462,23 @@ func (r *Router) matchingMountEntryByPath(ctx context.Context, path string, apiP
 
 // Route is used to route a given request
 func (r *Router) Route(ctx context.Context, req *logical.Request) (*logical.Response, error) {
-	resp, _, _, err := r.routeCommon(ctx, req, false)
+	resp, _, _, err := r.routeCommon(ctx, req, false, false)
 	return resp, err
 }
 
 // RouteExistenceCheck is used to route a given existence check request
 func (r *Router) RouteExistenceCheck(ctx context.Context, req *logical.Request) (*logical.Response, bool, bool, error) {
-	resp, ok, exists, err := r.routeCommon(ctx, req, true)
+	resp, ok, exists, err := r.routeCommon(ctx, req, true, false)
 	return resp, ok, exists, err
 }
 
-func (r *Router) routeCommon(ctx context.Context, req *logical.Request, existenceCheck bool) (*logical.Response, bool, bool, error) {
+// RouteInitialize is used to route a given initialization request
+func (r *Router) RouteInitialize(ctx context.Context, req *logical.Request) error {
+	_, _, _, err := r.routeCommon(ctx, req, false, true)
+	return err
+}
+
+func (r *Router) routeCommon(ctx context.Context, req *logical.Request, existenceCheck bool, initialize bool) (*logical.Response, bool, bool, error) {
 	ns, err := namespace.FromContext(ctx)
 	if err != nil {
 		return nil, false, false, err
@@ -661,6 +667,8 @@ func (r *Router) routeCommon(ctx context.Context, req *logical.Request, existenc
 	if existenceCheck {
 		ok, exists, err := re.backend.HandleExistenceCheck(ctx, req)
 		return nil, ok, exists, err
+	} else if initialize {
+		return nil, false, false, re.backend.Initialize(ctx, req)
 	} else {
 		resp, err := re.backend.HandleRequest(ctx, req)
 		if resp != nil {

From a3d10a5ef92e4e4b0e3076c33c67500c09012744 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Mon, 1 Jul 2019 09:55:17 -0400
Subject: [PATCH 10/53] use InitializeRequest to call Initialize()

---
 builtin/credential/aws/path_role.go |  4 +---
 builtin/plugin/backend.go           |  2 +-
 sdk/framework/backend.go            |  6 +++---
 sdk/logical/logical.go              |  2 +-
 sdk/logical/request.go              |  8 ++++++++
 sdk/plugin/grpc_backend_client.go   |  2 +-
 sdk/plugin/middleware.go            |  6 +++---
 vault/auth.go                       | 11 +++++------
 vault/router.go                     | 14 +++-----------
 vault/router_testing.go             |  2 +-
 vault/testing.go                    |  2 +-
 11 files changed, 28 insertions(+), 31 deletions(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index cf1481e0b7a6..beed3579b289 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -322,7 +322,7 @@ func (b *backend) setRole(ctx context.Context, s logical.Storage, roleName strin
 }
 
 // initialize is used to initialize the AWS roles
-func (b *backend) initialize(ctx context.Context, req *logical.Request) error {
+func (b *backend) initialize(ctx context.Context, s logical.Storage) error {
 
 	// Initialize only if we are either:
 	//   (1) A local mount.
@@ -350,8 +350,6 @@ func (b *backend) initialize(ctx context.Context, req *logical.Request) error {
 		}
 		b.isInitialized = true
 
-		s := req.Storage
-
 		// kick off the role upgrader
 		go func() {
 			defer atomic.StoreUint32(b.initializeCASGuard, 0)
diff --git a/builtin/plugin/backend.go b/builtin/plugin/backend.go
index 0f9932df4979..f51ef2e0634a 100644
--- a/builtin/plugin/backend.go
+++ b/builtin/plugin/backend.go
@@ -239,7 +239,7 @@ func (b *PluginBackend) HandleExistenceCheck(ctx context.Context, req *logical.R
 }
 
 // Initialize is a thin wrapper implementation of Initialize that includes automatic plugin reload.
-func (b *PluginBackend) Initialize(ctx context.Context, req *logical.Request) error {
+func (b *PluginBackend) Initialize(ctx context.Context, req *logical.InitializationRequest) error {
 	b.RLock()
 	canary := b.canary
 
diff --git a/sdk/framework/backend.go b/sdk/framework/backend.go
index 607bb0b94a0a..45fa67ba6bfa 100644
--- a/sdk/framework/backend.go
+++ b/sdk/framework/backend.go
@@ -114,12 +114,12 @@ type InvalidateFunc func(context.Context, string)
 
 // InitializeFunc is the callback, which if set, will be invoked via
 // Initialize() just after a plugin has been mounted.
-type InitializeFunc func(context.Context, *logical.Request) error
+type InitializeFunc func(context.Context, logical.Storage) error
 
 // Initialize is the logical.Backend implementation.
-func (b *Backend) Initialize(ctx context.Context, req *logical.Request) error {
+func (b *Backend) Initialize(ctx context.Context, req *logical.InitializationRequest) error {
 	if b.InitializeFunc != nil {
-		return b.InitializeFunc(ctx, req)
+		return b.InitializeFunc(ctx, req.Storage)
 	}
 	return nil
 }
diff --git a/sdk/logical/logical.go b/sdk/logical/logical.go
index b468af728524..db8831535a8e 100644
--- a/sdk/logical/logical.go
+++ b/sdk/logical/logical.go
@@ -40,7 +40,7 @@ func (b BackendType) String() string {
 type Backend interface {
 
 	// Initialize is used to initialize a plugin after it has been mounted.
-	Initialize(context.Context, *Request) error
+	Initialize(context.Context, *InitializationRequest) error
 
 	// HandleRequest is used to handle a request and generate a response.
 	// The backends must check the operation type and handle appropriately.
diff --git a/sdk/logical/request.go b/sdk/logical/request.go
index e030d7ac28d5..98d56cec8ffe 100644
--- a/sdk/logical/request.go
+++ b/sdk/logical/request.go
@@ -303,3 +303,11 @@ const (
 )
 
 type MFACreds map[string][]string
+
+// InitializationRequest stores the parameters and context of an Initialize()
+// call being made to a logical.Backend.
+type InitializationRequest struct {
+
+	// Storage can be used to durably store and retrieve state.
+	Storage Storage
+}
diff --git a/sdk/plugin/grpc_backend_client.go b/sdk/plugin/grpc_backend_client.go
index eb923d84d104..85eb4ae4c0ac 100644
--- a/sdk/plugin/grpc_backend_client.go
+++ b/sdk/plugin/grpc_backend_client.go
@@ -44,7 +44,7 @@ type backendGRPCPluginClient struct {
 	doneCtx    context.Context
 }
 
-func (b *backendGRPCPluginClient) Initialize(ctx context.Context, req *logical.Request) error {
+func (b *backendGRPCPluginClient) Initialize(ctx context.Context, req *logical.InitializationRequest) error {
 	panic("TODO")
 }
 
diff --git a/sdk/plugin/middleware.go b/sdk/plugin/middleware.go
index b7388d7bce73..04a6f4c5000a 100644
--- a/sdk/plugin/middleware.go
+++ b/sdk/plugin/middleware.go
@@ -19,12 +19,12 @@ type backendTracingMiddleware struct {
 // Validate the backendTracingMiddle object satisfies the backend interface
 var _ logical.Backend = &backendTracingMiddleware{}
 
-func (b *backendTracingMiddleware) Initialize(ctx context.Context, req *logical.Request) (err error) {
+func (b *backendTracingMiddleware) Initialize(ctx context.Context, req *logical.InitializationRequest) (err error) {
 	defer func(then time.Time) {
-		b.logger.Trace("initialize", "path", req.Path, "status", "finished", "err", err, "took", time.Since(then))
+		b.logger.Trace("initialize", "status", "finished", "err", err, "took", time.Since(then))
 	}(time.Now())
 
-	b.logger.Trace("initialize", "path", req.Path, "status", "started")
+	b.logger.Trace("initialize", "status", "started")
 	return b.next.Initialize(ctx, req)
 }
 
diff --git a/vault/auth.go b/vault/auth.go
index 9f9f73a19bc4..a0f075737796 100644
--- a/vault/auth.go
+++ b/vault/auth.go
@@ -186,12 +186,11 @@ func (c *Core) enableCredentialInternal(ctx context.Context, entry *MountEntry,
 	}
 
 	// Initialize
-	req := &logical.Request{
-		Path: entry.Table + "/" + entry.Path,
-	}
-	err = c.router.RouteInitialize(ctx, req)
-	if err != nil {
-		return err
+	if !nilMount {
+		err := backend.Initialize(ctx, &logical.InitializationRequest{Storage: view})
+		if err != nil {
+			return err
+		}
 	}
 
 	if c.logger.IsInfo() {
diff --git a/vault/router.go b/vault/router.go
index f001906e2238..1eccb3ff93f3 100644
--- a/vault/router.go
+++ b/vault/router.go
@@ -462,23 +462,17 @@ func (r *Router) matchingMountEntryByPath(ctx context.Context, path string, apiP
 
 // Route is used to route a given request
 func (r *Router) Route(ctx context.Context, req *logical.Request) (*logical.Response, error) {
-	resp, _, _, err := r.routeCommon(ctx, req, false, false)
+	resp, _, _, err := r.routeCommon(ctx, req, false)
 	return resp, err
 }
 
 // RouteExistenceCheck is used to route a given existence check request
 func (r *Router) RouteExistenceCheck(ctx context.Context, req *logical.Request) (*logical.Response, bool, bool, error) {
-	resp, ok, exists, err := r.routeCommon(ctx, req, true, false)
+	resp, ok, exists, err := r.routeCommon(ctx, req, true)
 	return resp, ok, exists, err
 }
 
-// RouteInitialize is used to route a given initialization request
-func (r *Router) RouteInitialize(ctx context.Context, req *logical.Request) error {
-	_, _, _, err := r.routeCommon(ctx, req, false, true)
-	return err
-}
-
-func (r *Router) routeCommon(ctx context.Context, req *logical.Request, existenceCheck bool, initialize bool) (*logical.Response, bool, bool, error) {
+func (r *Router) routeCommon(ctx context.Context, req *logical.Request, existenceCheck bool) (*logical.Response, bool, bool, error) {
 	ns, err := namespace.FromContext(ctx)
 	if err != nil {
 		return nil, false, false, err
@@ -667,8 +661,6 @@ func (r *Router) routeCommon(ctx context.Context, req *logical.Request, existenc
 	if existenceCheck {
 		ok, exists, err := re.backend.HandleExistenceCheck(ctx, req)
 		return nil, ok, exists, err
-	} else if initialize {
-		return nil, false, false, re.backend.Initialize(ctx, req)
 	} else {
 		resp, err := re.backend.HandleRequest(ctx, req)
 		if resp != nil {
diff --git a/vault/router_testing.go b/vault/router_testing.go
index bea608a3190b..499f971ec97f 100644
--- a/vault/router_testing.go
+++ b/vault/router_testing.go
@@ -103,7 +103,7 @@ func (n *NoopBackend) Logger() log.Logger {
 	return log.NewNullLogger()
 }
 
-func (n *NoopBackend) Initialize(ctx context.Context, req *logical.Request) error {
+func (n *NoopBackend) Initialize(ctx context.Context, req *logical.InitializationRequest) error {
 	return nil
 }
 
diff --git a/vault/testing.go b/vault/testing.go
index 1c45ec050110..5030db405d6d 100644
--- a/vault/testing.go
+++ b/vault/testing.go
@@ -694,7 +694,7 @@ func (n *rawHTTP) Cleanup(ctx context.Context) {
 	// noop
 }
 
-func (n *rawHTTP) Initialize(ctx context.Context, req *logical.Request) error {
+func (n *rawHTTP) Initialize(ctx context.Context, req *logical.InitializationRequest) error {
 	return nil
 }
 

From ae25817542e8c6e6b183c8697319d4c9d4e86a44 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Mon, 1 Jul 2019 12:12:07 -0400
Subject: [PATCH 11/53] improve unit testing for framework.Backend

---
 sdk/framework/backend_test.go | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/sdk/framework/backend_test.go b/sdk/framework/backend_test.go
index 24d2a60d3440..c8b0a6da88c5 100644
--- a/sdk/framework/backend_test.go
+++ b/sdk/framework/backend_test.go
@@ -642,3 +642,18 @@ func TestFieldSchemaDefaultOrZero(t *testing.T) {
 		}
 	}
 }
+
+func TestInitializeBackend(t *testing.T) {
+
+	var inited bool
+	backend := &Backend{InitializeFunc: func(context.Context, logical.Storage) error {
+		inited = true
+		return nil
+	}}
+
+	backend.Initialize(nil, &logical.InitializationRequest{Storage: nil})
+
+	if !inited {
+		t.Fatal("backend should be open")
+	}
+}

From 274ed9d4782ed840a4d3ff77bb7e8c05b5a050b3 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Mon, 1 Jul 2019 13:04:39 -0400
Subject: [PATCH 12/53] call logical.Backend.Initialize() from all of the
 places that it needs to be called.

---
 vault/auth.go           |  9 ++++++
 vault/auth_test.go      | 36 +++++++++++++++++++----
 vault/mount.go          | 17 +++++++++++
 vault/mount_test.go     | 64 +++++++++++++++++++++++++++++++++++++++++
 vault/router_testing.go |  3 ++
 5 files changed, 123 insertions(+), 6 deletions(-)

diff --git a/vault/auth.go b/vault/auth.go
index a0f075737796..c5e731b82fab 100644
--- a/vault/auth.go
+++ b/vault/auth.go
@@ -686,6 +686,15 @@ func (c *Core) setupCredentials(ctx context.Context) error {
 
 		// Populate cache
 		NamespaceByID(ctx, entry.NamespaceID, c)
+
+		// Initialize
+		if !nilMount {
+			err := backend.Initialize(ctx, &logical.InitializationRequest{Storage: view})
+			if err != nil {
+				c.logger.Error("failed to initialize auth entry", "path", entry.Path, "error", err)
+				return errLoadAuthFailed
+			}
+		}
 	}
 
 	if persistNeeded {
diff --git a/vault/auth_test.go b/vault/auth_test.go
index 701b44a1e420..c1df89c784cf 100644
--- a/vault/auth_test.go
+++ b/vault/auth_test.go
@@ -67,11 +67,14 @@ func TestCore_DefaultAuthTable(t *testing.T) {
 }
 
 func TestCore_EnableCredential(t *testing.T) {
+
+	backend := &NoopBackend{
+		BackendType: logical.TypeCredential,
+	}
+
 	c, keys, _ := TestCoreUnsealed(t)
 	c.credentialBackends["noop"] = func(context.Context, *logical.BackendConfig) (logical.Backend, error) {
-		return &NoopBackend{
-			BackendType: logical.TypeCredential,
-		}, nil
+		return backend, nil
 	}
 
 	me := &MountEntry{
@@ -84,6 +87,10 @@ func TestCore_EnableCredential(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
+	if !backend.isInitialized {
+		t.Fatalf("backend is not initialized")
+	}
+
 	match := c.router.MatchingMount(namespace.RootContext(nil), "auth/foo/bar")
 	if match != "auth/foo/" {
 		t.Fatalf("missing mount, match: %q", match)
@@ -122,11 +129,22 @@ func TestCore_EnableCredential(t *testing.T) {
 // entries, and that upon reading the entries from both are recombined
 // correctly
 func TestCore_EnableCredential_Local(t *testing.T) {
+
+	counter := 0
+	backends := []*NoopBackend{
+		&NoopBackend{
+			BackendType: logical.TypeCredential,
+		},
+		&NoopBackend{
+			BackendType: logical.TypeCredential,
+		},
+	}
+
 	c, _, _ := TestCoreUnsealed(t)
 	c.credentialBackends["noop"] = func(context.Context, *logical.BackendConfig) (logical.Backend, error) {
-		return &NoopBackend{
-			BackendType: logical.TypeCredential,
-		}, nil
+		b := backends[counter]
+		counter++
+		return b, nil
 	}
 
 	c.auth = &MountTable{
@@ -161,6 +179,12 @@ func TestCore_EnableCredential_Local(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	for _, b := range backends {
+		if !b.isInitialized {
+			t.Fatalf("backend is not initialized")
+		}
+	}
+
 	rawLocal, err := c.barrier.Get(context.Background(), coreLocalAuthConfigPath)
 	if err != nil {
 		t.Fatal(err)
diff --git a/vault/mount.go b/vault/mount.go
index 052c4da892fc..e0055e7772a6 100644
--- a/vault/mount.go
+++ b/vault/mount.go
@@ -511,6 +511,14 @@ func (c *Core) mountInternal(ctx context.Context, entry *MountEntry, updateStora
 		return err
 	}
 
+	// Initialize
+	if !nilMount {
+		err := backend.Initialize(ctx, &logical.InitializationRequest{Storage: view})
+		if err != nil {
+			return err
+		}
+	}
+
 	if c.logger.IsInfo() {
 		c.logger.Info("successful mount", "namespace", entry.Namespace().Path, "path", entry.Path, "type", entry.Type)
 	}
@@ -1131,6 +1139,15 @@ func (c *Core) setupMounts(ctx context.Context) error {
 			return errLoadMountsFailed
 		}
 
+		// Initialize
+		if !nilMount {
+			err := backend.Initialize(ctx, &logical.InitializationRequest{Storage: view})
+			if err != nil {
+				c.logger.Error("failed to initialize mount entry", "path", entry.Path, "error", err)
+				return errLoadMountsFailed
+			}
+		}
+
 		if c.logger.IsInfo() {
 			c.logger.Info("successfully mounted backend", "type", entry.Type, "path", entry.Path)
 		}
diff --git a/vault/mount_test.go b/vault/mount_test.go
index 9dee69a5e29d..bc174fc471f9 100644
--- a/vault/mount_test.go
+++ b/vault/mount_test.go
@@ -424,6 +424,9 @@ func TestCore_Remount_Cleanup(t *testing.T) {
 	if err := c.mount(namespace.RootContext(nil), me); err != nil {
 		t.Fatalf("err: %v", err)
 	}
+	if !noop.isInitialized {
+		t.Fatalf("backend is not initialized")
+	}
 
 	// Store the view
 	view := c.router.MatchingStorageByAPIPath(namespace.RootContext(nil), "test/")
@@ -735,3 +738,64 @@ func TestSingletonMountTableFunc(t *testing.T) {
 		t.Fatal("unexpected entry type for auth")
 	}
 }
+
+func TestCore_SetupMounts_Initialize(t *testing.T) {
+	c, _, _ := TestCoreUnsealed(t)
+
+	counter := 0
+	backends := []*NoopBackend{
+		&NoopBackend{
+			BackendType: logical.TypeLogical,
+		},
+		&NoopBackend{
+			BackendType: logical.TypeLogical,
+		},
+	}
+
+	c.logicalBackends["noop"] = func(context.Context, *logical.BackendConfig) (logical.Backend, error) {
+		b := backends[counter]
+		counter++
+		return b, nil
+	}
+
+	c.mounts = &MountTable{
+		Type: mountTableType,
+		Entries: []*MountEntry{
+			&MountEntry{
+				Table:            mountTableType,
+				Path:             "noop/",
+				Type:             "noop",
+				UUID:             "abcd",
+				Accessor:         "kv-abcd",
+				BackendAwareUUID: "abcde",
+				NamespaceID:      namespace.RootNamespaceID,
+				namespace:        namespace.RootNamespace,
+			},
+			&MountEntry{
+				Table:            mountTableType,
+				Path:             "noop2/",
+				Type:             "noop",
+				UUID:             "bcde",
+				Accessor:         "kv-bcde",
+				BackendAwareUUID: "bcdea",
+				NamespaceID:      namespace.RootNamespaceID,
+				namespace:        namespace.RootNamespace,
+			},
+		},
+	}
+
+	// Both should set up successfully
+	err := c.setupMounts(namespace.RootContext(nil))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(c.mounts.Entries) != 2 {
+		t.Fatalf("expected two entries, got %d", len(c.mounts.Entries))
+	}
+
+	for _, b := range backends {
+		if !b.isInitialized {
+			t.Fatalf("backend is not initialized")
+		}
+	}
+}
diff --git a/vault/router_testing.go b/vault/router_testing.go
index 499f971ec97f..965d21a33e4f 100644
--- a/vault/router_testing.go
+++ b/vault/router_testing.go
@@ -25,6 +25,8 @@ type NoopBackend struct {
 	DefaultLeaseTTL time.Duration
 	MaxLeaseTTL     time.Duration
 	BackendType     logical.BackendType
+
+	isInitialized bool
 }
 
 func NoopBackendFactory(_ context.Context, _ *logical.BackendConfig) (logical.Backend, error) {
@@ -104,6 +106,7 @@ func (n *NoopBackend) Logger() log.Logger {
 }
 
 func (n *NoopBackend) Initialize(ctx context.Context, req *logical.InitializationRequest) error {
+	n.isInitialized = true
 	return nil
 }
 

From 397dd6f06bbfecd700b3a5872f7e556edc7bdc9b Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Mon, 1 Jul 2019 13:51:41 -0400
Subject: [PATCH 13/53] implement backend.proto changes for
 logical.Backend.Initialize()

---
 sdk/plugin/grpc_backend_client.go |  25 +-
 sdk/plugin/grpc_backend_server.go |  15 +
 sdk/plugin/grpc_backend_test.go   |  10 +
 sdk/plugin/pb/backend.pb.go       | 574 ++++++++++++++++--------------
 sdk/plugin/pb/backend.proto       |  13 +
 5 files changed, 363 insertions(+), 274 deletions(-)

diff --git a/sdk/plugin/grpc_backend_client.go b/sdk/plugin/grpc_backend_client.go
index 85eb4ae4c0ac..be598de5cb3e 100644
--- a/sdk/plugin/grpc_backend_client.go
+++ b/sdk/plugin/grpc_backend_client.go
@@ -44,8 +44,29 @@ type backendGRPCPluginClient struct {
 	doneCtx    context.Context
 }
 
-func (b *backendGRPCPluginClient) Initialize(ctx context.Context, req *logical.InitializationRequest) error {
-	panic("TODO")
+func (b *backendGRPCPluginClient) Initialize(ctx context.Context, _ *logical.InitializationRequest) error {
+
+	if b.metadataMode {
+		return ErrClientInMetadataMode
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+	quitCh := pluginutil.CtxCancelIfCanceled(cancel, b.doneCtx)
+	defer close(quitCh)
+	defer cancel()
+
+	reply, err := b.client.Initialize(ctx, &pb.InitializeArgs{}, largeMsgGRPCCallOpts...)
+	if err != nil {
+		if b.doneCtx.Err() != nil {
+			return ErrPluginShutdown
+		}
+		return err
+	}
+	if reply.Err != nil {
+		return pb.ProtoErrToErr(reply.Err)
+	}
+
+	return nil
 }
 
 func (b *backendGRPCPluginClient) HandleRequest(ctx context.Context, req *logical.Request) (*logical.Response, error) {
diff --git a/sdk/plugin/grpc_backend_server.go b/sdk/plugin/grpc_backend_server.go
index 4bbb045f31eb..e0688a92740a 100644
--- a/sdk/plugin/grpc_backend_server.go
+++ b/sdk/plugin/grpc_backend_server.go
@@ -84,6 +84,21 @@ func (b *backendGRPCPluginServer) HandleRequest(ctx context.Context, args *pb.Ha
 	}, nil
 }
 
+func (b *backendGRPCPluginServer) Initialize(ctx context.Context, _ *pb.InitializeArgs) (*pb.InitializeReply, error) {
+	if pluginutil.InMetadataMode() {
+		return &pb.InitializeReply{}, ErrServerInMetadataMode
+	}
+
+	req := &logical.InitializationRequest{
+		Storage: newGRPCStorageClient(b.brokeredClient)}
+
+	respErr := b.backend.Initialize(ctx, req)
+
+	return &pb.InitializeReply{
+		Err: pb.ErrToProtoErr(respErr),
+	}, nil
+}
+
 func (b *backendGRPCPluginServer) SpecialPaths(ctx context.Context, args *pb.Empty) (*pb.SpecialPathsReply, error) {
 	paths := b.backend.SpecialPaths()
 	if paths == nil {
diff --git a/sdk/plugin/grpc_backend_test.go b/sdk/plugin/grpc_backend_test.go
index 2911b289a2af..e332a9c226bf 100644
--- a/sdk/plugin/grpc_backend_test.go
+++ b/sdk/plugin/grpc_backend_test.go
@@ -137,6 +137,16 @@ func TestGRPCBackendPlugin_Setup(t *testing.T) {
 	defer cleanup()
 }
 
+func TestGRPCBackendPlugin_Initialize(t *testing.T) {
+	b, cleanup := testGRPCBackend(t)
+	defer cleanup()
+
+	err := b.Initialize(context.Background(), &logical.InitializationRequest{})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
 func testGRPCBackend(t *testing.T) (logical.Backend, func()) {
 	// Create a mock provider
 	pluginMap := map[string]gplugin.Plugin{
diff --git a/sdk/plugin/pb/backend.pb.go b/sdk/plugin/pb/backend.pb.go
index bfa10aaf699e..5a392b32aeeb 100644
--- a/sdk/plugin/pb/backend.pb.go
+++ b/sdk/plugin/pb/backend.pb.go
@@ -10,8 +10,6 @@ import (
 	timestamp "github.com/golang/protobuf/ptypes/timestamp"
 	logical "github.com/hashicorp/vault/sdk/logical"
 	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
 	math "math"
 )
 
@@ -1372,6 +1370,78 @@ func (m *HandleRequestReply) GetErr() *ProtoError {
 	return nil
 }
 
+// InitializeArgs is the args for Initialize method.
+type InitializeArgs struct {
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *InitializeArgs) Reset()         { *m = InitializeArgs{} }
+func (m *InitializeArgs) String() string { return proto.CompactTextString(m) }
+func (*InitializeArgs) ProtoMessage()    {}
+func (*InitializeArgs) Descriptor() ([]byte, []int) {
+	return fileDescriptor_4dbf1dfe0c11846b, []int{14}
+}
+
+func (m *InitializeArgs) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_InitializeArgs.Unmarshal(m, b)
+}
+func (m *InitializeArgs) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_InitializeArgs.Marshal(b, m, deterministic)
+}
+func (m *InitializeArgs) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_InitializeArgs.Merge(m, src)
+}
+func (m *InitializeArgs) XXX_Size() int {
+	return xxx_messageInfo_InitializeArgs.Size(m)
+}
+func (m *InitializeArgs) XXX_DiscardUnknown() {
+	xxx_messageInfo_InitializeArgs.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_InitializeArgs proto.InternalMessageInfo
+
+// InitializeReply is the reply for Initialize method.
+type InitializeReply struct {
+	Err                  *ProtoError `sentinel:"" protobuf:"bytes,1,opt,name=err,proto3" json:"err,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}    `json:"-"`
+	XXX_unrecognized     []byte      `json:"-"`
+	XXX_sizecache        int32       `json:"-"`
+}
+
+func (m *InitializeReply) Reset()         { *m = InitializeReply{} }
+func (m *InitializeReply) String() string { return proto.CompactTextString(m) }
+func (*InitializeReply) ProtoMessage()    {}
+func (*InitializeReply) Descriptor() ([]byte, []int) {
+	return fileDescriptor_4dbf1dfe0c11846b, []int{15}
+}
+
+func (m *InitializeReply) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_InitializeReply.Unmarshal(m, b)
+}
+func (m *InitializeReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_InitializeReply.Marshal(b, m, deterministic)
+}
+func (m *InitializeReply) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_InitializeReply.Merge(m, src)
+}
+func (m *InitializeReply) XXX_Size() int {
+	return xxx_messageInfo_InitializeReply.Size(m)
+}
+func (m *InitializeReply) XXX_DiscardUnknown() {
+	xxx_messageInfo_InitializeReply.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_InitializeReply proto.InternalMessageInfo
+
+func (m *InitializeReply) GetErr() *ProtoError {
+	if m != nil {
+		return m.Err
+	}
+	return nil
+}
+
 // SpecialPathsReply is the reply for SpecialPaths method.
 type SpecialPathsReply struct {
 	Paths                *Paths   `sentinel:"" protobuf:"bytes,1,opt,name=paths,proto3" json:"paths,omitempty"`
@@ -1384,7 +1454,7 @@ func (m *SpecialPathsReply) Reset()         { *m = SpecialPathsReply{} }
 func (m *SpecialPathsReply) String() string { return proto.CompactTextString(m) }
 func (*SpecialPathsReply) ProtoMessage()    {}
 func (*SpecialPathsReply) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{14}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{16}
 }
 
 func (m *SpecialPathsReply) XXX_Unmarshal(b []byte) error {
@@ -1425,7 +1495,7 @@ func (m *HandleExistenceCheckArgs) Reset()         { *m = HandleExistenceCheckAr
 func (m *HandleExistenceCheckArgs) String() string { return proto.CompactTextString(m) }
 func (*HandleExistenceCheckArgs) ProtoMessage()    {}
 func (*HandleExistenceCheckArgs) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{15}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{17}
 }
 
 func (m *HandleExistenceCheckArgs) XXX_Unmarshal(b []byte) error {
@@ -1474,7 +1544,7 @@ func (m *HandleExistenceCheckReply) Reset()         { *m = HandleExistenceCheckR
 func (m *HandleExistenceCheckReply) String() string { return proto.CompactTextString(m) }
 func (*HandleExistenceCheckReply) ProtoMessage()    {}
 func (*HandleExistenceCheckReply) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{16}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{18}
 }
 
 func (m *HandleExistenceCheckReply) XXX_Unmarshal(b []byte) error {
@@ -1530,7 +1600,7 @@ func (m *SetupArgs) Reset()         { *m = SetupArgs{} }
 func (m *SetupArgs) String() string { return proto.CompactTextString(m) }
 func (*SetupArgs) ProtoMessage()    {}
 func (*SetupArgs) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{17}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{19}
 }
 
 func (m *SetupArgs) XXX_Unmarshal(b []byte) error {
@@ -1584,7 +1654,7 @@ func (m *SetupReply) Reset()         { *m = SetupReply{} }
 func (m *SetupReply) String() string { return proto.CompactTextString(m) }
 func (*SetupReply) ProtoMessage()    {}
 func (*SetupReply) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{18}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{20}
 }
 
 func (m *SetupReply) XXX_Unmarshal(b []byte) error {
@@ -1624,7 +1694,7 @@ func (m *TypeReply) Reset()         { *m = TypeReply{} }
 func (m *TypeReply) String() string { return proto.CompactTextString(m) }
 func (*TypeReply) ProtoMessage()    {}
 func (*TypeReply) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{19}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{21}
 }
 
 func (m *TypeReply) XXX_Unmarshal(b []byte) error {
@@ -1663,7 +1733,7 @@ func (m *InvalidateKeyArgs) Reset()         { *m = InvalidateKeyArgs{} }
 func (m *InvalidateKeyArgs) String() string { return proto.CompactTextString(m) }
 func (*InvalidateKeyArgs) ProtoMessage()    {}
 func (*InvalidateKeyArgs) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{20}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{22}
 }
 
 func (m *InvalidateKeyArgs) XXX_Unmarshal(b []byte) error {
@@ -1704,7 +1774,7 @@ func (m *StorageEntry) Reset()         { *m = StorageEntry{} }
 func (m *StorageEntry) String() string { return proto.CompactTextString(m) }
 func (*StorageEntry) ProtoMessage()    {}
 func (*StorageEntry) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{21}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{23}
 }
 
 func (m *StorageEntry) XXX_Unmarshal(b []byte) error {
@@ -1757,7 +1827,7 @@ func (m *StorageListArgs) Reset()         { *m = StorageListArgs{} }
 func (m *StorageListArgs) String() string { return proto.CompactTextString(m) }
 func (*StorageListArgs) ProtoMessage()    {}
 func (*StorageListArgs) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{22}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{24}
 }
 
 func (m *StorageListArgs) XXX_Unmarshal(b []byte) error {
@@ -1797,7 +1867,7 @@ func (m *StorageListReply) Reset()         { *m = StorageListReply{} }
 func (m *StorageListReply) String() string { return proto.CompactTextString(m) }
 func (*StorageListReply) ProtoMessage()    {}
 func (*StorageListReply) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{23}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{25}
 }
 
 func (m *StorageListReply) XXX_Unmarshal(b []byte) error {
@@ -1843,7 +1913,7 @@ func (m *StorageGetArgs) Reset()         { *m = StorageGetArgs{} }
 func (m *StorageGetArgs) String() string { return proto.CompactTextString(m) }
 func (*StorageGetArgs) ProtoMessage()    {}
 func (*StorageGetArgs) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{24}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{26}
 }
 
 func (m *StorageGetArgs) XXX_Unmarshal(b []byte) error {
@@ -1883,7 +1953,7 @@ func (m *StorageGetReply) Reset()         { *m = StorageGetReply{} }
 func (m *StorageGetReply) String() string { return proto.CompactTextString(m) }
 func (*StorageGetReply) ProtoMessage()    {}
 func (*StorageGetReply) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{25}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{27}
 }
 
 func (m *StorageGetReply) XXX_Unmarshal(b []byte) error {
@@ -1929,7 +1999,7 @@ func (m *StoragePutArgs) Reset()         { *m = StoragePutArgs{} }
 func (m *StoragePutArgs) String() string { return proto.CompactTextString(m) }
 func (*StoragePutArgs) ProtoMessage()    {}
 func (*StoragePutArgs) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{26}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{28}
 }
 
 func (m *StoragePutArgs) XXX_Unmarshal(b []byte) error {
@@ -1968,7 +2038,7 @@ func (m *StoragePutReply) Reset()         { *m = StoragePutReply{} }
 func (m *StoragePutReply) String() string { return proto.CompactTextString(m) }
 func (*StoragePutReply) ProtoMessage()    {}
 func (*StoragePutReply) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{27}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{29}
 }
 
 func (m *StoragePutReply) XXX_Unmarshal(b []byte) error {
@@ -2007,7 +2077,7 @@ func (m *StorageDeleteArgs) Reset()         { *m = StorageDeleteArgs{} }
 func (m *StorageDeleteArgs) String() string { return proto.CompactTextString(m) }
 func (*StorageDeleteArgs) ProtoMessage()    {}
 func (*StorageDeleteArgs) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{28}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{30}
 }
 
 func (m *StorageDeleteArgs) XXX_Unmarshal(b []byte) error {
@@ -2046,7 +2116,7 @@ func (m *StorageDeleteReply) Reset()         { *m = StorageDeleteReply{} }
 func (m *StorageDeleteReply) String() string { return proto.CompactTextString(m) }
 func (*StorageDeleteReply) ProtoMessage()    {}
 func (*StorageDeleteReply) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{29}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{31}
 }
 
 func (m *StorageDeleteReply) XXX_Unmarshal(b []byte) error {
@@ -2085,7 +2155,7 @@ func (m *TTLReply) Reset()         { *m = TTLReply{} }
 func (m *TTLReply) String() string { return proto.CompactTextString(m) }
 func (*TTLReply) ProtoMessage()    {}
 func (*TTLReply) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{30}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{32}
 }
 
 func (m *TTLReply) XXX_Unmarshal(b []byte) error {
@@ -2125,7 +2195,7 @@ func (m *SudoPrivilegeArgs) Reset()         { *m = SudoPrivilegeArgs{} }
 func (m *SudoPrivilegeArgs) String() string { return proto.CompactTextString(m) }
 func (*SudoPrivilegeArgs) ProtoMessage()    {}
 func (*SudoPrivilegeArgs) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{31}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{33}
 }
 
 func (m *SudoPrivilegeArgs) XXX_Unmarshal(b []byte) error {
@@ -2171,7 +2241,7 @@ func (m *SudoPrivilegeReply) Reset()         { *m = SudoPrivilegeReply{} }
 func (m *SudoPrivilegeReply) String() string { return proto.CompactTextString(m) }
 func (*SudoPrivilegeReply) ProtoMessage()    {}
 func (*SudoPrivilegeReply) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{32}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{34}
 }
 
 func (m *SudoPrivilegeReply) XXX_Unmarshal(b []byte) error {
@@ -2210,7 +2280,7 @@ func (m *TaintedReply) Reset()         { *m = TaintedReply{} }
 func (m *TaintedReply) String() string { return proto.CompactTextString(m) }
 func (*TaintedReply) ProtoMessage()    {}
 func (*TaintedReply) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{33}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{35}
 }
 
 func (m *TaintedReply) XXX_Unmarshal(b []byte) error {
@@ -2249,7 +2319,7 @@ func (m *CachingDisabledReply) Reset()         { *m = CachingDisabledReply{} }
 func (m *CachingDisabledReply) String() string { return proto.CompactTextString(m) }
 func (*CachingDisabledReply) ProtoMessage()    {}
 func (*CachingDisabledReply) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{34}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{36}
 }
 
 func (m *CachingDisabledReply) XXX_Unmarshal(b []byte) error {
@@ -2288,7 +2358,7 @@ func (m *ReplicationStateReply) Reset()         { *m = ReplicationStateReply{} }
 func (m *ReplicationStateReply) String() string { return proto.CompactTextString(m) }
 func (*ReplicationStateReply) ProtoMessage()    {}
 func (*ReplicationStateReply) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{35}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{37}
 }
 
 func (m *ReplicationStateReply) XXX_Unmarshal(b []byte) error {
@@ -2329,7 +2399,7 @@ func (m *ResponseWrapDataArgs) Reset()         { *m = ResponseWrapDataArgs{} }
 func (m *ResponseWrapDataArgs) String() string { return proto.CompactTextString(m) }
 func (*ResponseWrapDataArgs) ProtoMessage()    {}
 func (*ResponseWrapDataArgs) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{36}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{38}
 }
 
 func (m *ResponseWrapDataArgs) XXX_Unmarshal(b []byte) error {
@@ -2383,7 +2453,7 @@ func (m *ResponseWrapDataReply) Reset()         { *m = ResponseWrapDataReply{} }
 func (m *ResponseWrapDataReply) String() string { return proto.CompactTextString(m) }
 func (*ResponseWrapDataReply) ProtoMessage()    {}
 func (*ResponseWrapDataReply) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{37}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{39}
 }
 
 func (m *ResponseWrapDataReply) XXX_Unmarshal(b []byte) error {
@@ -2429,7 +2499,7 @@ func (m *MlockEnabledReply) Reset()         { *m = MlockEnabledReply{} }
 func (m *MlockEnabledReply) String() string { return proto.CompactTextString(m) }
 func (*MlockEnabledReply) ProtoMessage()    {}
 func (*MlockEnabledReply) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{38}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{40}
 }
 
 func (m *MlockEnabledReply) XXX_Unmarshal(b []byte) error {
@@ -2468,7 +2538,7 @@ func (m *LocalMountReply) Reset()         { *m = LocalMountReply{} }
 func (m *LocalMountReply) String() string { return proto.CompactTextString(m) }
 func (*LocalMountReply) ProtoMessage()    {}
 func (*LocalMountReply) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{39}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{41}
 }
 
 func (m *LocalMountReply) XXX_Unmarshal(b []byte) error {
@@ -2507,7 +2577,7 @@ func (m *EntityInfoArgs) Reset()         { *m = EntityInfoArgs{} }
 func (m *EntityInfoArgs) String() string { return proto.CompactTextString(m) }
 func (*EntityInfoArgs) ProtoMessage()    {}
 func (*EntityInfoArgs) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{40}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{42}
 }
 
 func (m *EntityInfoArgs) XXX_Unmarshal(b []byte) error {
@@ -2547,7 +2617,7 @@ func (m *EntityInfoReply) Reset()         { *m = EntityInfoReply{} }
 func (m *EntityInfoReply) String() string { return proto.CompactTextString(m) }
 func (*EntityInfoReply) ProtoMessage()    {}
 func (*EntityInfoReply) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{41}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{43}
 }
 
 func (m *EntityInfoReply) XXX_Unmarshal(b []byte) error {
@@ -2594,7 +2664,7 @@ func (m *PluginEnvReply) Reset()         { *m = PluginEnvReply{} }
 func (m *PluginEnvReply) String() string { return proto.CompactTextString(m) }
 func (*PluginEnvReply) ProtoMessage()    {}
 func (*PluginEnvReply) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{42}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{44}
 }
 
 func (m *PluginEnvReply) XXX_Unmarshal(b []byte) error {
@@ -2641,7 +2711,7 @@ func (m *Connection) Reset()         { *m = Connection{} }
 func (m *Connection) String() string { return proto.CompactTextString(m) }
 func (*Connection) ProtoMessage()    {}
 func (*Connection) Descriptor() ([]byte, []int) {
-	return fileDescriptor_4dbf1dfe0c11846b, []int{43}
+	return fileDescriptor_4dbf1dfe0c11846b, []int{45}
 }
 
 func (m *Connection) XXX_Unmarshal(b []byte) error {
@@ -2688,6 +2758,8 @@ func init() {
 	proto.RegisterType((*RequestWrapInfo)(nil), "pb.RequestWrapInfo")
 	proto.RegisterType((*HandleRequestArgs)(nil), "pb.HandleRequestArgs")
 	proto.RegisterType((*HandleRequestReply)(nil), "pb.HandleRequestReply")
+	proto.RegisterType((*InitializeArgs)(nil), "pb.InitializeArgs")
+	proto.RegisterType((*InitializeReply)(nil), "pb.InitializeReply")
 	proto.RegisterType((*SpecialPathsReply)(nil), "pb.SpecialPathsReply")
 	proto.RegisterType((*HandleExistenceCheckArgs)(nil), "pb.HandleExistenceCheckArgs")
 	proto.RegisterType((*HandleExistenceCheckReply)(nil), "pb.HandleExistenceCheckReply")
@@ -2724,165 +2796,167 @@ func init() {
 func init() { proto.RegisterFile("sdk/plugin/pb/backend.proto", fileDescriptor_4dbf1dfe0c11846b) }
 
 var fileDescriptor_4dbf1dfe0c11846b = []byte{
-	// 2519 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x59, 0xdb, 0x72, 0x1b, 0xc7,
-	0xd1, 0x2e, 0x00, 0xc4, 0xa9, 0x71, 0x22, 0x46, 0xb4, 0xfe, 0x15, 0x24, 0xff, 0x82, 0xd7, 0x91,
-	0x0c, 0x33, 0x36, 0x68, 0xd1, 0x71, 0x2c, 0x27, 0x65, 0xa7, 0x68, 0x8a, 0x96, 0x19, 0x93, 0x36,
-	0x6b, 0x09, 0xc7, 0x39, 0x55, 0xc1, 0x83, 0xdd, 0x21, 0xb8, 0xc5, 0xc5, 0xee, 0x66, 0x76, 0x96,
-	0x22, 0xae, 0xf2, 0x16, 0x79, 0x8d, 0xdc, 0xe6, 0x2e, 0x77, 0x29, 0x57, 0xee, 0xf3, 0x0a, 0xb9,
-	0xcc, 0x33, 0xa4, 0xa6, 0x67, 0xf6, 0x04, 0x80, 0x96, 0x5c, 0xe5, 0xdc, 0xcd, 0x74, 0xf7, 0x9c,
-	0x7a, 0xbe, 0xfe, 0xba, 0x67, 0x17, 0xee, 0x47, 0xce, 0xd5, 0x5e, 0xe8, 0xc5, 0x73, 0xd7, 0xdf,
-	0x0b, 0x67, 0x7b, 0x33, 0x6a, 0x5f, 0x31, 0xdf, 0x19, 0x87, 0x3c, 0x10, 0x01, 0x29, 0x87, 0xb3,
-	0xc1, 0xc3, 0x79, 0x10, 0xcc, 0x3d, 0xb6, 0x87, 0x92, 0x59, 0x7c, 0xb1, 0x27, 0xdc, 0x05, 0x8b,
-	0x04, 0x5d, 0x84, 0xca, 0x68, 0x30, 0x90, 0x33, 0x78, 0xc1, 0xdc, 0xb5, 0xa9, 0xb7, 0xe7, 0x3a,
-	0xcc, 0x17, 0xae, 0x58, 0x6a, 0x9d, 0x91, 0xd7, 0xa9, 0x55, 0x94, 0xc6, 0xac, 0x43, 0xf5, 0x68,
-	0x11, 0x8a, 0xa5, 0x39, 0x84, 0xda, 0xe7, 0x8c, 0x3a, 0x8c, 0x93, 0xbb, 0x50, 0xbb, 0xc4, 0x96,
-	0x51, 0x1a, 0x56, 0x46, 0x4d, 0x4b, 0xf7, 0xcc, 0x3f, 0x00, 0x9c, 0xc9, 0x31, 0x47, 0x9c, 0x07,
-	0x9c, 0xdc, 0x83, 0x06, 0xe3, 0x7c, 0x2a, 0x96, 0x21, 0x33, 0x4a, 0xc3, 0xd2, 0xa8, 0x63, 0xd5,
-	0x19, 0xe7, 0x93, 0x65, 0xc8, 0xc8, 0xff, 0x81, 0x6c, 0x4e, 0x17, 0xd1, 0xdc, 0x28, 0x0f, 0x4b,
-	0x72, 0x06, 0xc6, 0xf9, 0x69, 0x34, 0x4f, 0xc6, 0xd8, 0x81, 0xc3, 0x8c, 0xca, 0xb0, 0x34, 0xaa,
-	0xe0, 0x98, 0xc3, 0xc0, 0x61, 0xe6, 0x5f, 0x4a, 0x50, 0x3d, 0xa3, 0xe2, 0x32, 0x22, 0x04, 0xb6,
-	0x78, 0x10, 0x08, 0xbd, 0x38, 0xb6, 0xc9, 0x08, 0x7a, 0xb1, 0x4f, 0x63, 0x71, 0x29, 0x4f, 0x65,
-	0x53, 0xc1, 0x1c, 0xa3, 0x8c, 0xea, 0x55, 0x31, 0x79, 0x13, 0x3a, 0x5e, 0x60, 0x53, 0x6f, 0x1a,
-	0x89, 0x80, 0xd3, 0xb9, 0x5c, 0x47, 0xda, 0xb5, 0x51, 0x78, 0xae, 0x64, 0x64, 0x17, 0xfa, 0x11,
-	0xa3, 0xde, 0xf4, 0x05, 0xa7, 0x61, 0x6a, 0xb8, 0xa5, 0x26, 0x94, 0x8a, 0x6f, 0x38, 0x0d, 0xb5,
-	0xad, 0xf9, 0xf7, 0x1a, 0xd4, 0x2d, 0xf6, 0xa7, 0x98, 0x45, 0x82, 0x74, 0xa1, 0xec, 0x3a, 0x78,
-	0xda, 0xa6, 0x55, 0x76, 0x1d, 0x32, 0x06, 0x62, 0xb1, 0xd0, 0x93, 0x4b, 0xbb, 0x81, 0x7f, 0xe8,
-	0xc5, 0x91, 0x60, 0x5c, 0x9f, 0x79, 0x83, 0x86, 0x3c, 0x80, 0x66, 0x10, 0x32, 0x8e, 0x32, 0x74,
-	0x40, 0xd3, 0xca, 0x04, 0xf2, 0xe0, 0x21, 0x15, 0x97, 0xc6, 0x16, 0x2a, 0xb0, 0x2d, 0x65, 0x0e,
-	0x15, 0xd4, 0xa8, 0x2a, 0x99, 0x6c, 0x13, 0x13, 0x6a, 0x11, 0xb3, 0x39, 0x13, 0x46, 0x6d, 0x58,
-	0x1a, 0xb5, 0xf6, 0x61, 0x1c, 0xce, 0xc6, 0xe7, 0x28, 0xb1, 0xb4, 0x86, 0x3c, 0x80, 0x2d, 0xe9,
-	0x17, 0xa3, 0x8e, 0x16, 0x0d, 0x69, 0x71, 0x10, 0x8b, 0x4b, 0x0b, 0xa5, 0x64, 0x1f, 0xea, 0xea,
-	0x4e, 0x23, 0xa3, 0x31, 0xac, 0x8c, 0x5a, 0xfb, 0x86, 0x34, 0xd0, 0xa7, 0x1c, 0x2b, 0x18, 0x44,
-	0x47, 0xbe, 0xe0, 0x4b, 0x2b, 0x31, 0x24, 0x6f, 0x40, 0xdb, 0xf6, 0x5c, 0xe6, 0x8b, 0xa9, 0x08,
-	0xae, 0x98, 0x6f, 0x34, 0x71, 0x47, 0x2d, 0x25, 0x9b, 0x48, 0x11, 0xd9, 0x87, 0xd7, 0xf2, 0x26,
-	0x53, 0x6a, 0xdb, 0x2c, 0x8a, 0x02, 0x6e, 0x00, 0xda, 0xde, 0xc9, 0xd9, 0x1e, 0x68, 0x95, 0x9c,
-	0xd6, 0x71, 0xa3, 0xd0, 0xa3, 0xcb, 0xa9, 0x4f, 0x17, 0xcc, 0x68, 0xa9, 0x69, 0xb5, 0xec, 0x4b,
-	0xba, 0x60, 0xe4, 0x21, 0xb4, 0x16, 0x41, 0xec, 0x8b, 0x69, 0x18, 0xb8, 0xbe, 0x30, 0xda, 0x68,
-	0x01, 0x28, 0x3a, 0x93, 0x12, 0xf2, 0x3a, 0xa8, 0x9e, 0x02, 0x63, 0x47, 0xf9, 0x15, 0x25, 0x08,
-	0xc7, 0x47, 0xd0, 0x55, 0xea, 0x74, 0x3f, 0x5d, 0x34, 0xe9, 0xa0, 0x34, 0xdd, 0xc9, 0x7b, 0xd0,
-	0x44, 0x3c, 0xb8, 0xfe, 0x45, 0x60, 0xf4, 0xd0, 0x6f, 0x77, 0x72, 0x6e, 0x91, 0x98, 0x38, 0xf6,
-	0x2f, 0x02, 0xab, 0xf1, 0x42, 0xb7, 0xc8, 0xc7, 0x70, 0xbf, 0x70, 0x5e, 0xce, 0x16, 0xd4, 0xf5,
-	0x5d, 0x7f, 0x3e, 0x8d, 0x23, 0x16, 0x19, 0xdb, 0x88, 0x70, 0x23, 0x77, 0x6a, 0x2b, 0x31, 0xf8,
-	0x3a, 0x62, 0x11, 0xb9, 0x0f, 0x4d, 0x15, 0xa4, 0x53, 0xd7, 0x31, 0xfa, 0xb8, 0xa5, 0x86, 0x12,
-	0x1c, 0x3b, 0xe4, 0x2d, 0xe8, 0x85, 0x81, 0xe7, 0xda, 0xcb, 0x69, 0x70, 0xcd, 0x38, 0x77, 0x1d,
-	0x66, 0x90, 0x61, 0x69, 0xd4, 0xb0, 0xba, 0x4a, 0xfc, 0x95, 0x96, 0x6e, 0x0a, 0x8d, 0x3b, 0x68,
-	0xb8, 0x16, 0x1a, 0x63, 0x00, 0x3b, 0xf0, 0x7d, 0x66, 0x23, 0xfc, 0x76, 0xf0, 0x84, 0x5d, 0x79,
-	0xc2, 0xc3, 0x54, 0x6a, 0xe5, 0x2c, 0x06, 0x9f, 0x41, 0x3b, 0x0f, 0x05, 0xb2, 0x0d, 0x95, 0x2b,
-	0xb6, 0xd4, 0xf0, 0x97, 0x4d, 0x32, 0x84, 0xea, 0x35, 0xf5, 0x62, 0x86, 0x90, 0xd7, 0x40, 0x54,
-	0x43, 0x2c, 0xa5, 0xf8, 0x45, 0xf9, 0x69, 0xc9, 0xfc, 0x77, 0x15, 0xb6, 0x24, 0xf8, 0xc8, 0x07,
-	0xd0, 0xf1, 0x18, 0x8d, 0xd8, 0x34, 0x08, 0xe5, 0x02, 0x11, 0x4e, 0xd5, 0xda, 0xdf, 0x96, 0xc3,
-	0x4e, 0xa4, 0xe2, 0x2b, 0x25, 0xb7, 0xda, 0x5e, 0xae, 0x27, 0x43, 0xda, 0xf5, 0x05, 0xe3, 0x3e,
-	0xf5, 0xa6, 0x18, 0x0c, 0x2a, 0xc0, 0xda, 0x89, 0xf0, 0x99, 0x0c, 0x8a, 0x55, 0x1c, 0x55, 0xd6,
-	0x71, 0x34, 0x80, 0x06, 0xfa, 0xce, 0x65, 0x91, 0x0e, 0xf6, 0xb4, 0x4f, 0xf6, 0xa1, 0xb1, 0x60,
-	0x82, 0xea, 0x58, 0x93, 0x21, 0x71, 0x37, 0x89, 0x99, 0xf1, 0xa9, 0x56, 0xa8, 0x80, 0x48, 0xed,
-	0xd6, 0x22, 0xa2, 0xb6, 0x1e, 0x11, 0x03, 0x68, 0xa4, 0xa0, 0xab, 0xab, 0x1b, 0x4e, 0xfa, 0x92,
-	0x66, 0x43, 0xc6, 0xdd, 0xc0, 0x31, 0x1a, 0x08, 0x14, 0xdd, 0x93, 0x24, 0xe9, 0xc7, 0x0b, 0x05,
-	0xa1, 0xa6, 0x22, 0x49, 0x3f, 0x5e, 0xac, 0x23, 0x06, 0x56, 0x10, 0xf3, 0x13, 0xa8, 0x52, 0xcf,
-	0xa5, 0x11, 0x86, 0x90, 0xbc, 0x59, 0xcd, 0xf7, 0xe3, 0x03, 0x29, 0xb5, 0x94, 0x92, 0xbc, 0x0f,
-	0x9d, 0x39, 0x0f, 0xe2, 0x70, 0x8a, 0x5d, 0x16, 0x19, 0x6d, 0x3c, 0xed, 0xaa, 0x75, 0x1b, 0x8d,
-	0x0e, 0x94, 0x8d, 0x8c, 0xc0, 0x59, 0x10, 0xfb, 0xce, 0xd4, 0x76, 0x1d, 0x1e, 0x19, 0x1d, 0x74,
-	0x1e, 0xa0, 0xe8, 0x50, 0x4a, 0x64, 0x88, 0xa9, 0x10, 0x48, 0x1d, 0xdc, 0x45, 0x9b, 0x0e, 0x4a,
-	0xcf, 0x12, 0x2f, 0xff, 0x14, 0xfa, 0x49, 0x62, 0xca, 0x2c, 0x7b, 0x68, 0xb9, 0x9d, 0x28, 0x52,
-	0xe3, 0x11, 0x6c, 0xb3, 0x1b, 0x49, 0xa1, 0xae, 0x98, 0x2e, 0xe8, 0xcd, 0x54, 0x08, 0x4f, 0x87,
-	0x54, 0x37, 0x91, 0x9f, 0xd2, 0x9b, 0x89, 0xf0, 0x64, 0xfc, 0xab, 0xd5, 0x31, 0xfe, 0xfb, 0x98,
-	0x8c, 0x9a, 0x28, 0xc1, 0xf8, 0xdf, 0x85, 0xbe, 0x1f, 0x4c, 0x1d, 0x76, 0x41, 0x63, 0x4f, 0xa8,
-	0x75, 0x97, 0x3a, 0x98, 0x7a, 0x7e, 0xf0, 0x4c, 0xc9, 0x71, 0xd9, 0xe5, 0xe0, 0x97, 0xd0, 0x29,
-	0x5c, 0xf7, 0x06, 0xd0, 0xef, 0xe4, 0x41, 0xdf, 0xcc, 0x03, 0xfd, 0x9f, 0x5b, 0x00, 0x78, 0xef,
-	0x6a, 0xe8, 0x6a, 0xb6, 0xc8, 0x83, 0xa1, 0xbc, 0x01, 0x0c, 0x94, 0x33, 0x5f, 0x68, 0xe0, 0xea,
-	0xde, 0xf7, 0x62, 0x36, 0xc9, 0x17, 0xd5, 0x5c, 0xbe, 0x78, 0x07, 0xb6, 0x24, 0x3e, 0x8d, 0x5a,
-	0x46, 0xeb, 0xd9, 0x8e, 0x10, 0xc9, 0x0a, 0xc5, 0x68, 0xb5, 0x16, 0x34, 0xf5, 0xf5, 0xa0, 0xc9,
-	0xa3, 0xb1, 0x51, 0x44, 0xe3, 0x9b, 0xd0, 0xb1, 0x39, 0xc3, 0xdc, 0x35, 0x95, 0xc5, 0x88, 0x46,
-	0x6b, 0x3b, 0x11, 0x4e, 0xdc, 0x05, 0x93, 0xfe, 0x93, 0x17, 0x07, 0xa8, 0x92, 0xcd, 0x8d, 0xf7,
-	0xda, 0xda, 0x78, 0xaf, 0x58, 0x09, 0x78, 0x4c, 0x33, 0x3e, 0xb6, 0x73, 0x51, 0xd3, 0x29, 0x44,
-	0x4d, 0x21, 0x34, 0xba, 0x2b, 0xa1, 0xb1, 0x82, 0xdf, 0xde, 0x1a, 0x7e, 0xdf, 0x80, 0xb6, 0x74,
-	0x40, 0x14, 0x52, 0x9b, 0xc9, 0x09, 0xb6, 0x95, 0x23, 0x52, 0xd9, 0xb1, 0x83, 0xd1, 0x1e, 0xcf,
-	0x66, 0xcb, 0xcb, 0xc0, 0x63, 0x19, 0x61, 0xb7, 0x52, 0xd9, 0xb1, 0x23, 0xf7, 0x8b, 0x08, 0x24,
-	0x88, 0x40, 0x6c, 0x0f, 0x3e, 0x84, 0x66, 0xea, 0xf5, 0x1f, 0x04, 0xa6, 0xbf, 0x96, 0xa0, 0x9d,
-	0x27, 0x45, 0x39, 0x78, 0x32, 0x39, 0xc1, 0xc1, 0x15, 0x4b, 0x36, 0x65, 0x39, 0xc1, 0x99, 0xcf,
-	0x5e, 0xd0, 0x99, 0xa7, 0x26, 0x68, 0x58, 0x99, 0x40, 0x6a, 0x5d, 0xdf, 0xe6, 0x6c, 0x91, 0xa0,
-	0xaa, 0x62, 0x65, 0x02, 0xf2, 0x11, 0x80, 0x1b, 0x45, 0x31, 0x53, 0x37, 0xb7, 0x85, 0x94, 0x31,
-	0x18, 0xab, 0x1a, 0x73, 0x9c, 0xd4, 0x98, 0xe3, 0x49, 0x52, 0x63, 0x5a, 0x4d, 0xb4, 0xc6, 0x2b,
-	0xbd, 0x0b, 0x35, 0x79, 0x41, 0x93, 0x13, 0x44, 0x5e, 0xc5, 0xd2, 0x3d, 0xf3, 0xcf, 0x50, 0x53,
-	0x55, 0xc8, 0xff, 0x94, 0xe8, 0xef, 0x41, 0x43, 0xcd, 0xed, 0x3a, 0x3a, 0x56, 0xea, 0xd8, 0x3f,
-	0x76, 0xcc, 0xef, 0xca, 0xd0, 0xb0, 0x58, 0x14, 0x06, 0x7e, 0xc4, 0x72, 0x55, 0x52, 0xe9, 0xa5,
-	0x55, 0x52, 0x79, 0x63, 0x95, 0x94, 0xd4, 0x5e, 0x95, 0x5c, 0xed, 0x35, 0x80, 0x06, 0x67, 0x8e,
-	0xcb, 0x99, 0x2d, 0x74, 0x9d, 0x96, 0xf6, 0xa5, 0xee, 0x05, 0xe5, 0x32, 0xbd, 0x47, 0x98, 0x43,
-	0x9a, 0x56, 0xda, 0x27, 0x4f, 0xf2, 0xc5, 0x85, 0x2a, 0xdb, 0x76, 0x54, 0x71, 0xa1, 0xb6, 0xbb,
-	0xa1, 0xba, 0x78, 0x3f, 0x2b, 0xd2, 0xea, 0x18, 0xcd, 0xf7, 0xf2, 0x03, 0x36, 0x57, 0x69, 0x3f,
-	0x5a, 0xce, 0xfe, 0xae, 0x0c, 0xdb, 0xab, 0x7b, 0xdb, 0x80, 0xc0, 0x1d, 0xa8, 0xaa, 0xdc, 0xa7,
-	0xe1, 0x2b, 0xd6, 0xb2, 0x5e, 0x65, 0x85, 0xe8, 0x7e, 0xb5, 0x4a, 0x1a, 0x2f, 0x87, 0x5e, 0x91,
-	0x50, 0xde, 0x86, 0x6d, 0xe9, 0xa2, 0x90, 0x39, 0x59, 0x3d, 0xa7, 0x18, 0xb0, 0xa7, 0xe5, 0x69,
-	0x45, 0xb7, 0x0b, 0xfd, 0xc4, 0x34, 0xe3, 0x86, 0x5a, 0xc1, 0xf6, 0x28, 0xa1, 0x88, 0xbb, 0x50,
-	0xbb, 0x08, 0xf8, 0x82, 0x0a, 0x4d, 0x82, 0xba, 0x57, 0x20, 0x39, 0x64, 0xdb, 0x86, 0xc2, 0x64,
-	0x22, 0x94, 0x6f, 0x16, 0x49, 0x3e, 0xe9, 0x7b, 0x02, 0x59, 0xb0, 0x61, 0x35, 0x92, 0x77, 0x84,
-	0xf9, 0x5b, 0xe8, 0xad, 0x94, 0x90, 0x1b, 0x1c, 0x99, 0x2d, 0x5f, 0x2e, 0x2c, 0x5f, 0x98, 0xb9,
-	0xb2, 0x32, 0xf3, 0xef, 0xa0, 0xff, 0x39, 0xf5, 0x1d, 0x8f, 0xe9, 0xf9, 0x0f, 0xf8, 0x3c, 0x92,
-	0xc9, 0x50, 0xbf, 0x68, 0xa6, 0x3a, 0xfb, 0x74, 0xac, 0xa6, 0x96, 0x1c, 0x3b, 0xe4, 0x11, 0xd4,
-	0xb9, 0xb2, 0xd6, 0x00, 0x68, 0xe5, 0x6a, 0x5c, 0x2b, 0xd1, 0x99, 0xdf, 0x02, 0x29, 0x4c, 0x2d,
-	0x1f, 0x33, 0x4b, 0x32, 0x92, 0xe8, 0x57, 0xa0, 0xd0, 0x51, 0xd5, 0xce, 0x63, 0xd2, 0x4a, 0xb5,
-	0x64, 0x08, 0x15, 0xc6, 0xb9, 0x5e, 0x02, 0x8b, 0xcc, 0xec, 0xe9, 0x68, 0x49, 0x95, 0xf9, 0x33,
-	0xe8, 0x9f, 0x87, 0xcc, 0x76, 0xa9, 0x87, 0xcf, 0x3e, 0xb5, 0xc0, 0x43, 0xa8, 0x4a, 0x27, 0x27,
-	0x84, 0xd1, 0xc4, 0x81, 0xa8, 0x56, 0x72, 0xf3, 0x5b, 0x30, 0xd4, 0xbe, 0x8e, 0x6e, 0xdc, 0x48,
-	0x30, 0xdf, 0x66, 0x87, 0x97, 0xcc, 0xbe, 0xfa, 0x11, 0x4f, 0x7e, 0x0d, 0xf7, 0x36, 0xad, 0x90,
-	0xec, 0xaf, 0x65, 0xcb, 0xde, 0xf4, 0x42, 0xe6, 0x0e, 0x5c, 0xa3, 0x61, 0x01, 0x8a, 0x3e, 0x93,
-	0x12, 0x79, 0x8f, 0x4c, 0x8e, 0x8b, 0x34, 0x1f, 0xeb, 0x5e, 0xe2, 0x8f, 0xca, 0xed, 0xfe, 0xf8,
-	0x5b, 0x09, 0x9a, 0xe7, 0x4c, 0xc4, 0x21, 0x9e, 0xe5, 0x3e, 0x34, 0x67, 0x3c, 0xb8, 0x62, 0x3c,
-	0x3b, 0x4a, 0x43, 0x09, 0x8e, 0x1d, 0xf2, 0x04, 0x6a, 0x87, 0x81, 0x7f, 0xe1, 0xce, 0xf1, 0x11,
-	0xac, 0x89, 0x21, 0x1d, 0x3b, 0x56, 0x3a, 0x45, 0x0c, 0xda, 0x90, 0x0c, 0xa1, 0xa5, 0x3f, 0x29,
-	0x7c, 0xfd, 0xf5, 0xf1, 0xb3, 0xa4, 0x3a, 0xce, 0x89, 0x06, 0x1f, 0x41, 0x2b, 0x37, 0xf0, 0x07,
-	0xa5, 0xaa, 0xff, 0x07, 0xc0, 0xd5, 0x95, 0x8f, 0xb6, 0xd5, 0x51, 0xf5, 0x48, 0x79, 0xb4, 0x87,
-	0xd0, 0x94, 0x85, 0x98, 0x52, 0x27, 0x49, 0xb2, 0x94, 0x25, 0x49, 0xf3, 0x11, 0xf4, 0x8f, 0xfd,
-	0x6b, 0xea, 0xb9, 0x0e, 0x15, 0xec, 0x0b, 0xb6, 0x44, 0x17, 0xac, 0xed, 0xc0, 0x3c, 0x87, 0xb6,
-	0x7e, 0x95, 0xbf, 0xd2, 0x1e, 0xdb, 0x7a, 0x8f, 0xdf, 0x1f, 0x44, 0x6f, 0x43, 0x4f, 0x4f, 0x7a,
-	0xe2, 0xea, 0x10, 0x92, 0x35, 0x06, 0x67, 0x17, 0xee, 0x8d, 0x9e, 0x5a, 0xf7, 0xcc, 0xa7, 0xb0,
-	0x9d, 0x33, 0x4d, 0x8f, 0x73, 0xc5, 0x96, 0x51, 0xf2, 0xb5, 0x42, 0xb6, 0x13, 0x0f, 0x94, 0x33,
-	0x0f, 0x98, 0xd0, 0xd5, 0x23, 0x9f, 0x33, 0x71, 0xcb, 0xe9, 0xbe, 0x48, 0x37, 0xf2, 0x9c, 0xe9,
-	0xc9, 0x1f, 0x43, 0x95, 0xc9, 0x93, 0xe6, 0xf3, 0x67, 0xde, 0x03, 0x96, 0x52, 0x6f, 0x58, 0xf0,
-	0x69, 0xba, 0xe0, 0x59, 0xac, 0x16, 0x7c, 0xc5, 0xb9, 0xcc, 0x37, 0xd3, 0x6d, 0x9c, 0xc5, 0xe2,
-	0xb6, 0x1b, 0x7d, 0x04, 0x7d, 0x6d, 0xf4, 0x8c, 0x79, 0x4c, 0xb0, 0x5b, 0x8e, 0xf4, 0x18, 0x48,
-	0xc1, 0xec, 0xb6, 0xe9, 0x1e, 0x40, 0x63, 0x32, 0x39, 0x49, 0xb5, 0x45, 0x6e, 0x34, 0x3f, 0x86,
-	0xfe, 0x79, 0xec, 0x04, 0x67, 0xdc, 0xbd, 0x76, 0x3d, 0x36, 0x57, 0x8b, 0x25, 0xc5, 0x6f, 0x29,
-	0x57, 0xfc, 0x6e, 0xcc, 0x46, 0xe6, 0x08, 0x48, 0x61, 0x78, 0x7a, 0x6f, 0x51, 0xec, 0x04, 0x3a,
-	0x84, 0xb1, 0x6d, 0x8e, 0xa0, 0x3d, 0xa1, 0xb2, 0xd8, 0x70, 0x94, 0x8d, 0x01, 0x75, 0xa1, 0xfa,
-	0xda, 0x2c, 0xe9, 0x9a, 0xfb, 0xb0, 0x73, 0x48, 0xed, 0x4b, 0xd7, 0x9f, 0x3f, 0x73, 0x23, 0x59,
-	0x6d, 0xe9, 0x11, 0x03, 0x68, 0x38, 0x5a, 0xa0, 0x87, 0xa4, 0x7d, 0xf3, 0x5d, 0x78, 0x2d, 0xf7,
-	0x49, 0xe8, 0x5c, 0xd0, 0xc4, 0x1f, 0x3b, 0x50, 0x8d, 0x64, 0x0f, 0x47, 0x54, 0x2d, 0xd5, 0x31,
-	0xbf, 0x84, 0x9d, 0x7c, 0x02, 0x96, 0xb5, 0x4f, 0x72, 0x70, 0xac, 0x4a, 0x4a, 0xb9, 0xaa, 0x44,
-	0xfb, 0xac, 0x9c, 0xe5, 0x93, 0x6d, 0xa8, 0xfc, 0xfa, 0x9b, 0x89, 0x06, 0xbb, 0x6c, 0x9a, 0x7f,
-	0x94, 0xcb, 0x17, 0xe7, 0x53, 0xcb, 0x17, 0x4a, 0x93, 0xd2, 0x2b, 0x95, 0x26, 0xeb, 0x78, 0x7b,
-	0x17, 0xfa, 0xa7, 0x5e, 0x60, 0x5f, 0x1d, 0xf9, 0x39, 0x6f, 0x18, 0x50, 0x67, 0x7e, 0xde, 0x19,
-	0x49, 0xd7, 0x7c, 0x0b, 0x7a, 0x27, 0x81, 0x4d, 0xbd, 0xd3, 0x20, 0xf6, 0x45, 0xea, 0x05, 0xfc,
-	0x46, 0xa7, 0x4d, 0x55, 0xc7, 0x7c, 0x17, 0xba, 0x3a, 0x45, 0xfb, 0x17, 0x41, 0xc2, 0x8c, 0x59,
-	0x32, 0x2f, 0x15, 0x0b, 0x7d, 0xf3, 0x04, 0x7a, 0x99, 0xb9, 0x9a, 0xf7, 0x2d, 0xa8, 0x29, 0xb5,
-	0x3e, 0x5b, 0x2f, 0x7d, 0xe9, 0x2a, 0x4b, 0x4b, 0xab, 0x37, 0x1c, 0x6a, 0x01, 0xdd, 0x33, 0xfc,
-	0x56, 0x7a, 0xe4, 0x5f, 0xab, 0xc9, 0x8e, 0x81, 0xa8, 0xaf, 0xa7, 0x53, 0xe6, 0x5f, 0xbb, 0x3c,
-	0xf0, 0xb1, 0xb8, 0x2e, 0xe9, 0x12, 0x26, 0x99, 0x38, 0x1d, 0x94, 0x58, 0x58, 0xfd, 0x70, 0x55,
-	0xb4, 0xd1, 0x87, 0x90, 0x7d, 0x89, 0x91, 0xa9, 0x86, 0xb3, 0x45, 0x20, 0xd8, 0x94, 0x3a, 0x4e,
-	0x12, 0x2d, 0xa0, 0x44, 0x07, 0x8e, 0xc3, 0xf7, 0xff, 0x53, 0x86, 0xfa, 0xa7, 0x8a, 0xc0, 0xc9,
-	0x27, 0xd0, 0x29, 0xa4, 0x6b, 0xf2, 0x1a, 0x96, 0x75, 0xab, 0xc5, 0xc1, 0xe0, 0xee, 0x9a, 0x58,
-	0x9d, 0xeb, 0x3d, 0x68, 0xe7, 0x93, 0x31, 0xc1, 0xc4, 0x8b, 0xdf, 0x85, 0x07, 0x38, 0xd3, 0x7a,
-	0xa6, 0x3e, 0x87, 0x9d, 0x4d, 0x69, 0x92, 0x3c, 0xc8, 0x56, 0x58, 0x4f, 0xd1, 0x83, 0xd7, 0x6f,
-	0xd3, 0x26, 0xe9, 0xb5, 0x7e, 0xe8, 0x31, 0xea, 0xc7, 0x61, 0x7e, 0x07, 0x59, 0x93, 0x3c, 0x81,
-	0x4e, 0x21, 0x51, 0xa8, 0x73, 0xae, 0xe5, 0x8e, 0xfc, 0x90, 0xc7, 0x50, 0xc5, 0xe4, 0x44, 0x3a,
-	0x85, 0x2c, 0x39, 0xe8, 0xa6, 0x5d, 0xb5, 0xf6, 0x10, 0xb6, 0xf0, 0x6b, 0x41, 0x6e, 0x61, 0x1c,
-	0x91, 0x66, 0xae, 0xfd, 0x7f, 0x95, 0xa0, 0x9e, 0x7c, 0x41, 0x7e, 0x02, 0x5b, 0x32, 0x07, 0x90,
-	0x3b, 0x39, 0x1a, 0x4d, 0xf2, 0xc7, 0x60, 0x67, 0x45, 0xa8, 0x16, 0x18, 0x43, 0xe5, 0x39, 0x13,
-	0x84, 0xe4, 0x94, 0x3a, 0x19, 0x0c, 0xee, 0x14, 0x65, 0xa9, 0xfd, 0x59, 0x5c, 0xb4, 0xd7, 0x5c,
-	0x5e, 0xb0, 0x4f, 0x59, 0xfa, 0x43, 0xa8, 0x29, 0x96, 0x55, 0x4e, 0x59, 0xe3, 0x67, 0x75, 0xf9,
-	0xeb, 0x7c, 0xbc, 0xff, 0x8f, 0x2d, 0x80, 0xf3, 0x65, 0x24, 0xd8, 0xe2, 0x37, 0x2e, 0x7b, 0x41,
-	0x76, 0xa1, 0xa7, 0xbf, 0x89, 0xe0, 0x53, 0x4d, 0xb2, 0x49, 0xce, 0x27, 0x58, 0xf0, 0xa5, 0x64,
-	0xfd, 0x18, 0x5a, 0xa7, 0xf4, 0xe6, 0xe5, 0x76, 0x9f, 0x40, 0xa7, 0xc0, 0xc1, 0x7a, 0x8b, 0xab,
-	0xac, 0xae, 0xb7, 0xb8, 0xce, 0xd6, 0x8f, 0xa1, 0xae, 0x99, 0x39, 0xbf, 0x06, 0xe6, 0xb0, 0x02,
-	0x63, 0xff, 0x1c, 0x7a, 0x2b, 0xbc, 0x9c, 0xb7, 0xc7, 0xcf, 0x21, 0x1b, 0x79, 0xfb, 0xa9, 0x7c,
-	0xed, 0x14, 0xb9, 0x39, 0x3f, 0x50, 0xbf, 0xbc, 0x36, 0x91, 0xf7, 0xf3, 0xe2, 0x3b, 0x09, 0x9f,
-	0xa8, 0xc6, 0x2a, 0x7d, 0x26, 0xe4, 0x3d, 0xb8, 0xb7, 0x49, 0x93, 0x86, 0x60, 0x9e, 0x41, 0xd7,
-	0x42, 0x70, 0x9d, 0x5e, 0xdf, 0x01, 0xc8, 0x48, 0x34, 0x6f, 0x8f, 0xf0, 0x58, 0xe5, 0xd7, 0x0f,
-	0x00, 0x32, 0x6a, 0x54, 0xa8, 0x2a, 0x32, 0xab, 0x1a, 0xb6, 0x4a, 0x9f, 0xbb, 0xd0, 0x4c, 0xe9,
-	0x2c, 0xbf, 0x06, 0x4e, 0x50, 0x64, 0xc7, 0x4f, 0x77, 0x7f, 0x3f, 0x9a, 0xbb, 0xe2, 0x32, 0x9e,
-	0x8d, 0xed, 0x60, 0xb1, 0x77, 0x49, 0xa3, 0x4b, 0xd7, 0x0e, 0x78, 0xb8, 0x77, 0x2d, 0xc1, 0xb4,
-	0x57, 0xf8, 0xc1, 0x35, 0xab, 0xe1, 0x43, 0xef, 0xfd, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xbd,
-	0xc6, 0x6e, 0xfa, 0xf8, 0x1a, 0x00, 0x00,
+	// 2556 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x59, 0x4b, 0x73, 0xe3, 0xc6,
+	0x11, 0x2e, 0x92, 0xe2, 0xab, 0xf9, 0x9e, 0x95, 0x37, 0x58, 0xee, 0x3a, 0x2b, 0xc3, 0xd9, 0x35,
+	0xad, 0xd8, 0x94, 0x57, 0x1b, 0xc7, 0xeb, 0xa4, 0xec, 0x94, 0xac, 0x95, 0xd7, 0x8a, 0x25, 0x5b,
+	0x05, 0xd1, 0x71, 0x5e, 0x55, 0xf4, 0x10, 0x18, 0x51, 0x28, 0x81, 0x00, 0x32, 0x18, 0x68, 0xc5,
+	0x5c, 0xf2, 0x2f, 0xf2, 0x0f, 0x72, 0xce, 0x35, 0xb7, 0xdc, 0x52, 0xae, 0xdc, 0xf3, 0x17, 0xf2,
+	0x3b, 0x52, 0xd3, 0x33, 0x78, 0x91, 0x94, 0x1f, 0x55, 0xce, 0x6d, 0xa6, 0xbb, 0xe7, 0xd5, 0xf3,
+	0xf5, 0xd7, 0x3d, 0x00, 0xdc, 0x8f, 0x9c, 0xab, 0xbd, 0xd0, 0x8b, 0xe7, 0xae, 0xbf, 0x17, 0xce,
+	0xf6, 0x66, 0xd4, 0xbe, 0x62, 0xbe, 0x33, 0x0e, 0x79, 0x20, 0x02, 0x52, 0x0e, 0x67, 0xc3, 0x87,
+	0xf3, 0x20, 0x98, 0x7b, 0x6c, 0x0f, 0x25, 0xb3, 0xf8, 0x62, 0x4f, 0xb8, 0x0b, 0x16, 0x09, 0xba,
+	0x08, 0x95, 0xd1, 0x70, 0x28, 0x67, 0xf0, 0x82, 0xb9, 0x6b, 0x53, 0x6f, 0xcf, 0x75, 0x98, 0x2f,
+	0x5c, 0xb1, 0xd4, 0x3a, 0x23, 0xaf, 0x53, 0xab, 0x28, 0x8d, 0x59, 0x87, 0xea, 0xd1, 0x22, 0x14,
+	0x4b, 0x73, 0x07, 0x6a, 0x9f, 0x30, 0xea, 0x30, 0x4e, 0xee, 0x42, 0xed, 0x12, 0x5b, 0x46, 0x69,
+	0xa7, 0x32, 0x6a, 0x5a, 0xba, 0x67, 0xfe, 0x01, 0xe0, 0x4c, 0x8e, 0x39, 0xe2, 0x3c, 0xe0, 0xe4,
+	0x1e, 0x34, 0x18, 0xe7, 0x53, 0xb1, 0x0c, 0x99, 0x51, 0xda, 0x29, 0x8d, 0x3a, 0x56, 0x9d, 0x71,
+	0x3e, 0x59, 0x86, 0x8c, 0xfc, 0x08, 0x64, 0x73, 0xba, 0x88, 0xe6, 0x46, 0x79, 0xa7, 0x24, 0x67,
+	0x60, 0x9c, 0x9f, 0x46, 0xf3, 0x64, 0x8c, 0x1d, 0x38, 0xcc, 0xa8, 0xec, 0x94, 0x46, 0x15, 0x1c,
+	0x73, 0x18, 0x38, 0xcc, 0xfc, 0x6b, 0x09, 0xaa, 0x67, 0x54, 0x5c, 0x46, 0x84, 0xc0, 0x16, 0x0f,
+	0x02, 0xa1, 0x17, 0xc7, 0x36, 0x19, 0x41, 0x2f, 0xf6, 0x69, 0x2c, 0x2e, 0xe5, 0xa9, 0x6c, 0x2a,
+	0x98, 0x63, 0x94, 0x51, 0xbd, 0x2a, 0x26, 0xaf, 0x43, 0xc7, 0x0b, 0x6c, 0xea, 0x4d, 0x23, 0x11,
+	0x70, 0x3a, 0x97, 0xeb, 0x48, 0xbb, 0x36, 0x0a, 0xcf, 0x95, 0x8c, 0xec, 0xc2, 0x20, 0x62, 0xd4,
+	0x9b, 0xbe, 0xe4, 0x34, 0x4c, 0x0d, 0xb7, 0xd4, 0x84, 0x52, 0xf1, 0x25, 0xa7, 0xa1, 0xb6, 0x35,
+	0xff, 0x59, 0x83, 0xba, 0xc5, 0xfe, 0x14, 0xb3, 0x48, 0x90, 0x2e, 0x94, 0x5d, 0x07, 0x4f, 0xdb,
+	0xb4, 0xca, 0xae, 0x43, 0xc6, 0x40, 0x2c, 0x16, 0x7a, 0x72, 0x69, 0x37, 0xf0, 0x0f, 0xbd, 0x38,
+	0x12, 0x8c, 0xeb, 0x33, 0x6f, 0xd0, 0x90, 0x07, 0xd0, 0x0c, 0x42, 0xc6, 0x51, 0x86, 0x0e, 0x68,
+	0x5a, 0x99, 0x40, 0x1e, 0x3c, 0xa4, 0xe2, 0xd2, 0xd8, 0x42, 0x05, 0xb6, 0xa5, 0xcc, 0xa1, 0x82,
+	0x1a, 0x55, 0x25, 0x93, 0x6d, 0x62, 0x42, 0x2d, 0x62, 0x36, 0x67, 0xc2, 0xa8, 0xed, 0x94, 0x46,
+	0xad, 0x7d, 0x18, 0x87, 0xb3, 0xf1, 0x39, 0x4a, 0x2c, 0xad, 0x21, 0x0f, 0x60, 0x4b, 0xfa, 0xc5,
+	0xa8, 0xa3, 0x45, 0x43, 0x5a, 0x1c, 0xc4, 0xe2, 0xd2, 0x42, 0x29, 0xd9, 0x87, 0xba, 0xba, 0xd3,
+	0xc8, 0x68, 0xec, 0x54, 0x46, 0xad, 0x7d, 0x43, 0x1a, 0xe8, 0x53, 0x8e, 0x15, 0x0c, 0xa2, 0x23,
+	0x5f, 0xf0, 0xa5, 0x95, 0x18, 0x92, 0xd7, 0xa0, 0x6d, 0x7b, 0x2e, 0xf3, 0xc5, 0x54, 0x04, 0x57,
+	0xcc, 0x37, 0x9a, 0xb8, 0xa3, 0x96, 0x92, 0x4d, 0xa4, 0x88, 0xec, 0xc3, 0x2b, 0x79, 0x93, 0x29,
+	0xb5, 0x6d, 0x16, 0x45, 0x01, 0x37, 0x00, 0x6d, 0xef, 0xe4, 0x6c, 0x0f, 0xb4, 0x4a, 0x4e, 0xeb,
+	0xb8, 0x51, 0xe8, 0xd1, 0xe5, 0xd4, 0xa7, 0x0b, 0x66, 0xb4, 0xd4, 0xb4, 0x5a, 0xf6, 0x19, 0x5d,
+	0x30, 0xf2, 0x10, 0x5a, 0x8b, 0x20, 0xf6, 0xc5, 0x34, 0x0c, 0x5c, 0x5f, 0x18, 0x6d, 0xb4, 0x00,
+	0x14, 0x9d, 0x49, 0x09, 0x79, 0x15, 0x54, 0x4f, 0x81, 0xb1, 0xa3, 0xfc, 0x8a, 0x12, 0x84, 0xe3,
+	0x23, 0xe8, 0x2a, 0x75, 0xba, 0x9f, 0x2e, 0x9a, 0x74, 0x50, 0x9a, 0xee, 0xe4, 0x1d, 0x68, 0x22,
+	0x1e, 0x5c, 0xff, 0x22, 0x30, 0x7a, 0xe8, 0xb7, 0x3b, 0x39, 0xb7, 0x48, 0x4c, 0x1c, 0xfb, 0x17,
+	0x81, 0xd5, 0x78, 0xa9, 0x5b, 0xe4, 0x03, 0xb8, 0x5f, 0x38, 0x2f, 0x67, 0x0b, 0xea, 0xfa, 0xae,
+	0x3f, 0x9f, 0xc6, 0x11, 0x8b, 0x8c, 0x3e, 0x22, 0xdc, 0xc8, 0x9d, 0xda, 0x4a, 0x0c, 0xbe, 0x88,
+	0x58, 0x44, 0xee, 0x43, 0x53, 0x05, 0xe9, 0xd4, 0x75, 0x8c, 0x01, 0x6e, 0xa9, 0xa1, 0x04, 0xc7,
+	0x0e, 0x79, 0x03, 0x7a, 0x61, 0xe0, 0xb9, 0xf6, 0x72, 0x1a, 0x5c, 0x33, 0xce, 0x5d, 0x87, 0x19,
+	0x64, 0xa7, 0x34, 0x6a, 0x58, 0x5d, 0x25, 0xfe, 0x5c, 0x4b, 0x37, 0x85, 0xc6, 0x1d, 0x34, 0x5c,
+	0x0b, 0x8d, 0x31, 0x80, 0x1d, 0xf8, 0x3e, 0xb3, 0x11, 0x7e, 0xdb, 0x78, 0xc2, 0xae, 0x3c, 0xe1,
+	0x61, 0x2a, 0xb5, 0x72, 0x16, 0xc3, 0x8f, 0xa1, 0x9d, 0x87, 0x02, 0xe9, 0x43, 0xe5, 0x8a, 0x2d,
+	0x35, 0xfc, 0x65, 0x93, 0xec, 0x40, 0xf5, 0x9a, 0x7a, 0x31, 0x43, 0xc8, 0x6b, 0x20, 0xaa, 0x21,
+	0x96, 0x52, 0xfc, 0xa2, 0xfc, 0xac, 0x64, 0xfe, 0xb7, 0x0a, 0x5b, 0x12, 0x7c, 0xe4, 0x5d, 0xe8,
+	0x78, 0x8c, 0x46, 0x6c, 0x1a, 0x84, 0x72, 0x81, 0x08, 0xa7, 0x6a, 0xed, 0xf7, 0xe5, 0xb0, 0x13,
+	0xa9, 0xf8, 0x5c, 0xc9, 0xad, 0xb6, 0x97, 0xeb, 0xc9, 0x90, 0x76, 0x7d, 0xc1, 0xb8, 0x4f, 0xbd,
+	0x29, 0x06, 0x83, 0x0a, 0xb0, 0x76, 0x22, 0x7c, 0x2e, 0x83, 0x62, 0x15, 0x47, 0x95, 0x75, 0x1c,
+	0x0d, 0xa1, 0x81, 0xbe, 0x73, 0x59, 0xa4, 0x83, 0x3d, 0xed, 0x93, 0x7d, 0x68, 0x2c, 0x98, 0xa0,
+	0x3a, 0xd6, 0x64, 0x48, 0xdc, 0x4d, 0x62, 0x66, 0x7c, 0xaa, 0x15, 0x2a, 0x20, 0x52, 0xbb, 0xb5,
+	0x88, 0xa8, 0xad, 0x47, 0xc4, 0x10, 0x1a, 0x29, 0xe8, 0xea, 0xea, 0x86, 0x93, 0xbe, 0xa4, 0xd9,
+	0x90, 0x71, 0x37, 0x70, 0x8c, 0x06, 0x02, 0x45, 0xf7, 0x24, 0x49, 0xfa, 0xf1, 0x42, 0x41, 0xa8,
+	0xa9, 0x48, 0xd2, 0x8f, 0x17, 0xeb, 0x88, 0x81, 0x15, 0xc4, 0xfc, 0x04, 0xaa, 0xd4, 0x73, 0x69,
+	0x84, 0x21, 0x24, 0x6f, 0x56, 0xf3, 0xfd, 0xf8, 0x40, 0x4a, 0x2d, 0xa5, 0x24, 0x4f, 0xa1, 0x33,
+	0xe7, 0x41, 0x1c, 0x4e, 0xb1, 0xcb, 0x22, 0xa3, 0x8d, 0xa7, 0x5d, 0xb5, 0x6e, 0xa3, 0xd1, 0x81,
+	0xb2, 0x91, 0x11, 0x38, 0x0b, 0x62, 0xdf, 0x99, 0xda, 0xae, 0xc3, 0x23, 0xa3, 0x83, 0xce, 0x03,
+	0x14, 0x1d, 0x4a, 0x89, 0x0c, 0x31, 0x15, 0x02, 0xa9, 0x83, 0xbb, 0x68, 0xd3, 0x41, 0xe9, 0x59,
+	0xe2, 0xe5, 0x9f, 0xc2, 0x20, 0x49, 0x4c, 0x99, 0x65, 0x0f, 0x2d, 0xfb, 0x89, 0x22, 0x35, 0x1e,
+	0x41, 0x9f, 0xdd, 0x48, 0x0a, 0x75, 0xc5, 0x74, 0x41, 0x6f, 0xa6, 0x42, 0x78, 0x3a, 0xa4, 0xba,
+	0x89, 0xfc, 0x94, 0xde, 0x4c, 0x84, 0x27, 0xe3, 0x5f, 0xad, 0x8e, 0xf1, 0x3f, 0xc0, 0x64, 0xd4,
+	0x44, 0x09, 0xc6, 0xff, 0x2e, 0x0c, 0xfc, 0x60, 0xea, 0xb0, 0x0b, 0x1a, 0x7b, 0x42, 0xad, 0xbb,
+	0xd4, 0xc1, 0xd4, 0xf3, 0x83, 0xe7, 0x4a, 0x8e, 0xcb, 0x2e, 0x87, 0xbf, 0x84, 0x4e, 0xe1, 0xba,
+	0x37, 0x80, 0x7e, 0x3b, 0x0f, 0xfa, 0x66, 0x1e, 0xe8, 0xff, 0xde, 0x02, 0xc0, 0x7b, 0x57, 0x43,
+	0x57, 0xb3, 0x45, 0x1e, 0x0c, 0xe5, 0x0d, 0x60, 0xa0, 0x9c, 0xf9, 0x42, 0x03, 0x57, 0xf7, 0xbe,
+	0x11, 0xb3, 0x49, 0xbe, 0xa8, 0xe6, 0xf2, 0xc5, 0x5b, 0xb0, 0x25, 0xf1, 0x69, 0xd4, 0x32, 0x5a,
+	0xcf, 0x76, 0x84, 0x48, 0x56, 0x28, 0x46, 0xab, 0xb5, 0xa0, 0xa9, 0xaf, 0x07, 0x4d, 0x1e, 0x8d,
+	0x8d, 0x22, 0x1a, 0x5f, 0x87, 0x8e, 0xcd, 0x19, 0xe6, 0xae, 0xa9, 0x2c, 0x46, 0x34, 0x5a, 0xdb,
+	0x89, 0x70, 0xe2, 0x2e, 0x98, 0xf4, 0x9f, 0xbc, 0x38, 0x40, 0x95, 0x6c, 0x6e, 0xbc, 0xd7, 0xd6,
+	0xc6, 0x7b, 0xc5, 0x4a, 0xc0, 0x63, 0x9a, 0xf1, 0xb1, 0x9d, 0x8b, 0x9a, 0x4e, 0x21, 0x6a, 0x0a,
+	0xa1, 0xd1, 0x5d, 0x09, 0x8d, 0x15, 0xfc, 0xf6, 0xd6, 0xf0, 0xfb, 0x1a, 0xb4, 0xa5, 0x03, 0xa2,
+	0x90, 0xda, 0x4c, 0x4e, 0xd0, 0x57, 0x8e, 0x48, 0x65, 0xc7, 0x0e, 0x46, 0x7b, 0x3c, 0x9b, 0x2d,
+	0x2f, 0x03, 0x8f, 0x65, 0x84, 0xdd, 0x4a, 0x65, 0xc7, 0x8e, 0xdc, 0x2f, 0x22, 0x90, 0x20, 0x02,
+	0xb1, 0x3d, 0x7c, 0x0f, 0x9a, 0xa9, 0xd7, 0xbf, 0x17, 0x98, 0xfe, 0x5e, 0x82, 0x76, 0x9e, 0x14,
+	0xe5, 0xe0, 0xc9, 0xe4, 0x04, 0x07, 0x57, 0x2c, 0xd9, 0x94, 0xe5, 0x04, 0x67, 0x3e, 0x7b, 0x49,
+	0x67, 0x9e, 0x9a, 0xa0, 0x61, 0x65, 0x02, 0xa9, 0x75, 0x7d, 0x9b, 0xb3, 0x45, 0x82, 0xaa, 0x8a,
+	0x95, 0x09, 0xc8, 0xfb, 0x00, 0x6e, 0x14, 0xc5, 0x4c, 0xdd, 0xdc, 0x16, 0x52, 0xc6, 0x70, 0xac,
+	0x6a, 0xcc, 0x71, 0x52, 0x63, 0x8e, 0x27, 0x49, 0x8d, 0x69, 0x35, 0xd1, 0x1a, 0xaf, 0xf4, 0x2e,
+	0xd4, 0xe4, 0x05, 0x4d, 0x4e, 0x10, 0x79, 0x15, 0x4b, 0xf7, 0xcc, 0xbf, 0x40, 0x4d, 0x55, 0x21,
+	0xff, 0x57, 0xa2, 0xbf, 0x07, 0x0d, 0x35, 0xb7, 0xeb, 0xe8, 0x58, 0xa9, 0x63, 0xff, 0xd8, 0x31,
+	0xbf, 0x2e, 0x43, 0xc3, 0x62, 0x51, 0x18, 0xf8, 0x11, 0xcb, 0x55, 0x49, 0xa5, 0x6f, 0xad, 0x92,
+	0xca, 0x1b, 0xab, 0xa4, 0xa4, 0xf6, 0xaa, 0xe4, 0x6a, 0xaf, 0x21, 0x34, 0x38, 0x73, 0x5c, 0xce,
+	0x6c, 0xa1, 0xeb, 0xb4, 0xb4, 0x2f, 0x75, 0x2f, 0x29, 0x97, 0xe9, 0x3d, 0xc2, 0x1c, 0xd2, 0xb4,
+	0xd2, 0x3e, 0x79, 0x92, 0x2f, 0x2e, 0x54, 0xd9, 0xb6, 0xad, 0x8a, 0x0b, 0xb5, 0xdd, 0x0d, 0xd5,
+	0xc5, 0xd3, 0xac, 0x48, 0xab, 0x63, 0x34, 0xdf, 0xcb, 0x0f, 0xd8, 0x5c, 0xa5, 0xfd, 0x60, 0x39,
+	0xfb, 0xeb, 0x32, 0xf4, 0x57, 0xf7, 0xb6, 0x01, 0x81, 0xdb, 0x50, 0x55, 0xb9, 0x4f, 0xc3, 0x57,
+	0xac, 0x65, 0xbd, 0xca, 0x0a, 0xd1, 0xfd, 0x6a, 0x95, 0x34, 0xbe, 0x1d, 0x7a, 0x45, 0x42, 0x79,
+	0x13, 0xfa, 0xd2, 0x45, 0x21, 0x73, 0xb2, 0x7a, 0x4e, 0x31, 0x60, 0x4f, 0xcb, 0xd3, 0x8a, 0x6e,
+	0x17, 0x06, 0x89, 0x69, 0xc6, 0x0d, 0xb5, 0x82, 0xed, 0x51, 0x42, 0x11, 0x77, 0xa1, 0x76, 0x11,
+	0xf0, 0x05, 0x15, 0x9a, 0x04, 0x75, 0xaf, 0x40, 0x72, 0xc8, 0xb6, 0x0d, 0x85, 0xc9, 0x44, 0x28,
+	0xdf, 0x2c, 0x92, 0x7c, 0xd2, 0xf7, 0x04, 0xb2, 0x60, 0xc3, 0x6a, 0x24, 0xef, 0x08, 0xf3, 0xb7,
+	0xd0, 0x5b, 0x29, 0x21, 0x37, 0x38, 0x32, 0x5b, 0xbe, 0x5c, 0x58, 0xbe, 0x30, 0x73, 0x65, 0x65,
+	0xe6, 0xdf, 0xc1, 0xe0, 0x13, 0xea, 0x3b, 0x1e, 0xd3, 0xf3, 0x1f, 0xf0, 0x79, 0x24, 0x93, 0xa1,
+	0x7e, 0xd1, 0x4c, 0x75, 0xf6, 0xe9, 0x58, 0x4d, 0x2d, 0x39, 0x76, 0xc8, 0x23, 0xa8, 0x73, 0x65,
+	0xad, 0x01, 0xd0, 0xca, 0xd5, 0xb8, 0x56, 0xa2, 0x33, 0xbf, 0x02, 0x52, 0x98, 0x5a, 0x3e, 0x66,
+	0x96, 0x64, 0x24, 0xd1, 0xaf, 0x40, 0xa1, 0xa3, 0xaa, 0x9d, 0xc7, 0xa4, 0x95, 0x6a, 0xc9, 0x0e,
+	0x54, 0x18, 0xe7, 0x7a, 0x09, 0x2c, 0x32, 0xb3, 0xa7, 0xa3, 0x25, 0x55, 0x66, 0x1f, 0xba, 0xc7,
+	0xbe, 0x2b, 0x5c, 0xea, 0xb9, 0x7f, 0x66, 0x72, 0xe7, 0xe6, 0x53, 0xe8, 0x65, 0x12, 0xb5, 0xa0,
+	0x9e, 0xa6, 0x74, 0xfb, 0x34, 0x3f, 0x83, 0xc1, 0x79, 0xc8, 0x6c, 0x97, 0x7a, 0xf8, 0x7a, 0x54,
+	0xc3, 0x1e, 0x42, 0x55, 0xde, 0x55, 0xc2, 0x3b, 0x4d, 0x1c, 0x88, 0x6a, 0x25, 0x37, 0xbf, 0x02,
+	0x43, 0x1d, 0xef, 0xe8, 0xc6, 0x8d, 0x04, 0xf3, 0x6d, 0x76, 0x78, 0xc9, 0xec, 0xab, 0x1f, 0xd0,
+	0x81, 0xd7, 0x70, 0x6f, 0xd3, 0x0a, 0xc9, 0xfe, 0x5a, 0xb6, 0xec, 0x4d, 0x2f, 0x64, 0x0a, 0xc2,
+	0x35, 0x1a, 0x16, 0xa0, 0xe8, 0x63, 0x29, 0x91, 0x70, 0x60, 0x72, 0x5c, 0xa4, 0x69, 0x5d, 0xf7,
+	0x12, 0x7f, 0x54, 0x6e, 0xf7, 0xc7, 0x3f, 0x4a, 0xd0, 0x3c, 0x67, 0x22, 0x0e, 0xf1, 0x2c, 0xf7,
+	0xa1, 0x39, 0xe3, 0xc1, 0x15, 0xe3, 0xd9, 0x51, 0x1a, 0x4a, 0x70, 0xec, 0x90, 0x27, 0x50, 0x3b,
+	0x0c, 0xfc, 0x0b, 0x77, 0x8e, 0x6f, 0x69, 0xcd, 0x2f, 0xe9, 0xd8, 0xb1, 0xd2, 0x29, 0x7e, 0xd1,
+	0x86, 0x64, 0x07, 0x5a, 0xfa, 0xcb, 0xc4, 0x17, 0x5f, 0x1c, 0x3f, 0x4f, 0x8a, 0xec, 0x9c, 0x68,
+	0xf8, 0x3e, 0xb4, 0x72, 0x03, 0xbf, 0x57, 0xc6, 0xfb, 0x31, 0x00, 0xae, 0xae, 0x7c, 0xd4, 0xcf,
+	0xae, 0xbe, 0xa9, 0x8e, 0xf6, 0x10, 0x9a, 0xb2, 0x9e, 0x53, 0xea, 0x24, 0xd7, 0x96, 0xb2, 0x5c,
+	0x6b, 0x3e, 0x82, 0xc1, 0xb1, 0x7f, 0x4d, 0x3d, 0xd7, 0xa1, 0x82, 0x7d, 0xca, 0x96, 0xe8, 0x82,
+	0xb5, 0x1d, 0x98, 0xe7, 0xd0, 0xd6, 0x8f, 0xfb, 0xef, 0xb4, 0xc7, 0xb6, 0xde, 0xe3, 0x37, 0xc7,
+	0xe2, 0x9b, 0xd0, 0xd3, 0x93, 0x9e, 0xb8, 0x3a, 0x12, 0x65, 0xa9, 0xc2, 0xd9, 0x85, 0x7b, 0xa3,
+	0xa7, 0xd6, 0x3d, 0xf3, 0x19, 0xf4, 0x73, 0xa6, 0xe9, 0x71, 0xae, 0xd8, 0x32, 0x4a, 0x3e, 0x7a,
+	0xc8, 0x76, 0xe2, 0x81, 0x72, 0xe6, 0x01, 0x13, 0xba, 0x7a, 0xe4, 0x0b, 0x26, 0x6e, 0x39, 0xdd,
+	0xa7, 0xe9, 0x46, 0x5e, 0x30, 0x3d, 0xf9, 0x63, 0xa8, 0x32, 0x79, 0xd2, 0x7c, 0x1a, 0xce, 0x7b,
+	0xc0, 0x52, 0xea, 0x0d, 0x0b, 0x3e, 0x4b, 0x17, 0x3c, 0x8b, 0xd5, 0x82, 0xdf, 0x71, 0x2e, 0xf3,
+	0xf5, 0x74, 0x1b, 0x67, 0xb1, 0xb8, 0xed, 0x46, 0x1f, 0xc1, 0x40, 0x1b, 0x3d, 0x67, 0x1e, 0x13,
+	0xec, 0x96, 0x23, 0x3d, 0x06, 0x52, 0x30, 0xbb, 0x6d, 0xba, 0x07, 0xd0, 0x98, 0x4c, 0x4e, 0x52,
+	0x6d, 0x91, 0x62, 0xcd, 0x0f, 0x60, 0x70, 0x1e, 0x3b, 0xc1, 0x19, 0x77, 0xaf, 0x5d, 0x8f, 0xcd,
+	0xd5, 0x62, 0x49, 0x0d, 0x5d, 0xca, 0xd5, 0xd0, 0x1b, 0x93, 0x9a, 0x39, 0x02, 0x52, 0x18, 0x9e,
+	0xde, 0x5b, 0x14, 0x3b, 0x81, 0x0e, 0x61, 0x6c, 0x9b, 0x23, 0x68, 0x4f, 0xa8, 0xac, 0x59, 0x1c,
+	0x65, 0x63, 0x40, 0x5d, 0xa8, 0xbe, 0x36, 0x4b, 0xba, 0xe6, 0x3e, 0x6c, 0x1f, 0x52, 0xfb, 0xd2,
+	0xf5, 0xe7, 0xcf, 0xdd, 0x48, 0x16, 0x6d, 0x7a, 0xc4, 0x10, 0x1a, 0x8e, 0x16, 0xe8, 0x21, 0x69,
+	0xdf, 0x7c, 0x1b, 0x5e, 0xc9, 0x7d, 0x59, 0x3a, 0x17, 0x34, 0xf1, 0xc7, 0x36, 0x54, 0x23, 0xd9,
+	0xc3, 0x11, 0x55, 0x4b, 0x75, 0xcc, 0xcf, 0x60, 0x3b, 0x9f, 0xc7, 0x65, 0x09, 0x95, 0x1c, 0x1c,
+	0x8b, 0x9b, 0x52, 0xae, 0xb8, 0xd1, 0x3e, 0x2b, 0x67, 0x69, 0xa9, 0x0f, 0x95, 0x5f, 0x7f, 0x39,
+	0xd1, 0x60, 0x97, 0x4d, 0xf3, 0x8f, 0x72, 0xf9, 0xe2, 0x7c, 0x6a, 0xf9, 0x42, 0x85, 0x53, 0xfa,
+	0x4e, 0x15, 0xce, 0x3a, 0xde, 0xde, 0x86, 0xc1, 0xa9, 0x17, 0xd8, 0x57, 0x47, 0x7e, 0xce, 0x1b,
+	0x06, 0xd4, 0x99, 0x9f, 0x77, 0x46, 0xd2, 0x35, 0xdf, 0x80, 0xde, 0x49, 0x60, 0x53, 0xef, 0x34,
+	0x88, 0x7d, 0x91, 0x7a, 0x01, 0x3f, 0xf5, 0x69, 0x53, 0xd5, 0x31, 0xdf, 0x86, 0xae, 0xce, 0xf4,
+	0xfe, 0x45, 0x90, 0x30, 0x63, 0x56, 0x13, 0x94, 0x8a, 0xef, 0x05, 0xf3, 0x04, 0x7a, 0x99, 0xb9,
+	0x9a, 0xf7, 0x0d, 0xa8, 0x29, 0xb5, 0x3e, 0x5b, 0x2f, 0x7d, 0x30, 0x2b, 0x4b, 0x4b, 0xab, 0x37,
+	0x1c, 0x6a, 0x01, 0xdd, 0x33, 0xfc, 0xe4, 0x7a, 0xe4, 0x5f, 0xab, 0xc9, 0x8e, 0x81, 0xa8, 0x8f,
+	0xb0, 0x53, 0xe6, 0x5f, 0xbb, 0x3c, 0xf0, 0xb1, 0x46, 0x2f, 0xe9, 0x4a, 0x28, 0x99, 0x38, 0x1d,
+	0x94, 0x58, 0x58, 0x83, 0x70, 0x55, 0xb4, 0xd1, 0x87, 0x90, 0x7d, 0xd0, 0x91, 0xa9, 0x86, 0xb3,
+	0x45, 0x20, 0xd8, 0x94, 0x3a, 0x4e, 0x12, 0x2d, 0xa0, 0x44, 0x07, 0x8e, 0xc3, 0xf7, 0xff, 0x56,
+	0x81, 0xfa, 0x47, 0x8a, 0xc0, 0xc9, 0x87, 0xd0, 0x29, 0x64, 0x7d, 0xf2, 0x0a, 0x56, 0x87, 0xab,
+	0x35, 0xc6, 0xf0, 0xee, 0x9a, 0x58, 0x9d, 0xeb, 0x1d, 0x68, 0xe7, 0x93, 0x31, 0xc1, 0xc4, 0x8b,
+	0x9f, 0x97, 0x87, 0x38, 0xd3, 0x7a, 0xa6, 0x3e, 0x87, 0xed, 0x4d, 0x69, 0x92, 0x3c, 0xc8, 0x56,
+	0x58, 0x4f, 0xd1, 0xc3, 0x57, 0x6f, 0xd3, 0x26, 0xe9, 0xb5, 0x7e, 0xe8, 0x31, 0xea, 0xc7, 0x61,
+	0x7e, 0x07, 0x59, 0x93, 0x3c, 0x81, 0x4e, 0x21, 0x51, 0xa8, 0x73, 0xae, 0xe5, 0x8e, 0xfc, 0x90,
+	0xc7, 0x50, 0xc5, 0xe4, 0x44, 0x3a, 0x85, 0x2c, 0x39, 0xec, 0xa6, 0x5d, 0xb5, 0xf6, 0xbb, 0x00,
+	0x59, 0x11, 0x43, 0x88, 0x9a, 0x37, 0x5f, 0xe6, 0x0c, 0xef, 0x14, 0x65, 0x49, 0xa1, 0xb3, 0x85,
+	0xdf, 0x2a, 0x72, 0xfb, 0xc5, 0x85, 0xd2, 0x84, 0xb7, 0xff, 0x9f, 0x12, 0xd4, 0x93, 0xef, 0xd7,
+	0x4f, 0x60, 0x4b, 0xa6, 0x0e, 0x72, 0x27, 0xc7, 0xbe, 0x49, 0xda, 0x19, 0x6e, 0xaf, 0x08, 0xd5,
+	0x02, 0x63, 0xa8, 0xbc, 0x60, 0x42, 0x6d, 0xa8, 0x98, 0x43, 0x86, 0x77, 0x8a, 0xb2, 0xd4, 0xfe,
+	0x2c, 0x2e, 0xda, 0xeb, 0x14, 0x50, 0xb0, 0x4f, 0xc9, 0xfd, 0x3d, 0xa8, 0x29, 0x72, 0x56, 0xbe,
+	0x5c, 0xa3, 0x75, 0x85, 0x99, 0x75, 0x1a, 0xdf, 0xff, 0xd7, 0x16, 0xc0, 0xf9, 0x32, 0x12, 0x6c,
+	0xf1, 0x1b, 0x97, 0xbd, 0x24, 0xbb, 0xd0, 0xd3, 0x5f, 0x64, 0xf0, 0xa1, 0x28, 0x49, 0x28, 0xe7,
+	0x13, 0x2c, 0x37, 0x53, 0x8e, 0x7f, 0x0c, 0xad, 0x53, 0x7a, 0xf3, 0xed, 0x76, 0x1f, 0x42, 0xa7,
+	0x40, 0xdd, 0x7a, 0x8b, 0xab, 0xc9, 0x40, 0x6f, 0x71, 0x9d, 0xe4, 0x1f, 0x43, 0x5d, 0x13, 0x7a,
+	0x7e, 0x0d, 0x4c, 0x7d, 0x05, 0xa2, 0xff, 0x39, 0xf4, 0x56, 0xe8, 0x3c, 0x6f, 0x8f, 0x1f, 0x63,
+	0x36, 0xd2, 0xfd, 0x33, 0xf9, 0xd6, 0x2a, 0x52, 0x7a, 0x7e, 0xa0, 0x7e, 0xf7, 0x6d, 0xe2, 0xfc,
+	0x17, 0xc5, 0x57, 0x1a, 0x3e, 0x90, 0x8d, 0x55, 0xd6, 0x4d, 0x38, 0x7f, 0x78, 0x6f, 0x93, 0x26,
+	0x8d, 0xdc, 0x3c, 0xf1, 0xae, 0x45, 0xee, 0x3a, 0x2b, 0xbf, 0x05, 0x90, 0x71, 0x6f, 0xde, 0x1e,
+	0xe1, 0xb1, 0x4a, 0xcb, 0xef, 0x02, 0x64, 0x8c, 0xaa, 0x50, 0x55, 0x24, 0x64, 0x35, 0x6c, 0x95,
+	0x75, 0x77, 0xa1, 0x99, 0xb2, 0x60, 0x7e, 0x0d, 0x9c, 0xa0, 0x48, 0xaa, 0x1f, 0xed, 0xfe, 0x7e,
+	0x34, 0x77, 0xc5, 0x65, 0x3c, 0x1b, 0xdb, 0xc1, 0x62, 0xef, 0x92, 0x46, 0x97, 0xae, 0x1d, 0xf0,
+	0x70, 0xef, 0x5a, 0x82, 0x69, 0xaf, 0xf0, 0x7b, 0x6d, 0x56, 0xc3, 0x67, 0xe6, 0xd3, 0xff, 0x05,
+	0x00, 0x00, 0xff, 0xff, 0xc4, 0xd7, 0xaa, 0x59, 0x76, 0x1b, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -2927,6 +3001,9 @@ type BackendClient interface {
 	// broker_id to create a connection back to Vault for use with the Storage
 	// and SystemView clients.
 	Setup(ctx context.Context, in *SetupArgs, opts ...grpc.CallOption) (*SetupReply, error)
+	// Initialize is invoked just after mounting a backend to allow it to
+	// handle any initialization tasks that need to be performed.
+	Initialize(ctx context.Context, in *InitializeArgs, opts ...grpc.CallOption) (*InitializeReply, error)
 	// Type returns the BackendType for the particular backend
 	Type(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*TypeReply, error)
 }
@@ -2993,6 +3070,15 @@ func (c *backendClient) Setup(ctx context.Context, in *SetupArgs, opts ...grpc.C
 	return out, nil
 }
 
+func (c *backendClient) Initialize(ctx context.Context, in *InitializeArgs, opts ...grpc.CallOption) (*InitializeReply, error) {
+	out := new(InitializeReply)
+	err := c.cc.Invoke(ctx, "/pb.Backend/Initialize", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func (c *backendClient) Type(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*TypeReply, error) {
 	out := new(TypeReply)
 	err := c.cc.Invoke(ctx, "/pb.Backend/Type", in, out, opts...)
@@ -3034,36 +3120,13 @@ type BackendServer interface {
 	// broker_id to create a connection back to Vault for use with the Storage
 	// and SystemView clients.
 	Setup(context.Context, *SetupArgs) (*SetupReply, error)
+	// Initialize is invoked just after mounting a backend to allow it to
+	// handle any initialization tasks that need to be performed.
+	Initialize(context.Context, *InitializeArgs) (*InitializeReply, error)
 	// Type returns the BackendType for the particular backend
 	Type(context.Context, *Empty) (*TypeReply, error)
 }
 
-// UnimplementedBackendServer can be embedded to have forward compatible implementations.
-type UnimplementedBackendServer struct {
-}
-
-func (*UnimplementedBackendServer) HandleRequest(ctx context.Context, req *HandleRequestArgs) (*HandleRequestReply, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method HandleRequest not implemented")
-}
-func (*UnimplementedBackendServer) SpecialPaths(ctx context.Context, req *Empty) (*SpecialPathsReply, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method SpecialPaths not implemented")
-}
-func (*UnimplementedBackendServer) HandleExistenceCheck(ctx context.Context, req *HandleExistenceCheckArgs) (*HandleExistenceCheckReply, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method HandleExistenceCheck not implemented")
-}
-func (*UnimplementedBackendServer) Cleanup(ctx context.Context, req *Empty) (*Empty, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Cleanup not implemented")
-}
-func (*UnimplementedBackendServer) InvalidateKey(ctx context.Context, req *InvalidateKeyArgs) (*Empty, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method InvalidateKey not implemented")
-}
-func (*UnimplementedBackendServer) Setup(ctx context.Context, req *SetupArgs) (*SetupReply, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Setup not implemented")
-}
-func (*UnimplementedBackendServer) Type(ctx context.Context, req *Empty) (*TypeReply, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Type not implemented")
-}
-
 func RegisterBackendServer(s *grpc.Server, srv BackendServer) {
 	s.RegisterService(&_Backend_serviceDesc, srv)
 }
@@ -3176,6 +3239,24 @@ func _Backend_Setup_Handler(srv interface{}, ctx context.Context, dec func(inter
 	return interceptor(ctx, in, info, handler)
 }
 
+func _Backend_Initialize_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(InitializeArgs)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(BackendServer).Initialize(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/pb.Backend/Initialize",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(BackendServer).Initialize(ctx, req.(*InitializeArgs))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 func _Backend_Type_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(Empty)
 	if err := dec(in); err != nil {
@@ -3222,6 +3303,10 @@ var _Backend_serviceDesc = grpc.ServiceDesc{
 			MethodName: "Setup",
 			Handler:    _Backend_Setup_Handler,
 		},
+		{
+			MethodName: "Initialize",
+			Handler:    _Backend_Initialize_Handler,
+		},
 		{
 			MethodName: "Type",
 			Handler:    _Backend_Type_Handler,
@@ -3293,23 +3378,6 @@ type StorageServer interface {
 	Delete(context.Context, *StorageDeleteArgs) (*StorageDeleteReply, error)
 }
 
-// UnimplementedStorageServer can be embedded to have forward compatible implementations.
-type UnimplementedStorageServer struct {
-}
-
-func (*UnimplementedStorageServer) List(ctx context.Context, req *StorageListArgs) (*StorageListReply, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method List not implemented")
-}
-func (*UnimplementedStorageServer) Get(ctx context.Context, req *StorageGetArgs) (*StorageGetReply, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Get not implemented")
-}
-func (*UnimplementedStorageServer) Put(ctx context.Context, req *StoragePutArgs) (*StoragePutReply, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Put not implemented")
-}
-func (*UnimplementedStorageServer) Delete(ctx context.Context, req *StorageDeleteArgs) (*StorageDeleteReply, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Delete not implemented")
-}
-
 func RegisterStorageServer(s *grpc.Server, srv StorageServer) {
 	s.RegisterService(&_Storage_serviceDesc, srv)
 }
@@ -3600,44 +3668,6 @@ type SystemViewServer interface {
 	PluginEnv(context.Context, *Empty) (*PluginEnvReply, error)
 }
 
-// UnimplementedSystemViewServer can be embedded to have forward compatible implementations.
-type UnimplementedSystemViewServer struct {
-}
-
-func (*UnimplementedSystemViewServer) DefaultLeaseTTL(ctx context.Context, req *Empty) (*TTLReply, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method DefaultLeaseTTL not implemented")
-}
-func (*UnimplementedSystemViewServer) MaxLeaseTTL(ctx context.Context, req *Empty) (*TTLReply, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method MaxLeaseTTL not implemented")
-}
-func (*UnimplementedSystemViewServer) SudoPrivilege(ctx context.Context, req *SudoPrivilegeArgs) (*SudoPrivilegeReply, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method SudoPrivilege not implemented")
-}
-func (*UnimplementedSystemViewServer) Tainted(ctx context.Context, req *Empty) (*TaintedReply, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Tainted not implemented")
-}
-func (*UnimplementedSystemViewServer) CachingDisabled(ctx context.Context, req *Empty) (*CachingDisabledReply, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method CachingDisabled not implemented")
-}
-func (*UnimplementedSystemViewServer) ReplicationState(ctx context.Context, req *Empty) (*ReplicationStateReply, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method ReplicationState not implemented")
-}
-func (*UnimplementedSystemViewServer) ResponseWrapData(ctx context.Context, req *ResponseWrapDataArgs) (*ResponseWrapDataReply, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method ResponseWrapData not implemented")
-}
-func (*UnimplementedSystemViewServer) MlockEnabled(ctx context.Context, req *Empty) (*MlockEnabledReply, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method MlockEnabled not implemented")
-}
-func (*UnimplementedSystemViewServer) LocalMount(ctx context.Context, req *Empty) (*LocalMountReply, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method LocalMount not implemented")
-}
-func (*UnimplementedSystemViewServer) EntityInfo(ctx context.Context, req *EntityInfoArgs) (*EntityInfoReply, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method EntityInfo not implemented")
-}
-func (*UnimplementedSystemViewServer) PluginEnv(ctx context.Context, req *Empty) (*PluginEnvReply, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method PluginEnv not implemented")
-}
-
 func RegisterSystemViewServer(s *grpc.Server, srv SystemViewServer) {
 	s.RegisterService(&_SystemView_serviceDesc, srv)
 }
diff --git a/sdk/plugin/pb/backend.proto b/sdk/plugin/pb/backend.proto
index ca86c9c13928..d513aed1294f 100644
--- a/sdk/plugin/pb/backend.proto
+++ b/sdk/plugin/pb/backend.proto
@@ -355,6 +355,15 @@ message HandleRequestReply {
 	ProtoError err = 2;
 }
 
+// InitializeArgs is the args for Initialize method.
+message InitializeArgs {
+}
+
+// InitializeReply is the reply for Initialize method.
+message InitializeReply {
+	ProtoError err = 1;
+}
+
 // SpecialPathsReply is the reply for SpecialPaths method.
 message SpecialPathsReply {
 	Paths paths = 1;
@@ -434,6 +443,10 @@ service Backend {
 	// and SystemView clients.
 	rpc Setup(SetupArgs) returns (SetupReply);
 
+    // Initialize is invoked just after mounting a backend to allow it to
+	// handle any initialization tasks that need to be performed.
+	rpc Initialize(InitializeArgs) returns (InitializeReply);
+
    	// Type returns the BackendType for the particular backend
 	rpc Type(Empty) returns (TypeReply);
 }

From e38ffab505371d5e88f4a24cc7d238eef46fddfd Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Mon, 1 Jul 2019 15:53:54 -0400
Subject: [PATCH 14/53] persist current role storage version when upgrading aws
 roles

---
 builtin/credential/aws/path_role.go      | 67 ++++++++++++++++++++----
 builtin/credential/aws/path_role_test.go | 50 ++++++++++++++++--
 2 files changed, 105 insertions(+), 12 deletions(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index beed3579b289..7c4d1259ada8 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strconv"
 	"strings"
 	"sync/atomic"
 	"time"
@@ -331,11 +332,6 @@ func (b *backend) initialize(ctx context.Context, s logical.Storage) error {
 
 		logger := b.Logger().Named("initialize")
 
-		// TODO we need to figure out a way to persist the fact that this has
-		// already been run in the past for a given currentRoleStorageVersion,
-		// so that we only ever run it once even if the vault server is
-		// restarted.
-
 		// grab the guard
 		if !atomic.CompareAndSwapUint32(b.initializeCASGuard, 0, 1) {
 			logger.Error("initializational already in progress")
@@ -358,12 +354,33 @@ func (b *backend) initialize(ctx context.Context, s logical.Storage) error {
 			ctx = context.Background()
 
 			logger.Info("starting initialization")
-			err := b.updateUpgradableRoleEntries(ctx, s)
-			if err == nil {
-				logger.Info("initialization succeeded")
-			} else {
+
+			// check if we've already upgraded to the current role storage version
+			version, err := b.upgradedRoleStorageVersion(ctx, s)
+			if err != nil {
+				logger.Error("error running initialization", "error", err)
+				return
+			}
+			if version == currentRoleStorageVersion {
+				logger.Info("initialization has already been performed for storage version", currentRoleStorageVersion)
+				return
+			}
+
+			// perform the upgrade
+			err = b.updateUpgradableRoleEntries(ctx, s)
+			if err != nil {
 				logger.Error("error running initialization", "error", err)
+				return
 			}
+
+			// save the current role storage version
+			err = b.setUpgradedRoleStorageVersion(ctx, s, currentRoleStorageVersion)
+			if err != nil {
+				logger.Error("error running initialization", "error", err)
+				return
+			}
+
+			logger.Info("initialization succeeded")
 		}()
 
 		return nil
@@ -407,6 +424,38 @@ func (b *backend) updateUpgradableRoleEntry(ctx context.Context, s logical.Stora
 	return err
 }
 
+// upgradedRoleStorageVersion returns the value of the roleStorageVersion that
+// we have already upgraded to, or -1 if the upgrade has never been run.
+func (b *backend) upgradedRoleStorageVersion(ctx context.Context, s logical.Storage) (int, error) {
+	b.roleMutex.Lock()
+	defer b.roleMutex.Unlock()
+
+	entry, err := s.Get(ctx, "config/role-storage-version")
+	if err != nil {
+		return -1, err
+	}
+
+	// the upgrade has never been run
+	if entry == nil {
+		return -1, nil
+	}
+
+	// return the upgraded value
+	return strconv.Atoi(string(entry.Value))
+}
+
+// setUpgradedRoleStorageVersion returns the value of the roleStorageVersion that
+// we have already upgraded to, or -1 if the upgrade has never been run
+func (b *backend) setUpgradedRoleStorageVersion(ctx context.Context, s logical.Storage, version int) error {
+	b.roleMutex.Lock()
+	defer b.roleMutex.Unlock()
+
+	return s.Put(ctx, &logical.StorageEntry{
+		Key:   "config/role-storage-version",
+		Value: []byte(strconv.Itoa(version)),
+	})
+}
+
 // If needed, updates the role entry and returns a bool indicating if it was updated
 // (and thus needs to be persisted)
 func (b *backend) upgradeRole(ctx context.Context, s logical.Storage, roleEntry *awsRoleEntry) (bool, error) {
diff --git a/builtin/credential/aws/path_role_test.go b/builtin/credential/aws/path_role_test.go
index a56bce3048fb..f94086392eb4 100644
--- a/builtin/credential/aws/path_role_test.go
+++ b/builtin/credential/aws/path_role_test.go
@@ -801,7 +801,7 @@ func TestRoleEntryUpgradeV(t *testing.T) {
 	}
 }
 
-func TestUpdateUpgradableRoleEntries(t *testing.T) {
+func TestInitialize(t *testing.T) {
 
 	config := logical.TestBackendConfig()
 	storage := &logical.InmemStorage{}
@@ -858,8 +858,9 @@ func TestUpdateUpgradableRoleEntries(t *testing.T) {
 		}
 	}
 
-	// upgrade all the entries
-	err = b.updateUpgradableRoleEntries(ctx, storage)
+	// upgrade all the entries, and wait for the goroutine to finish
+	err = b.initialize(ctx, storage)
+	time.Sleep(time.Second)
 
 	// read the entries from storage
 	after := make([]testData, 0)
@@ -905,6 +906,49 @@ func TestUpdateUpgradableRoleEntries(t *testing.T) {
 	if diff := deep.Equal(expected, after); diff != nil {
 		t.Fatal(diff)
 	}
+
+	// run it againg -- nothing will happen
+	err = b.initialize(ctx, storage)
+}
+
+func TestUpgradedRoleStorageVersion(t *testing.T) {
+
+	config := logical.TestBackendConfig()
+	storage := &logical.InmemStorage{}
+	config.StorageView = storage
+	b, err := Backend(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ctx := context.Background()
+	err = b.Setup(ctx, config)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	//-------------------------
+
+	version, err := b.upgradedRoleStorageVersion(ctx, storage)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if -1 != version {
+		t.Fatalf("expected version %d, got %d", -1, version)
+	}
+
+	err = b.setUpgradedRoleStorageVersion(ctx, storage, 42)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	version, err = b.upgradedRoleStorageVersion(ctx, storage)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if 42 != version {
+		t.Fatalf("expected version %d, got %d", 42, version)
+	}
 }
 
 func resolveArnToFakeUniqueId(ctx context.Context, s logical.Storage, arn string) (string, error) {

From f68254bc202add23821af61241262590c6655d5d Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Mon, 1 Jul 2019 15:57:47 -0400
Subject: [PATCH 15/53] format comments correctly

---
 builtin/credential/aws/path_role.go | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index 7c4d1259ada8..7da0e118e797 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -355,14 +355,16 @@ func (b *backend) initialize(ctx context.Context, s logical.Storage) error {
 
 			logger.Info("starting initialization")
 
-			// check if we've already upgraded to the current role storage version
+			// check if we've already upgraded to the current role storage
+			// version
 			version, err := b.upgradedRoleStorageVersion(ctx, s)
 			if err != nil {
 				logger.Error("error running initialization", "error", err)
 				return
 			}
 			if version == currentRoleStorageVersion {
-				logger.Info("initialization has already been performed for storage version", currentRoleStorageVersion)
+				logger.Info("skipping initialization -- has already been performed for storage version",
+					currentRoleStorageVersion)
 				return
 			}
 
@@ -444,8 +446,8 @@ func (b *backend) upgradedRoleStorageVersion(ctx context.Context, s logical.Stor
 	return strconv.Atoi(string(entry.Value))
 }
 
-// setUpgradedRoleStorageVersion returns the value of the roleStorageVersion that
-// we have already upgraded to, or -1 if the upgrade has never been run
+// setUpgradedRoleStorageVersion returns the value of the roleStorageVersion
+// that we have already upgraded to, or -1 if the upgrade has never been run
 func (b *backend) setUpgradedRoleStorageVersion(ctx context.Context, s logical.Storage, version int) error {
 	b.roleMutex.Lock()
 	defer b.roleMutex.Unlock()

From 07e47225c8a05c8731a403703c2c17664c556ff1 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Mon, 1 Jul 2019 16:25:29 -0400
Subject: [PATCH 16/53] improve comments

---
 builtin/credential/aws/backend_test.go   | 10 ++++++----
 builtin/credential/aws/path_role_test.go |  2 +-
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/builtin/credential/aws/backend_test.go b/builtin/credential/aws/backend_test.go
index 12bc114bf247..4bed75f85374 100644
--- a/builtin/credential/aws/backend_test.go
+++ b/builtin/credential/aws/backend_test.go
@@ -1819,11 +1819,13 @@ func generateRenewRequest(s logical.Storage, auth *logical.Auth) *logical.Reques
 	return renewReq
 }
 
-func TestBackend_Initialize(t *testing.T) {
+func TestBackend_E2E_Initialize(t *testing.T) {
 
-	// TODO this test is not very good. it just outputs some log entries showing
-	// that initialization ran.  lets find a way to verify that the initializer
-	// actually did something, or at least somehow trap the fact that it ran.
+	// All this test does is load up the aws plugin, so that if we are in
+	// verbose mode we can observe in the log output that the initialization
+	// process ran:
+	//
+	//    go test -v ./builtin/credential/aws/ -run TestBackend_E2E_Initialize
 
 	logger := logging.NewVaultLogger(hclog.Trace)
 	coreConfig := &vault.CoreConfig{
diff --git a/builtin/credential/aws/path_role_test.go b/builtin/credential/aws/path_role_test.go
index f94086392eb4..e5e6600b0d0e 100644
--- a/builtin/credential/aws/path_role_test.go
+++ b/builtin/credential/aws/path_role_test.go
@@ -801,7 +801,7 @@ func TestRoleEntryUpgradeV(t *testing.T) {
 	}
 }
 
-func TestInitialize(t *testing.T) {
+func TestRoleInitialize(t *testing.T) {
 
 	config := logical.TestBackendConfig()
 	storage := &logical.InmemStorage{}

From 7bca47165e982c5aad00209daaf8a6856b9e2849 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Mon, 1 Jul 2019 17:55:52 -0400
Subject: [PATCH 17/53] use postUnseal funcs to initialize backends

---
 vault/auth.go           |  14 +++--
 vault/auth_test.go      |  10 ----
 vault/mount.go          |  11 ++--
 vault/mount_test.go     | 117 +++++++++++++++++++---------------------
 vault/router_testing.go |  32 +++++++++--
 5 files changed, 95 insertions(+), 89 deletions(-)

diff --git a/vault/auth.go b/vault/auth.go
index c5e731b82fab..9b4d24366b4e 100644
--- a/vault/auth.go
+++ b/vault/auth.go
@@ -689,11 +689,12 @@ func (c *Core) setupCredentials(ctx context.Context) error {
 
 		// Initialize
 		if !nilMount {
-			err := backend.Initialize(ctx, &logical.InitializationRequest{Storage: view})
-			if err != nil {
-				c.logger.Error("failed to initialize auth entry", "path", entry.Path, "error", err)
-				return errLoadAuthFailed
-			}
+			c.postUnsealFuncs = append(c.postUnsealFuncs, func() {
+				err := backend.Initialize(ctx, &logical.InitializationRequest{Storage: view})
+				if err != nil {
+					c.logger.Error("failed to initialize auth entry", "path", entry.Path, "error", err)
+				}
+			})
 		}
 	}
 
@@ -702,9 +703,6 @@ func (c *Core) setupCredentials(ctx context.Context) error {
 		return c.persistAuth(ctx, c.auth, nil)
 	}
 
-	// TODO do we need to check for AWS auth builtin backed, and initialize it?
-	// If so, is this the right place to do that?
-
 	return nil
 }
 
diff --git a/vault/auth_test.go b/vault/auth_test.go
index c1df89c784cf..eb37dc488c9d 100644
--- a/vault/auth_test.go
+++ b/vault/auth_test.go
@@ -87,10 +87,6 @@ func TestCore_EnableCredential(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 
-	if !backend.isInitialized {
-		t.Fatalf("backend is not initialized")
-	}
-
 	match := c.router.MatchingMount(namespace.RootContext(nil), "auth/foo/bar")
 	if match != "auth/foo/" {
 		t.Fatalf("missing mount, match: %q", match)
@@ -179,12 +175,6 @@ func TestCore_EnableCredential_Local(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	for _, b := range backends {
-		if !b.isInitialized {
-			t.Fatalf("backend is not initialized")
-		}
-	}
-
 	rawLocal, err := c.barrier.Get(context.Background(), coreLocalAuthConfigPath)
 	if err != nil {
 		t.Fatal(err)
diff --git a/vault/mount.go b/vault/mount.go
index e0055e7772a6..d8f8cd678744 100644
--- a/vault/mount.go
+++ b/vault/mount.go
@@ -1141,11 +1141,12 @@ func (c *Core) setupMounts(ctx context.Context) error {
 
 		// Initialize
 		if !nilMount {
-			err := backend.Initialize(ctx, &logical.InitializationRequest{Storage: view})
-			if err != nil {
-				c.logger.Error("failed to initialize mount entry", "path", entry.Path, "error", err)
-				return errLoadMountsFailed
-			}
+			c.postUnsealFuncs = append(c.postUnsealFuncs, func() {
+				err := backend.Initialize(ctx, &logical.InitializationRequest{Storage: view})
+				if err != nil {
+					c.logger.Error("failed to initialize mount entry", "path", entry.Path, "error", err)
+				}
+			})
 		}
 
 		if c.logger.IsInfo() {
diff --git a/vault/mount_test.go b/vault/mount_test.go
index bc174fc471f9..185fdfa8eab0 100644
--- a/vault/mount_test.go
+++ b/vault/mount_test.go
@@ -424,9 +424,6 @@ func TestCore_Remount_Cleanup(t *testing.T) {
 	if err := c.mount(namespace.RootContext(nil), me); err != nil {
 		t.Fatalf("err: %v", err)
 	}
-	if !noop.isInitialized {
-		t.Fatalf("backend is not initialized")
-	}
 
 	// Store the view
 	view := c.router.MatchingStorageByAPIPath(namespace.RootContext(nil), "test/")
@@ -739,63 +736,57 @@ func TestSingletonMountTableFunc(t *testing.T) {
 	}
 }
 
-func TestCore_SetupMounts_Initialize(t *testing.T) {
-	c, _, _ := TestCoreUnsealed(t)
-
-	counter := 0
-	backends := []*NoopBackend{
-		&NoopBackend{
-			BackendType: logical.TypeLogical,
-		},
-		&NoopBackend{
-			BackendType: logical.TypeLogical,
-		},
-	}
-
-	c.logicalBackends["noop"] = func(context.Context, *logical.BackendConfig) (logical.Backend, error) {
-		b := backends[counter]
-		counter++
-		return b, nil
-	}
-
-	c.mounts = &MountTable{
-		Type: mountTableType,
-		Entries: []*MountEntry{
-			&MountEntry{
-				Table:            mountTableType,
-				Path:             "noop/",
-				Type:             "noop",
-				UUID:             "abcd",
-				Accessor:         "kv-abcd",
-				BackendAwareUUID: "abcde",
-				NamespaceID:      namespace.RootNamespaceID,
-				namespace:        namespace.RootNamespace,
-			},
-			&MountEntry{
-				Table:            mountTableType,
-				Path:             "noop2/",
-				Type:             "noop",
-				UUID:             "bcde",
-				Accessor:         "kv-bcde",
-				BackendAwareUUID: "bcdea",
-				NamespaceID:      namespace.RootNamespaceID,
-				namespace:        namespace.RootNamespace,
-			},
-		},
-	}
-
-	// Both should set up successfully
-	err := c.setupMounts(namespace.RootContext(nil))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(c.mounts.Entries) != 2 {
-		t.Fatalf("expected two entries, got %d", len(c.mounts.Entries))
-	}
-
-	for _, b := range backends {
-		if !b.isInitialized {
-			t.Fatalf("backend is not initialized")
-		}
-	}
-}
+//func TestCore_SetupMounts_Initialize(t *testing.T) {
+//	c, _, _ := TestCoreUnsealed(t)
+//
+//	counter := 0
+//	backends := []*NoopBackend{
+//		&NoopBackend{
+//			BackendType: logical.TypeLogical,
+//		},
+//		&NoopBackend{
+//			BackendType: logical.TypeLogical,
+//		},
+//	}
+//
+//	c.logicalBackends["noop"] = func(context.Context, *logical.BackendConfig) (logical.Backend, error) {
+//		b := backends[counter]
+//		counter++
+//		return b, nil
+//	}
+//
+//	c.mounts = &MountTable{
+//		Type: mountTableType,
+//		Entries: []*MountEntry{
+//			&MountEntry{
+//				Table:            mountTableType,
+//				Path:             "noop/",
+//				Type:             "noop",
+//				UUID:             "abcd",
+//				Accessor:         "kv-abcd",
+//				BackendAwareUUID: "abcde",
+//				NamespaceID:      namespace.RootNamespaceID,
+//				namespace:        namespace.RootNamespace,
+//			},
+//			&MountEntry{
+//				Table:            mountTableType,
+//				Path:             "noop2/",
+//				Type:             "noop",
+//				UUID:             "bcde",
+//				Accessor:         "kv-bcde",
+//				BackendAwareUUID: "bcdea",
+//				NamespaceID:      namespace.RootNamespaceID,
+//				namespace:        namespace.RootNamespace,
+//			},
+//		},
+//	}
+//
+//	// Both should set up successfully
+//	err := c.setupMounts(namespace.RootContext(nil))
+//	if err != nil {
+//		t.Fatal(err)
+//	}
+//	if len(c.mounts.Entries) != 2 {
+//		t.Fatalf("expected two entries, got %d", len(c.mounts.Entries))
+//	}
+//}
diff --git a/vault/router_testing.go b/vault/router_testing.go
index 965d21a33e4f..161c025f70bc 100644
--- a/vault/router_testing.go
+++ b/vault/router_testing.go
@@ -2,6 +2,7 @@ package vault
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"sync"
 	"time"
@@ -25,8 +26,6 @@ type NoopBackend struct {
 	DefaultLeaseTTL time.Duration
 	MaxLeaseTTL     time.Duration
 	BackendType     logical.BackendType
-
-	isInitialized bool
 }
 
 func NoopBackendFactory(_ context.Context, _ *logical.BackendConfig) (logical.Backend, error) {
@@ -106,7 +105,6 @@ func (n *NoopBackend) Logger() log.Logger {
 }
 
 func (n *NoopBackend) Initialize(ctx context.Context, req *logical.InitializationRequest) error {
-	n.isInitialized = true
 	return nil
 }
 
@@ -116,3 +114,31 @@ func (n *NoopBackend) Type() logical.BackendType {
 	}
 	return n.BackendType
 }
+
+// InitializableBackend is a backend that knows whether it has been initialized
+// properly.
+type InitializableBackend struct {
+	*NoopBackend
+	isInitialized bool
+}
+
+func (b *InitializableBackend) Initialize(ctx context.Context, req *logical.InitializationRequest) error {
+	if b.isInitialized {
+		return errors.New("already initialized")
+	}
+
+	//debug.PrintStack()
+	// do a dummy write to storage, to prove that the storage is not readonly
+	println(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
+	entry := &logical.StorageEntry{
+		Key:   "initialize/zork",
+		Value: []byte("quux"),
+	}
+	err := req.Storage.Put(ctx, entry)
+	if err != nil {
+		return err
+	}
+
+	b.isInitialized = true
+	return nil
+}

From bffa3f5ae262b943b97c92cd5e94345c4dd54458 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Mon, 1 Jul 2019 18:02:39 -0400
Subject: [PATCH 18/53] simplify test suite

---
 vault/auth_test.go      | 26 +++++--------------
 vault/router_testing.go | 55 ++++++++++++++++++++---------------------
 2 files changed, 33 insertions(+), 48 deletions(-)

diff --git a/vault/auth_test.go b/vault/auth_test.go
index eb37dc488c9d..701b44a1e420 100644
--- a/vault/auth_test.go
+++ b/vault/auth_test.go
@@ -67,14 +67,11 @@ func TestCore_DefaultAuthTable(t *testing.T) {
 }
 
 func TestCore_EnableCredential(t *testing.T) {
-
-	backend := &NoopBackend{
-		BackendType: logical.TypeCredential,
-	}
-
 	c, keys, _ := TestCoreUnsealed(t)
 	c.credentialBackends["noop"] = func(context.Context, *logical.BackendConfig) (logical.Backend, error) {
-		return backend, nil
+		return &NoopBackend{
+			BackendType: logical.TypeCredential,
+		}, nil
 	}
 
 	me := &MountEntry{
@@ -125,22 +122,11 @@ func TestCore_EnableCredential(t *testing.T) {
 // entries, and that upon reading the entries from both are recombined
 // correctly
 func TestCore_EnableCredential_Local(t *testing.T) {
-
-	counter := 0
-	backends := []*NoopBackend{
-		&NoopBackend{
-			BackendType: logical.TypeCredential,
-		},
-		&NoopBackend{
-			BackendType: logical.TypeCredential,
-		},
-	}
-
 	c, _, _ := TestCoreUnsealed(t)
 	c.credentialBackends["noop"] = func(context.Context, *logical.BackendConfig) (logical.Backend, error) {
-		b := backends[counter]
-		counter++
-		return b, nil
+		return &NoopBackend{
+			BackendType: logical.TypeCredential,
+		}, nil
 	}
 
 	c.auth = &MountTable{
diff --git a/vault/router_testing.go b/vault/router_testing.go
index 161c025f70bc..b5b18258802e 100644
--- a/vault/router_testing.go
+++ b/vault/router_testing.go
@@ -2,7 +2,6 @@ package vault
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"sync"
 	"time"
@@ -115,30 +114,30 @@ func (n *NoopBackend) Type() logical.BackendType {
 	return n.BackendType
 }
 
-// InitializableBackend is a backend that knows whether it has been initialized
-// properly.
-type InitializableBackend struct {
-	*NoopBackend
-	isInitialized bool
-}
-
-func (b *InitializableBackend) Initialize(ctx context.Context, req *logical.InitializationRequest) error {
-	if b.isInitialized {
-		return errors.New("already initialized")
-	}
-
-	//debug.PrintStack()
-	// do a dummy write to storage, to prove that the storage is not readonly
-	println(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
-	entry := &logical.StorageEntry{
-		Key:   "initialize/zork",
-		Value: []byte("quux"),
-	}
-	err := req.Storage.Put(ctx, entry)
-	if err != nil {
-		return err
-	}
-
-	b.isInitialized = true
-	return nil
-}
+//// InitializableBackend is a backend that knows whether it has been initialized
+//// properly.
+//type InitializableBackend struct {
+//	*NoopBackend
+//	isInitialized bool
+//}
+//
+//func (b *InitializableBackend) Initialize(ctx context.Context, req *logical.InitializationRequest) error {
+//	if b.isInitialized {
+//		return errors.New("already initialized")
+//	}
+//
+//	//debug.PrintStack()
+//	// do a dummy write to storage, to prove that the storage is not readonly
+//	println(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
+//	entry := &logical.StorageEntry{
+//		Key:   "initialize/zork",
+//		Value: []byte("quux"),
+//	}
+//	err := req.Storage.Put(ctx, entry)
+//	if err != nil {
+//		return err
+//	}
+//
+//	b.isInitialized = true
+//	return nil
+//}

From 851610c61ad7ba56aabcc353f2db45a806d6ddcb Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Mon, 1 Jul 2019 18:59:14 -0400
Subject: [PATCH 19/53] improve test suite

---
 vault/auth.go           |   5 +-
 vault/auth_test.go      |  69 +++++++++++++++++++++++
 vault/mount.go          |   4 ++
 vault/mount_test.go     | 122 ++++++++++++++++++++++------------------
 vault/router_testing.go |  53 +++++++++--------
 5 files changed, 171 insertions(+), 82 deletions(-)

diff --git a/vault/auth.go b/vault/auth.go
index 9b4d24366b4e..ca7b852c1cbb 100644
--- a/vault/auth.go
+++ b/vault/auth.go
@@ -185,8 +185,11 @@ func (c *Core) enableCredentialInternal(ctx context.Context, entry *MountEntry,
 		return err
 	}
 
-	// Initialize
 	if !nilMount {
+		// restore the original readOnlyErr, so we can write to the view in
+		// Initialize() if necessary
+		view.setReadOnlyErr(origViewReadOnlyErr)
+		// Initialize
 		err := backend.Initialize(ctx, &logical.InitializationRequest{Storage: view})
 		if err != nil {
 			return err
diff --git a/vault/auth_test.go b/vault/auth_test.go
index 701b44a1e420..38a0a94f09d0 100644
--- a/vault/auth_test.go
+++ b/vault/auth_test.go
@@ -424,3 +424,72 @@ func verifyDefaultAuthTable(t *testing.T, table *MountTable) {
 		}
 	}
 }
+
+func TestCore_CredentialInitialize(t *testing.T) {
+	{
+		backend := &InitializableBackend{
+			&NoopBackend{
+				BackendType: logical.TypeCredential,
+			}, false}
+
+		c, _, _ := TestCoreUnsealed(t)
+		c.credentialBackends["initable"] = func(context.Context, *logical.BackendConfig) (logical.Backend, error) {
+			return backend, nil
+		}
+
+		me := &MountEntry{
+			Table: credentialTableType,
+			Path:  "foo/",
+			Type:  "initable",
+		}
+		err := c.enableCredential(namespace.RootContext(nil), me)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		if !backend.isInitialized {
+			t.Fatal("backend is not initialized")
+		}
+	}
+	{
+		backend := &InitializableBackend{
+			&NoopBackend{
+				BackendType: logical.TypeCredential,
+			}, false}
+
+		c, _, _ := TestCoreUnsealed(t)
+		c.credentialBackends["initable"] = func(context.Context, *logical.BackendConfig) (logical.Backend, error) {
+			return backend, nil
+		}
+
+		c.auth = &MountTable{
+			Type: credentialTableType,
+			Entries: []*MountEntry{
+				&MountEntry{
+					Table:            credentialTableType,
+					Path:             "foo/",
+					Type:             "initable",
+					UUID:             "abcd",
+					Accessor:         "initable-abcd",
+					BackendAwareUUID: "abcde",
+					NamespaceID:      namespace.RootNamespaceID,
+					namespace:        namespace.RootNamespace,
+				},
+			},
+		}
+
+		err := c.setupCredentials(context.Background())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// run the postUnseal funcs, so that the backend will be inited
+		for _, f := range c.postUnsealFuncs {
+			f()
+		}
+
+		if !backend.isInitialized {
+			t.Fatal("backend is not initialized")
+		}
+	}
+}
diff --git a/vault/mount.go b/vault/mount.go
index d8f8cd678744..818d0644030a 100644
--- a/vault/mount.go
+++ b/vault/mount.go
@@ -513,6 +513,10 @@ func (c *Core) mountInternal(ctx context.Context, entry *MountEntry, updateStora
 
 	// Initialize
 	if !nilMount {
+		// restore the original readOnlyErr, so we can write to the view in
+		// Initialize() if necessary
+		view.setReadOnlyErr(origReadOnlyErr)
+		// Initialize
 		err := backend.Initialize(ctx, &logical.InitializationRequest{Storage: view})
 		if err != nil {
 			return err
diff --git a/vault/mount_test.go b/vault/mount_test.go
index 185fdfa8eab0..701afb74a83b 100644
--- a/vault/mount_test.go
+++ b/vault/mount_test.go
@@ -736,57 +736,71 @@ func TestSingletonMountTableFunc(t *testing.T) {
 	}
 }
 
-//func TestCore_SetupMounts_Initialize(t *testing.T) {
-//	c, _, _ := TestCoreUnsealed(t)
-//
-//	counter := 0
-//	backends := []*NoopBackend{
-//		&NoopBackend{
-//			BackendType: logical.TypeLogical,
-//		},
-//		&NoopBackend{
-//			BackendType: logical.TypeLogical,
-//		},
-//	}
-//
-//	c.logicalBackends["noop"] = func(context.Context, *logical.BackendConfig) (logical.Backend, error) {
-//		b := backends[counter]
-//		counter++
-//		return b, nil
-//	}
-//
-//	c.mounts = &MountTable{
-//		Type: mountTableType,
-//		Entries: []*MountEntry{
-//			&MountEntry{
-//				Table:            mountTableType,
-//				Path:             "noop/",
-//				Type:             "noop",
-//				UUID:             "abcd",
-//				Accessor:         "kv-abcd",
-//				BackendAwareUUID: "abcde",
-//				NamespaceID:      namespace.RootNamespaceID,
-//				namespace:        namespace.RootNamespace,
-//			},
-//			&MountEntry{
-//				Table:            mountTableType,
-//				Path:             "noop2/",
-//				Type:             "noop",
-//				UUID:             "bcde",
-//				Accessor:         "kv-bcde",
-//				BackendAwareUUID: "bcdea",
-//				NamespaceID:      namespace.RootNamespaceID,
-//				namespace:        namespace.RootNamespace,
-//			},
-//		},
-//	}
-//
-//	// Both should set up successfully
-//	err := c.setupMounts(namespace.RootContext(nil))
-//	if err != nil {
-//		t.Fatal(err)
-//	}
-//	if len(c.mounts.Entries) != 2 {
-//		t.Fatalf("expected two entries, got %d", len(c.mounts.Entries))
-//	}
-//}
+func TestCore_MountInitialize(t *testing.T) {
+	{
+		backend := &InitializableBackend{
+			&NoopBackend{
+				BackendType: logical.TypeLogical,
+			}, false}
+
+		c, _, _ := TestCoreUnsealed(t)
+		c.logicalBackends["initable"] = func(context.Context, *logical.BackendConfig) (logical.Backend, error) {
+			return backend, nil
+		}
+
+		// Mount the noop backend
+		me := &MountEntry{
+			Table: mountTableType,
+			Path:  "foo/",
+			Type:  "initable",
+		}
+		if err := c.mount(namespace.RootContext(nil), me); err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		if !backend.isInitialized {
+			t.Fatal("backend is not initialized")
+		}
+	}
+	{
+		backend := &InitializableBackend{
+			&NoopBackend{
+				BackendType: logical.TypeLogical,
+			}, false}
+
+		c, _, _ := TestCoreUnsealed(t)
+		c.logicalBackends["initable"] = func(context.Context, *logical.BackendConfig) (logical.Backend, error) {
+			return backend, nil
+		}
+
+		c.mounts = &MountTable{
+			Type: mountTableType,
+			Entries: []*MountEntry{
+				&MountEntry{
+					Table:            mountTableType,
+					Path:             "foo/",
+					Type:             "initable",
+					UUID:             "abcd",
+					Accessor:         "initable-abcd",
+					BackendAwareUUID: "abcde",
+					NamespaceID:      namespace.RootNamespaceID,
+					namespace:        namespace.RootNamespace,
+				},
+			},
+		}
+
+		err := c.setupMounts(namespace.RootContext(nil))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// run the postUnseal funcs, so that the backend will be inited
+		for _, f := range c.postUnsealFuncs {
+			f()
+		}
+
+		if !backend.isInitialized {
+			t.Fatal("backend is not initialized")
+		}
+	}
+}
diff --git a/vault/router_testing.go b/vault/router_testing.go
index b5b18258802e..db1ff4b159d5 100644
--- a/vault/router_testing.go
+++ b/vault/router_testing.go
@@ -2,6 +2,7 @@ package vault
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"sync"
 	"time"
@@ -114,30 +115,28 @@ func (n *NoopBackend) Type() logical.BackendType {
 	return n.BackendType
 }
 
-//// InitializableBackend is a backend that knows whether it has been initialized
-//// properly.
-//type InitializableBackend struct {
-//	*NoopBackend
-//	isInitialized bool
-//}
-//
-//func (b *InitializableBackend) Initialize(ctx context.Context, req *logical.InitializationRequest) error {
-//	if b.isInitialized {
-//		return errors.New("already initialized")
-//	}
-//
-//	//debug.PrintStack()
-//	// do a dummy write to storage, to prove that the storage is not readonly
-//	println(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
-//	entry := &logical.StorageEntry{
-//		Key:   "initialize/zork",
-//		Value: []byte("quux"),
-//	}
-//	err := req.Storage.Put(ctx, entry)
-//	if err != nil {
-//		return err
-//	}
-//
-//	b.isInitialized = true
-//	return nil
-//}
+// InitializableBackend is a backend that knows whether it has been initialized
+// properly.
+type InitializableBackend struct {
+	*NoopBackend
+	isInitialized bool
+}
+
+func (b *InitializableBackend) Initialize(ctx context.Context, req *logical.InitializationRequest) error {
+	if b.isInitialized {
+		return errors.New("already initialized")
+	}
+
+	// do a dummy write, to prove that the storage is not readonly
+	entry := &logical.StorageEntry{
+		Key:   "initialize/zork",
+		Value: []byte("quux"),
+	}
+	err := req.Storage.Put(ctx, entry)
+	if err != nil {
+		return err
+	}
+
+	b.isInitialized = true
+	return nil
+}

From 4f36e95cde218cb76957da80c63c8e35a528b356 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Tue, 2 Jul 2019 08:27:36 -0400
Subject: [PATCH 20/53] simplify logic in aws role upgrade

---
 builtin/credential/aws/path_role.go      | 50 ++++++++++++------------
 builtin/credential/aws/path_role_test.go | 13 ++++--
 2 files changed, 34 insertions(+), 29 deletions(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index b4e409cd3cce..4e9e9485f095 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -334,7 +334,7 @@ func (b *backend) initialize(ctx context.Context, s logical.Storage) error {
 
 		// grab the guard
 		if !atomic.CompareAndSwapUint32(b.initializeCASGuard, 0, 1) {
-			logger.Error("initializational already in progress")
+			logger.Error("initialization already in progress")
 			return nil
 		}
 
@@ -355,31 +355,14 @@ func (b *backend) initialize(ctx context.Context, s logical.Storage) error {
 
 			logger.Info("starting initialization")
 
-			// check if we've already upgraded to the current role storage
-			// version
-			version, err := b.upgradedRoleStorageVersion(ctx, s)
+			updated, err := b.updateUpgradableRoleEntries(ctx, s)
 			if err != nil {
 				logger.Error("error running initialization", "error", err)
 				return
 			}
-			if version == currentRoleStorageVersion {
-				logger.Info("skipping initialization -- has already been performed for storage version",
+			if !updated {
+				logger.Info("upgrade has already been performed for role storage version",
 					currentRoleStorageVersion)
-				return
-			}
-
-			// perform the upgrade
-			err = b.updateUpgradableRoleEntries(ctx, s)
-			if err != nil {
-				logger.Error("error running initialization", "error", err)
-				return
-			}
-
-			// save the current role storage version
-			err = b.setUpgradedRoleStorageVersion(ctx, s, currentRoleStorageVersion)
-			if err != nil {
-				logger.Error("error running initialization", "error", err)
-				return
 			}
 
 			logger.Info("initialization succeeded")
@@ -389,11 +372,22 @@ func (b *backend) initialize(ctx context.Context, s logical.Storage) error {
 	}
 
 	return nil
+
 }
 
 // updateUpgradableRoleEntries upgrades and persists all of the role entries
 // that are in need of being upgraded.
-func (b *backend) updateUpgradableRoleEntries(ctx context.Context, s logical.Storage) error {
+func (b *backend) updateUpgradableRoleEntries(ctx context.Context, s logical.Storage) (bool, error) {
+
+	// check if we've already upgraded to the current role storage
+	// version
+	version, err := b.upgradedRoleStorageVersion(ctx, s)
+	if err != nil {
+		return false, err
+	}
+	if version == currentRoleStorageVersion {
+		return false, nil
+	}
 
 	// Read all the role names.  We don't need to grab the mutex here for
 	// listing. The reason is that if we're in the process of creating a new
@@ -401,18 +395,24 @@ func (b *backend) updateUpgradableRoleEntries(ctx context.Context, s logical.Sto
 	// and will properly use the latest role structure.
 	roleNames, err := s.List(ctx, "role/")
 	if err != nil {
-		return err
+		return false, err
 	}
 
 	// Upgrade the roles as necessary.
 	for _, roleName := range roleNames {
 		err := b.updateUpgradableRoleEntry(ctx, s, roleName)
 		if err != nil {
-			return err
+			return false, err
 		}
 	}
 
-	return nil
+	// save the current role storage version
+	err = b.setUpgradedRoleStorageVersion(ctx, s, currentRoleStorageVersion)
+	if err != nil {
+		return false, err
+	}
+
+	return true, nil
 }
 
 // updateUpgradableRoleEntry uses the write lock to call roleInternal(), which
diff --git a/builtin/credential/aws/path_role_test.go b/builtin/credential/aws/path_role_test.go
index ca1f7db89ef8..b89dd92e640c 100644
--- a/builtin/credential/aws/path_role_test.go
+++ b/builtin/credential/aws/path_role_test.go
@@ -5,6 +5,7 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/go-test/deep"
 	"github.com/hashicorp/vault/sdk/helper/policyutil"
@@ -915,8 +916,14 @@ func TestRoleInitialize(t *testing.T) {
 		t.Fatal(diff)
 	}
 
-	// run it againg -- nothing will happen
-	err = b.initialize(ctx, storage)
+	// run it again -- nothing will happen
+	updated, err := b.updateUpgradableRoleEntries(ctx, storage)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if updated {
+		t.Fatalf("expected no updates")
+	}
 }
 
 func TestUpgradedRoleStorageVersion(t *testing.T) {
@@ -935,8 +942,6 @@ func TestUpgradedRoleStorageVersion(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	//-------------------------
-
 	version, err := b.upgradedRoleStorageVersion(ctx, storage)
 	if err != nil {
 		t.Fatal(err)

From ccb86b5bcc06f89dddd8e48ed357729a824bb498 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Tue, 2 Jul 2019 10:42:41 -0400
Subject: [PATCH 21/53] simplify aws credential initialization logic

---
 builtin/credential/aws/backend.go   |  7 -------
 builtin/credential/aws/path_role.go | 24 ++----------------------
 2 files changed, 2 insertions(+), 29 deletions(-)

diff --git a/builtin/credential/aws/backend.go b/builtin/credential/aws/backend.go
index 90e39cb0086b..4f4d254474f0 100644
--- a/builtin/credential/aws/backend.go
+++ b/builtin/credential/aws/backend.go
@@ -34,12 +34,6 @@ type backend struct {
 	// Lock to make changes to any of the backend's configuration endpoints.
 	configMutex sync.RWMutex
 
-	// Guards the initialize function
-	initializeCASGuard *uint32
-
-	// check whether initialize has already been called
-	isInitialized bool
-
 	// Lock to make changes to role entries
 	roleMutex sync.Mutex
 
@@ -97,7 +91,6 @@ func Backend(conf *logical.BackendConfig) (*backend, error) {
 		EC2ClientsMap:         make(map[string]map[string]*ec2.EC2),
 		IAMClientsMap:         make(map[string]map[string]*iam.IAM),
 		iamUserIdToArnCache:   cache.New(7*24*time.Hour, 24*time.Hour),
-		initializeCASGuard:    new(uint32),
 		tidyBlacklistCASGuard: new(uint32),
 		tidyWhitelistCASGuard: new(uint32),
 		roleCache:             cache.New(cache.NoExpiration, cache.NoExpiration),
diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index 4e9e9485f095..cd8e6ee016b6 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
-	"sync/atomic"
 	"time"
 
 	"github.com/hashicorp/errwrap"
@@ -327,32 +326,16 @@ func (b *backend) initialize(ctx context.Context, s logical.Storage) error {
 
 	// Initialize only if we are either:
 	//   (1) A local mount.
-	//   (2) Are _not_ a replicated standby cluster.
+	//   (2) Are _not_ a replicated secondary cluster or a performance standby node.
 	if b.System().LocalMount() || !b.System().ReplicationState().HasState(consts.ReplicationPerformanceSecondary|consts.ReplicationPerformanceStandby) {
 
-		logger := b.Logger().Named("initialize")
-
-		// grab the guard
-		if !atomic.CompareAndSwapUint32(b.initializeCASGuard, 0, 1) {
-			logger.Error("initialization already in progress")
-			return nil
-		}
-
-		// check if this method has already been called
-		if b.isInitialized {
-			atomic.StoreUint32(b.initializeCASGuard, 0)
-			logger.Error("already initialized")
-			return nil
-		}
-		b.isInitialized = true
-
 		// kick off the role upgrader
 		go func() {
-			defer atomic.StoreUint32(b.initializeCASGuard, 0)
 
 			// Don't cancel when the original client request goes away
 			ctx = context.Background()
 
+			logger := b.Logger().Named("initialize")
 			logger.Info("starting initialization")
 
 			updated, err := b.updateUpgradableRoleEntries(ctx, s)
@@ -367,12 +350,9 @@ func (b *backend) initialize(ctx context.Context, s logical.Storage) error {
 
 			logger.Info("initialization succeeded")
 		}()
-
-		return nil
 	}
 
 	return nil
-
 }
 
 // updateUpgradableRoleEntries upgrades and persists all of the role entries

From 1f95d4b278f44641a4616c750870e75b8e469364 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Tue, 2 Jul 2019 11:35:28 -0400
Subject: [PATCH 22/53] simplify logic in aws role upgrade

---
 builtin/credential/aws/path_role.go      | 117 +++++++++--------------
 builtin/credential/aws/path_role_test.go |  40 +++-----
 2 files changed, 56 insertions(+), 101 deletions(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index cd8e6ee016b6..cb3c3d0bd30b 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"strconv"
 	"strings"
 	"time"
 
@@ -329,50 +328,58 @@ func (b *backend) initialize(ctx context.Context, s logical.Storage) error {
 	//   (2) Are _not_ a replicated secondary cluster or a performance standby node.
 	if b.System().LocalMount() || !b.System().ReplicationState().HasState(consts.ReplicationPerformanceSecondary|consts.ReplicationPerformanceStandby) {
 
-		// kick off the role upgrader
-		go func() {
+		logger := b.Logger().Named("initialize")
+		logger.Info("starting initialization")
 
-			// Don't cancel when the original client request goes away
-			ctx = context.Background()
-
-			logger := b.Logger().Named("initialize")
-			logger.Info("starting initialization")
-
-			updated, err := b.updateUpgradableRoleEntries(ctx, s)
-			if err != nil {
-				logger.Error("error running initialization", "error", err)
-				return
-			}
-			if !updated {
-				logger.Info("upgrade has already been performed for role storage version",
-					currentRoleStorageVersion)
-			}
+		upgraded, err := b.upgrade(ctx, s)
+		if err != nil {
+			logger.Error("error running initialization", "error", err)
+			return err
+		}
+		if upgraded {
+			logger.Info("an upgrade was performed during initialization")
+		}
 
-			logger.Info("initialization succeeded")
-		}()
+		logger.Info("initialization succeeded")
 	}
 
 	return nil
 }
 
-// updateUpgradableRoleEntries upgrades and persists all of the role entries
-// that are in need of being upgraded.
-func (b *backend) updateUpgradableRoleEntries(ctx context.Context, s logical.Storage) (bool, error) {
+// roleStorageVersion stores the latest role storage version that we have
+// upgraded to.
+type roleStorageVersion struct {
+	Version int `json:"version"`
+}
+
+// upgrade and persist all of the role entries that are in need of being
+// upgraded.
+func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error) {
+
+	b.roleMutex.Lock()
+	defer b.roleMutex.Unlock()
 
-	// check if we've already upgraded to the current role storage
-	// version
-	version, err := b.upgradedRoleStorageVersion(ctx, s)
+	// find the current role-storage-version
+	version := -1
+	entry, err := s.Get(ctx, "config/role-storage-version")
 	if err != nil {
 		return false, err
 	}
+	if entry != nil {
+		var rsv roleStorageVersion
+		err = entry.DecodeJSON(&rsv)
+		if err != nil {
+			return false, err
+		}
+		version = rsv.Version
+	}
+
+	// check if we need to upgrade
 	if version == currentRoleStorageVersion {
 		return false, nil
 	}
 
-	// Read all the role names.  We don't need to grab the mutex here for
-	// listing. The reason is that if we're in the process of creating a new
-	// mount, it will already happen on the active node (since it's a write)
-	// and will properly use the latest role structure.
+	// Read all the role names.
 	roleNames, err := s.List(ctx, "role/")
 	if err != nil {
 		return false, err
@@ -380,62 +387,24 @@ func (b *backend) updateUpgradableRoleEntries(ctx context.Context, s logical.Sto
 
 	// Upgrade the roles as necessary.
 	for _, roleName := range roleNames {
-		err := b.updateUpgradableRoleEntry(ctx, s, roleName)
+		_, err := b.roleInternal(ctx, s, roleName)
 		if err != nil {
 			return false, err
 		}
 	}
 
 	// save the current role storage version
-	err = b.setUpgradedRoleStorageVersion(ctx, s, currentRoleStorageVersion)
+	rsv := roleStorageVersion{Version: currentRoleStorageVersion}
+	entry, err = logical.StorageEntryJSON("config/role-storage-version", &rsv)
 	if err != nil {
 		return false, err
 	}
-
-	return true, nil
-}
-
-// updateUpgradableRoleEntry uses the write lock to call roleInternal(), which
-// will do an upgrade and save it if need be.
-func (b *backend) updateUpgradableRoleEntry(ctx context.Context, s logical.Storage, roleName string) error {
-
-	b.roleMutex.Lock()
-	defer b.roleMutex.Unlock()
-
-	_, err := b.roleInternal(ctx, s, roleName)
-	return err
-}
-
-// upgradedRoleStorageVersion returns the value of the roleStorageVersion that
-// we have already upgraded to, or -1 if the upgrade has never been run.
-func (b *backend) upgradedRoleStorageVersion(ctx context.Context, s logical.Storage) (int, error) {
-	b.roleMutex.Lock()
-	defer b.roleMutex.Unlock()
-
-	entry, err := s.Get(ctx, "config/role-storage-version")
+	err = s.Put(ctx, entry)
 	if err != nil {
-		return -1, err
-	}
-
-	// the upgrade has never been run
-	if entry == nil {
-		return -1, nil
+		return false, err
 	}
 
-	// return the upgraded value
-	return strconv.Atoi(string(entry.Value))
-}
-
-// setUpgradedRoleStorageVersion returns the value of the roleStorageVersion
-// that we have already upgraded to, or -1 if the upgrade has never been run
-func (b *backend) setUpgradedRoleStorageVersion(ctx context.Context, s logical.Storage, version int) error {
-	b.roleMutex.Lock()
-	defer b.roleMutex.Unlock()
-
-	return s.Put(ctx, &logical.StorageEntry{
-		Key:   "config/role-storage-version",
-		Value: []byte(strconv.Itoa(version)),
-	})
+	return true, nil
 }
 
 // If needed, updates the role entry and returns a bool indicating if it was updated
diff --git a/builtin/credential/aws/path_role_test.go b/builtin/credential/aws/path_role_test.go
index b89dd92e640c..5ed6a0267983 100644
--- a/builtin/credential/aws/path_role_test.go
+++ b/builtin/credential/aws/path_role_test.go
@@ -916,51 +916,37 @@ func TestRoleInitialize(t *testing.T) {
 		t.Fatal(diff)
 	}
 
+	// TODO check storage version
+
 	// run it again -- nothing will happen
-	updated, err := b.updateUpgradableRoleEntries(ctx, storage)
+	upgraded, err := b.upgrade(ctx, storage)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if updated {
-		t.Fatalf("expected no updates")
+	if upgraded {
+		t.Fatalf("expected no upgrade")
 	}
 }
 
-func TestUpgradedRoleStorageVersion(t *testing.T) {
-
-	config := logical.TestBackendConfig()
-	storage := &logical.InmemStorage{}
-	config.StorageView = storage
-	b, err := Backend(config)
-	if err != nil {
-		t.Fatal(err)
-	}
+func TestRoleStorageVersion(t *testing.T) {
 
-	ctx := context.Background()
-	err = b.Setup(ctx, config)
-	if err != nil {
-		t.Fatal(err)
+	before := roleStorageVersion{
+		Version: 42,
 	}
 
-	version, err := b.upgradedRoleStorageVersion(ctx, storage)
+	entry, err := logical.StorageEntryJSON("config/role-storage-version", &before)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if -1 != version {
-		t.Fatalf("expected version %d, got %d", -1, version)
-	}
 
-	err = b.setUpgradedRoleStorageVersion(ctx, storage, 42)
+	var after roleStorageVersion
+	err = entry.DecodeJSON(&after)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	version, err = b.upgradedRoleStorageVersion(ctx, storage)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if 42 != version {
-		t.Fatalf("expected version %d, got %d", 42, version)
+	if diff := deep.Equal(before, after); diff != nil {
+		t.Fatal(diff)
 	}
 }
 

From bdef02112cd23a0dab67dd7676831b1c4c6298bb Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Tue, 2 Jul 2019 11:49:44 -0400
Subject: [PATCH 23/53] use the core's activeContext for initialization

---
 builtin/credential/aws/path_role.go | 4 ++++
 vault/auth.go                       | 4 ++--
 vault/mount.go                      | 5 ++---
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index cb3c3d0bd30b..336a05c56e14 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -387,6 +387,10 @@ func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error)
 
 	// Upgrade the roles as necessary.
 	for _, roleName := range roleNames {
+		// make context hasn't been canceled
+		if ctx.Err() != nil {
+			return false, err
+		}
 		_, err := b.roleInternal(ctx, s, roleName)
 		if err != nil {
 			return false, err
diff --git a/vault/auth.go b/vault/auth.go
index ca7b852c1cbb..29ccd0970554 100644
--- a/vault/auth.go
+++ b/vault/auth.go
@@ -189,8 +189,8 @@ func (c *Core) enableCredentialInternal(ctx context.Context, entry *MountEntry,
 		// restore the original readOnlyErr, so we can write to the view in
 		// Initialize() if necessary
 		view.setReadOnlyErr(origViewReadOnlyErr)
-		// Initialize
-		err := backend.Initialize(ctx, &logical.InitializationRequest{Storage: view})
+		// initialize, using the core's active context.
+		err := backend.Initialize(c.activeContext, &logical.InitializationRequest{Storage: view})
 		if err != nil {
 			return err
 		}
diff --git a/vault/mount.go b/vault/mount.go
index 818d0644030a..ab9e547b9ebf 100644
--- a/vault/mount.go
+++ b/vault/mount.go
@@ -511,13 +511,12 @@ func (c *Core) mountInternal(ctx context.Context, entry *MountEntry, updateStora
 		return err
 	}
 
-	// Initialize
 	if !nilMount {
 		// restore the original readOnlyErr, so we can write to the view in
 		// Initialize() if necessary
 		view.setReadOnlyErr(origReadOnlyErr)
-		// Initialize
-		err := backend.Initialize(ctx, &logical.InitializationRequest{Storage: view})
+		// initialize, using the core's active context.
+		err := backend.Initialize(c.activeContext, &logical.InitializationRequest{Storage: view})
 		if err != nil {
 			return err
 		}

From 65df479e57b2fd0a10003ab517ec37859115cbb6 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Tue, 2 Jul 2019 13:07:55 -0400
Subject: [PATCH 24/53] refactor builtin/plugin/Backend

---
 builtin/credential/aws/path_role.go |   4 +-
 builtin/plugin/backend.go           | 130 +++++++---------------------
 2 files changed, 34 insertions(+), 100 deletions(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index 336a05c56e14..d1e90d54584b 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -387,7 +387,7 @@ func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error)
 
 	// Upgrade the roles as necessary.
 	for _, roleName := range roleNames {
-		// make context hasn't been canceled
+		// make sure the context hasn't been canceled
 		if ctx.Err() != nil {
 			return false, err
 		}
@@ -397,7 +397,7 @@ func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error)
 		}
 	}
 
-	// save the current role storage version
+	// save the current role-storage-version
 	rsv := roleStorageVersion{Version: currentRoleStorageVersion}
 	entry, err = logical.StorageEntryJSON("config/role-storage-version", &rsv)
 	if err != nil {
diff --git a/builtin/plugin/backend.go b/builtin/plugin/backend.go
index f51ef2e0634a..88bd3292fca8 100644
--- a/builtin/plugin/backend.go
+++ b/builtin/plugin/backend.go
@@ -137,8 +137,8 @@ func (b *PluginBackend) startBackend(ctx context.Context) error {
 	return nil
 }
 
-// HandleRequest is a thin wrapper implementation of HandleRequest that includes automatic plugin reload.
-func (b *PluginBackend) HandleRequest(ctx context.Context, req *logical.Request) (*logical.Response, error) {
+// lazyLoad lazy-loads the backend before running a method
+func (b *PluginBackend) lazyLoadBackend(ctx context.Context, methodWrapper func() error) error {
 	b.RLock()
 	canary := b.canary
 
@@ -152,14 +152,16 @@ func (b *PluginBackend) HandleRequest(ctx context.Context, req *logical.Request)
 			err := b.startBackend(ctx)
 			if err != nil {
 				b.Unlock()
-				return nil, err
+				return err
 			}
 		}
 		b.Unlock()
 		b.RLock()
 	}
-	resp, err := b.Backend.HandleRequest(ctx, req)
+
+	err := methodWrapper()
 	b.RUnlock()
+
 	// Need to compare string value for case were err comes from plugin RPC
 	// and is returned as plugin.BasicError type.
 	if err != nil &&
@@ -170,120 +172,52 @@ func (b *PluginBackend) HandleRequest(ctx context.Context, req *logical.Request)
 			err := b.reloadBackend(ctx)
 			if err != nil {
 				b.Unlock()
-				return nil, err
+				return err
 			}
 			b.canary, err = uuid.GenerateUUID()
 			if err != nil {
 				b.Unlock()
-				return nil, err
+				return err
 			}
 		}
 		b.Unlock()
 
-		// Try request once more
+		// Try once more
 		b.RLock()
 		defer b.RUnlock()
-		return b.Backend.HandleRequest(ctx, req)
+		return methodWrapper()
 	}
-	return resp, err
+	return err
 }
 
-// HandleExistenceCheck is a thin wrapper implementation of HandleExistenceCheck that includes automatic plugin reload.
-func (b *PluginBackend) HandleExistenceCheck(ctx context.Context, req *logical.Request) (bool, bool, error) {
-	b.RLock()
-	canary := b.canary
+// HandleRequest is a thin wrapper implementation of HandleRequest that includes automatic plugin reload.
+func (b *PluginBackend) HandleRequest(ctx context.Context, req *logical.Request) (resp *logical.Response, err error) {
 
-	// Lazy-load backend
-	if !b.loaded {
-		// Upgrade lock
-		b.RUnlock()
-		b.Lock()
-		// Check once more after lock swap
-		if !b.loaded {
-			err := b.startBackend(ctx)
-			if err != nil {
-				b.Unlock()
-				return false, false, err
-			}
-		}
-		b.Unlock()
-		b.RLock()
-	}
+	err = b.lazyLoadBackend(ctx, func() error {
+		var merr error
+		resp, merr = b.Backend.HandleRequest(ctx, req)
+		return merr
+	})
 
-	checkFound, exists, err := b.Backend.HandleExistenceCheck(ctx, req)
-	b.RUnlock()
-	if err != nil &&
-		(err.Error() == rpc.ErrShutdown.Error() || err == bplugin.ErrPluginShutdown) {
-		// Reload plugin if it's an rpc.ErrShutdown
-		b.Lock()
-		if b.canary == canary {
-			err := b.reloadBackend(ctx)
-			if err != nil {
-				b.Unlock()
-				return false, false, err
-			}
-			b.canary, err = uuid.GenerateUUID()
-			if err != nil {
-				b.Unlock()
-				return false, false, err
-			}
-		}
-		b.Unlock()
+	return
+}
 
-		// Try request once more
-		b.RLock()
-		defer b.RUnlock()
-		return b.Backend.HandleExistenceCheck(ctx, req)
-	}
-	return checkFound, exists, err
+// HandleExistenceCheck is a thin wrapper implementation of HandleExistenceCheck that includes automatic plugin reload.
+func (b *PluginBackend) HandleExistenceCheck(ctx context.Context, req *logical.Request) (checkFound bool, exists bool, err error) {
+
+	err = b.lazyLoadBackend(ctx, func() error {
+		var merr error
+		checkFound, exists, merr = b.Backend.HandleExistenceCheck(ctx, req)
+		return merr
+	})
+
+	return
 }
 
 // Initialize is a thin wrapper implementation of Initialize that includes automatic plugin reload.
 func (b *PluginBackend) Initialize(ctx context.Context, req *logical.InitializationRequest) error {
-	b.RLock()
-	canary := b.canary
 
-	// Lazy-load backend
-	if !b.loaded {
-		// Upgrade lock
-		b.RUnlock()
-		b.Lock()
-		// Check once more after lock swap
-		if !b.loaded {
-			err := b.startBackend(ctx)
-			if err != nil {
-				b.Unlock()
-				return err
-			}
-		}
-		b.Unlock()
-		b.RLock()
-	}
-
-	err := b.Backend.Initialize(ctx, req)
-	b.RUnlock()
-	if err != nil &&
-		(err.Error() == rpc.ErrShutdown.Error() || err == bplugin.ErrPluginShutdown) {
-		// Reload plugin if it's an rpc.ErrShutdown
-		b.Lock()
-		if b.canary == canary {
-			err := b.reloadBackend(ctx)
-			if err != nil {
-				b.Unlock()
-				return err
-			}
-			b.canary, err = uuid.GenerateUUID()
-			if err != nil {
-				b.Unlock()
-				return err
-			}
-		}
-		b.Unlock()
-
-		// Try request once more
-		b.RLock()
-		defer b.RUnlock()
+	return b.lazyLoadBackend(ctx, func() error {
 		return b.Backend.Initialize(ctx, req)
-	}
-	return err
+	})
 }

From ce5e3cf09d3bdfba1b407816946fbbbba038f121 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Tue, 2 Jul 2019 13:58:11 -0400
Subject: [PATCH 25/53] use a goroutine to upgrade the aws roles

---
 builtin/credential/aws/path_role.go | 32 ++++++++++++++++++-----------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index d1e90d54584b..94f8dd7387c2 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -331,16 +331,27 @@ func (b *backend) initialize(ctx context.Context, s logical.Storage) error {
 		logger := b.Logger().Named("initialize")
 		logger.Info("starting initialization")
 
-		upgraded, err := b.upgrade(ctx, s)
-		if err != nil {
-			logger.Error("error running initialization", "error", err)
-			return err
-		}
-		if upgraded {
-			logger.Info("an upgrade was performed during initialization")
-		}
+		go func() {
+			// The vault will become unsealed while this goroutine is running,
+			// so we could see some role requests block until the lock is
+			// released.  However we'd rather see those requests block (and
+			// potentially start timing out) than allow a non-upgraded role to
+			// be fetched.
+			b.roleMutex.Lock()
+			defer b.roleMutex.Unlock()
+
+			upgraded, err := b.upgrade(ctx, s)
+			if err != nil {
+				logger.Error("error running initialization", "error", err)
+				return
+			}
+			if upgraded {
+				logger.Info("an upgrade was performed during initialization")
+			}
+
+			logger.Info("initialization succeeded")
+		}()
 
-		logger.Info("initialization succeeded")
 	}
 
 	return nil
@@ -356,9 +367,6 @@ type roleStorageVersion struct {
 // upgraded.
 func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error) {
 
-	b.roleMutex.Lock()
-	defer b.roleMutex.Unlock()
-
 	// find the current role-storage-version
 	version := -1
 	entry, err := s.Get(ctx, "config/role-storage-version")

From 5401e63f7ffd042457de4bb842d7864456e5ce39 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Tue, 2 Jul 2019 14:48:36 -0400
Subject: [PATCH 26/53] misc improvements and cleanup

---
 builtin/credential/aws/path_role.go      | 4 +++-
 builtin/credential/aws/path_role_test.go | 2 +-
 sdk/framework/backend.go                 | 4 ++--
 sdk/framework/backend_test.go            | 2 +-
 sdk/plugin/grpc_backend_client.go        | 3 +--
 sdk/plugin/grpc_backend_server.go        | 3 ++-
 sdk/plugin/pb/backend.proto              | 2 +-
 7 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index 94f8dd7387c2..b60ebae2a882 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -321,13 +321,15 @@ func (b *backend) setRole(ctx context.Context, s logical.Storage, roleName strin
 }
 
 // initialize is used to initialize the AWS roles
-func (b *backend) initialize(ctx context.Context, s logical.Storage) error {
+func (b *backend) initialize(ctx context.Context, req *logical.InitializationRequest) error {
 
 	// Initialize only if we are either:
 	//   (1) A local mount.
 	//   (2) Are _not_ a replicated secondary cluster or a performance standby node.
 	if b.System().LocalMount() || !b.System().ReplicationState().HasState(consts.ReplicationPerformanceSecondary|consts.ReplicationPerformanceStandby) {
 
+		s := req.Storage
+
 		logger := b.Logger().Named("initialize")
 		logger.Info("starting initialization")
 
diff --git a/builtin/credential/aws/path_role_test.go b/builtin/credential/aws/path_role_test.go
index 5ed6a0267983..2340483a76cf 100644
--- a/builtin/credential/aws/path_role_test.go
+++ b/builtin/credential/aws/path_role_test.go
@@ -868,7 +868,7 @@ func TestRoleInitialize(t *testing.T) {
 	}
 
 	// upgrade all the entries, and wait for the goroutine to finish
-	err = b.initialize(ctx, storage)
+	err = b.initialize(ctx, &logical.InitializationRequest{storage})
 	time.Sleep(time.Second)
 
 	// read the entries from storage
diff --git a/sdk/framework/backend.go b/sdk/framework/backend.go
index 45fa67ba6bfa..48a87a53ac92 100644
--- a/sdk/framework/backend.go
+++ b/sdk/framework/backend.go
@@ -114,12 +114,12 @@ type InvalidateFunc func(context.Context, string)
 
 // InitializeFunc is the callback, which if set, will be invoked via
 // Initialize() just after a plugin has been mounted.
-type InitializeFunc func(context.Context, logical.Storage) error
+type InitializeFunc func(context.Context, *logical.InitializationRequest) error
 
 // Initialize is the logical.Backend implementation.
 func (b *Backend) Initialize(ctx context.Context, req *logical.InitializationRequest) error {
 	if b.InitializeFunc != nil {
-		return b.InitializeFunc(ctx, req.Storage)
+		return b.InitializeFunc(ctx, req)
 	}
 	return nil
 }
diff --git a/sdk/framework/backend_test.go b/sdk/framework/backend_test.go
index c8b0a6da88c5..567557aef446 100644
--- a/sdk/framework/backend_test.go
+++ b/sdk/framework/backend_test.go
@@ -646,7 +646,7 @@ func TestFieldSchemaDefaultOrZero(t *testing.T) {
 func TestInitializeBackend(t *testing.T) {
 
 	var inited bool
-	backend := &Backend{InitializeFunc: func(context.Context, logical.Storage) error {
+	backend := &Backend{InitializeFunc: func(context.Context, *logical.InitializationRequest) error {
 		inited = true
 		return nil
 	}}
diff --git a/sdk/plugin/grpc_backend_client.go b/sdk/plugin/grpc_backend_client.go
index be598de5cb3e..6cf3ea53e096 100644
--- a/sdk/plugin/grpc_backend_client.go
+++ b/sdk/plugin/grpc_backend_client.go
@@ -45,9 +45,8 @@ type backendGRPCPluginClient struct {
 }
 
 func (b *backendGRPCPluginClient) Initialize(ctx context.Context, _ *logical.InitializationRequest) error {
-
 	if b.metadataMode {
-		return ErrClientInMetadataMode
+		return nil
 	}
 
 	ctx, cancel := context.WithCancel(ctx)
diff --git a/sdk/plugin/grpc_backend_server.go b/sdk/plugin/grpc_backend_server.go
index e0688a92740a..b895a9cf7cec 100644
--- a/sdk/plugin/grpc_backend_server.go
+++ b/sdk/plugin/grpc_backend_server.go
@@ -90,7 +90,8 @@ func (b *backendGRPCPluginServer) Initialize(ctx context.Context, _ *pb.Initiali
 	}
 
 	req := &logical.InitializationRequest{
-		Storage: newGRPCStorageClient(b.brokeredClient)}
+		Storage: newGRPCStorageClient(b.brokeredClient),
+	}
 
 	respErr := b.backend.Initialize(ctx, req)
 
diff --git a/sdk/plugin/pb/backend.proto b/sdk/plugin/pb/backend.proto
index d513aed1294f..326fcdba9d1a 100644
--- a/sdk/plugin/pb/backend.proto
+++ b/sdk/plugin/pb/backend.proto
@@ -443,7 +443,7 @@ service Backend {
 	// and SystemView clients.
 	rpc Setup(SetupArgs) returns (SetupReply);
 
-    // Initialize is invoked just after mounting a backend to allow it to
+	// Initialize is invoked just after mounting a backend to allow it to
 	// handle any initialization tasks that need to be performed.
 	rpc Initialize(InitializeArgs) returns (InitializeReply);
 

From e67f828da4822c0affb92279ddb19c9e744111f8 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Tue, 2 Jul 2019 15:00:30 -0400
Subject: [PATCH 27/53] do not run AWS role upgrade on DR Secondary

---
 builtin/credential/aws/path_role.go | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index b60ebae2a882..2708f6b93b37 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -325,8 +325,9 @@ func (b *backend) initialize(ctx context.Context, req *logical.InitializationReq
 
 	// Initialize only if we are either:
 	//   (1) A local mount.
-	//   (2) Are _not_ a replicated secondary cluster or a performance standby node.
-	if b.System().LocalMount() || !b.System().ReplicationState().HasState(consts.ReplicationPerformanceSecondary|consts.ReplicationPerformanceStandby) {
+	//   (2) Are _NOT_ a replicated secondary cluster, or a performance standby node, or a DR secondary.
+	if b.System().LocalMount() || !b.System().ReplicationState().HasState(
+		consts.ReplicationPerformanceSecondary|consts.ReplicationPerformanceStandby|consts.ReplicationDRSecondary) {
 
 		s := req.Storage
 

From e574a14f0c5cc6e4676088ee4a3060f66ae5da24 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Tue, 2 Jul 2019 15:44:30 -0400
Subject: [PATCH 28/53] always call logical.Backend.Initialize() when loading a
 plugin.

---
 builtin/plugin/backend.go | 30 ++++++++++++++++--------------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/builtin/plugin/backend.go b/builtin/plugin/backend.go
index 88bd3292fca8..d3ae0cf900bc 100644
--- a/builtin/plugin/backend.go
+++ b/builtin/plugin/backend.go
@@ -91,13 +91,8 @@ type PluginBackend struct {
 	loaded bool
 }
 
-func (b *PluginBackend) reloadBackend(ctx context.Context) error {
-	b.Logger().Debug("reloading plugin backend", "plugin", b.config.Config["plugin_name"])
-	return b.startBackend(ctx)
-}
-
 // startBackend starts a plugin backend
-func (b *PluginBackend) startBackend(ctx context.Context) error {
+func (b *PluginBackend) startBackend(ctx context.Context, storage logical.Storage) error {
 	pluginName := b.config.Config["plugin_name"]
 	pluginType, err := consts.ParsePluginType(b.config.Config["plugin_type"])
 	if err != nil {
@@ -134,11 +129,14 @@ func (b *PluginBackend) startBackend(ctx context.Context) error {
 	b.Backend = nb
 	b.loaded = true
 
-	return nil
+	// call Initialize() explicitly here.
+	return b.Backend.Initialize(ctx, &logical.InitializationRequest{
+		Storage: storage,
+	})
 }
 
 // lazyLoad lazy-loads the backend before running a method
-func (b *PluginBackend) lazyLoadBackend(ctx context.Context, methodWrapper func() error) error {
+func (b *PluginBackend) lazyLoadBackend(ctx context.Context, storage logical.Storage, methodWrapper func() error) error {
 	b.RLock()
 	canary := b.canary
 
@@ -149,7 +147,7 @@ func (b *PluginBackend) lazyLoadBackend(ctx context.Context, methodWrapper func(
 		b.Lock()
 		// Check once more after lock swap
 		if !b.loaded {
-			err := b.startBackend(ctx)
+			err := b.startBackend(ctx, storage)
 			if err != nil {
 				b.Unlock()
 				return err
@@ -169,7 +167,8 @@ func (b *PluginBackend) lazyLoadBackend(ctx context.Context, methodWrapper func(
 		// Reload plugin if it's an rpc.ErrShutdown
 		b.Lock()
 		if b.canary == canary {
-			err := b.reloadBackend(ctx)
+			b.Logger().Debug("reloading plugin backend", "plugin", b.config.Config["plugin_name"])
+			err := b.startBackend(ctx, storage)
 			if err != nil {
 				b.Unlock()
 				return err
@@ -193,7 +192,7 @@ func (b *PluginBackend) lazyLoadBackend(ctx context.Context, methodWrapper func(
 // HandleRequest is a thin wrapper implementation of HandleRequest that includes automatic plugin reload.
 func (b *PluginBackend) HandleRequest(ctx context.Context, req *logical.Request) (resp *logical.Response, err error) {
 
-	err = b.lazyLoadBackend(ctx, func() error {
+	err = b.lazyLoadBackend(ctx, req.Storage, func() error {
 		var merr error
 		resp, merr = b.Backend.HandleRequest(ctx, req)
 		return merr
@@ -205,7 +204,7 @@ func (b *PluginBackend) HandleRequest(ctx context.Context, req *logical.Request)
 // HandleExistenceCheck is a thin wrapper implementation of HandleExistenceCheck that includes automatic plugin reload.
 func (b *PluginBackend) HandleExistenceCheck(ctx context.Context, req *logical.Request) (checkFound bool, exists bool, err error) {
 
-	err = b.lazyLoadBackend(ctx, func() error {
+	err = b.lazyLoadBackend(ctx, req.Storage, func() error {
 		var merr error
 		checkFound, exists, merr = b.Backend.HandleExistenceCheck(ctx, req)
 		return merr
@@ -217,7 +216,10 @@ func (b *PluginBackend) HandleExistenceCheck(ctx context.Context, req *logical.R
 // Initialize is a thin wrapper implementation of Initialize that includes automatic plugin reload.
 func (b *PluginBackend) Initialize(ctx context.Context, req *logical.InitializationRequest) error {
 
-	return b.lazyLoadBackend(ctx, func() error {
-		return b.Backend.Initialize(ctx, req)
+	return b.lazyLoadBackend(ctx, req.Storage, func() error {
+		// We are guaranteed to be lazy-loaded here, so we will do a noop
+		// function.  b.Backend.Initialize() gets called from startBackend(),
+		// and we know startBackend() will be called
+		return nil
 	})
 }

From dc7db932631e1e1f158898ca6d0ce5d8581dc45d Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Tue, 2 Jul 2019 17:58:39 -0400
Subject: [PATCH 29/53] improve comments

---
 builtin/plugin/backend.go | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/builtin/plugin/backend.go b/builtin/plugin/backend.go
index d3ae0cf900bc..58c2f0379865 100644
--- a/builtin/plugin/backend.go
+++ b/builtin/plugin/backend.go
@@ -216,10 +216,11 @@ func (b *PluginBackend) HandleExistenceCheck(ctx context.Context, req *logical.R
 // Initialize is a thin wrapper implementation of Initialize that includes automatic plugin reload.
 func (b *PluginBackend) Initialize(ctx context.Context, req *logical.InitializationRequest) error {
 
+	// This method is only ever called just after mounting, so we know that the
+	// cally to lazyLoadBackend() will call startBackend().  Since
+	// startBackend() calls Initialize() on the underlying logical.Backend, the
+	// method wrapper that we pass in here is a no-op
 	return b.lazyLoadBackend(ctx, req.Storage, func() error {
-		// We are guaranteed to be lazy-loaded here, so we will do a noop
-		// function.  b.Backend.Initialize() gets called from startBackend(),
-		// and we know startBackend() will be called
 		return nil
 	})
 }

From 9e74fb3e2bcf512e104105bf8d7d77d412bb145d Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Tue, 2 Jul 2019 19:59:50 -0400
Subject: [PATCH 30/53] on standbys and DR secondaries we do not want to run
 any kind of upgrade logic

---
 builtin/credential/aws/path_role.go | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index 2708f6b93b37..7dcfd03cf299 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -323,11 +323,15 @@ func (b *backend) setRole(ctx context.Context, s logical.Storage, roleName strin
 // initialize is used to initialize the AWS roles
 func (b *backend) initialize(ctx context.Context, req *logical.InitializationRequest) error {
 
+	// on standbys and DR secondaries we do not want to run any kind of upgrade logic
+	if b.System().ReplicationState().HasState(consts.ReplicationPerformanceStandby | consts.ReplicationDRSecondary) {
+		return nil
+	}
+
 	// Initialize only if we are either:
 	//   (1) A local mount.
-	//   (2) Are _NOT_ a replicated secondary cluster, or a performance standby node, or a DR secondary.
-	if b.System().LocalMount() || !b.System().ReplicationState().HasState(
-		consts.ReplicationPerformanceSecondary|consts.ReplicationPerformanceStandby|consts.ReplicationDRSecondary) {
+	//   (2) Are _NOT_ a replicated performance secondary
+	if b.System().LocalMount() || !b.System().ReplicationState().HasState(consts.ReplicationPerformanceSecondary) {
 
 		s := req.Storage
 

From 11dbf68dd38de0888d88ecb998f93cf48421c574 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Wed, 3 Jul 2019 10:34:35 -0400
Subject: [PATCH 31/53] fix awsVersion struct

---
 builtin/credential/aws/backend_test.go   | 15 ++++++++++++---
 builtin/credential/aws/path_role.go      | 22 +++++++++++-----------
 builtin/credential/aws/path_role_test.go |  8 ++++----
 builtin/plugin/backend.go                |  2 +-
 4 files changed, 28 insertions(+), 19 deletions(-)

diff --git a/builtin/credential/aws/backend_test.go b/builtin/credential/aws/backend_test.go
index 4bed75f85374..0336f99751af 100644
--- a/builtin/credential/aws/backend_test.go
+++ b/builtin/credential/aws/backend_test.go
@@ -1821,12 +1821,21 @@ func generateRenewRequest(s logical.Storage, auth *logical.Auth) *logical.Reques
 
 func TestBackend_E2E_Initialize(t *testing.T) {
 
-	// All this test does is load up the aws plugin, so that if we are in
+	// TODO: All this test does is load up the aws plugin, so that if we are in
 	// verbose mode we can observe in the log output that the initialization
 	// process ran:
 	//
 	//    go test -v ./builtin/credential/aws/ -run TestBackend_E2E_Initialize
-
+	//
+	// We need to modify this test so that it:
+	//     loads the plugin
+	//     writes some roles
+	//     "downgrades"the roles manually by directly modifying storage
+	//     somehow re-triggers an Initialize(), which will upgrade the roles
+	//     read from storage to confirm that the roles are upgraded
+	//     read from storage to confirm that the 'config/version' entry is there
+
+	// set up a test cluster
 	logger := logging.NewVaultLogger(hclog.Trace)
 	coreConfig := &vault.CoreConfig{
 		Logger: logger,
@@ -1844,7 +1853,7 @@ func TestBackend_E2E_Initialize(t *testing.T) {
 	vault.TestWaitActive(t, cluster.Cores[0].Core)
 	client := cluster.Cores[0].Client
 
-	// Setup Vault
+	// load the auth plugin
 	if err := client.Sys().EnableAuthWithOptions("aws", &api.EnableAuthOptions{
 		Type: "aws",
 	}); err != nil {
diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index 4b7e22b8d4e9..047d898c8797 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -355,7 +355,7 @@ func (b *backend) initialize(ctx context.Context, req *logical.InitializationReq
 				logger.Info("an upgrade was performed during initialization")
 			}
 
-			logger.Info("initialization succeeded")
+			logger.Trace("initialization succeeded")
 		}()
 
 	}
@@ -363,29 +363,29 @@ func (b *backend) initialize(ctx context.Context, req *logical.InitializationReq
 	return nil
 }
 
-// roleStorageVersion stores the latest role storage version that we have
+// awsVersion stores the latest role storage version that we have
 // upgraded to.
-type roleStorageVersion struct {
-	Version int `json:"version"`
+type awsVersion struct {
+	RoleVersion int `json:"role_verison"`
 }
 
 // upgrade and persist all of the role entries that are in need of being
 // upgraded.
 func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error) {
 
-	// find the current role-storage-version
+	// find the current version
 	version := -1
-	entry, err := s.Get(ctx, "config/role-storage-version")
+	entry, err := s.Get(ctx, "config/version")
 	if err != nil {
 		return false, err
 	}
 	if entry != nil {
-		var rsv roleStorageVersion
+		var rsv awsVersion
 		err = entry.DecodeJSON(&rsv)
 		if err != nil {
 			return false, err
 		}
-		version = rsv.Version
+		version = rsv.RoleVersion
 	}
 
 	// check if we need to upgrade
@@ -411,9 +411,9 @@ func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error)
 		}
 	}
 
-	// save the current role-storage-version
-	rsv := roleStorageVersion{Version: currentRoleStorageVersion}
-	entry, err = logical.StorageEntryJSON("config/role-storage-version", &rsv)
+	// save the current version
+	rsv := awsVersion{RoleVersion: currentRoleStorageVersion}
+	entry, err = logical.StorageEntryJSON("config/version", &rsv)
 	if err != nil {
 		return false, err
 	}
diff --git a/builtin/credential/aws/path_role_test.go b/builtin/credential/aws/path_role_test.go
index 2340483a76cf..4d3fe110fea2 100644
--- a/builtin/credential/aws/path_role_test.go
+++ b/builtin/credential/aws/path_role_test.go
@@ -930,16 +930,16 @@ func TestRoleInitialize(t *testing.T) {
 
 func TestRoleStorageVersion(t *testing.T) {
 
-	before := roleStorageVersion{
-		Version: 42,
+	before := awsVersion{
+		RoleVersion: 42,
 	}
 
-	entry, err := logical.StorageEntryJSON("config/role-storage-version", &before)
+	entry, err := logical.StorageEntryJSON("config/version", &before)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	var after roleStorageVersion
+	var after awsVersion
 	err = entry.DecodeJSON(&after)
 	if err != nil {
 		t.Fatal(err)
diff --git a/builtin/plugin/backend.go b/builtin/plugin/backend.go
index 58c2f0379865..dbefa9dc0fdf 100644
--- a/builtin/plugin/backend.go
+++ b/builtin/plugin/backend.go
@@ -217,7 +217,7 @@ func (b *PluginBackend) HandleExistenceCheck(ctx context.Context, req *logical.R
 func (b *PluginBackend) Initialize(ctx context.Context, req *logical.InitializationRequest) error {
 
 	// This method is only ever called just after mounting, so we know that the
-	// cally to lazyLoadBackend() will call startBackend().  Since
+	// call to lazyLoadBackend() will call startBackend().  Since
 	// startBackend() calls Initialize() on the underlying logical.Backend, the
 	// method wrapper that we pass in here is a no-op
 	return b.lazyLoadBackend(ctx, req.Storage, func() error {

From 5187bd2f211163c06aed901ec220b0df59af9fe5 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Wed, 3 Jul 2019 11:01:15 -0400
Subject: [PATCH 32/53] clarify aws version upgrade

---
 builtin/credential/aws/path_role.go | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index 047d898c8797..f0389d9ef48d 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -363,18 +363,17 @@ func (b *backend) initialize(ctx context.Context, req *logical.InitializationReq
 	return nil
 }
 
-// awsVersion stores the latest role storage version that we have
+// awsVersion stores info about the the latest aws version that we have
 // upgraded to.
 type awsVersion struct {
 	RoleVersion int `json:"role_verison"`
 }
 
-// upgrade and persist all of the role entries that are in need of being
-// upgraded.
+// upgrade aws version
 func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error) {
 
-	// find the current version
-	version := -1
+	// find the persisted role version
+	roleVersion := -1
 	entry, err := s.Get(ctx, "config/version")
 	if err != nil {
 		return false, err
@@ -385,11 +384,13 @@ func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error)
 		if err != nil {
 			return false, err
 		}
-		version = rsv.RoleVersion
+		roleVersion = rsv.RoleVersion
 	}
 
-	// check if we need to upgrade
-	if version == currentRoleStorageVersion {
+	// for now, we only need to upgrade if the persisted role version
+	// has been superseded. This may change in the future.
+	needToUpgrade := roleVersion < currentRoleStorageVersion
+	if !needToUpgrade {
 		return false, nil
 	}
 

From c5e63e42ec4cebb18e7396a0f9f0f262b7df0afe Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Wed, 3 Jul 2019 13:14:59 -0400
Subject: [PATCH 33/53] make the upgrade logic for aws auth more explicit

---
 builtin/credential/aws/path_role.go | 55 +++++++++++++++++++----------
 1 file changed, 37 insertions(+), 18 deletions(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index f0389d9ef48d..ac240817f00f 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -354,8 +354,6 @@ func (b *backend) initialize(ctx context.Context, req *logical.InitializationReq
 			if upgraded {
 				logger.Info("an upgrade was performed during initialization")
 			}
-
-			logger.Trace("initialization succeeded")
 		}()
 
 	}
@@ -372,24 +370,11 @@ type awsVersion struct {
 // upgrade aws version
 func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error) {
 
-	// find the persisted role version
-	roleVersion := -1
-	entry, err := s.Get(ctx, "config/version")
+	// check if we should upgrade
+	needToUpgrade, err := b.shouldUprade(ctx, s)
 	if err != nil {
 		return false, err
 	}
-	if entry != nil {
-		var rsv awsVersion
-		err = entry.DecodeJSON(&rsv)
-		if err != nil {
-			return false, err
-		}
-		roleVersion = rsv.RoleVersion
-	}
-
-	// for now, we only need to upgrade if the persisted role version
-	// has been superseded. This may change in the future.
-	needToUpgrade := roleVersion < currentRoleStorageVersion
 	if !needToUpgrade {
 		return false, nil
 	}
@@ -414,7 +399,7 @@ func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error)
 
 	// save the current version
 	rsv := awsVersion{RoleVersion: currentRoleStorageVersion}
-	entry, err = logical.StorageEntryJSON("config/version", &rsv)
+	entry, err := logical.StorageEntryJSON("config/version", &rsv)
 	if err != nil {
 		return false, err
 	}
@@ -426,6 +411,40 @@ func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error)
 	return true, nil
 }
 
+// shouldUprade checks if we need to upgrade
+func (b *backend) shouldUprade(ctx context.Context, s logical.Storage) (bool, error) {
+
+	entry, err := s.Get(ctx, "config/version")
+	if err != nil {
+		return false, err
+	}
+
+	// if there is no persisted version, we need to upgrade
+	if entry == nil {
+		return true, nil
+	}
+
+	var version awsVersion
+	err = entry.DecodeJSON(&version)
+	if err != nil {
+		return false, err
+	}
+
+	// for now, we only need to upgrade if the role version has been
+	// superseded. This may change in the future.
+	switch version.RoleVersion {
+	case 0:
+	case 1:
+	case 2:
+		return true, nil
+
+	case currentRoleStorageVersion:
+		return false, nil
+
+	}
+	return false, fmt.Errorf("unrecognized role version: %q", version.RoleVersion)
+}
+
 // If needed, updates the role entry and returns a bool indicating if it was updated
 // (and thus needs to be persisted)
 func (b *backend) upgradeRole(ctx context.Context, s logical.Storage, roleEntry *awsRoleEntry) (bool, error) {

From c652d554b96156a34edb33b24a9a89851668ece5 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Wed, 3 Jul 2019 14:51:30 -0400
Subject: [PATCH 34/53] aws upgrade is now called from a switch

---
 builtin/credential/aws/path_role.go | 85 ++++++++++++++---------------
 1 file changed, 41 insertions(+), 44 deletions(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index ac240817f00f..571a032838e4 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -367,33 +367,64 @@ type awsVersion struct {
 	RoleVersion int `json:"role_verison"`
 }
 
-// upgrade aws version
+// upgrade does an upgrade, if necessary
 func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error) {
 
-	// check if we should upgrade
-	needToUpgrade, err := b.shouldUprade(ctx, s)
+	entry, err := s.Get(ctx, "config/version")
 	if err != nil {
 		return false, err
 	}
-	if !needToUpgrade {
+
+	// if there is no persisted version, we need to upgrade
+	if entry == nil {
+		err = b.upgradeRoles(ctx, s)
+		if err != nil {
+			return false, err
+		}
+		return true, nil
+	}
+
+	var version awsVersion
+	err = entry.DecodeJSON(&version)
+	if err != nil {
+		return false, err
+	}
+
+	// upgrade if persisted roleVersion is out of date
+	switch version.RoleVersion {
+	case 0:
+	case 1:
+	case 2:
+		err = b.upgradeRoles(ctx, s)
+		if err != nil {
+			return false, err
+		}
+
+	case currentRoleStorageVersion:
 		return false, nil
+
 	}
+	return false, fmt.Errorf("unrecognized role version: %q", version.RoleVersion)
+}
+
+// upgradeRoles upgrades the various aws roles
+func (b *backend) upgradeRoles(ctx context.Context, s logical.Storage) error {
 
 	// Read all the role names.
 	roleNames, err := s.List(ctx, "role/")
 	if err != nil {
-		return false, err
+		return err
 	}
 
 	// Upgrade the roles as necessary.
 	for _, roleName := range roleNames {
 		// make sure the context hasn't been canceled
 		if ctx.Err() != nil {
-			return false, err
+			return err
 		}
 		_, err := b.roleInternal(ctx, s, roleName)
 		if err != nil {
-			return false, err
+			return err
 		}
 	}
 
@@ -401,48 +432,14 @@ func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error)
 	rsv := awsVersion{RoleVersion: currentRoleStorageVersion}
 	entry, err := logical.StorageEntryJSON("config/version", &rsv)
 	if err != nil {
-		return false, err
+		return err
 	}
 	err = s.Put(ctx, entry)
 	if err != nil {
-		return false, err
-	}
-
-	return true, nil
-}
-
-// shouldUprade checks if we need to upgrade
-func (b *backend) shouldUprade(ctx context.Context, s logical.Storage) (bool, error) {
-
-	entry, err := s.Get(ctx, "config/version")
-	if err != nil {
-		return false, err
-	}
-
-	// if there is no persisted version, we need to upgrade
-	if entry == nil {
-		return true, nil
-	}
-
-	var version awsVersion
-	err = entry.DecodeJSON(&version)
-	if err != nil {
-		return false, err
+		return err
 	}
 
-	// for now, we only need to upgrade if the role version has been
-	// superseded. This may change in the future.
-	switch version.RoleVersion {
-	case 0:
-	case 1:
-	case 2:
-		return true, nil
-
-	case currentRoleStorageVersion:
-		return false, nil
-
-	}
-	return false, fmt.Errorf("unrecognized role version: %q", version.RoleVersion)
+	return nil
 }
 
 // If needed, updates the role entry and returns a bool indicating if it was updated

From 0344a8730dad72c4d92f9947bcf6951d7211f6cf Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Wed, 3 Jul 2019 15:23:48 -0400
Subject: [PATCH 35/53] fix fallthrough bug

---
 builtin/credential/aws/path_role.go      |  5 ++-
 builtin/credential/aws/path_role_test.go | 50 ++++++++++++++++++++----
 2 files changed, 47 insertions(+), 8 deletions(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index 571a032838e4..240f92068a7f 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -393,18 +393,21 @@ func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error)
 	// upgrade if persisted roleVersion is out of date
 	switch version.RoleVersion {
 	case 0:
+		fallthrough
 	case 1:
+		fallthrough
 	case 2:
 		err = b.upgradeRoles(ctx, s)
 		if err != nil {
 			return false, err
 		}
+		return true, nil
 
 	case currentRoleStorageVersion:
 		return false, nil
 
 	}
-	return false, fmt.Errorf("unrecognized role version: %q", version.RoleVersion)
+	return false, fmt.Errorf("unrecognized role version: %d", version.RoleVersion)
 }
 
 // upgradeRoles upgrades the various aws roles
diff --git a/builtin/credential/aws/path_role_test.go b/builtin/credential/aws/path_role_test.go
index 4d3fe110fea2..cdcde24c1282 100644
--- a/builtin/credential/aws/path_role_test.go
+++ b/builtin/credential/aws/path_role_test.go
@@ -5,7 +5,6 @@ import (
 	"reflect"
 	"strings"
 	"testing"
-	"time"
 
 	"github.com/go-test/deep"
 	"github.com/hashicorp/vault/sdk/helper/policyutil"
@@ -867,9 +866,14 @@ func TestRoleInitialize(t *testing.T) {
 		}
 	}
 
-	// upgrade all the entries, and wait for the goroutine to finish
-	err = b.initialize(ctx, &logical.InitializationRequest{storage})
-	time.Sleep(time.Second)
+	// upgrade all the entries
+	upgraded, err := b.upgrade(ctx, storage)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !upgraded {
+		t.Fatalf("expected upgrade")
+	}
 
 	// read the entries from storage
 	after := make([]testData, 0)
@@ -916,16 +920,48 @@ func TestRoleInitialize(t *testing.T) {
 		t.Fatal(diff)
 	}
 
-	// TODO check storage version
-
 	// run it again -- nothing will happen
-	upgraded, err := b.upgrade(ctx, storage)
+	upgraded, err = b.upgrade(ctx, storage)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if upgraded {
 		t.Fatalf("expected no upgrade")
 	}
+
+	// make sure saved role version is correct
+	entry, err := storage.Get(ctx, "config/version")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var version awsVersion
+	err = entry.DecodeJSON(&version)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if version.RoleVersion != currentRoleStorageVersion {
+		t.Fatalf("expected version %d, got  %d", currentRoleStorageVersion, version.RoleVersion)
+	}
+
+	// stomp on the saved version
+	version.RoleVersion = 0
+	e2, err := logical.StorageEntryJSON("config/version", version)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = storage.Put(ctx, e2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// run it again -- now an upgrade will happen
+	upgraded, err = b.upgrade(ctx, storage)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !upgraded {
+		t.Fatalf("expected upgrade")
+	}
 }
 
 func TestRoleStorageVersion(t *testing.T) {

From 52dae9584f2b3373189804adb15d3a0521cd29ac Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Wed, 3 Jul 2019 15:53:57 -0400
Subject: [PATCH 36/53] simplify logic

---
 builtin/credential/aws/path_role.go | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index 240f92068a7f..525624e31f89 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -374,23 +374,16 @@ func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error)
 	if err != nil {
 		return false, err
 	}
-
-	// if there is no persisted version, we need to upgrade
+	var version awsVersion
 	if entry == nil {
-		err = b.upgradeRoles(ctx, s)
+		version.RoleVersion = 0
+	} else {
+		err = entry.DecodeJSON(&version)
 		if err != nil {
 			return false, err
 		}
-		return true, nil
-	}
-
-	var version awsVersion
-	err = entry.DecodeJSON(&version)
-	if err != nil {
-		return false, err
 	}
 
-	// upgrade if persisted roleVersion is out of date
 	switch version.RoleVersion {
 	case 0:
 		fallthrough

From c3bdfac1f7c2a490705b4f0b6825e6142d51aa0e Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Wed, 3 Jul 2019 15:59:23 -0400
Subject: [PATCH 37/53] simplify logic

---
 builtin/credential/aws/path_role.go | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index 525624e31f89..c96209b5b337 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -394,13 +394,25 @@ func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error)
 		if err != nil {
 			return false, err
 		}
-		return true, nil
-
 	case currentRoleStorageVersion:
 		return false, nil
 
+	default:
+		return false, fmt.Errorf("unrecognized role version: %d", version.RoleVersion)
+	}
+
+	// save the current version
+	rsv := awsVersion{RoleVersion: currentRoleStorageVersion}
+	entry, err = logical.StorageEntryJSON("config/version", &rsv)
+	if err != nil {
+		return false, err
+	}
+	err = s.Put(ctx, entry)
+	if err != nil {
+		return false, err
 	}
-	return false, fmt.Errorf("unrecognized role version: %d", version.RoleVersion)
+
+	return true, nil
 }
 
 // upgradeRoles upgrades the various aws roles
@@ -424,17 +436,6 @@ func (b *backend) upgradeRoles(ctx context.Context, s logical.Storage) error {
 		}
 	}
 
-	// save the current version
-	rsv := awsVersion{RoleVersion: currentRoleStorageVersion}
-	entry, err := logical.StorageEntryJSON("config/version", &rsv)
-	if err != nil {
-		return err
-	}
-	err = s.Put(ctx, entry)
-	if err != nil {
-		return err
-	}
-
 	return nil
 }
 

From 94e707b230ebf90a8b65f123dad047be3a4f59a0 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Wed, 3 Jul 2019 16:23:29 -0400
Subject: [PATCH 38/53] rename things

---
 builtin/credential/aws/path_role.go      | 12 +++++-------
 builtin/credential/aws/path_role_test.go |  8 ++++----
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index c96209b5b337..714f11342fdc 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -364,7 +364,7 @@ func (b *backend) initialize(ctx context.Context, req *logical.InitializationReq
 // awsVersion stores info about the the latest aws version that we have
 // upgraded to.
 type awsVersion struct {
-	RoleVersion int `json:"role_verison"`
+	Version int `json:"version"`
 }
 
 // upgrade does an upgrade, if necessary
@@ -375,16 +375,14 @@ func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error)
 		return false, err
 	}
 	var version awsVersion
-	if entry == nil {
-		version.RoleVersion = 0
-	} else {
+	if entry != nil {
 		err = entry.DecodeJSON(&version)
 		if err != nil {
 			return false, err
 		}
 	}
 
-	switch version.RoleVersion {
+	switch version.Version {
 	case 0:
 		fallthrough
 	case 1:
@@ -398,11 +396,11 @@ func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error)
 		return false, nil
 
 	default:
-		return false, fmt.Errorf("unrecognized role version: %d", version.RoleVersion)
+		return false, fmt.Errorf("unrecognized role version: %d", version.Version)
 	}
 
 	// save the current version
-	rsv := awsVersion{RoleVersion: currentRoleStorageVersion}
+	rsv := awsVersion{Version: currentRoleStorageVersion}
 	entry, err = logical.StorageEntryJSON("config/version", &rsv)
 	if err != nil {
 		return false, err
diff --git a/builtin/credential/aws/path_role_test.go b/builtin/credential/aws/path_role_test.go
index cdcde24c1282..4ed4132bd338 100644
--- a/builtin/credential/aws/path_role_test.go
+++ b/builtin/credential/aws/path_role_test.go
@@ -939,12 +939,12 @@ func TestRoleInitialize(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if version.RoleVersion != currentRoleStorageVersion {
-		t.Fatalf("expected version %d, got  %d", currentRoleStorageVersion, version.RoleVersion)
+	if version.Version != currentRoleStorageVersion {
+		t.Fatalf("expected version %d, got  %d", currentRoleStorageVersion, version.Version)
 	}
 
 	// stomp on the saved version
-	version.RoleVersion = 0
+	version.Version = 0
 	e2, err := logical.StorageEntryJSON("config/version", version)
 	if err != nil {
 		t.Fatal(err)
@@ -967,7 +967,7 @@ func TestRoleInitialize(t *testing.T) {
 func TestRoleStorageVersion(t *testing.T) {
 
 	before := awsVersion{
-		RoleVersion: 42,
+		Version: 42,
 	}
 
 	entry, err := logical.StorageEntryJSON("config/version", &before)

From d5de690585283f699bee57046b0a2ccf7c0b2e27 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Wed, 3 Jul 2019 17:03:42 -0400
Subject: [PATCH 39/53] introduce currentAwsVersion const to track aws version

---
 builtin/credential/aws/path_role.go      | 10 ++++------
 builtin/credential/aws/path_role_test.go |  6 +++---
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index 714f11342fdc..667fdf013aab 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -367,6 +367,8 @@ type awsVersion struct {
 	Version int `json:"version"`
 }
 
+const currentAwsVersion = 1
+
 // upgrade does an upgrade, if necessary
 func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error) {
 
@@ -384,15 +386,11 @@ func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error)
 
 	switch version.Version {
 	case 0:
-		fallthrough
-	case 1:
-		fallthrough
-	case 2:
 		err = b.upgradeRoles(ctx, s)
 		if err != nil {
 			return false, err
 		}
-	case currentRoleStorageVersion:
+	case currentAwsVersion:
 		return false, nil
 
 	default:
@@ -400,7 +398,7 @@ func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error)
 	}
 
 	// save the current version
-	rsv := awsVersion{Version: currentRoleStorageVersion}
+	rsv := awsVersion{Version: currentAwsVersion}
 	entry, err = logical.StorageEntryJSON("config/version", &rsv)
 	if err != nil {
 		return false, err
diff --git a/builtin/credential/aws/path_role_test.go b/builtin/credential/aws/path_role_test.go
index 4ed4132bd338..08ce12200654 100644
--- a/builtin/credential/aws/path_role_test.go
+++ b/builtin/credential/aws/path_role_test.go
@@ -939,8 +939,8 @@ func TestRoleInitialize(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if version.Version != currentRoleStorageVersion {
-		t.Fatalf("expected version %d, got  %d", currentRoleStorageVersion, version.Version)
+	if version.Version != currentAwsVersion {
+		t.Fatalf("expected version %d, got  %d", currentAwsVersion, version.Version)
 	}
 
 	// stomp on the saved version
@@ -964,7 +964,7 @@ func TestRoleInitialize(t *testing.T) {
 	}
 }
 
-func TestRoleStorageVersion(t *testing.T) {
+func TestAwsVersion(t *testing.T) {
 
 	before := awsVersion{
 		Version: 42,

From 4aea057e689fe98d1efce920e5cb6cf51e5deabc Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Wed, 3 Jul 2019 17:15:57 -0400
Subject: [PATCH 40/53] improve comments

---
 builtin/credential/aws/path_role.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index 667fdf013aab..77d996d9033e 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -367,6 +367,8 @@ type awsVersion struct {
 	Version int `json:"version"`
 }
 
+// currentAwsVersion stores the latest version that we have upgraded to.
+// Note that this is tracked independently from currentRoleStorageVersion.
 const currentAwsVersion = 1
 
 // upgrade does an upgrade, if necessary

From e86858541dcb526186dc25fafaa14c1f8f9792d2 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Thu, 4 Jul 2019 07:34:06 -0400
Subject: [PATCH 41/53] rearrange things once more

---
 builtin/credential/aws/path_role.go | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index 77d996d9033e..468fcb57bfc6 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -386,6 +386,7 @@ func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error)
 		}
 	}
 
+	upgraded := version.Version < currentAwsVersion
 	switch version.Version {
 	case 0:
 		err = b.upgradeRoles(ctx, s)
@@ -393,24 +394,25 @@ func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error)
 			return false, err
 		}
 	case currentAwsVersion:
-		return false, nil
-
+		break
 	default:
 		return false, fmt.Errorf("unrecognized role version: %d", version.Version)
 	}
 
 	// save the current version
-	rsv := awsVersion{Version: currentAwsVersion}
-	entry, err = logical.StorageEntryJSON("config/version", &rsv)
-	if err != nil {
-		return false, err
-	}
-	err = s.Put(ctx, entry)
-	if err != nil {
-		return false, err
+	if upgraded {
+		cv := awsVersion{Version: currentAwsVersion}
+		entry, err = logical.StorageEntryJSON("config/version", &cv)
+		if err != nil {
+			return false, err
+		}
+		err = s.Put(ctx, entry)
+		if err != nil {
+			return false, err
+		}
 	}
 
-	return true, nil
+	return upgraded, nil
 }
 
 // upgradeRoles upgrades the various aws roles

From d38f38b2b1677034c24e1b5c1dd18a1e94b1e2f7 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Thu, 4 Jul 2019 07:38:41 -0400
Subject: [PATCH 42/53] conglomerate things into one function

---
 builtin/credential/aws/path_role.go | 39 +++++++++++------------------
 1 file changed, 14 insertions(+), 25 deletions(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index 468fcb57bfc6..8bcd28054c1c 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -389,10 +389,23 @@ func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error)
 	upgraded := version.Version < currentAwsVersion
 	switch version.Version {
 	case 0:
-		err = b.upgradeRoles(ctx, s)
+		// Read all the role names.
+		roleNames, err := s.List(ctx, "role/")
 		if err != nil {
 			return false, err
 		}
+
+		// Upgrade the roles as necessary.
+		for _, roleName := range roleNames {
+			// make sure the context hasn't been canceled
+			if ctx.Err() != nil {
+				return false, err
+			}
+			_, err := b.roleInternal(ctx, s, roleName)
+			if err != nil {
+				return false, err
+			}
+		}
 	case currentAwsVersion:
 		break
 	default:
@@ -415,30 +428,6 @@ func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error)
 	return upgraded, nil
 }
 
-// upgradeRoles upgrades the various aws roles
-func (b *backend) upgradeRoles(ctx context.Context, s logical.Storage) error {
-
-	// Read all the role names.
-	roleNames, err := s.List(ctx, "role/")
-	if err != nil {
-		return err
-	}
-
-	// Upgrade the roles as necessary.
-	for _, roleName := range roleNames {
-		// make sure the context hasn't been canceled
-		if ctx.Err() != nil {
-			return err
-		}
-		_, err := b.roleInternal(ctx, s, roleName)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
 // If needed, updates the role entry and returns a bool indicating if it was updated
 // (and thus needs to be persisted)
 func (b *backend) upgradeRole(ctx context.Context, s logical.Storage, roleEntry *awsRoleEntry) (bool, error) {

From 0e1150a7a58302169a8a5bf45530882cace1a8ad Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Wed, 3 Jul 2019 12:53:03 -0400
Subject: [PATCH 43/53] stub out aws auth initialize e2e test

---
 builtin/credential/aws/backend_test.go | 71 ++++++++++++++++++++------
 1 file changed, 55 insertions(+), 16 deletions(-)

diff --git a/builtin/credential/aws/backend_test.go b/builtin/credential/aws/backend_test.go
index 0336f99751af..f7686af3d3d5 100644
--- a/builtin/credential/aws/backend_test.go
+++ b/builtin/credential/aws/backend_test.go
@@ -1821,20 +1821,6 @@ func generateRenewRequest(s logical.Storage, auth *logical.Auth) *logical.Reques
 
 func TestBackend_E2E_Initialize(t *testing.T) {
 
-	// TODO: All this test does is load up the aws plugin, so that if we are in
-	// verbose mode we can observe in the log output that the initialization
-	// process ran:
-	//
-	//    go test -v ./builtin/credential/aws/ -run TestBackend_E2E_Initialize
-	//
-	// We need to modify this test so that it:
-	//     loads the plugin
-	//     writes some roles
-	//     "downgrades"the roles manually by directly modifying storage
-	//     somehow re-triggers an Initialize(), which will upgrade the roles
-	//     read from storage to confirm that the roles are upgraded
-	//     read from storage to confirm that the 'config/version' entry is there
-
 	// set up a test cluster
 	logger := logging.NewVaultLogger(hclog.Trace)
 	coreConfig := &vault.CoreConfig{
@@ -1850,8 +1836,9 @@ func TestBackend_E2E_Initialize(t *testing.T) {
 	cluster.Start()
 	defer cluster.Cleanup()
 
-	vault.TestWaitActive(t, cluster.Cores[0].Core)
-	client := cluster.Cores[0].Client
+	core := cluster.Cores[0]
+	vault.TestWaitActive(t, core.Core)
+	client := core.Client
 
 	// load the auth plugin
 	if err := client.Sys().EnableAuthWithOptions("aws", &api.EnableAuthOptions{
@@ -1859,4 +1846,56 @@ func TestBackend_E2E_Initialize(t *testing.T) {
 	}); err != nil {
 		t.Fatal(err)
 	}
+
+	// create some role entries
+	type testRole struct {
+		name string
+		data map[string]interface{}
+	}
+	before := []testRole{
+		{
+			name: "test-role-0",
+			data: map[string]interface{}{
+				"auth_type":               "iam",
+				"policies":                "default",
+				"bound_iam_principal_arn": "arn:aws:iam::000000000001:role/my_role_prefix",
+			},
+		},
+	}
+
+	for _, tr := range before {
+		_, err := client.Logical().Write("auth/aws/role/"+tr.name, tr.data)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	//			data: map[string]interface{}{
+	//			"bound_iam_principal_arn": os.Getenv(envVarAwsTestRoleArn)[:25] + "*",
+	//
+	//// put the entries in storage
+	//for _, role := range before {
+	//	//client.Logical().Write("auth/aws/roles/foo", map[string]interface{}{...
+	//	client.Logical().Write("auth/aws/roles/foo", nil)
+
+	//if _, err := client.Logical().Write("auth/aws/role/test", ); err != nil {
+	//	fmt.Println(err)
+	//	t.Fatal(err)
+	//}
+
+	//	//err = b.setRole(ctx, storage, role.name, role.entry)
+	//	//if err != nil {
+	//	//	t.Fatal(err)
+	//	//}
+	//}
+
+	//---------------------------------------------------
+
+	// "downgrade" some of the roles by directly modifying storage
+
+	// re-trigger Initialize(), which will upgrade the roles
+
+	// read from storage to confirm that the roles are upgraded
+
+	// read from storage to confirm that the 'config/version' entry is there
 }

From e8da5e552d2c585f85cf0418a28f2a369c75f361 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Wed, 3 Jul 2019 14:42:31 -0400
Subject: [PATCH 44/53] improve aws auth initialize e2e test

---
 builtin/credential/aws/backend_test.go | 90 +++++++++++++++++---------
 1 file changed, 60 insertions(+), 30 deletions(-)

diff --git a/builtin/credential/aws/backend_test.go b/builtin/credential/aws/backend_test.go
index f7686af3d3d5..8a6bb13286ee 100644
--- a/builtin/credential/aws/backend_test.go
+++ b/builtin/credential/aws/backend_test.go
@@ -3,6 +3,7 @@ package awsauth
 import (
 	"context"
 	"encoding/base64"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -22,6 +23,7 @@ import (
 	"github.com/hashicorp/vault/sdk/helper/logging"
 	"github.com/hashicorp/vault/sdk/helper/policyutil"
 	"github.com/hashicorp/vault/sdk/logical"
+	"github.com/hashicorp/vault/sdk/physical"
 	"github.com/hashicorp/vault/vault"
 )
 
@@ -482,13 +484,13 @@ func TestBackend_ConfigClient(t *testing.T) {
 
 	stepCreate := logicaltest.TestStep{
 		Operation: logical.CreateOperation,
-		Path:      "config/client",
+		Path:      "config/core.Client",
 		Data:      data,
 	}
 
 	stepUpdate := logicaltest.TestStep{
 		Operation: logical.UpdateOperation,
-		Path:      "config/client",
+		Path:      "config/core.Client",
 		Data:      data,
 	}
 
@@ -497,7 +499,7 @@ func TestBackend_ConfigClient(t *testing.T) {
 	}
 	stepInvalidAccessKey := logicaltest.TestStep{
 		Operation: logical.UpdateOperation,
-		Path:      "config/client",
+		Path:      "config/core.Client",
 		Data:      data3,
 		ErrorOk:   true,
 	}
@@ -507,7 +509,7 @@ func TestBackend_ConfigClient(t *testing.T) {
 	}
 	stepInvalidSecretKey := logicaltest.TestStep{
 		Operation: logical.UpdateOperation,
-		Path:      "config/client",
+		Path:      "config/core.Client",
 		Data:      data4,
 		ErrorOk:   true,
 	}
@@ -526,23 +528,23 @@ func TestBackend_ConfigClient(t *testing.T) {
 	// test existence check returning false
 	checkFound, exists, err := b.HandleExistenceCheck(context.Background(), &logical.Request{
 		Operation: logical.CreateOperation,
-		Path:      "config/client",
+		Path:      "config/core.Client",
 		Storage:   storage,
 	})
 	if err != nil {
 		t.Fatal(err)
 	}
 	if !checkFound {
-		t.Fatal("existence check not found for path 'config/client'")
+		t.Fatal("existence check not found for path 'config/core.Client'")
 	}
 	if exists {
-		t.Fatal("existence check should have returned 'false' for 'config/client'")
+		t.Fatal("existence check should have returned 'false' for 'config/core.Client'")
 	}
 
 	// create an entry
 	configClientCreateRequest := &logical.Request{
 		Operation: logical.UpdateOperation,
-		Path:      "config/client",
+		Path:      "config/core.Client",
 		Data:      data,
 		Storage:   storage,
 	}
@@ -554,17 +556,17 @@ func TestBackend_ConfigClient(t *testing.T) {
 	//test existence check returning true
 	checkFound, exists, err = b.HandleExistenceCheck(context.Background(), &logical.Request{
 		Operation: logical.CreateOperation,
-		Path:      "config/client",
+		Path:      "config/core.Client",
 		Storage:   storage,
 	})
 	if err != nil {
 		t.Fatal(err)
 	}
 	if !checkFound {
-		t.Fatal("existence check not found for path 'config/client'")
+		t.Fatal("existence check not found for path 'config/core.Client'")
 	}
 	if !exists {
-		t.Fatal("existence check should have returned 'true' for 'config/client'")
+		t.Fatal("existence check should have returned 'true' for 'config/core.Client'")
 	}
 
 	endpointData := map[string]interface{}{
@@ -575,7 +577,7 @@ func TestBackend_ConfigClient(t *testing.T) {
 
 	endpointReq := &logical.Request{
 		Operation: logical.UpdateOperation,
-		Path:      "config/client",
+		Path:      "config/core.Client",
 		Storage:   storage,
 		Data:      endpointData,
 	}
@@ -1110,7 +1112,7 @@ func TestBackendAcc_LoginWithInstanceIdentityDocAndWhitelistIdentity(t *testing.
 		_, err = b.HandleRequest(context.Background(), &logical.Request{
 			Operation: logical.UpdateOperation,
 			Storage:   storage,
-			Path:      "config/client",
+			Path:      "config/core.Client",
 			Data:      clientConfig,
 		})
 		if err != nil {
@@ -1120,7 +1122,7 @@ func TestBackendAcc_LoginWithInstanceIdentityDocAndWhitelistIdentity(t *testing.
 
 	loginInput := map[string]interface{}{
 		"pkcs7": pkcs7,
-		"nonce": "vault-client-nonce",
+		"nonce": "vault-core.Client-nonce",
 	}
 
 	parsedIdentityDoc, err := b.parseIdentityDocument(context.Background(), storage, pkcs7)
@@ -1249,17 +1251,17 @@ func TestBackendAcc_LoginWithInstanceIdentityDocAndWhitelistIdentity(t *testing.
 
 	_, ok := resp.Auth.Metadata["nonce"]
 	if ok {
-		t.Fatalf("client nonce should not have been returned")
+		t.Fatalf("core.Client nonce should not have been returned")
 	}
 
-	loginInput["nonce"] = "changed-vault-client-nonce"
+	loginInput["nonce"] = "changed-vault-core.Client-nonce"
 	// try to login again with changed nonce
 	resp, err = b.HandleRequest(context.Background(), loginRequest)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if resp == nil || !resp.IsError() {
-		t.Fatalf("login attempt should have failed due to client nonce mismatch")
+		t.Fatalf("login attempt should have failed due to core.Client nonce mismatch")
 	}
 
 	// Check if a whitelist identity entry is created after the login.
@@ -1507,7 +1509,7 @@ func TestBackendAcc_LoginWithCallerIdentity(t *testing.T) {
 
 	// Test setup largely done
 	// At this point, we're going to:
-	// 1. Configure the client to require our test header value
+	// 1. Configure the core.Client to require our test header value
 	// 2. Configure identity to use the ARN for the alias
 	// 3. Configure two different roles:
 	//    a. One bound to our test user
@@ -1526,7 +1528,7 @@ func TestBackendAcc_LoginWithCallerIdentity(t *testing.T) {
 	}
 	clientRequest := &logical.Request{
 		Operation: logical.UpdateOperation,
-		Path:      "config/client",
+		Path:      "config/core.Client",
 		Storage:   storage,
 		Data:      clientConfigData,
 	}
@@ -1838,10 +1840,9 @@ func TestBackend_E2E_Initialize(t *testing.T) {
 
 	core := cluster.Cores[0]
 	vault.TestWaitActive(t, core.Core)
-	client := core.Client
 
 	// load the auth plugin
-	if err := client.Sys().EnableAuthWithOptions("aws", &api.EnableAuthOptions{
+	if err := core.Client.Sys().EnableAuthWithOptions("aws", &api.EnableAuthOptions{
 		Type: "aws",
 	}); err != nil {
 		t.Fatal(err)
@@ -1856,15 +1857,14 @@ func TestBackend_E2E_Initialize(t *testing.T) {
 		{
 			name: "test-role-0",
 			data: map[string]interface{}{
-				"auth_type":               "iam",
-				"policies":                "default",
-				"bound_iam_principal_arn": "arn:aws:iam::000000000001:role/my_role_prefix",
+				"auth_type":       "ec2",
+				"policies":        "default",
+				"bound_subnet_id": "subnet-abcdef",
 			},
 		},
 	}
-
 	for _, tr := range before {
-		_, err := client.Logical().Write("auth/aws/role/"+tr.name, tr.data)
+		_, err := core.Client.Logical().Write("auth/aws/role/"+tr.name, tr.data)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -1875,10 +1875,10 @@ func TestBackend_E2E_Initialize(t *testing.T) {
 	//
 	//// put the entries in storage
 	//for _, role := range before {
-	//	//client.Logical().Write("auth/aws/roles/foo", map[string]interface{}{...
-	//	client.Logical().Write("auth/aws/roles/foo", nil)
+	//	//core.Client.Logical().Write("auth/aws/roles/foo", map[string]interface{}{...
+	//	core.Client.Logical().Write("auth/aws/roles/foo", nil)
 
-	//if _, err := client.Logical().Write("auth/aws/role/test", ); err != nil {
+	//if _, err := core.Client.Logical().Write("auth/aws/role/test", ); err != nil {
 	//	fmt.Println(err)
 	//	t.Fatal(err)
 	//}
@@ -1891,7 +1891,37 @@ func TestBackend_E2E_Initialize(t *testing.T) {
 
 	//---------------------------------------------------
 
-	// "downgrade" some of the roles by directly modifying storage
+	// fetch the auth backend's uuid in storage
+	ctx := context.Background()
+	uuid, err := core.UnderlyingStorage.List(ctx, "auth/")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// "downgrade" some of the roles by directly modifying them in storage
+	for _, tr := range before {
+
+		key := "auth/" + uuid[0] + "role/" + tr.name
+		entry, err := core.UnderlyingStorage.Get(ctx, key)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		fmt.Printf("storage %T\n", core.UnderlyingStorage)
+		fmt.Printf("storage backend %T\n", core.UnderlyingStorage.(*physical.StorageEncoding).Backend)
+		fmt.Printf("entry %T\n", entry)
+		fmt.Printf("key %s\n", entry.Key)
+		fmt.Printf("value %s\n", hex.EncodeToString(entry.Value))
+
+		//roleEntry := new(awsRoleEntry)
+		//err = jsonutil.DecodeJSON(entry.Value, roleEntry)
+		//if err != nil {
+		//	t.Fatal(err)
+		//}
+		//if err := entry.DecodeJSON(role); err != nil {
+		//	t.Fatal(err)
+		//}
+	}
 
 	// re-trigger Initialize(), which will upgrade the roles
 

From 8e3e3f939785eaf75b1e7eb7869f39b238c6c393 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Thu, 4 Jul 2019 10:26:30 -0400
Subject: [PATCH 45/53] finish aws auth initialize e2e test

---
 builtin/credential/aws/backend_e2e_test.go | 128 +++++++++++++++++++++
 builtin/credential/aws/backend_test.go     | 116 -------------------
 2 files changed, 128 insertions(+), 116 deletions(-)
 create mode 100644 builtin/credential/aws/backend_e2e_test.go

diff --git a/builtin/credential/aws/backend_e2e_test.go b/builtin/credential/aws/backend_e2e_test.go
new file mode 100644
index 000000000000..f35f87b15634
--- /dev/null
+++ b/builtin/credential/aws/backend_e2e_test.go
@@ -0,0 +1,128 @@
+package awsauth
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	hclog "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/vault/api"
+	vaulthttp "github.com/hashicorp/vault/http"
+	"github.com/hashicorp/vault/sdk/helper/logging"
+	"github.com/hashicorp/vault/sdk/logical"
+	"github.com/hashicorp/vault/vault"
+)
+
+func TestBackend_E2E_Initialize(t *testing.T) {
+
+	ctx := context.Background()
+
+	// Set up the cluster.  This will trigger an Initialize(); we sleep briefly
+	// awaiting its completion.
+	cluster, awsStoragePath := setupAwsTestCluster(t, ctx)
+	defer cluster.Cleanup()
+	time.Sleep(time.Second)
+	core := cluster.Cores[0]
+
+	// Make sure that the upgrade happened, by fishing the 'config/version'
+	// entry out of storage.  We can't use core.Client.Logical().Read() to do
+	// this, because 'config/version' hasn't been exposed as a path.
+	// TODO: should we expose 'config/version' as a path?
+	version, err := core.UnderlyingStorage.Get(ctx, awsStoragePath+"config/version")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if version == nil {
+		t.Fatalf("no config found")
+	}
+
+	// Nuke the version, so we can pretend that Initialize() has never been run
+	if err := core.UnderlyingStorage.Delete(ctx, awsStoragePath+"config/version"); err != nil {
+		t.Fatal(err)
+	}
+	version, err = core.UnderlyingStorage.Get(ctx, awsStoragePath+"config/version")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if version != nil {
+		t.Fatalf("version found")
+	}
+
+	// Create a role
+	data := map[string]interface{}{
+		"auth_type":       "ec2",
+		"policies":        "default",
+		"bound_subnet_id": "subnet-abcdef"}
+	if _, err := core.Client.Logical().Write("auth/aws/role/test-role", data); err != nil {
+		t.Fatal(err)
+	}
+	role, err := core.Client.Logical().Read("auth/aws/role/test-role")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if role == nil {
+		t.Fatalf("no role found")
+	}
+
+	// There should _still_ be no config version
+	version, err = core.UnderlyingStorage.Get(ctx, awsStoragePath+"config/version")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if version != nil {
+		t.Fatalf("version found")
+	}
+
+	// Seal, and then Unseal. This will once again trigger an Initialize(), only this time
+	// there will be a role for it to upgrade.
+	core.Seal(t)
+	cluster.UnsealCores(t)
+	time.Sleep(time.Second)
+
+	// Now the config version should be there again
+	version, err = core.UnderlyingStorage.Get(ctx, awsStoragePath+"config/version")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if version == nil {
+		t.Fatalf("no version found")
+	}
+}
+
+func setupAwsTestCluster(t *testing.T, ctx context.Context) (*vault.TestCluster, string) {
+
+	logger := logging.NewVaultLogger(hclog.Trace)
+	coreConfig := &vault.CoreConfig{
+		Logger: logger,
+		CredentialBackends: map[string]logical.Factory{
+			"aws": Factory,
+		},
+	}
+	cluster := vault.NewTestCluster(t, coreConfig, &vault.TestClusterOptions{
+		NumCores:    1,
+		HandlerFunc: vaulthttp.Handler,
+	})
+	cluster.Start()
+
+	if len(cluster.Cores) != 1 {
+		t.Fatalf("expected exactly one core")
+	}
+
+	core := cluster.Cores[0]
+	vault.TestWaitActive(t, core.Core)
+
+	// load the auth plugin
+	if err := core.Client.Sys().EnableAuthWithOptions("aws", &api.EnableAuthOptions{
+		Type: "aws",
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	// fetch the aws backend's uuid in storage
+	authUuids, err := core.UnderlyingStorage.List(ctx, "auth/")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return cluster, "auth/" + authUuids[0]
+}
diff --git a/builtin/credential/aws/backend_test.go b/builtin/credential/aws/backend_test.go
index 8a6bb13286ee..4ce9265c6381 100644
--- a/builtin/credential/aws/backend_test.go
+++ b/builtin/credential/aws/backend_test.go
@@ -3,7 +3,6 @@ package awsauth
 import (
 	"context"
 	"encoding/base64"
-	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -15,16 +14,10 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/sts"
-	hclog "github.com/hashicorp/go-hclog"
-	"github.com/hashicorp/vault/api"
 	logicaltest "github.com/hashicorp/vault/helper/testhelpers/logical"
-	vaulthttp "github.com/hashicorp/vault/http"
 	"github.com/hashicorp/vault/sdk/framework"
-	"github.com/hashicorp/vault/sdk/helper/logging"
 	"github.com/hashicorp/vault/sdk/helper/policyutil"
 	"github.com/hashicorp/vault/sdk/logical"
-	"github.com/hashicorp/vault/sdk/physical"
-	"github.com/hashicorp/vault/vault"
 )
 
 const testVaultHeaderValue = "VaultAcceptanceTesting"
@@ -1820,112 +1813,3 @@ func generateRenewRequest(s logical.Storage, auth *logical.Auth) *logical.Reques
 
 	return renewReq
 }
-
-func TestBackend_E2E_Initialize(t *testing.T) {
-
-	// set up a test cluster
-	logger := logging.NewVaultLogger(hclog.Trace)
-	coreConfig := &vault.CoreConfig{
-		Logger: logger,
-		CredentialBackends: map[string]logical.Factory{
-			"aws": Factory,
-		},
-	}
-	cluster := vault.NewTestCluster(t, coreConfig, &vault.TestClusterOptions{
-		NumCores:    1,
-		HandlerFunc: vaulthttp.Handler,
-	})
-	cluster.Start()
-	defer cluster.Cleanup()
-
-	core := cluster.Cores[0]
-	vault.TestWaitActive(t, core.Core)
-
-	// load the auth plugin
-	if err := core.Client.Sys().EnableAuthWithOptions("aws", &api.EnableAuthOptions{
-		Type: "aws",
-	}); err != nil {
-		t.Fatal(err)
-	}
-
-	// create some role entries
-	type testRole struct {
-		name string
-		data map[string]interface{}
-	}
-	before := []testRole{
-		{
-			name: "test-role-0",
-			data: map[string]interface{}{
-				"auth_type":       "ec2",
-				"policies":        "default",
-				"bound_subnet_id": "subnet-abcdef",
-			},
-		},
-	}
-	for _, tr := range before {
-		_, err := core.Client.Logical().Write("auth/aws/role/"+tr.name, tr.data)
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	//			data: map[string]interface{}{
-	//			"bound_iam_principal_arn": os.Getenv(envVarAwsTestRoleArn)[:25] + "*",
-	//
-	//// put the entries in storage
-	//for _, role := range before {
-	//	//core.Client.Logical().Write("auth/aws/roles/foo", map[string]interface{}{...
-	//	core.Client.Logical().Write("auth/aws/roles/foo", nil)
-
-	//if _, err := core.Client.Logical().Write("auth/aws/role/test", ); err != nil {
-	//	fmt.Println(err)
-	//	t.Fatal(err)
-	//}
-
-	//	//err = b.setRole(ctx, storage, role.name, role.entry)
-	//	//if err != nil {
-	//	//	t.Fatal(err)
-	//	//}
-	//}
-
-	//---------------------------------------------------
-
-	// fetch the auth backend's uuid in storage
-	ctx := context.Background()
-	uuid, err := core.UnderlyingStorage.List(ctx, "auth/")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// "downgrade" some of the roles by directly modifying them in storage
-	for _, tr := range before {
-
-		key := "auth/" + uuid[0] + "role/" + tr.name
-		entry, err := core.UnderlyingStorage.Get(ctx, key)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		fmt.Printf("storage %T\n", core.UnderlyingStorage)
-		fmt.Printf("storage backend %T\n", core.UnderlyingStorage.(*physical.StorageEncoding).Backend)
-		fmt.Printf("entry %T\n", entry)
-		fmt.Printf("key %s\n", entry.Key)
-		fmt.Printf("value %s\n", hex.EncodeToString(entry.Value))
-
-		//roleEntry := new(awsRoleEntry)
-		//err = jsonutil.DecodeJSON(entry.Value, roleEntry)
-		//if err != nil {
-		//	t.Fatal(err)
-		//}
-		//if err := entry.DecodeJSON(role); err != nil {
-		//	t.Fatal(err)
-		//}
-	}
-
-	// re-trigger Initialize(), which will upgrade the roles
-
-	// read from storage to confirm that the roles are upgraded
-
-	// read from storage to confirm that the 'config/version' entry is there
-}

From b1603a341acad8d2351e12f7db99831a6e1606dc Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Thu, 4 Jul 2019 12:01:04 -0400
Subject: [PATCH 46/53] tinker with aws auth initialize e2e test

---
 builtin/credential/aws/backend_e2e_test.go | 36 ++++++++++++----------
 1 file changed, 20 insertions(+), 16 deletions(-)

diff --git a/builtin/credential/aws/backend_e2e_test.go b/builtin/credential/aws/backend_e2e_test.go
index f35f87b15634..7cd6860b5806 100644
--- a/builtin/credential/aws/backend_e2e_test.go
+++ b/builtin/credential/aws/backend_e2e_test.go
@@ -19,16 +19,26 @@ func TestBackend_E2E_Initialize(t *testing.T) {
 
 	// Set up the cluster.  This will trigger an Initialize(); we sleep briefly
 	// awaiting its completion.
-	cluster, awsStoragePath := setupAwsTestCluster(t, ctx)
+	cluster := setupAwsTestCluster(t, ctx)
 	defer cluster.Cleanup()
 	time.Sleep(time.Second)
 	core := cluster.Cores[0]
 
+	// fetch the auth aws auth's path in storage
+	authPaths, err := core.UnderlyingStorage.List(ctx, "auth/")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(authPaths) != 1 {
+		t.Fatalf("expected exactly one auth path")
+	}
+	awsPath := "auth/" + authPaths[0]
+
 	// Make sure that the upgrade happened, by fishing the 'config/version'
 	// entry out of storage.  We can't use core.Client.Logical().Read() to do
 	// this, because 'config/version' hasn't been exposed as a path.
 	// TODO: should we expose 'config/version' as a path?
-	version, err := core.UnderlyingStorage.Get(ctx, awsStoragePath+"config/version")
+	version, err := core.UnderlyingStorage.Get(ctx, awsPath+"config/version")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -37,10 +47,10 @@ func TestBackend_E2E_Initialize(t *testing.T) {
 	}
 
 	// Nuke the version, so we can pretend that Initialize() has never been run
-	if err := core.UnderlyingStorage.Delete(ctx, awsStoragePath+"config/version"); err != nil {
+	if err := core.UnderlyingStorage.Delete(ctx, awsPath+"config/version"); err != nil {
 		t.Fatal(err)
 	}
-	version, err = core.UnderlyingStorage.Get(ctx, awsStoragePath+"config/version")
+	version, err = core.UnderlyingStorage.Get(ctx, awsPath+"config/version")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -65,7 +75,7 @@ func TestBackend_E2E_Initialize(t *testing.T) {
 	}
 
 	// There should _still_ be no config version
-	version, err = core.UnderlyingStorage.Get(ctx, awsStoragePath+"config/version")
+	version, err = core.UnderlyingStorage.Get(ctx, awsPath+"config/version")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -80,7 +90,7 @@ func TestBackend_E2E_Initialize(t *testing.T) {
 	time.Sleep(time.Second)
 
 	// Now the config version should be there again
-	version, err = core.UnderlyingStorage.Get(ctx, awsStoragePath+"config/version")
+	version, err = core.UnderlyingStorage.Get(ctx, awsPath+"config/version")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -89,8 +99,9 @@ func TestBackend_E2E_Initialize(t *testing.T) {
 	}
 }
 
-func setupAwsTestCluster(t *testing.T, ctx context.Context) (*vault.TestCluster, string) {
+func setupAwsTestCluster(t *testing.T, ctx context.Context) *vault.TestCluster {
 
+	// create a cluster with the aws auth backend built-in
 	logger := logging.NewVaultLogger(hclog.Trace)
 	coreConfig := &vault.CoreConfig{
 		Logger: logger,
@@ -102,12 +113,11 @@ func setupAwsTestCluster(t *testing.T, ctx context.Context) (*vault.TestCluster,
 		NumCores:    1,
 		HandlerFunc: vaulthttp.Handler,
 	})
-	cluster.Start()
 
+	cluster.Start()
 	if len(cluster.Cores) != 1 {
 		t.Fatalf("expected exactly one core")
 	}
-
 	core := cluster.Cores[0]
 	vault.TestWaitActive(t, core.Core)
 
@@ -118,11 +128,5 @@ func setupAwsTestCluster(t *testing.T, ctx context.Context) (*vault.TestCluster,
 		t.Fatal(err)
 	}
 
-	// fetch the aws backend's uuid in storage
-	authUuids, err := core.UnderlyingStorage.List(ctx, "auth/")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	return cluster, "auth/" + authUuids[0]
+	return cluster
 }

From 9a13a4437ec3c357bd88587e00c15f9aa59050a9 Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Thu, 4 Jul 2019 12:06:55 -0400
Subject: [PATCH 47/53] tinker with aws auth initialize e2e test

---
 builtin/credential/aws/backend_e2e_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/builtin/credential/aws/backend_e2e_test.go b/builtin/credential/aws/backend_e2e_test.go
index 7cd6860b5806..d0dded9f9e00 100644
--- a/builtin/credential/aws/backend_e2e_test.go
+++ b/builtin/credential/aws/backend_e2e_test.go
@@ -83,8 +83,8 @@ func TestBackend_E2E_Initialize(t *testing.T) {
 		t.Fatalf("version found")
 	}
 
-	// Seal, and then Unseal. This will once again trigger an Initialize(), only this time
-	// there will be a role for it to upgrade.
+	// Seal, and then Unseal. This will once again trigger an Initialize(),
+	// only this time there will be a role for it to upgrade.
 	core.Seal(t)
 	cluster.UnsealCores(t)
 	time.Sleep(time.Second)

From 7a2044e862dc0ac16b0a9dfb969c35e8d34effdc Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Fri, 5 Jul 2019 08:08:19 -0400
Subject: [PATCH 48/53] tinker with aws auth initialize e2e test

---
 builtin/credential/aws/backend_e2e_test.go | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/builtin/credential/aws/backend_e2e_test.go b/builtin/credential/aws/backend_e2e_test.go
index d0dded9f9e00..6be9414e693c 100644
--- a/builtin/credential/aws/backend_e2e_test.go
+++ b/builtin/credential/aws/backend_e2e_test.go
@@ -24,15 +24,16 @@ func TestBackend_E2E_Initialize(t *testing.T) {
 	time.Sleep(time.Second)
 	core := cluster.Cores[0]
 
-	// fetch the auth aws auth's path in storage
-	authPaths, err := core.UnderlyingStorage.List(ctx, "auth/")
+	// Fetch the aws auth's path in storage.  This is a uuid that is different
+	// every time we run the test
+	authUuids, err := core.UnderlyingStorage.List(ctx, "auth/")
 	if err != nil {
 		t.Fatal(err)
 	}
-	if len(authPaths) != 1 {
+	if len(authUuids) != 1 {
 		t.Fatalf("expected exactly one auth path")
 	}
-	awsPath := "auth/" + authPaths[0]
+	awsPath := "auth/" + authUuids[0]
 
 	// Make sure that the upgrade happened, by fishing the 'config/version'
 	// entry out of storage.  We can't use core.Client.Logical().Read() to do
@@ -84,7 +85,7 @@ func TestBackend_E2E_Initialize(t *testing.T) {
 	}
 
 	// Seal, and then Unseal. This will once again trigger an Initialize(),
-	// only this time there will be a role for it to upgrade.
+	// only this time there will be a role present during the upgrade.
 	core.Seal(t)
 	cluster.UnsealCores(t)
 	time.Sleep(time.Second)

From 840aa890eaf774c9e72028d3986cb833e51c51fd Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Fri, 5 Jul 2019 13:32:11 -0400
Subject: [PATCH 49/53] fix typo in test suite

---
 builtin/credential/aws/backend_test.go | 38 +++++++++++++-------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/builtin/credential/aws/backend_test.go b/builtin/credential/aws/backend_test.go
index 4ce9265c6381..b8b947350b31 100644
--- a/builtin/credential/aws/backend_test.go
+++ b/builtin/credential/aws/backend_test.go
@@ -477,13 +477,13 @@ func TestBackend_ConfigClient(t *testing.T) {
 
 	stepCreate := logicaltest.TestStep{
 		Operation: logical.CreateOperation,
-		Path:      "config/core.Client",
+		Path:      "config/client",
 		Data:      data,
 	}
 
 	stepUpdate := logicaltest.TestStep{
 		Operation: logical.UpdateOperation,
-		Path:      "config/core.Client",
+		Path:      "config/client",
 		Data:      data,
 	}
 
@@ -492,7 +492,7 @@ func TestBackend_ConfigClient(t *testing.T) {
 	}
 	stepInvalidAccessKey := logicaltest.TestStep{
 		Operation: logical.UpdateOperation,
-		Path:      "config/core.Client",
+		Path:      "config/client",
 		Data:      data3,
 		ErrorOk:   true,
 	}
@@ -502,7 +502,7 @@ func TestBackend_ConfigClient(t *testing.T) {
 	}
 	stepInvalidSecretKey := logicaltest.TestStep{
 		Operation: logical.UpdateOperation,
-		Path:      "config/core.Client",
+		Path:      "config/client",
 		Data:      data4,
 		ErrorOk:   true,
 	}
@@ -521,23 +521,23 @@ func TestBackend_ConfigClient(t *testing.T) {
 	// test existence check returning false
 	checkFound, exists, err := b.HandleExistenceCheck(context.Background(), &logical.Request{
 		Operation: logical.CreateOperation,
-		Path:      "config/core.Client",
+		Path:      "config/client",
 		Storage:   storage,
 	})
 	if err != nil {
 		t.Fatal(err)
 	}
 	if !checkFound {
-		t.Fatal("existence check not found for path 'config/core.Client'")
+		t.Fatal("existence check not found for path 'config/client'")
 	}
 	if exists {
-		t.Fatal("existence check should have returned 'false' for 'config/core.Client'")
+		t.Fatal("existence check should have returned 'false' for 'config/client'")
 	}
 
 	// create an entry
 	configClientCreateRequest := &logical.Request{
 		Operation: logical.UpdateOperation,
-		Path:      "config/core.Client",
+		Path:      "config/client",
 		Data:      data,
 		Storage:   storage,
 	}
@@ -549,17 +549,17 @@ func TestBackend_ConfigClient(t *testing.T) {
 	//test existence check returning true
 	checkFound, exists, err = b.HandleExistenceCheck(context.Background(), &logical.Request{
 		Operation: logical.CreateOperation,
-		Path:      "config/core.Client",
+		Path:      "config/client",
 		Storage:   storage,
 	})
 	if err != nil {
 		t.Fatal(err)
 	}
 	if !checkFound {
-		t.Fatal("existence check not found for path 'config/core.Client'")
+		t.Fatal("existence check not found for path 'config/client'")
 	}
 	if !exists {
-		t.Fatal("existence check should have returned 'true' for 'config/core.Client'")
+		t.Fatal("existence check should have returned 'true' for 'config/client'")
 	}
 
 	endpointData := map[string]interface{}{
@@ -570,7 +570,7 @@ func TestBackend_ConfigClient(t *testing.T) {
 
 	endpointReq := &logical.Request{
 		Operation: logical.UpdateOperation,
-		Path:      "config/core.Client",
+		Path:      "config/client",
 		Storage:   storage,
 		Data:      endpointData,
 	}
@@ -1105,7 +1105,7 @@ func TestBackendAcc_LoginWithInstanceIdentityDocAndWhitelistIdentity(t *testing.
 		_, err = b.HandleRequest(context.Background(), &logical.Request{
 			Operation: logical.UpdateOperation,
 			Storage:   storage,
-			Path:      "config/core.Client",
+			Path:      "config/client",
 			Data:      clientConfig,
 		})
 		if err != nil {
@@ -1115,7 +1115,7 @@ func TestBackendAcc_LoginWithInstanceIdentityDocAndWhitelistIdentity(t *testing.
 
 	loginInput := map[string]interface{}{
 		"pkcs7": pkcs7,
-		"nonce": "vault-core.Client-nonce",
+		"nonce": "vault-client-nonce",
 	}
 
 	parsedIdentityDoc, err := b.parseIdentityDocument(context.Background(), storage, pkcs7)
@@ -1244,17 +1244,17 @@ func TestBackendAcc_LoginWithInstanceIdentityDocAndWhitelistIdentity(t *testing.
 
 	_, ok := resp.Auth.Metadata["nonce"]
 	if ok {
-		t.Fatalf("core.Client nonce should not have been returned")
+		t.Fatalf("client nonce should not have been returned")
 	}
 
-	loginInput["nonce"] = "changed-vault-core.Client-nonce"
+	loginInput["nonce"] = "changed-vault-client-nonce"
 	// try to login again with changed nonce
 	resp, err = b.HandleRequest(context.Background(), loginRequest)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if resp == nil || !resp.IsError() {
-		t.Fatalf("login attempt should have failed due to core.Client nonce mismatch")
+		t.Fatalf("login attempt should have failed due to client nonce mismatch")
 	}
 
 	// Check if a whitelist identity entry is created after the login.
@@ -1502,7 +1502,7 @@ func TestBackendAcc_LoginWithCallerIdentity(t *testing.T) {
 
 	// Test setup largely done
 	// At this point, we're going to:
-	// 1. Configure the core.Client to require our test header value
+	// 1. Configure the client to require our test header value
 	// 2. Configure identity to use the ARN for the alias
 	// 3. Configure two different roles:
 	//    a. One bound to our test user
@@ -1521,7 +1521,7 @@ func TestBackendAcc_LoginWithCallerIdentity(t *testing.T) {
 	}
 	clientRequest := &logical.Request{
 		Operation: logical.UpdateOperation,
-		Path:      "config/core.Client",
+		Path:      "config/client",
 		Storage:   storage,
 		Data:      clientConfigData,
 	}

From 9d2275c56a276a5910f1c4b815d1f991ed8c2a1e Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Fri, 5 Jul 2019 13:38:53 -0400
Subject: [PATCH 50/53] simplify logic a tad

---
 builtin/credential/aws/path_role.go | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index 8bcd28054c1c..48b839ac208a 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -406,16 +406,18 @@ func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error)
 				return false, err
 			}
 		}
+		fallthrough
+
 	case currentAwsVersion:
-		break
+
 	default:
 		return false, fmt.Errorf("unrecognized role version: %d", version.Version)
 	}
 
 	// save the current version
 	if upgraded {
-		cv := awsVersion{Version: currentAwsVersion}
-		entry, err = logical.StorageEntryJSON("config/version", &cv)
+		version.Version = currentAwsVersion
+		entry, err = logical.StorageEntryJSON("config/version", &version)
 		if err != nil {
 			return false, err
 		}

From fcdae78c5822c1a57f824e4201b29e6af95e2c7a Mon Sep 17 00:00:00 2001
From: Mike Jarmy <mjarmy@gmail.com>
Date: Fri, 5 Jul 2019 18:18:44 -0400
Subject: [PATCH 51/53] rearrange assignment

---
 builtin/credential/aws/path_role.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index 48b839ac208a..84514e6d90fc 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -409,6 +409,7 @@ func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error)
 		fallthrough
 
 	case currentAwsVersion:
+		version.Version = currentAwsVersion
 
 	default:
 		return false, fmt.Errorf("unrecognized role version: %d", version.Version)
@@ -416,7 +417,6 @@ func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error)
 
 	// save the current version
 	if upgraded {
-		version.Version = currentAwsVersion
 		entry, err = logical.StorageEntryJSON("config/version", &version)
 		if err != nil {
 			return false, err

From a8afc7453994f99c73a03e923a286ae93690b6ad Mon Sep 17 00:00:00 2001
From: Brian Kassouf <briankassouf@users.noreply.github.com>
Date: Fri, 5 Jul 2019 16:06:58 -0700
Subject: [PATCH 52/53] Fix a few lifecycle related issues in #7025 (#7075)

---
 builtin/credential/aws/backend.go   | 11 +++++++++++
 builtin/credential/aws/path_role.go |  8 +++++---
 sdk/plugin/grpc_backend_client.go   | 10 ++++++++++
 3 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/builtin/credential/aws/backend.go b/builtin/credential/aws/backend.go
index 4f4d254474f0..ce185dff445a 100644
--- a/builtin/credential/aws/backend.go
+++ b/builtin/credential/aws/backend.go
@@ -81,6 +81,10 @@ type backend struct {
 	roleCache *cache.Cache
 
 	resolveArnToUniqueIDFunc func(context.Context, logical.Storage, string) (string, error)
+
+	// upgradeCancelFunc is used to cancel the context used in the upgrade
+	// function
+	upgradeCancelFunc context.CancelFunc
 }
 
 func Backend(conf *logical.BackendConfig) (*backend, error) {
@@ -137,6 +141,7 @@ func Backend(conf *logical.BackendConfig) (*backend, error) {
 		Invalidate:     b.invalidate,
 		InitializeFunc: b.initialize,
 		BackendType:    logical.TypeCredential,
+		Clean:          b.cleanup,
 	}
 
 	return b, nil
@@ -206,6 +211,12 @@ func (b *backend) periodicFunc(ctx context.Context, req *logical.Request) error
 	return nil
 }
 
+func (b *backend) cleanup(ctx context.Context) {
+	if b.upgradeCancelFunc != nil {
+		b.upgradeCancelFunc()
+	}
+}
+
 func (b *backend) invalidate(ctx context.Context, key string) {
 	switch {
 	case key == "config/client":
diff --git a/builtin/credential/aws/path_role.go b/builtin/credential/aws/path_role.go
index 84514e6d90fc..04adaa4d7b3f 100644
--- a/builtin/credential/aws/path_role.go
+++ b/builtin/credential/aws/path_role.go
@@ -335,7 +335,10 @@ func (b *backend) initialize(ctx context.Context, req *logical.InitializationReq
 		s := req.Storage
 
 		logger := b.Logger().Named("initialize")
-		logger.Info("starting initialization")
+		logger.Debug("starting initialization")
+
+		var upgradeCtx context.Context
+		upgradeCtx, b.upgradeCancelFunc = context.WithCancel(context.Background())
 
 		go func() {
 			// The vault will become unsealed while this goroutine is running,
@@ -346,7 +349,7 @@ func (b *backend) initialize(ctx context.Context, req *logical.InitializationReq
 			b.roleMutex.Lock()
 			defer b.roleMutex.Unlock()
 
-			upgraded, err := b.upgrade(ctx, s)
+			upgraded, err := b.upgrade(upgradeCtx, s)
 			if err != nil {
 				logger.Error("error running initialization", "error", err)
 				return
@@ -373,7 +376,6 @@ const currentAwsVersion = 1
 
 // upgrade does an upgrade, if necessary
 func (b *backend) upgrade(ctx context.Context, s logical.Storage) (bool, error) {
-
 	entry, err := s.Get(ctx, "config/version")
 	if err != nil {
 		return false, err
diff --git a/sdk/plugin/grpc_backend_client.go b/sdk/plugin/grpc_backend_client.go
index 6cf3ea53e096..8e0acc1bbcff 100644
--- a/sdk/plugin/grpc_backend_client.go
+++ b/sdk/plugin/grpc_backend_client.go
@@ -7,6 +7,8 @@ import (
 	"sync/atomic"
 
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 
 	log "github.com/hashicorp/go-hclog"
 	plugin "github.com/hashicorp/go-plugin"
@@ -59,6 +61,14 @@ func (b *backendGRPCPluginClient) Initialize(ctx context.Context, _ *logical.Ini
 		if b.doneCtx.Err() != nil {
 			return ErrPluginShutdown
 		}
+
+		// If the plugin doesn't have Initialize implemented we should not fail
+		// the initalize call; otherwise this could halt startup of vault.
+		grpcStatus, ok := status.FromError(err)
+		if ok && grpcStatus.Code() == codes.Unimplemented {
+			return nil
+		}
+
 		return err
 	}
 	if reply.Err != nil {

From f50bf2f429d56bbf662559fa816d4bc1fcaeaaee Mon Sep 17 00:00:00 2001
From: Brian Kassouf <bkassouf@hashicorp.com>
Date: Fri, 5 Jul 2019 16:47:28 -0700
Subject: [PATCH 53/53] Fix panic when plugin fails to load

---
 vault/auth.go  | 5 +++++
 vault/mount.go | 5 +++++
 2 files changed, 10 insertions(+)

diff --git a/vault/auth.go b/vault/auth.go
index 29ccd0970554..a1ca19762f2e 100644
--- a/vault/auth.go
+++ b/vault/auth.go
@@ -693,6 +693,11 @@ func (c *Core) setupCredentials(ctx context.Context) error {
 		// Initialize
 		if !nilMount {
 			c.postUnsealFuncs = append(c.postUnsealFuncs, func() {
+				if backend == nil {
+					c.logger.Error("skipping initialization on nil backend", "path", entry.Path)
+					return
+				}
+
 				err := backend.Initialize(ctx, &logical.InitializationRequest{Storage: view})
 				if err != nil {
 					c.logger.Error("failed to initialize auth entry", "path", entry.Path, "error", err)
diff --git a/vault/mount.go b/vault/mount.go
index 39e133934451..e2604bdebc24 100644
--- a/vault/mount.go
+++ b/vault/mount.go
@@ -1145,6 +1145,11 @@ func (c *Core) setupMounts(ctx context.Context) error {
 		// Initialize
 		if !nilMount {
 			c.postUnsealFuncs = append(c.postUnsealFuncs, func() {
+				if backend == nil {
+					c.logger.Error("skipping initialization on nil backend", "path", entry.Path)
+					return
+				}
+
 				err := backend.Initialize(ctx, &logical.InitializationRequest{Storage: view})
 				if err != nil {
 					c.logger.Error("failed to initialize mount entry", "path", entry.Path, "error", err)