SSCT Tokens Feature [OSS] (#14109)

* port SSCT OSS

* port header hmac key to ent and generate token proto without make command

* remove extra nil check in request handling

* add changelog

* add comment to router.go

* change test var to use length constants

* remove local index is 0 check and extra defer which can be removed after use of ExternalID

Makefile

@@ -194,6 +194,7 @@ proto: bootstrap
 	protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative sdk/database/dbplugin/*.proto
 	protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative sdk/database/dbplugin/v5/proto/*.proto
 	protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative sdk/plugin/pb/*.proto
+	protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative vault/tokens/token.proto
 	protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative sdk/helper/pluginutil/*.proto
 
 	# No additional sed expressions should be added to this list. Going forward

changelog/14109.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+Server Side Consistent Tokens: Service tokens now use SSC token format and token prefixes are updated."
+```

command/commands.go

@@ -81,6 +81,10 @@ const (
 	// path to a license file on disk
 	EnvVaultLicensePath = "VAULT_LICENSE_PATH"
 
+	// DisableSSCTokens is an env var used to disable index bearing
+	// token functionality
+	DisableSSCTokens = "VAULT_DISABLE_SERVER_SIDE_CONSISTENT_TOKENS"
+
 	// flagNameAddress is the flag used in the base command to read in the
 	// address of the Vault server.
 	flagNameAddress = "address"

command/login_test.go

@@ -13,6 +13,11 @@ import (
 	"github.com/hashicorp/vault/vault"
 )
 
+// minTokenLengthExternal is the minimum size of SSC
+// tokens we are currently handing out to end users, without any
+// namespace information
+const minTokenLengthExternal = 91
+
 func testLoginCommand(tb testing.TB) (*cli.MockUi, *LoginCommand) {
 	tb.Helper()
 
@@ -82,7 +87,7 @@ func TestLoginCommand_Run(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		if l, exp := len(storedToken), vault.TokenLength+2; l != exp {
+		if l, exp := len(storedToken), minTokenLengthExternal+vault.TokenPrefixLength; l != exp {
 			t.Errorf("expected token to be %d characters, was %d: %q", exp, l, storedToken)
 		}
 	})
@@ -209,7 +214,7 @@ func TestLoginCommand_Run(t *testing.T) {
 
 		// Verify only the token was printed
 		token := ui.OutputWriter.String()
-		if l, exp := len(token), vault.TokenLength+2; l != exp {
+		if l, exp := len(token), minTokenLengthExternal+vault.TokenPrefixLength; l != exp {
 			t.Errorf("expected token to be %d characters, was %d: %q", exp, l, token)
 		}
 

command/operator_generate_root_test.go

@@ -435,7 +435,7 @@ func TestOperatorGenerateRootCommand_Run(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		if l, exp := len(token), vault.TokenLength+2; l != exp {
+		if l, exp := len(token), vault.TokenLength+vault.TokenPrefixLength; l != exp {
 			t.Errorf("expected %d to be %d: %s", l, exp, token)
 		}
 	})
@@ -521,7 +521,7 @@ func TestOperatorGenerateRootCommand_Run(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		if l, exp := len(token), vault.TokenLength+2; l != exp {
+		if l, exp := len(token), vault.TokenLength+vault.TokenPrefixLength; l != exp {
 			t.Errorf("expected %d to be %d: %s", l, exp, token)
 		}
 	})

command/operator_init_test.go

@@ -333,7 +333,7 @@ func TestOperatorInitCommand_Run(t *testing.T) {
 		root := match[0][1]
 		decryptedRoot := testPGPDecrypt(t, pgpkeys.TestPrivKey1, root)
 
-		if l, exp := len(decryptedRoot), vault.TokenLength+2; l != exp {
+		if l, exp := len(decryptedRoot), vault.TokenLength+vault.TokenPrefixLength; l != exp {
 			t.Errorf("expected %d to be %d", l, exp)
 		}
 	})

command/server.go

@@ -1163,6 +1163,15 @@ func (c *ServerCommand) Run(args []string) int {
 	if envLicense := os.Getenv(EnvVaultLicense); envLicense != "" {
 		config.License = envLicense
 	}
+	if disableSSC := os.Getenv(DisableSSCTokens); disableSSC != "" {
+		var err error
+		config.DisableSSCTokens, err = strconv.ParseBool(disableSSC)
+		if err != nil {
+			c.UI.Warn(wrapAtLength("WARNING! failed to parse " +
+				"VAULT_DISABLE_SERVER_SIDE_CONSISTENT_TOKENS env var: " +
+				"setting to default value false"))
+		}
+	}
 
 	// If mlockall(2) isn't supported, show a warning. We disable this in dev
 	// because it is quite scary to see when first using Vault. We also disable
@@ -2502,6 +2511,7 @@ func createCoreConfig(c *ServerCommand, config *server.Config, backend physical.
 		EnableResponseHeaderRaftNodeID: config.EnableResponseHeaderRaftNodeID,
 		License:                        config.License,
 		LicensePath:                    config.LicensePath,
+		DisableSSCTokens:               config.DisableSSCTokens,
 	}
 	if c.flagDev {
 		coreConfig.EnableRaw = true

command/server/config.go

@@ -83,8 +83,9 @@ type Config struct {
 	EnableResponseHeaderRaftNodeID    bool        `hcl:"-"`
 	EnableResponseHeaderRaftNodeIDRaw interface{} `hcl:"enable_response_header_raft_node_id"`
 
-	License     string `hcl:"-"`
-	LicensePath string `hcl:"license_path"`
+	License          string `hcl:"-"`
+	LicensePath      string `hcl:"license_path"`
+	DisableSSCTokens bool   `hcl:"-"`
 }
 
 const (

helper/namespace/namespace.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"errors"
 	"strings"
+
+	"github.com/hashicorp/vault/sdk/helper/consts"
 )
 
 type contextValues struct{}
@@ -105,6 +107,12 @@ func SplitIDFromString(input string) (string, string) {
 	case strings.HasPrefix(input, "s."):
 		prefix = "s."
 		input = input[2:]
+	case strings.HasPrefix(input, consts.BatchTokenPrefix):
+		prefix = consts.BatchTokenPrefix
+		input = input[4:]
+	case strings.HasPrefix(input, consts.ServiceTokenPrefix):
+		prefix = consts.ServiceTokenPrefix
+		input = input[4:]
 
 	case slashIdx > 0:
 		// Leases will never have a b./s. to start

http/sys_generate_root.go

@@ -64,14 +64,20 @@ func handleSysGenerateRootAttemptGet(core *vault.Core, w http.ResponseWriter, r
 		respondError(w, http.StatusInternalServerError, err)
 		return
 	}
+	var otpLength int
+	if core.DisableSSCTokens() {
+		otpLength = vault.TokenLength + vault.OldTokenPrefixLength
+	} else {
+		otpLength = vault.TokenLength + vault.TokenPrefixLength
+	}
 
 	// Format the status
 	status := &GenerateRootStatusResponse{
 		Started:   false,
 		Progress:  progress,
 		Required:  sealConfig.SecretThreshold,
 		Complete:  false,
-		OTPLength: vault.TokenLength + 2,
+		OTPLength: otpLength,
 		OTP:       otp,
 	}
 	if generationConfig != nil {
@@ -98,7 +104,11 @@ func handleSysGenerateRootAttemptPut(core *vault.Core, w http.ResponseWriter, r
 	case len(req.PGPKey) > 0, len(req.OTP) > 0:
 	default:
 		genned = true
-		req.OTP, err = base62.Random(vault.TokenLength + 2)
+		if core.DisableSSCTokens() {
+			req.OTP, err = base62.Random(vault.TokenLength + vault.OldTokenPrefixLength)
+		} else {
+			req.OTP, err = base62.Random(vault.TokenLength + vault.TokenPrefixLength)
+		}
 		if err != nil {
 			respondError(w, http.StatusInternalServerError, err)
 			return

http/sys_generate_root_test.go

@@ -19,6 +19,8 @@ import (
 	"github.com/hashicorp/vault/vault"
 )
 
+var tokenLength string = fmt.Sprintf("%d", vault.TokenLength+vault.TokenPrefixLength)
+
 func TestSysGenerateRootAttempt_Status(t *testing.T) {
 	core, _, token := vault.TestCoreUnsealed(t)
 	ln, addr := TestServer(t, core)
@@ -40,7 +42,7 @@ func TestSysGenerateRootAttempt_Status(t *testing.T) {
 		"encoded_root_token": "",
 		"pgp_fingerprint":    "",
 		"nonce":              "",
-		"otp_length":         json.Number("26"),
+		"otp_length":         json.Number(tokenLength),
 	}
 	testResponseStatus(t, resp, 200)
 	testResponseBody(t, resp, &actual)
@@ -68,7 +70,7 @@ func TestSysGenerateRootAttempt_Setup_OTP(t *testing.T) {
 		"encoded_token":      "",
 		"encoded_root_token": "",
 		"pgp_fingerprint":    "",
-		"otp_length":         json.Number("26"),
+		"otp_length":         json.Number(tokenLength),
 	}
 	testResponseStatus(t, resp, 200)
 	testResponseBody(t, resp, &actual)
@@ -93,7 +95,7 @@ func TestSysGenerateRootAttempt_Setup_OTP(t *testing.T) {
 		"encoded_root_token": "",
 		"pgp_fingerprint":    "",
 		"otp":                "",
-		"otp_length":         json.Number("26"),
+		"otp_length":         json.Number(tokenLength),
 	}
 	testResponseStatus(t, resp, 200)
 	testResponseBody(t, resp, &actual)
@@ -129,7 +131,7 @@ func TestSysGenerateRootAttempt_Setup_PGP(t *testing.T) {
 		"encoded_root_token": "",
 		"pgp_fingerprint":    "816938b8a29146fbe245dd29e7cbaf8e011db793",
 		"otp":                "",
-		"otp_length":         json.Number("26"),
+		"otp_length":         json.Number(tokenLength),
 	}
 	testResponseStatus(t, resp, 200)
 	testResponseBody(t, resp, &actual)
@@ -159,7 +161,7 @@ func TestSysGenerateRootAttempt_Cancel(t *testing.T) {
 		"encoded_token":      "",
 		"encoded_root_token": "",
 		"pgp_fingerprint":    "",
-		"otp_length":         json.Number("26"),
+		"otp_length":         json.Number(tokenLength),
 	}
 	testResponseStatus(t, resp, 200)
 	testResponseBody(t, resp, &actual)
@@ -191,7 +193,7 @@ func TestSysGenerateRootAttempt_Cancel(t *testing.T) {
 		"pgp_fingerprint":    "",
 		"nonce":              "",
 		"otp":                "",
-		"otp_length":         json.Number("26"),
+		"otp_length":         json.Number(tokenLength),
 	}
 	testResponseStatus(t, resp, 200)
 	testResponseBody(t, resp, &actual)

sdk/helper/consts/token_consts.go

@@ -0,0 +1,10 @@
+package consts
+
+const (
+	ServiceTokenPrefix        = "hvs."
+	BatchTokenPrefix          = "hvb."
+	RecoveryTokenPrefix       = "hvr."
+	LegacyServiceTokenPrefix  = "s."
+	LegacyBatchTokenPrefix    = "b."
+	LegacyRecoveryTokenPrefix = "r."
+)

sdk/logical/request.go

@@ -220,6 +220,10 @@ type Request struct {
 	// this will be the sha256(sorted policies + namespace) associated with the
 	// client token.
 	ClientID string `json:"client_id" structs:"client_id" mapstructure:"client_id" sentinel:""`
+
+	// InboundSSCToken is the token that arrives on an inbound request, supplied
+	// by the vault user.
+	InboundSSCToken string
 }
 
 // Clone returns a deep copy of the request by using copystructure

sdk/logical/token.go

@@ -93,6 +93,10 @@ type TokenEntry struct {
 	// ID of this entry, generally a random UUID
 	ID string `json:"id" mapstructure:"id" structs:"id" sentinel:""`
 
+	// ExternalID is the ID of a newly created service
+	// token that will be returned to a user
+	ExternalID string `json:"-"`
+
 	// Accessor for this token, a random UUID
 	Accessor string `json:"accessor" mapstructure:"accessor" structs:"accessor" sentinel:""`
 

vault/audit_broker.go

@@ -91,6 +91,15 @@ func (a *AuditBroker) LogRequest(ctx context.Context, in *logical.LogInput, head
 	defer metrics.MeasureSince([]string{"audit", "log_request"}, time.Now())
 	a.RLock()
 	defer a.RUnlock()
+	if in.Request.InboundSSCToken != "" {
+		if in.Auth != nil {
+			reqAuthToken := in.Auth.ClientToken
+			in.Auth.ClientToken = in.Request.InboundSSCToken
+			defer func() {
+				in.Auth.ClientToken = reqAuthToken
+			}()
+		}
+	}
 
 	var retErr *multierror.Error
 
@@ -153,6 +162,15 @@ func (a *AuditBroker) LogResponse(ctx context.Context, in *logical.LogInput, hea
 	defer metrics.MeasureSince([]string{"audit", "log_response"}, time.Now())
 	a.RLock()
 	defer a.RUnlock()
+	if in.Request.InboundSSCToken != "" {
+		if in.Auth != nil {
+			reqAuthToken := in.Auth.ClientToken
+			in.Auth.ClientToken = in.Request.InboundSSCToken
+			defer func() {
+				in.Auth.ClientToken = reqAuthToken
+			}()
+		}
+	}
 
 	var retErr *multierror.Error
 

vault/core.go

@@ -74,6 +74,11 @@ const (
 	coreKeyringCanaryPath = "core/canary-keyring"
 
 	indexHeaderHMACKeyPath = "core/index-header-hmac-key"
+
+	// ForwardSSCTokenToActive is the value that must be set in the
+	// forwardToActive to trigger forwarding if a perf standby encounters
+	// an SSC Token that it does not have the WAL state for.
+	ForwardSSCTokenToActive = "new_token"
 )
 
 var (
@@ -576,6 +581,9 @@ type Core struct {
 	enableResponseHeaderHostname   bool
 	enableResponseHeaderRaftNodeID bool
 
+	// disableSSCTokens is used to disable server side consistent token creation/usage
+	disableSSCTokens bool
+
 	// versionTimestamps is a map of vault versions to timestamps when the version
 	// was first run. Note that because perf standbys should be upgraded first, and
 	// only the active node will actually write the new version timestamp, a perf
@@ -702,6 +710,9 @@ type CoreConfig struct {
 	// Whether to send headers in the HTTP response showing hostname or raft node ID
 	EnableResponseHeaderHostname   bool
 	EnableResponseHeaderRaftNodeID bool
+
+	// DisableSSCTokens is used to disable the use of server side consistent tokens
+	DisableSSCTokens bool
 }
 
 // GetServiceRegistration returns the config's ServiceRegistration, or nil if it does
@@ -844,6 +855,7 @@ func CreateCore(conf *CoreConfig) (*Core, error) {
 		disableAutopilot:               conf.DisableAutopilot,
 		enableResponseHeaderHostname:   conf.EnableResponseHeaderHostname,
 		enableResponseHeaderRaftNodeID: conf.EnableResponseHeaderRaftNodeID,
+		disableSSCTokens:               conf.DisableSSCTokens,
 	}
 	c.standbyStopCh.Store(make(chan struct{}))
 	atomic.StoreUint32(c.sealed, 1)
@@ -1098,6 +1110,11 @@ func (c *Core) RaftNodeIDHeaderEnabled() bool {
 	return c.enableResponseHeaderRaftNodeID
 }
 
+// DisableSSCTokens determines whether to use server side consistent tokens or not.
+func (c *Core) DisableSSCTokens() bool {
+	return c.disableSSCTokens
+}
+
 // Shutdown is invoked when the Vault instance is about to be terminated. It
 // should not be accessible as part of an API call as it will cause an availability
 // problem. It is only used to gracefully quit in the case of HA so that failover
@@ -2064,6 +2081,9 @@ func (s standardUnsealStrategy) unseal(ctx context.Context, logger log.Logger, c
 	if err := c.setupQuotas(ctx, false); err != nil {
 		return err
 	}
+	if err := c.setupHeaderHMACKey(ctx, false); err != nil {
+		return err
+	}
 	if !c.IsDRSecondary() {
 		if err := c.startRollback(); err != nil {
 			return err