Merge branch 'master' into alex/configurable-http-server-timeouts

.circleci/config.yml

@@ -36,6 +36,33 @@ jobs:
           paths:
             - ui/node_modules
 
+  go-mod-vendor:
+    docker:
+      - image: *GOLANG_IMAGE
+    working_directory: /go/src/github.com/hashicorp/vault
+    steps:
+      - checkout
+      - restore_cache:
+          key: go-vendor-modules-v1-{{ checksum "vendor/modules.txt" }}
+      - run:
+          name: Fix git url config
+          command: git config --local url."git@github.com:".insteadof https://github.com/
+      - run:
+          name: Check go mod vendor
+          command: |
+            GO111MODULE=on go mod vendor
+            out=$(git status vendor --porcelain)
+            if [ "$out" != "" ] ; then
+              echo "'go mod vendor' was not clean! Please check go modules for updates (notably api and sdk)"
+              echo "output was:"
+              echo "$out"
+              exit 1
+            fi
+      - save_cache:
+          key: go-vendor-modules-v1-{{ checksum "vendor/modules.txt" }}
+          paths:
+            - /root/.cache/go-build
+
   build-go-dev:
     docker:
       - image: *GOLANG_IMAGE
@@ -163,7 +190,10 @@ workflows:
   ci:
     jobs:
       - install-ui-dependencies
-      - build-go-dev
+      - go-mod-vendor
+      - build-go-dev:
+          requires:
+            - go-mod-vendor
       - test-ui:
           requires:
             - install-ui-dependencies

CHANGELOG.md

@@ -5,9 +5,22 @@ CHANGES:
  * autoseal/aws: The user-configured regions on the AWSKMS seal stanza 
    will now be preferred over regions set in the enclosing environment.
    This is a _breaking_ change.
+ * audit: Several more values in audit logs now are omitted if they are empty.
+   This helps reduce the size of audit log entries by not reproducing keys in
+   each entry that commonly don't contain any value, which can help in cases
+   where audit log entries are above the maximum UDP packet size and others.
+   See [GH-6387](https://github.com/hashicorp/vault/pull/6387) for details.
+ * Go Modules change: Vault now uses Go Modules to manage dependencies. As a
+   result to both reduce transitive dependencies for API library users and
+   plugin authors, and to work around various conflicts, we have moved various
+   helpers around, mostly under an `sdk/` submodule. A couple of functions have
+   also moved from plugin helper code to the `api/` submodule. If you are a
+   plugin author, take a look at some of our official plugins and the paths
+   they are importing for guidance.
 
 IMPROVEMENTS: 
 
+ * auth/jwt: A JWKS endpoint may now be configured for signature verification [JWT-43]
  * ui: KV v1 and v2 will now gracefully degrade allowing a write without read
    workflow in the UI [GH-6570]
 
@@ -21,6 +34,7 @@ BUG FIXES:
  * pki: fix a panic when a client submits a null value [GH-5679]
  * replication: Fix an issue causing startup problems if a namespace policy
    wasn't replicated properly
+ * storage/consul: recognize `https://` address even if schema not specified [GH-6602]
  * storage/dynamodb: Fix an issue where a deleted lock key in DynamoDB (HA) could cause
    constant switching of the active node [GH-6637]
  * storage/dynamodb: Eliminate a high-CPU condition that could occur if an error was

README.md

@@ -63,9 +63,11 @@ first need [Go](https://www.golang.org) installed on your machine (version
 1.12.1+ is *required*).
 
 For local dev first make sure Go is properly installed, including setting up a
-[GOPATH](https://golang.org/doc/code.html#GOPATH). Next, clone this repository
-into `$GOPATH/src/github.com/hashicorp/vault`. You can then download any
-required build tools by bootstrapping your environment:
+[GOPATH](https://golang.org/doc/code.html#GOPATH). Ensure that `$GOPATH/bin` is in
+your path as some distributions bundle old version of build tools. Next, clone this
+repository. Vault uses [Go Modules](https://github.com/golang/go/wiki/Modules),
+so it is recommended that you clone the repository ***outside*** of the GOPATH.
+You can then download any required build tools by bootstrapping your environment:
 
 ```sh
 $ make bootstrap

api/plugin_helpers.go

@@ -12,7 +12,16 @@ import (
 	squarejwt "gopkg.in/square/go-jose.v2/jwt"
 
 	"github.com/hashicorp/errwrap"
-	"github.com/hashicorp/vault/sdk/helper/pluginutil"
+)
+
+var (
+	// PluginMetadataModeEnv is an ENV name used to disable TLS communication
+	// to bootstrap mounting plugins.
+	PluginMetadataModeEnv = "VAULT_PLUGIN_METADATA_MODE"
+
+	// PluginUnwrapTokenEnv is the ENV name used to pass unwrap tokens to the
+	// plugin.
+	PluginUnwrapTokenEnv = "VAULT_UNWRAP_TOKEN"
 )
 
 // PluginAPIClientMeta is a helper that plugins can use to configure TLS connections
@@ -61,12 +70,12 @@ func (f *PluginAPIClientMeta) GetTLSConfig() *TLSConfig {
 // VaultPluginTLSProvider is run inside a plugin and retrieves the response
 // wrapped TLS certificate from vault. It returns a configured TLS Config.
 func VaultPluginTLSProvider(apiTLSConfig *TLSConfig) func() (*tls.Config, error) {
-	if os.Getenv(pluginutil.PluginMetadataModeEnv) == "true" {
+	if os.Getenv(PluginMetadataModeEnv) == "true" {
 		return nil
 	}
 
 	return func() (*tls.Config, error) {
-		unwrapToken := os.Getenv(pluginutil.PluginUnwrapTokenEnv)
+		unwrapToken := os.Getenv(PluginUnwrapTokenEnv)
 
 		parsedJWT, err := squarejwt.ParseSigned(unwrapToken)
 		if err != nil {

builtin/logical/pki/backend_test.go

@@ -432,7 +432,7 @@ func checkCertsAndPrivateKey(keyType string, key crypto.Signer, usage x509.KeyUs
 }
 
 func generateURLSteps(t *testing.T, caCert, caKey string, intdata, reqdata map[string]interface{}) []logicaltest.TestStep {
-	expected := urlEntries{
+	expected := certutil.URLEntries{
 		IssuingCertificates: []string{
 			"http://example.com/ca1",
 			"http://example.com/ca2",
@@ -499,7 +499,7 @@ func generateURLSteps(t *testing.T, caCert, caKey string, intdata, reqdata map[s
 				if resp.Data == nil {
 					return fmt.Errorf("no data returned")
 				}
-				var entries urlEntries
+				var entries certutil.URLEntries
 				err := mapstructure.Decode(resp.Data, &entries)
 				if err != nil {
 					return err
@@ -855,7 +855,7 @@ func generateRoleSteps(t *testing.T, useCSRs bool) []logicaltest.TestStep {
 			}
 			cert := parsedCertBundle.Certificate
 
-			expected := strutil.RemoveDuplicates(role.OU, true)
+			expected := strutil.RemoveDuplicatesStable(role.OU, true)
 			if !reflect.DeepEqual(cert.Subject.OrganizationalUnit, expected) {
 				return fmt.Errorf("error: returned certificate has OU of %s but %s was specified in the role", cert.Subject.OrganizationalUnit, expected)
 			}

builtin/logical/pki/ca_util.go

@@ -4,6 +4,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/vault/sdk/framework"
+	"github.com/hashicorp/vault/sdk/helper/certutil"
 	"github.com/hashicorp/vault/sdk/logical"
 )
 
@@ -53,7 +54,9 @@ func (b *backend) getGenerationParams(
 		return
 	}
 
-	errorResp = validateKeyTypeLength(role.KeyType, role.KeyBits)
+	if err := certutil.ValidateKeyTypeLength(role.KeyType, role.KeyBits); err != nil {
+		errorResp = logical.ErrorResponse(err.Error())
+	}
 
 	return
 }