Add HashiCorp Vault support (#655)

* feat: initial adding of vualt transit backend to sops initial work on integration feat(vault): added cli coomands working for vualt" fix(vault): fixed config with correct tests fix(vault): added vault to keygroup and to keyservice server fixed metadata load * feat(docs): added docs in README.md and in command help fix(doc): fix rst formatting" fix(doc): fix rst formatting * fix(vault): addressed typos and fixes from autrilla feat(cli): moved vault to hc-vault naming * fix(test): typo while rebasing * fix typos and imporve error messages for vault kms * rename package from vault to hcvault * refactor vault keysource url validation * add negative test cases for vault keysource * add hc vault transit config option via objects additional to URIs * remove vault_example.yml * streamline key name to snake case * rename `BackendPath` to `EnginePath` for hc vault * correction in hc-vault-transit commands Signed-off-by: vnzongzna <github@vaibhavk.in> * resolving conflict Signed-off-by: vnzongzna <github@vaibhavk.in> * Apply suggestions from code review Co-Authored-By: Adrian Utrilla <adrianutrilla@gmail.com> * allowing only hc_vault_transit_uri as input Co-Authored-By: gitirabassi Co-Authored-By: ldue Signed-off-by: vnzongzna <github@vaibhavk.in> Co-authored-by: gitirabassi <giacomo@tirabassi.eu> Co-authored-by: ldue <larsduennwald@gmail.com> Co-authored-by: Vaibhav Kaushik <vaibhavkaushik@vaibhavka-ltm1.internal.salesforce.com> Co-authored-by: Adrian Utrilla <adrianutrilla@gmail.com>
getsops · May 4, 2020 · e4abd87 · e4abd87
1 parent 8f93ee3
commit e4abd87
Show file tree

Hide file tree

Showing 15 changed files with 1,186 additions and 248 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,5 @@
 target
 Cargo.lock
 vendor/
+coverage.txt
+profile.out
diff --git a/README.rst b/README.rst
@@ -287,6 +287,66 @@ And decrypt it using::
 	 $ sops --decrypt test.enc.yaml
 
 
+Encrypting using Hashicorp Vault
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We assume you have an instance (or more) of Vault running and you have privileged access to it. For instructions on how to deploy a secure instance of Vault, refer to Hashicorp's official documentation.
+
+To easily deploy Vault locally: (DO NOT DO THIS FOR PRODUCTION!!!) 
+
+.. code:: bash
+
+	$ docker run -d -p8200:8200 vault:1.2.0 server -dev -dev-root-token-id=toor
+
+
+.. code:: bash
+
+	$ # Substitute this with the address Vault is running on
+	$ export VAULT_ADDR=http://127.0.0.1:8200 
+
+	$ # this may not be necessary in case you previously used `vault login` for production use
+	$ export VAULT_TOKEN=toor 
+	
+	$ # to check if Vault started and is configured correctly
+	$ vault status
+	Key             Value
+	---             -----
+	Seal Type       shamir
+	Initialized     true
+	Sealed          false
+	Total Shares    1
+	Threshold       1
+	Version         1.2.0
+	Cluster Name    vault-cluster-618cc902
+	Cluster ID      e532e461-e8f0-1352-8a41-fc7c11096908
+	HA Enabled      false
+
+	$ # It is required to enable a transit engine if not already done (It is suggested to create a transit engine specifically for sops, in which it is possible to have multiple keys with various permission levels)
+	$ vault secrets enable -path=sops transit
+	Success! Enabled the transit secrets engine at: sops/
+
+	$ # Then create one or more keys
+	$ vault write sops/keys/firstkey type=rsa-4096
+	Success! Data written to: sops/keys/firstkey
+
+	$ vault write sops/keys/secondkey type=rsa-2048
+	Success! Data written to: sops/keys/secondkey
+
+	$ vault write sops/keys/thirdkey type=chacha20-poly1305
+	Success! Data written to: sops/keys/thirdkey
+
+	$ sops --hc-vault-transit $VAULT_ADDR/v1/sops/keys/firstkey vault_example.yml
+
+	$ cat <<EOF > .sops.yaml
+	creation_rules:
+		- path_regex: \.dev\.yaml$
+		  hc_vault_transit_uri: "$VAULT_ADDR/v1/sops/keys/secondkey"
+		- path_regex: \.prod\.yaml$
+		  hc_vault_transit_uri: "$VAULT_ADDR/v1/sops/keys/thirdkey"
+	EOF
+
+	$ sops --verbose -e prod/raw.yaml > prod/encrypted.yaml
+
 Adding and removing keys
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -543,6 +603,7 @@ can manage the three sets of configurations for the three types of files:
 		- path_regex: \.prod\.yaml$
 		  kms: 'arn:aws:kms:us-west-2:361527076523:key/5052f06a-5d3f-489e-b86c-57201e06f31e+arn:aws:iam::361527076523:role/hiera-sops-prod,arn:aws:kms:eu-central-1:361527076523:key/cb1fab90-8d17-42a1-a9d8-334968904f94+arn:aws:iam::361527076523:role/hiera-sops-prod'
 		  pgp: 'FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4'
+		  hc_vault_uris: "http://localhost:8200/v1/sops/keys/thirdkey"
 
 		# gcp files using GCP KMS
 		- path_regex: \.gcp\.yaml$

diff --git a/cmd/sops/main.go b/cmd/sops/main.go
@@ -27,6 +27,7 @@ import (
 	"go.mozilla.org/sops/v3/cmd/sops/subcommand/updatekeys"
 	"go.mozilla.org/sops/v3/config"
 	"go.mozilla.org/sops/v3/gcpkms"
+	"go.mozilla.org/sops/v3/hcvault"
 	"go.mozilla.org/sops/v3/keys"
 	"go.mozilla.org/sops/v3/keyservice"
 	"go.mozilla.org/sops/v3/kms"
@@ -80,6 +81,14 @@ func main() {
    (you need to setup google application default credentials. See
     https://developers.google.com/identity/protocols/application-default-credentials)
 
+
+   To encrypt or decrypt a document with HashiCorp Vault's Transit Secret Engine, specify the
+   Vault key URI name in the --hc-vault-transit flag or in the SOPS_VAULT_URIS environment variable (eg. https://vault.example.org:8200/v1/transit/keys/dev
+      where 'https://vault.example.org:8200' is the vault server, 'transit' the enginePath, and 'dev' is the name of the key )
+   environment variable.
+   (you need to enable the Transit Secrets Engine in Vault. See
+      https://www.vaultproject.io/docs/secrets/transit/index.html)
+
    To encrypt or decrypt a document with Azure Key Vault, specify the
    Azure Key Vault key URL in the --azure-kv flag or in the SOPS_AZURE_KEYVAULT_URL
    environment variable.
@@ -93,11 +102,11 @@ func main() {
    To use multiple KMS or PGP keys, separate them by commas. For example:
        $ sops -p "10F2...0A, 85D...B3F21" file.yaml
 
-   The -p, -k, --gcp-kms and --azure-kv flags are only used to encrypt new documents. Editing
+   The -p, -k, --gcp-kms, --hc-vault-transit and --azure-kv flags are only used to encrypt new documents. Editing
    or decrypting existing documents can be done with "sops file" or
    "sops -d file" respectively. The KMS and PGP keys listed in the encrypted
    documents are used then. To manage master keys in existing documents, use
-   the "add-{kms,pgp,gcp-kms,azure-kv}" and "rm-{kms,pgp,gcp-kms,azure-kv}" flags.
+   the "add-{kms,pgp,gcp-kms,azure-kv,hc-vault-transit}" and "rm-{kms,pgp,gcp-kms,azure-kv,hc-vault-transit}" flags.
 
    To use a different GPG binary than the one in your PATH, set SOPS_GPG_EXEC.
    To use a GPG key server other than gpg.mozilla.org, set SOPS_GPG_KEYSERVER.
@@ -357,6 +366,10 @@ func main() {
 							Name:  "azure-kv",
 							Usage: "the Azure Key Vault key URL the new group should contain. Can be specified more than once",
 						},
+						cli.StringSliceFlag{
+							Name:  "hc-vault-transit",
+							Usage: "the full vault path to the key used to encrypt/decrypt. Make you choose and configure a key with encrption/decryption enabled (e.g. 'https://vault.example.org:8200/v1/transit/keys/dev'). Can be specified more than once",
+						},
 						cli.BoolFlag{
 							Name:  "in-place, i",
 							Usage: "write output back to the same file instead of stdout",
@@ -374,6 +387,7 @@ func main() {
 						pgpFps := c.StringSlice("pgp")
 						kmsArns := c.StringSlice("kms")
 						gcpKmses := c.StringSlice("gcp-kms")
+						vaultURIs := c.StringSlice("hc-vault-transit")
 						azkvs := c.StringSlice("azure-kv")
 						var group sops.KeyGroup
 						for _, fp := range pgpFps {
@@ -385,6 +399,14 @@ func main() {
 						for _, kms := range gcpKmses {
 							group = append(group, gcpkms.NewMasterKeyFromResourceID(kms))
 						}
+						for _, uri := range vaultURIs {
+							k, err := hcvault.NewMasterKeyFromURI(uri)
+							if err != nil {
+								log.WithError(err).Error("Failed to add key")
+								continue
+							}
+							group = append(group, k)
+						}
 						for _, url := range azkvs {
 							k, err := azkv.NewMasterKeyFromURL(url)
 							if err != nil {
@@ -507,6 +529,11 @@ func main() {
 			Usage:  "comma separated list of Azure Key Vault URLs",
 			EnvVar: "SOPS_AZURE_KEYVAULT_URLS",
 		},
+		cli.StringFlag{
+			Name:   "hc-vault-transit",
+			Usage:  "comma separated list of vault's key URI (e.g. 'https://vault.example.org:8200/v1/transit/keys/dev')",
+			EnvVar: "SOPS_VAULT_URIS",
+		},
 		cli.StringFlag{
 			Name:   "pgp, p",
 			Usage:  "comma separated list of PGP fingerprints",
@@ -556,6 +583,14 @@ func main() {
 			Name:  "rm-kms",
 			Usage: "remove the provided comma-separated list of KMS ARNs from the list of master keys on the given file",
 		},
+		cli.StringFlag{
+			Name:  "add-hc-vault-transit",
+			Usage: "add the provided comma-separated list of Vault's URI key to the list of master keys on the given file ( eg. https://vault.example.org:8200/v1/transit/keys/dev)",
+		},
+		cli.StringFlag{
+			Name:  "rm-hc-vault-transit",
+			Usage: "remove the provided comma-separated list of Vault's URI key from the list of master keys on the given file ( eg. https://vault.example.org:8200/v1/transit/keys/dev)",
+		},
 		cli.StringFlag{
 			Name:  "add-pgp",
 			Usage: "add the provided comma-separated list of PGP fingerprints to the list of master keys on the given file",
@@ -621,8 +656,8 @@ func main() {
 			return toExitError(err)
 		}
 		if _, err := os.Stat(fileName); os.IsNotExist(err) {
-			if c.String("add-kms") != "" || c.String("add-pgp") != "" || c.String("add-gcp-kms") != "" || c.String("add-azure-kv") != "" ||
-				c.String("rm-kms") != "" || c.String("rm-pgp") != "" || c.String("rm-gcp-kms") != "" || c.String("rm-azure-kv") != "" {
+			if c.String("add-kms") != "" || c.String("add-pgp") != "" || c.String("add-gcp-kms") != "" || c.String("add-hc-vault-transit") != "" || c.String("add-azure-kv") != "" ||
+				c.String("rm-kms") != "" || c.String("rm-pgp") != "" || c.String("rm-gcp-kms") != "" || c.String("rm-hc-vault-transit") != "" || c.String("rm-azure-kv") != "" {
 				return common.NewExitError("Error: cannot add or remove keys on non-existent files, use `--kms` and `--pgp` instead.", codes.CannotChangeKeysFromNonExistentFile)
 			}
 			if c.Bool("encrypt") || c.Bool("decrypt") || c.Bool("rotate") {
@@ -735,6 +770,13 @@ func main() {
 			for _, k := range azureKeys {
 				addMasterKeys = append(addMasterKeys, k)
 			}
+			hcVaultKeys, err := hcvault.NewMasterKeysFromURIs(c.String("add-hc-vault-transit"))
+			if err != nil {
+				return err
+			}
+			for _, k := range hcVaultKeys {
+				addMasterKeys = append(addMasterKeys, k)
+			}
 
 			var rmMasterKeys []keys.MasterKey
 			for _, k := range kms.MasterKeysFromArnString(c.String("rm-kms"), kmsEncryptionContext, c.String("aws-profile")) {
@@ -753,6 +795,14 @@ func main() {
 			for _, k := range azureKeys {
 				rmMasterKeys = append(rmMasterKeys, k)
 			}
+			hcVaultKeys, err = hcvault.NewMasterKeysFromURIs(c.String("rm-hc-vault-transit"))
+			if err != nil {
+				return err
+			}
+			for _, k := range hcVaultKeys {
+				rmMasterKeys = append(rmMasterKeys, k)
+			}
+
 			output, err = rotate(rotateOpts{
 				OutputStore:      outputStore,
 				InputStore:       inputStore,
@@ -946,6 +996,7 @@ func keyGroups(c *cli.Context, file string) ([]sops.KeyGroup, error) {
 	var pgpKeys []keys.MasterKey
 	var cloudKmsKeys []keys.MasterKey
 	var azkvKeys []keys.MasterKey
+	var hcVaultMkKeys []keys.MasterKey
 	kmsEncryptionContext := kms.ParseKMSContext(c.String("encryption-context"))
 	if c.String("encryption-context") != "" && kmsEncryptionContext == nil {
 		return nil, common.NewExitError("Invalid KMS encryption context format", codes.ErrorInvalidKMSEncryptionContextFormat)
@@ -969,12 +1020,21 @@ func keyGroups(c *cli.Context, file string) ([]sops.KeyGroup, error) {
 			azkvKeys = append(azkvKeys, k)
 		}
 	}
+	if c.String("hc-vault-transit") != "" {
+		hcVaultKeys, err := hcvault.NewMasterKeysFromURIs(c.String("hc-vault-transit"))
+		if err != nil {
+			return nil, err
+		}
+		for _, k := range hcVaultKeys {
+			hcVaultMkKeys = append(hcVaultMkKeys, k)
+		}
+	}
 	if c.String("pgp") != "" {
 		for _, k := range pgp.MasterKeysFromFingerprintString(c.String("pgp")) {
 			pgpKeys = append(pgpKeys, k)
 		}
 	}
-	if c.String("kms") == "" && c.String("pgp") == "" && c.String("gcp-kms") == "" && c.String("azure-kv") == "" {
+	if c.String("kms") == "" && c.String("pgp") == "" && c.String("gcp-kms") == "" && c.String("azure-kv") == "" && c.String("hc-vault-transit") == "" {
 		conf, err := loadConfig(c, file, kmsEncryptionContext)
 		// config file might just not be supplied, without any error
 		if conf == nil {
@@ -991,6 +1051,8 @@ func keyGroups(c *cli.Context, file string) ([]sops.KeyGroup, error) {
 	group = append(group, cloudKmsKeys...)
 	group = append(group, azkvKeys...)
 	group = append(group, pgpKeys...)
+	group = append(group, hcVaultMkKeys...)
+	log.Debugf("Master keys available:  %+v", group)
 	return []sops.KeyGroup{group}, nil
 }
 

diff --git a/config/config.go b/config/config.go
@@ -15,6 +15,7 @@ import (
 	"go.mozilla.org/sops/v3"
 	"go.mozilla.org/sops/v3/azkv"
 	"go.mozilla.org/sops/v3/gcpkms"
+	"go.mozilla.org/sops/v3/hcvault"
 	"go.mozilla.org/sops/v3/kms"
 	"go.mozilla.org/sops/v3/logging"
 	"go.mozilla.org/sops/v3/pgp"
@@ -69,6 +70,7 @@ type keyGroup struct {
 	KMS     []kmsKey
 	GCPKMS  []gcpKmsKey  `yaml:"gcp_kms"`
 	AzureKV []azureKVKey `yaml:"azure_keyvault"`
+	Vault   []string     `yaml:"hc_vault"`
 	PGP     []string
 }
 
@@ -110,6 +112,7 @@ type creationRule struct {
 	PGP               string
 	GCPKMS            string     `yaml:"gcp_kms"`
 	AzureKeyVault     string     `yaml:"azure_keyvault"`
+	VaultURI          string     `yaml:"hc_vault_transit_uri"`
 	KeyGroups         []keyGroup `yaml:"key_groups"`
 	ShamirThreshold   int        `yaml:"shamir_threshold"`
 	UnencryptedSuffix string     `yaml:"unencrypted_suffix"`
@@ -154,6 +157,13 @@ func getKeyGroupsFromCreationRule(cRule *creationRule, kmsEncryptionContext map[
 			for _, k := range group.AzureKV {
 				keyGroup = append(keyGroup, azkv.NewMasterKey(k.VaultURL, k.Key, k.Version))
 			}
+			for _, k := range group.Vault {
+				if masterKey, err := hcvault.NewMasterKeyFromURI(k); err == nil {
+					keyGroup = append(keyGroup, masterKey)
+				} else {
+					return nil, err
+				}
+			}
 			groups = append(groups, keyGroup)
 		}
 	} else {
@@ -174,6 +184,13 @@ func getKeyGroupsFromCreationRule(cRule *creationRule, kmsEncryptionContext map[
 		for _, k := range azureKeys {
 			keyGroup = append(keyGroup, k)
 		}
+		vaultKeys, err := hcvault.NewMasterKeysFromURIs(cRule.VaultURI)
+		if err != nil {
+			return nil, err
+		}
+		for _, k := range vaultKeys {
+			keyGroup = append(keyGroup, k)
+		}
 		groups = append(groups, keyGroup)
 	}
 	return groups, nil
@@ -250,7 +267,7 @@ func parseDestinationRuleForFile(conf *configFile, filePath string, kmsEncryptio
 	var dest publish.Destination
 	if dRule != nil {
 		if dRule.S3Bucket != "" && dRule.GCSBucket != "" && dRule.VaultPath != "" {
-			return nil, fmt.Errorf("error loading config: more than one destinations were found in a single destination rule, you can only use one per rule.")
+			return nil, fmt.Errorf("error loading config: more than one destinations were found in a single destination rule, you can only use one per rule")
 		}
 		if dRule.S3Bucket != "" {
 			dest = publish.NewS3Destination(dRule.S3Bucket, dRule.S3Prefix)