Skip to content

Commit

Permalink
Configure server side TLS via Vault if enabled (#6052)
Browse files Browse the repository at this point in the history 
* Read certs from vault for server TLS config

* Update CHANGELOG.md

* Refactor to allow easier testing

* Update formatting

* Lint modules_test.go

* Return mock from init if set in config
  • Loading branch information
@fayzal-g
fayzal-g committed Sep 22, 2023
1 parent f611c71 commit f39c938
Show file tree
Hide file tree
Showing 4 changed files with 178 additions and 2 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
* [ENHANCEMENT] Distributor: add experimental support for storing metadata when ingesting metrics via OTLP. This makes metrics description and type available when ingesting metrics via OTLP. Enable with `-distributor.enable-otlp-metadata-storage=true`. #5693 #6035
* [ENHANCEMENT] Ingester: added support for sampling errors, which can be enabled by setting `-ingester.error-sample-rate`. This way each error will be logged once in the configured number of times. All the discarded samples will still be tracked by the `cortex_discarded_samples_total` metric. #5584 #6014
* [ENHANCEMENT] Ruler: Fetch secrets used to configure TLS on the Alertmanager client from Vault when `-vault.enabled` is true. #5239
* [ENHANCEMENT] Fetch secrets used to configure server-side TLS from Vault when `-vault.enabled` is true. #6052.
* [BUGFIX] Query-frontend: Don't retry read requests rejected by the ingester due to utilization based read path limiting. #6032

### Mixin
Expand Down
48 changes: 46 additions & 2 deletions pkg/mimir/modules.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/opentracing/opentracing-go"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/config"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/rules"
Expand Down Expand Up @@ -176,11 +177,11 @@ func (t *Mimir) initVault() (services.Service, error) {
return nil, nil
}

vault, err := vault.NewVault(t.Cfg.Vault)
v, err := vault.NewVault(t.Cfg.Vault)
if err != nil {
return nil, err
}
t.Vault = vault
t.Vault = v

// Update Configs - KVStore
t.Cfg.MemberlistKV.TCPTransport.TLS.Reader = t.Vault
Expand Down Expand Up @@ -210,6 +211,49 @@ func (t *Mimir) initVault() (services.Service, error) {
t.Cfg.Alertmanager.AlertmanagerClient.GRPCClientConfig.TLS.Reader = t.Vault
t.Cfg.QueryScheduler.GRPCClientConfig.TLS.Reader = t.Vault

// Update the Server
updateServerTLSCfgFunc := func(vault *vault.Vault, tlsConfig *server.TLSConfig) error {
cert, err := vault.ReadSecret(tlsConfig.TLSCertPath)
if err != nil {
return err
}
tlsConfig.TLSCert = string(cert)
tlsConfig.TLSCertPath = ""

key, err := vault.ReadSecret(tlsConfig.TLSKeyPath)
if err != nil {
return err
}
tlsConfig.TLSKey = config.Secret(key)
tlsConfig.TLSKeyPath = ""

var ca []byte
if tlsConfig.ClientCAs != "" {
ca, err = vault.ReadSecret(tlsConfig.ClientCAs)
if err != nil {
return err
}
tlsConfig.ClientCAsText = string(ca)
tlsConfig.ClientCAs = ""
}

return nil
}

if len(t.Cfg.Server.HTTPTLSConfig.TLSCertPath) > 0 && len(t.Cfg.Server.HTTPTLSConfig.TLSKeyPath) > 0 {
err := updateServerTLSCfgFunc(t.Vault, &t.Cfg.Server.HTTPTLSConfig)
if err != nil {
return nil, err
}
}

if len(t.Cfg.Server.GRPCTLSConfig.TLSCertPath) > 0 && len(t.Cfg.Server.GRPCTLSConfig.TLSKeyPath) > 0 {
err := updateServerTLSCfgFunc(t.Vault, &t.Cfg.Server.GRPCTLSConfig)
if err != nil {
return nil, err
}
}

return nil, nil
}

Expand Down
123 changes: 123 additions & 0 deletions pkg/mimir/modules_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
package mimir

import (
"context"
"net/http/httptest"
"os"
"path/filepath"
Expand All @@ -14,9 +15,13 @@ import (
"github.com/gorilla/mux"
"github.com/grafana/dskit/flagext"
"github.com/grafana/dskit/server"
hashivault "github.com/hashicorp/vault/api"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/grafana/mimir/pkg/vault"
)

func changeTargetConfig(c *Config) {
Expand Down Expand Up @@ -242,3 +247,121 @@ func TestMultiKVSetup(t *testing.T) {
})
}
}

type mockKVStore struct {
values map[string]mockValue
}

type mockValue struct {
secret *hashivault.KVSecret
err error
}

func newMockKVStore() *mockKVStore {
return &mockKVStore{
values: map[string]mockValue{
"test/secret1": {
secret: &hashivault.KVSecret{
Data: map[string]interface{}{
"value": "foo1",
},
},
err: nil,
},
"test/secret2": {
secret: &hashivault.KVSecret{
Data: map[string]interface{}{
"value": "foo2",
},
},
err: nil,
},
"test/secret3": {
secret: &hashivault.KVSecret{
Data: map[string]interface{}{
"value": "foo3",
},
},
err: nil,
},
},
}
}

func (m *mockKVStore) Get(_ context.Context, path string) (*hashivault.KVSecret, error) {
return m.values[path].secret, m.values[path].err
}

func TestInitVault(t *testing.T) {
cfg := Config{
Server: server.Config{
HTTPTLSConfig: server.TLSConfig{
TLSCertPath: "test/secret1",
TLSKeyPath: "test/secret2",
ClientCAs: "test/secret3",
},
GRPCTLSConfig: server.TLSConfig{
TLSCertPath: "test/secret1",
TLSKeyPath: "test/secret2",
ClientCAs: "test/secret3",
},
},
Vault: vault.Config{
Enabled: true,
Mock: newMockKVStore(),
},
}

mimir := &Mimir{
Server: &server.Server{Registerer: prometheus.NewPedanticRegistry()},
Cfg: cfg,
}

_, err := mimir.initVault()
require.NoError(t, err)

// Check KVStore
require.NotNil(t, mimir.Cfg.MemberlistKV.TCPTransport.TLS.Reader)
require.NotNil(t, mimir.Cfg.Distributor.HATrackerConfig.KVStore.StoreConfig.Etcd.TLS.Reader)
require.NotNil(t, mimir.Cfg.Alertmanager.ShardingRing.Common.KVStore.StoreConfig.Etcd.TLS.Reader)
require.NotNil(t, mimir.Cfg.Compactor.ShardingRing.Common.KVStore.StoreConfig.Etcd.TLS.Reader)
require.NotNil(t, mimir.Cfg.Distributor.DistributorRing.Common.KVStore.StoreConfig.Etcd.TLS.Reader)
require.NotNil(t, mimir.Cfg.Ingester.IngesterRing.KVStore.StoreConfig.Etcd.TLS.Reader)
require.NotNil(t, mimir.Cfg.Ruler.Ring.Common.KVStore.StoreConfig.Etcd.TLS.Reader)
require.NotNil(t, mimir.Cfg.StoreGateway.ShardingRing.KVStore.StoreConfig.Etcd.TLS.Reader)
require.NotNil(t, mimir.Cfg.QueryScheduler.ServiceDiscovery.SchedulerRing.KVStore.StoreConfig.Etcd.TLS.Reader)
require.NotNil(t, mimir.Cfg.OverridesExporter.Ring.Common.KVStore.StoreConfig.Etcd.TLS.Reader)

// Check Redis Clients
require.NotNil(t, mimir.Cfg.BlocksStorage.BucketStore.IndexCache.BackendConfig.Redis.TLS.Reader)
require.NotNil(t, mimir.Cfg.BlocksStorage.BucketStore.ChunksCache.BackendConfig.Redis.TLS.Reader)
require.NotNil(t, mimir.Cfg.BlocksStorage.BucketStore.MetadataCache.BackendConfig.Redis.TLS.Reader)
require.NotNil(t, mimir.Cfg.Frontend.QueryMiddleware.ResultsCacheConfig.BackendConfig.Redis.TLS.Reader)

// Check GRPC Clients
require.NotNil(t, mimir.Cfg.IngesterClient.GRPCClientConfig.TLS.Reader)
require.NotNil(t, mimir.Cfg.Worker.GRPCClientConfig.TLS.Reader)
require.NotNil(t, mimir.Cfg.Querier.StoreGatewayClient.TLS.Reader)
require.NotNil(t, mimir.Cfg.Frontend.FrontendV2.GRPCClientConfig.TLS.Reader)
require.NotNil(t, mimir.Cfg.Ruler.ClientTLSConfig.TLS.Reader)
require.NotNil(t, mimir.Cfg.Ruler.Notifier.TLS.Reader)
require.NotNil(t, mimir.Cfg.Alertmanager.AlertmanagerClient.GRPCClientConfig.TLS.Reader)
require.NotNil(t, mimir.Cfg.QueryScheduler.GRPCClientConfig.TLS.Reader)

// Check Server
require.Empty(t, mimir.Cfg.Server.HTTPTLSConfig.TLSCertPath)
require.Empty(t, mimir.Cfg.Server.HTTPTLSConfig.TLSKeyPath)
require.Empty(t, mimir.Cfg.Server.HTTPTLSConfig.ClientCAs)

require.Empty(t, mimir.Cfg.Server.GRPCTLSConfig.TLSCertPath)
require.Empty(t, mimir.Cfg.Server.GRPCTLSConfig.TLSKeyPath)
require.Empty(t, mimir.Cfg.Server.HTTPTLSConfig.ClientCAs)

require.Equal(t, "foo1", mimir.Cfg.Server.HTTPTLSConfig.TLSCert)
require.Equal(t, config.Secret("foo2"), mimir.Cfg.Server.HTTPTLSConfig.TLSKey)
require.Equal(t, "foo3", mimir.Cfg.Server.HTTPTLSConfig.ClientCAsText)

require.Equal(t, "foo1", mimir.Cfg.Server.GRPCTLSConfig.TLSCert)
require.Equal(t, config.Secret("foo2"), mimir.Cfg.Server.GRPCTLSConfig.TLSKey)
require.Equal(t, "foo3", mimir.Cfg.Server.GRPCTLSConfig.ClientCAsText)
}
8 changes: 8 additions & 0 deletions pkg/vault/vault.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ type Config struct {
URL string `yaml:"url" category:"experimental"`
Token string `yaml:"token" category:"experimental"`
MountPath string `yaml:"mount_path" category:"experimental"`

Mock SecretsEngine `yaml:"-"`
}

func (cfg *Config) Validate() error {
Expand Down Expand Up @@ -56,6 +58,12 @@ type Vault struct {
}

func NewVault(cfg Config) (*Vault, error) {
if cfg.Mock != nil {
return &Vault{
KVStore: cfg.Mock,
}, nil
}

config := hashivault.DefaultConfig()
config.Address = cfg.URL

Expand Down

0 comments on commit f39c938

Please sign in to comment.