Sasha/cos2

Gopkg.toml

@@ -39,6 +39,10 @@ ignored = ["github.com/Sirupsen/logrus", "github.com/gravitational/license/gener
   name = "github.com/cenkalti/backoff"
   version = ">=1.0.0, <=19.0.0-g5d150e7"
 
+[[constraint]]
+  name = "github.com/johannesboyne/gofakes3"
+  branch = "master"
+
 [[constraint]]
   name = "github.com/armon/go-radix"
   branch = "master"

constants.go

@@ -327,6 +327,15 @@ const (
 	// Region is AWS region parameter
 	Region = "region"
 
+	// Endpoint is an optional Host for non-AWS S3
+	Endpoint = "endpoint"
+
+	// Insecure is an optional switch to use HTTP instead of HTTPS
+	Insecure = "insecure"
+
+	// DisableServerSideEncryption is an optional switch to opt out of SSE in case the provider does not support it
+	DisableServerSideEncryption = "disablesse"
+
 	// SchemeFile is a local disk file storage
 	SchemeFile = "file"
 

lib/backend/etcdbk/etcd.go

@@ -25,6 +25,7 @@ import (
 	"encoding/base64"
 	"io/ioutil"
 	"sort"
+	"strings"
 	"time"
 
 	"github.com/gravitational/teleport/lib/backend"
@@ -142,17 +143,30 @@ type EtcdBackend struct {
 
 // Config represents JSON config for etcd backend
 type Config struct {
-	Nodes       []string `json:"peers,omitempty"`
-	Key         string   `json:"prefix,omitempty"`
-	TLSKeyFile  string   `json:"tls_key_file,omitempty"`
-	TLSCertFile string   `json:"tls_cert_file,omitempty"`
-	TLSCAFile   string   `json:"tls_ca_file,omitempty"`
-	Insecure    bool     `json:"insecure,omitempty"`
+	// Nodes is a list of nodes
+	Nodes []string `json:"peers,omitempty"`
+	// Key is an optional prefix for etcd
+	Key string `json:"prefix,omitempty"`
+	// TLSKeyFile is a private key, implies mTLS client authentication
+	TLSKeyFile string `json:"tls_key_file,omitempty"`
+	// TLSCertFile is a client certificate implies mTLS client authentication
+	TLSCertFile string `json:"tls_cert_file,omitempty"`
+	// TLSCAFile is a trusted certificate authority certificate
+	TLSCAFile string `json:"tls_ca_file,omitempty"`
+	// Insecure turns off TLS
+	Insecure bool `json:"insecure,omitempty"`
 	// BufferSize is a default buffer size
 	// used to pull events
 	BufferSize int `json:"buffer_size,omitempty"`
 	// DialTimeout specifies dial timeout
 	DialTimeout time.Duration `json:"dial_timeout,omitempty"`
+	// Username is an optional username for HTTPS basic authentication
+	Username string `json:"username,omitempty"`
+	// Password is initalized from password file, and is not read from the config
+	Password string `json:"-"`
+	// PasswordFile is an optional password file for HTTPS basic authentication,
+	// expects path to a file
+	PasswordFile string `json:"password_file,omitempty"`
 }
 
 // GetName returns the name of etcd backend as it appears in 'storage/type' section
@@ -209,20 +223,14 @@ func New(ctx context.Context, params backend.Params) (*EtcdBackend, error) {
 // Validate checks if all the parameters are present/valid
 func (cfg *Config) Validate() error {
 	if len(cfg.Key) == 0 {
-		return trace.BadParameter(`etcd: missing "prefix" setting`)
+		return trace.BadParameter(`etcd: missing "prefix" parameter`)
 	}
 	if len(cfg.Nodes) == 0 {
-		return trace.BadParameter(`etcd: missing "peers" setting`)
+		return trace.BadParameter(`etcd: missing "peers" parameter`)
 	}
 	if cfg.Insecure == false {
-		if cfg.TLSKeyFile == "" {
-			return trace.BadParameter(`etcd: missing "tls_key_file" setting`)
-		}
-		if cfg.TLSCertFile == "" {
-			return trace.BadParameter(`etcd: missing "tls_cert_file" setting`)
-		}
 		if cfg.TLSCAFile == "" {
-			return trace.BadParameter(`etcd: missing "tls_ca_file" setting`)
+			return trace.BadParameter(`etcd: missing "tls_ca_file" parameter`)
 		}
 	}
 	if cfg.BufferSize == 0 {
@@ -231,6 +239,14 @@ func (cfg *Config) Validate() error {
 	if cfg.DialTimeout == 0 {
 		cfg.DialTimeout = defaults.DefaultDialTimeout
 	}
+	if cfg.PasswordFile != "" {
+		out, err := ioutil.ReadFile(cfg.PasswordFile)
+		if err != nil {
+			return trace.ConvertSystemError(err)
+		}
+		// trim newlines as passwords in files tend to have newlines
+		cfg.Password = strings.TrimSpace(string(out))
+	}
 	return nil
 }
 
@@ -251,38 +267,49 @@ func (b *EtcdBackend) CloseWatchers() {
 }
 
 func (b *EtcdBackend) reconnect() error {
-	clientCertPEM, err := ioutil.ReadFile(b.cfg.TLSCertFile)
-	if err != nil {
-		return trace.ConvertSystemError(err)
-	}
-	clientKeyPEM, err := ioutil.ReadFile(b.cfg.TLSKeyFile)
-	if err != nil {
-		return trace.ConvertSystemError(err)
-	}
-	caCertPEM, err := ioutil.ReadFile(b.cfg.TLSCAFile)
-	if err != nil {
-		return trace.ConvertSystemError(err)
-	}
 	tlsConfig := utils.TLSConfig(nil)
-	tlsCert, err := tls.X509KeyPair(clientCertPEM, clientKeyPEM)
-	if err != nil {
-		return trace.BadParameter("failed to parse private key: %v", err)
+
+	if b.cfg.TLSCertFile != "" {
+		clientCertPEM, err := ioutil.ReadFile(b.cfg.TLSCertFile)
+		if err != nil {
+			return trace.ConvertSystemError(err)
+		}
+		clientKeyPEM, err := ioutil.ReadFile(b.cfg.TLSKeyFile)
+		if err != nil {
+			return trace.ConvertSystemError(err)
+		}
+		tlsCert, err := tls.X509KeyPair(clientCertPEM, clientKeyPEM)
+		if err != nil {
+			return trace.BadParameter("failed to parse private key: %v", err)
+		}
+		tlsConfig.Certificates = []tls.Certificate{tlsCert}
+	}
+
+	var caCertPEM []byte
+	if b.cfg.TLSCAFile != "" {
+		var err error
+		caCertPEM, err = ioutil.ReadFile(b.cfg.TLSCAFile)
+		if err != nil {
+			return trace.ConvertSystemError(err)
+		}
 	}
+
 	certPool := x509.NewCertPool()
 	parsedCert, err := tlsca.ParseCertificatePEM(caCertPEM)
 	if err != nil {
 		return trace.Wrap(err, "failed to parse CA certificate")
 	}
 	certPool.AddCert(parsedCert)
 
-	tlsConfig.Certificates = []tls.Certificate{tlsCert}
 	tlsConfig.RootCAs = certPool
 	tlsConfig.ClientCAs = certPool
 
 	clt, err := clientv3.New(clientv3.Config{
 		Endpoints:   b.nodes,
 		TLS:         tlsConfig,
 		DialTimeout: b.cfg.DialTimeout,
+		Username:    b.cfg.Username,
+		Password:    b.cfg.Password,
 	})
 	if err != nil {
 		return trace.Wrap(err)

lib/events/s3sessions/s3handler.go

@@ -20,8 +20,10 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net/url"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -30,6 +32,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
 	awssession "github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
@@ -45,8 +48,45 @@ type Config struct {
 	Region string
 	// Path is an optional bucket path
 	Path string
+	// Host is an optional third party S3 compatible endpoint
+	Endpoint string
+	// Insecure is an optional switch to opt out of https connections
+	Insecure bool
+	//DisableServerSideEncryption is an optional switch to opt out of SSE in case the provider does not support it
+	DisableServerSideEncryption bool
 	// Session is an optional existing AWS client session
 	Session *awssession.Session
+	// Credentials if supplied are used in tests
+	Credentials *credentials.Credentials
+}
+
+// SetFromURL sets values on the Config from the supplied URI
+func (s *Config) SetFromURL(in *url.URL, inRegion string) error {
+	region := inRegion
+	if uriRegion := in.Query().Get(teleport.Region); uriRegion != "" {
+		region = uriRegion
+	}
+	if endpoint := in.Query().Get(teleport.Endpoint); endpoint != "" {
+		s.Endpoint = endpoint
+	}
+	if val := in.Query().Get(teleport.Insecure); val != "" {
+		insecure, err := strconv.ParseBool(val)
+		if err != nil {
+			return trace.BadParameter("failed to parse URI %q flag %q - %q, supported values are 'true' or 'false'", in.String(), teleport.Insecure, val)
+		}
+		s.Insecure = insecure
+	}
+	if val := in.Query().Get(teleport.DisableServerSideEncryption); val != "" {
+		disableServerSideEncryption, err := strconv.ParseBool(val)
+		if err != nil {
+			return trace.BadParameter("failed to parse URI %q flag %q - %q, supported values are 'true' or 'false'", in.String(), teleport.DisableServerSideEncryption, val)
+		}
+		s.DisableServerSideEncryption = disableServerSideEncryption
+	}
+	s.Region = region
+	s.Bucket = in.Host
+	s.Path = in.Path
+	return nil
 }
 
 // CheckAndSetDefaults checks and sets defaults
@@ -63,11 +103,21 @@ func (s *Config) CheckAndSetDefaults() error {
 		if err != nil {
 			return trace.Wrap(err)
 		}
-		// override the default environment (region + credentials) with the values
+		// override the default environment (region + Host + credentials) with the values
 		// from the YAML file:
 		if s.Region != "" {
 			sess.Config.Region = aws.String(s.Region)
 		}
+		if s.Endpoint != "" {
+			sess.Config.Endpoint = aws.String(s.Endpoint)
+			sess.Config.S3ForcePathStyle = aws.Bool(true)
+		}
+		if s.Insecure {
+			sess.Config.DisableSSL = aws.Bool(s.Insecure)
+		}
+		if s.Credentials != nil {
+			sess.Config.Credentials = s.Credentials
+		}
 		s.Session = sess
 	}
 	return nil
@@ -271,19 +321,21 @@ func (h *Handler) ensureBucket() error {
 	}
 
 	// Turn on server-side encryption for the bucket.
-	_, err = h.client.PutBucketEncryption(&s3.PutBucketEncryptionInput{
-		Bucket: aws.String(h.Bucket),
-		ServerSideEncryptionConfiguration: &s3.ServerSideEncryptionConfiguration{
-			Rules: []*s3.ServerSideEncryptionRule{&s3.ServerSideEncryptionRule{
-				ApplyServerSideEncryptionByDefault: &s3.ServerSideEncryptionByDefault{
-					SSEAlgorithm: aws.String(s3.ServerSideEncryptionAwsKms),
-				},
-			}},
-		},
-	})
-	err = ConvertS3Error(err, "failed to set versioning state for bucket %q", h.Bucket)
-	if err != nil {
-		return trace.Wrap(err)
+	if !h.DisableServerSideEncryption {
+		_, err = h.client.PutBucketEncryption(&s3.PutBucketEncryptionInput{
+			Bucket: aws.String(h.Bucket),
+			ServerSideEncryptionConfiguration: &s3.ServerSideEncryptionConfiguration{
+				Rules: []*s3.ServerSideEncryptionRule{&s3.ServerSideEncryptionRule{
+					ApplyServerSideEncryptionByDefault: &s3.ServerSideEncryptionByDefault{
+						SSEAlgorithm: aws.String(s3.ServerSideEncryptionAwsKms),
+					},
+				}},
+			},
+		})
+		err = ConvertS3Error(err, "failed to set versioning state for bucket %q", h.Bucket)
+		if err != nil {
+			return trace.Wrap(err)
+		}
 	}
 	return nil
 }