From 67ac979d4abec27f1b15c87189887531124fe92f Mon Sep 17 00:00:00 2001 From: Annanay Date: Thu, 9 Apr 2020 22:30:55 +0530 Subject: [PATCH 01/29] Checkpoint Signed-off-by: Annanay --- go.mod | 2 + go.sum | 3 - .../weaveworks/common/server/server.go | 67 ++++++++++++++++--- vendor/modules.txt | 2 +- 4 files changed, 62 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 027c77625f..66308542f0 100644 --- a/go.mod +++ b/go.mod @@ -71,3 +71,5 @@ replace git.apache.org/thrift.git => github.com/apache/thrift v0.0.0-20180902110 // Override reference that causes an error from Go proxy - see https://github.com/golang/go/issues/33558 replace k8s.io/client-go => k8s.io/client-go v0.0.0-20190620085101-78d2af792bab + +replace github.com/weaveworks/common => /Users/annanay/Desktop/git/go/src/github.com/weaveworks/common diff --git a/go.sum b/go.sum index e33fcfc275..67443afa5e 100644 --- a/go.sum +++ b/go.sum @@ -786,9 +786,6 @@ github.com/uber/jaeger-lib v1.5.1-0.20181102163054-1fc5c315e03c/go.mod h1:ComeND github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/weaveworks/common v0.0.0-20200206153930-760e36ae819a/go.mod h1:6enWAqfQBFrE8X/XdJwZr8IKgh1chStuFR0mjU/UOUw= -github.com/weaveworks/common v0.0.0-20200310113808-2708ba4e60a4 h1:H1CjeKf1q/bL7OBvb6KZclHsvnGRGr0Tsuy6y5rtFEc= -github.com/weaveworks/common v0.0.0-20200310113808-2708ba4e60a4/go.mod h1:6enWAqfQBFrE8X/XdJwZr8IKgh1chStuFR0mjU/UOUw= github.com/weaveworks/promrus v1.2.0 h1:jOLf6pe6/vss4qGHjXmGz4oDJQA+AOCqEL3FvvZGz7M= github.com/weaveworks/promrus v1.2.0/go.mod h1:SaE82+OJ91yqjrE1rsvBWVzNZKcHYFtMUyS1+Ogs/KA= github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs= diff --git a/vendor/github.com/weaveworks/common/server/server.go b/vendor/github.com/weaveworks/common/server/server.go index 748fa8f2c1..c1b584f64d 100644 --- a/vendor/github.com/weaveworks/common/server/server.go +++ b/vendor/github.com/weaveworks/common/server/server.go @@ -1,8 +1,11 @@ package server import ( + "crypto/tls" + "crypto/x509" "flag" "fmt" + "io/ioutil" "math" "net" "net/http" @@ -18,6 +21,7 @@ import ( "golang.org/x/net/context" "golang.org/x/net/netutil" "google.golang.org/grpc" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/keepalive" "github.com/weaveworks/common/httpgrpc" @@ -34,6 +38,9 @@ type Config struct { HTTPListenAddress string `yaml:"http_listen_address"` HTTPListenPort int `yaml:"http_listen_port"` HTTPConnLimit int `yaml:"http_listen_conn_limit"` + HTTPCertPath string `yaml:"http_cert_path"` + HTTPKeyPath string `yaml:"http_key_path"` + HTTPCAPath string `yaml:"http_ca_path"` GRPCListenAddress string `yaml:"grpc_listen_address"` GRPCListenPort int `yaml:"grpc_listen_port"` GRPCConnLimit int `yaml:"grpc_listen_conn_limit"` @@ -71,6 +78,9 @@ var infinty = time.Duration(math.MaxInt64) // RegisterFlags adds the flags required to config this to the given FlagSet func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.StringVar(&cfg.HTTPListenAddress, "server.http-listen-address", "", "HTTP server listen address.") + f.StringVar(&cfg.HTTPCertPath, "server.http-cert-path", "", "HTTP server cert path.") + f.StringVar(&cfg.HTTPKeyPath, "server.http-key-path", "", "HTTP server key path.") + f.StringVar(&cfg.HTTPKeyPath, "server.http-ca-path", "", "HTTP CA path.") f.IntVar(&cfg.HTTPListenPort, "server.http-listen-port", 80, "HTTP server listen port.") f.IntVar(&cfg.HTTPConnLimit, "server.http-conn-limit", 0, "Maximum number of simultaneous http connections, <=0 to disable") f.StringVar(&cfg.GRPCListenAddress, "server.grpc-listen-address", "", "gRPC server listen address.") @@ -128,6 +138,42 @@ func New(cfg Config) (*Server, error) { grpcListener = netutil.LimitListener(grpcListener, cfg.GRPCConnLimit) } + // If user doesn't supply a logging implementation, by default instantiate + // logrus. + log := cfg.Log + if log == nil { + log = logging.NewLogrus(cfg.LogLevel) + } + + // Setup TLS + var cert tls.Certificate + if cfg.HTTPCertPath != "" && cfg.HTTPKeyPath != "" { + cert, err = tls.LoadX509KeyPair(cfg.HTTPCertPath, cfg.HTTPKeyPath) + if err != nil { + log.Warnf("error loading cert %s or key %s, tls disabled", cfg.HTTPCertPath, cfg.HTTPKeyPath) + } + } + + var caCertPool *x509.CertPool + if cfg.HTTPCAPath != "" { + caCert, err := ioutil.ReadFile(cfg.HTTPCAPath) + if err != nil { + log.Warnf("error loading ca cert %s, tls disabled", cfg.HTTPCAPath) + } else { + caCertPool = x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + } + } + + var tlsConfig *tls.Config + if len(cert.Certificate) > 0 && caCertPool != nil { + tlsConfig = &tls.Config{ + Certificates: []tls.Certificate{cert}, + ClientAuth: tls.RequireAndVerifyClientCert, + ClientCAs: caCertPool, + } + } + // Prometheus histograms for requests. requestDuration := prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: cfg.MetricsNamespace, @@ -137,13 +183,6 @@ func New(cfg Config) (*Server, error) { }, []string{"method", "route", "status_code", "ws"}) prometheus.MustRegister(requestDuration) - // If user doesn't supply a logging implementation, by default instantiate - // logrus. - log := cfg.Log - if log == nil { - log = logging.NewLogrus(cfg.LogLevel) - } - log.WithField("http", httpListener.Addr()).WithField("grpc", grpcListener.Addr()).Infof("server listening on addresses") // Setup gRPC server @@ -186,6 +225,10 @@ func New(cfg Config) (*Server, error) { grpc.MaxConcurrentStreams(uint32(cfg.GPRCServerMaxConcurrentStreams)), } grpcOptions = append(grpcOptions, cfg.GRPCOptions...) + if tlsConfig != nil { + grpcCreds := credentials.NewTLS(tlsConfig) + grpcOptions = append(grpcOptions, grpc.Creds(grpcCreds)) + } grpcServer := grpc.NewServer(grpcOptions...) // Setup HTTP server @@ -218,6 +261,9 @@ func New(cfg Config) (*Server, error) { IdleTimeout: cfg.HTTPServerIdleTimeout, Handler: middleware.Merge(httpMiddleware...).Wrap(router), } + if tlsConfig != nil { + httpServer.TLSConfig = tlsConfig + } return &Server{ cfg: cfg, @@ -252,7 +298,12 @@ func (s *Server) Run() error { }() go func() { - err := s.HTTPServer.Serve(s.httpListener) + var err error + if s.HTTPServer.TLSConfig == nil { + err = s.HTTPServer.Serve(s.httpListener) + } else { + err = s.HTTPServer.ServeTLS(s.httpListener, s.cfg.HTTPCertPath, s.cfg.HTTPKeyPath) + } if err == http.ErrServerClosed { err = nil } diff --git a/vendor/modules.txt b/vendor/modules.txt index c23bc172ac..45ed3cce17 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -567,7 +567,7 @@ github.com/uber/jaeger-client-go/utils # github.com/uber/jaeger-lib v2.2.0+incompatible github.com/uber/jaeger-lib/metrics github.com/uber/jaeger-lib/metrics/prometheus -# github.com/weaveworks/common v0.0.0-20200310113808-2708ba4e60a4 +# github.com/weaveworks/common v0.0.0-20200310113808-2708ba4e60a4 => /Users/annanay/Desktop/git/go/src/github.com/weaveworks/common github.com/weaveworks/common/aws github.com/weaveworks/common/errors github.com/weaveworks/common/grpc From 828e96819ac7a40b3202ddd1931fd69f1d234148 Mon Sep 17 00:00:00 2001 From: Annanay Date: Mon, 13 Apr 2020 16:24:19 +0530 Subject: [PATCH 02/29] Add tls options to grpc client Signed-off-by: Annanay --- pkg/util/grpcclient/grpcclient.go | 42 ++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/pkg/util/grpcclient/grpcclient.go b/pkg/util/grpcclient/grpcclient.go index 2101dfec55..2b0afd3f7e 100644 --- a/pkg/util/grpcclient/grpcclient.go +++ b/pkg/util/grpcclient/grpcclient.go @@ -1,10 +1,15 @@ package grpcclient import ( + "crypto/tls" + "crypto/x509" "flag" + "io/ioutil" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" + "github.com/prometheus/common/log" "google.golang.org/grpc" + "google.golang.org/grpc/credentials" "github.com/cortexproject/cortex/pkg/util" ) @@ -19,6 +24,10 @@ type Config struct { BackoffOnRatelimits bool `yaml:"backoff_on_ratelimits"` BackoffConfig util.BackoffConfig `yaml:"backoff_config"` + + TLSCertPath string `yaml:"tls_cert_path"` + TLSKeyPath string `yaml:"tls_key_path"` + TLSCAPath string `yaml:"tls_ca_path"` } // RegisterFlags registers flags. @@ -31,6 +40,10 @@ func (cfg *Config) RegisterFlags(prefix string, f *flag.FlagSet) { f.BoolVar(&cfg.BackoffOnRatelimits, prefix+".backoff-on-ratelimits", false, "Enable backoff and retry when we hit ratelimits.") cfg.BackoffConfig.RegisterFlags(prefix, f) + + f.StringVar(&cfg.TLSCertPath, prefix+".tls-cert-path", "", "gRPC TLS cert path.") + f.StringVar(&cfg.TLSKeyPath, prefix+".tls-key-path", "", "gRPC TLS key path.") + f.StringVar(&cfg.TLSCAPath, prefix+".tls-ca-path", "", "gRPC TLS CA path.") } // CallOptions returns the config in terms of CallOptions. @@ -54,9 +67,32 @@ func (cfg *Config) DialOption(unaryClientInterceptors []grpc.UnaryClientIntercep unaryClientInterceptors = append([]grpc.UnaryClientInterceptor{NewRateLimiter(cfg)}, unaryClientInterceptors...) } - return []grpc.DialOption{ - grpc.WithDefaultCallOptions(cfg.CallOptions()...), + var opts []grpc.DialOption + if cfg.TLSCertPath != "" && cfg.TLSKeyPath != "" && cfg.TLSCAPath != "" { + clientCert, err := tls.LoadX509KeyPair(cfg.TLSCertPath, cfg.TLSKeyPath) + if err != nil { + log.Warnf("error loading cert %s or key %s, tls disabled", cfg.TLSCertPath, cfg.TLSKeyPath) + } + var caCertPool *x509.CertPool + caCert, err := ioutil.ReadFile(cfg.TLSCAPath) + if err != nil { + log.Warnf("error loading ca cert %s, tls disabled", cfg.TLSCAPath) + } else { + caCertPool = x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + } + if len(clientCert.Certificate) > 0 && caCertPool != nil { + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + Certificates: []tls.Certificate{clientCert}, + RootCAs: caCertPool, + } + opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))) + } + } + + return append(opts, grpc.WithDefaultCallOptions(cfg.CallOptions()...), grpc.WithUnaryInterceptor(grpc_middleware.ChainUnaryClient(unaryClientInterceptors...)), grpc.WithStreamInterceptor(grpc_middleware.ChainStreamClient(streamClientInterceptors...)), - } + ) } From 1ee9ddbed172f9dfd1d8b2792160353f1c54014e Mon Sep 17 00:00:00 2001 From: Annanay Date: Tue, 14 Apr 2020 15:56:33 +0530 Subject: [PATCH 03/29] Add new httpclient util package for use in all client configs Signed-off-by: Annanay --- cmd/query-tee/main.go | 16 +++---- cmd/query-tee/proxy.go | 5 +- cmd/query-tee/proxy_backend.go | 33 +++++++------ cmd/query-tee/proxy_backend_test.go | 3 +- pkg/configs/client/client.go | 40 ++++++---------- pkg/configs/client/configs_test.go | 3 +- pkg/util/httpclient/httpclient.go | 72 +++++++++++++++++++++++++++++ 7 files changed, 121 insertions(+), 51 deletions(-) create mode 100644 pkg/util/httpclient/httpclient.go diff --git a/cmd/query-tee/main.go b/cmd/query-tee/main.go index 1167dcede4..46c0d25c5d 100644 --- a/cmd/query-tee/main.go +++ b/cmd/query-tee/main.go @@ -3,7 +3,6 @@ package main import ( "flag" "os" - "time" "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" @@ -11,15 +10,16 @@ import ( "github.com/weaveworks/common/server" "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/httpclient" ) type Config struct { - ServerServicePort int - ServerMetricsPort int - BackendEndpoints string - PreferredBackend string - BackendReadTimeout time.Duration - LogLevel logging.Level + ServerServicePort int + ServerMetricsPort int + BackendEndpoints string + PreferredBackend string + ClientConfig httpclient.Config + LogLevel logging.Level } func main() { @@ -29,7 +29,7 @@ func main() { flag.IntVar(&cfg.ServerMetricsPort, "server.metrics-port", 9900, "The port where metrics are exposed.") flag.StringVar(&cfg.BackendEndpoints, "backend.endpoints", "", "Comma separated list of backend endpoints to query.") flag.StringVar(&cfg.PreferredBackend, "backend.preferred", "", "The hostname of the preferred backend when selecting the response to send back to the client.") - flag.DurationVar(&cfg.BackendReadTimeout, "backend.read-timeout", 90*time.Second, "The timeout when reading the response from a backend.") + cfg.ClientConfig.RegisterFlags(flag.CommandLine) cfg.LogLevel.RegisterFlags(flag.CommandLine) flag.Parse() diff --git a/cmd/query-tee/proxy.go b/cmd/query-tee/proxy.go index 9987d0cae8..0735594480 100644 --- a/cmd/query-tee/proxy.go +++ b/cmd/query-tee/proxy.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/gorilla/mux" @@ -69,7 +70,9 @@ func NewProxy(cfg Config, logger log.Logger, registerer prometheus.Registerer) ( preferred = preferredIdx == idx } - p.backends = append(p.backends, NewProxyBackend(name, u, cfg.BackendReadTimeout, preferred)) + clientConfig := cfg.ClientConfig + clientConfig.HTTPEndpoint = flagext.URLValue{URL: u} + p.backends = append(p.backends, NewProxyBackend(name, &clientConfig, preferred)) } // At least 1 backend is required diff --git a/cmd/query-tee/proxy_backend.go b/cmd/query-tee/proxy_backend.go index 86dee7d5a0..0c90f88c9d 100644 --- a/cmd/query-tee/proxy_backend.go +++ b/cmd/query-tee/proxy_backend.go @@ -10,6 +10,8 @@ import ( "time" "github.com/pkg/errors" + + "github.com/cortexproject/cortex/pkg/util/httpclient" ) // ProxyBackend holds the information of a single backend. @@ -25,26 +27,31 @@ type ProxyBackend struct { } // NewProxyBackend makes a new ProxyBackend -func NewProxyBackend(name string, endpoint *url.URL, timeout time.Duration, preferred bool) *ProxyBackend { +func NewProxyBackend(name string, config *httpclient.Config, preferred bool) *ProxyBackend { + roundTripper := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, // see https://github.com/golang/go/issues/13801 + IdleConnTimeout: 90 * time.Second, + } + if tlsConfig := config.GetTLSConfig(); tlsConfig != nil { + roundTripper.TLSClientConfig = tlsConfig + } + return &ProxyBackend{ name: name, - endpoint: endpoint, - timeout: timeout, + endpoint: config.HTTPEndpoint.URL, + timeout: config.HTTPClientTimeout, preferred: preferred, client: &http.Client{ CheckRedirect: func(_ *http.Request, _ []*http.Request) error { return errors.New("the query-tee proxy does not follow redirects") }, - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - }).DialContext, - MaxIdleConns: 100, - MaxIdleConnsPerHost: 100, // see https://github.com/golang/go/issues/13801 - IdleConnTimeout: 90 * time.Second, - }, + Transport: roundTripper, }, } } diff --git a/cmd/query-tee/proxy_backend_test.go b/cmd/query-tee/proxy_backend_test.go index d4f50a3d4a..3a3297cca4 100644 --- a/cmd/query-tee/proxy_backend_test.go +++ b/cmd/query-tee/proxy_backend_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/cortexproject/cortex/pkg/util/httpclient" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -61,7 +62,7 @@ func Test_ProxyBackend_createBackendRequest_HTTPBasicAuthentication(t *testing.T orig := httptest.NewRequest("GET", "/test", nil) orig.SetBasicAuth(testData.clientUser, testData.clientPass) - b := NewProxyBackend("test", u, time.Second, false) + b := NewProxyBackend("test", httpclient.ConfigFromURLAndTimeout(u, time.Second), false) r, err := b.createBackendRequest(orig) require.NoError(t, err) diff --git a/pkg/configs/client/client.go b/pkg/configs/client/client.go index a020bb54f0..5dfbcc24a7 100644 --- a/pkg/configs/client/client.go +++ b/pkg/configs/client/client.go @@ -3,13 +3,11 @@ package client import ( "context" "encoding/json" - "flag" "fmt" "net/http" - "net/url" - "time" "github.com/cortexproject/cortex/pkg/configs/userconfig" + "github.com/cortexproject/cortex/pkg/util/httpclient" "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" @@ -17,21 +15,8 @@ import ( "github.com/weaveworks/common/instrument" "github.com/cortexproject/cortex/pkg/util" - "github.com/cortexproject/cortex/pkg/util/flagext" ) -// Config says where we can find the ruler userconfig. -type Config struct { - ConfigsAPIURL flagext.URLValue `yaml:"configs_api_url"` - ClientTimeout time.Duration `yaml:"client_timeout"` // HTTP timeout duration for requests made to the Weave Cloud configs service. -} - -// RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet -func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { - f.Var(&cfg.ConfigsAPIURL, prefix+"configs.url", "URL of configs API server.") - f.DurationVar(&cfg.ClientTimeout, prefix+"configs.client-timeout", 5*time.Second, "Timeout for requests to Weave Cloud configs service.") -} - var configsRequestDuration = instrument.NewHistogramCollector(promauto.NewHistogramVec(prometheus.HistogramOpts{ Namespace: "cortex", Name: "configs_request_duration_seconds", @@ -50,17 +35,15 @@ type Client interface { } // New creates a new ConfigClient. -func New(cfg Config) (*ConfigDBClient, error) { +func New(cfg httpclient.Config) (*ConfigDBClient, error) { return &ConfigDBClient{ - URL: cfg.ConfigsAPIURL.URL, - Timeout: cfg.ClientTimeout, + Config: cfg, }, nil } // ConfigDBClient allows retrieving recording and alerting rules from the configs server. type ConfigDBClient struct { - URL *url.URL - Timeout time.Duration + Config httpclient.Config } // GetRules implements Client @@ -69,11 +52,11 @@ func (c ConfigDBClient) GetRules(ctx context.Context, since userconfig.ID) (map[ if since != 0 { suffix = fmt.Sprintf("?since=%d", since) } - endpoint := fmt.Sprintf("%s/private/api/prom/configs/rules%s", c.URL.String(), suffix) + endpoint := fmt.Sprintf("%s/private/api/prom/configs/rules%s", c.Config.HTTPEndpoint.URL.String(), suffix) var response *ConfigsResponse err := instrument.CollectedRequest(ctx, "GetRules", configsRequestDuration, instrument.ErrorCode, func(ctx context.Context) error { var err error - response, err = doRequest(endpoint, c.Timeout, since) + response, err = doRequest(endpoint, c.Config, since) return err }) if err != nil { @@ -95,23 +78,26 @@ func (c ConfigDBClient) GetAlerts(ctx context.Context, since userconfig.ID) (*Co if since != 0 { suffix = fmt.Sprintf("?since=%d", since) } - endpoint := fmt.Sprintf("%s/private/api/prom/configs/alertmanager%s", c.URL.String(), suffix) + endpoint := fmt.Sprintf("%s/private/api/prom/configs/alertmanager%s", c.Config.HTTPEndpoint.URL.String(), suffix) var response *ConfigsResponse err := instrument.CollectedRequest(ctx, "GetAlerts", configsRequestDuration, instrument.ErrorCode, func(ctx context.Context) error { var err error - response, err = doRequest(endpoint, c.Timeout, since) + response, err = doRequest(endpoint, c.Config, since) return err }) return response, err } -func doRequest(endpoint string, timeout time.Duration, since userconfig.ID) (*ConfigsResponse, error) { +func doRequest(endpoint string, clientConfig httpclient.Config, since userconfig.ID) (*ConfigsResponse, error) { req, err := http.NewRequest("GET", endpoint, nil) if err != nil { return nil, err } - client := &http.Client{Timeout: timeout} + client := &http.Client{Timeout: clientConfig.HTTPClientTimeout} + if tlsConfig := clientConfig.GetTLSConfig(); tlsConfig != nil { + client.Transport = &http.Transport{TLSClientConfig: tlsConfig} + } resp, err := client.Do(req) if err != nil { diff --git a/pkg/configs/client/configs_test.go b/pkg/configs/client/configs_test.go index 86b0427c7e..3982b79d93 100644 --- a/pkg/configs/client/configs_test.go +++ b/pkg/configs/client/configs_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/cortexproject/cortex/pkg/configs/userconfig" + "github.com/cortexproject/cortex/pkg/util/httpclient" "github.com/stretchr/testify/require" @@ -35,7 +36,7 @@ func TestDoRequest(t *testing.T) { })) defer server.Close() - resp, err := doRequest(server.URL, 1*time.Second, 0) + resp, err := doRequest(server.URL, httpclient.Config{HTTPClientTimeout: 1 * time.Second}, 0) assert.Nil(t, err) expected := ConfigsResponse{Configs: map[string]userconfig.View{ diff --git a/pkg/util/httpclient/httpclient.go b/pkg/util/httpclient/httpclient.go new file mode 100644 index 0000000000..b2cb3cc5c6 --- /dev/null +++ b/pkg/util/httpclient/httpclient.go @@ -0,0 +1,72 @@ +package httpclient + +import ( + "crypto/tls" + "crypto/x509" + "flag" + "io/ioutil" + "net/url" + "time" + + "github.com/cortexproject/cortex/pkg/util/flagext" + "github.com/prometheus/common/log" +) + +// Config is the config for the HTTP client +type Config struct { + HTTPEndpoint flagext.URLValue `yaml:"http_endpoint"` + HTTPClientTimeout time.Duration `yaml:"http_client_timeout"` + + TLSCertPath string `yaml:"tls_cert_path"` + TLSKeyPath string `yaml:"tls_key_path"` + TLSCAPath string `yaml:"tls_ca_path"` +} + +// RegisterFlags registers flags. +func (cfg *Config) RegisterFlags(f *flag.FlagSet) { + cfg.RegisterFlagsWithPrefix("", f) +} + +func ConfigFromURLAndTimeout(u *url.URL, timeout time.Duration) *Config { + return &Config{ + HTTPEndpoint: flagext.URLValue{URL: u}, + HTTPClientTimeout: timeout, + } +} + +// RegisterFlagsWithPrefix registers flags with prefix. +func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { + f.Var(&cfg.HTTPEndpoint, prefix+".http-endpoint", "Endpoint to connect to") + f.DurationVar(&cfg.HTTPClientTimeout, prefix+".http-client-timeout", 5*time.Second, "Timeout for connecting to the endpoint.") + + f.StringVar(&cfg.TLSCertPath, prefix+".tls-cert-path", "", "HTTP TLS cert path.") + f.StringVar(&cfg.TLSKeyPath, prefix+".tls-key-path", "", "HTTP TLS key path.") + f.StringVar(&cfg.TLSCAPath, prefix+".tls-ca-path", "", "HTTP TLS CA path.") +} + +// GetTLSConfig initialises tls.Config from config options. +func (cfg *Config) GetTLSConfig() *tls.Config { + if cfg.TLSCertPath != "" && cfg.TLSKeyPath != "" && cfg.TLSCAPath != "" { + clientCert, err := tls.LoadX509KeyPair(cfg.TLSCertPath, cfg.TLSKeyPath) + if err != nil { + log.Warnf("error loading cert %s or key %s, tls disabled", cfg.TLSCertPath, cfg.TLSKeyPath) + } + + var caCertPool *x509.CertPool + caCert, err := ioutil.ReadFile(cfg.TLSCAPath) + if err != nil { + log.Warnf("error loading ca cert %s, tls disabled", cfg.TLSCAPath) + } else { + caCertPool = x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + } + if len(clientCert.Certificate) > 0 && caCertPool != nil { + return &tls.Config{ + InsecureSkipVerify: true, + Certificates: []tls.Certificate{clientCert}, + RootCAs: caCertPool, + } + } + } + return nil +} From b93af112bbf30daab99c8094293b1375343c5a48 Mon Sep 17 00:00:00 2001 From: Annanay Date: Wed, 22 Apr 2020 17:00:38 +0530 Subject: [PATCH 04/29] Change all grpc clients to use grpcclient Signed-off-by: Annanay --- cmd/query-tee/proxy_backend.go | 27 +- go.mod | 20 +- go.sum | 127 +++ pkg/configs/client/client.go | 6 +- pkg/ingester/client/client.go | 3 +- pkg/querier/frontend/worker.go | 3 +- pkg/querier/store_gateway_client.go | 3 +- pkg/ruler/ruler.go | 6 +- pkg/ruler/storage.go | 6 +- .../github.com/aws/aws-sdk-go/aws/arn/arn.go | 7 +- .../aws-sdk-go/aws/corehandlers/handlers.go | 2 +- .../stscreds/assume_role_provider.go | 17 +- .../aws/aws-sdk-go/aws/csm/reporter.go | 2 +- .../aws/ec2metadata/token_provider.go | 2 +- .../aws/aws-sdk-go/aws/endpoints/defaults.go | 190 ++++- .../aws/aws-sdk-go/aws/endpoints/endpoints.go | 6 +- .../aws/aws-sdk-go/aws/request/handlers.go | 21 + .../aws/aws-sdk-go/aws/request/request.go | 16 + .../aws/aws-sdk-go/aws/request/retryer.go | 7 +- .../aws/aws-sdk-go/aws/signer/v4/stream.go | 63 ++ .../aws/aws-sdk-go/aws/signer/v4/v4.go | 105 ++- vendor/github.com/aws/aws-sdk-go/aws/types.go | 34 + .../github.com/aws/aws-sdk-go/aws/version.go | 2 +- .../private/protocol/eventstream/debug.go | 2 +- .../private/protocol/eventstream/decode.go | 33 +- .../private/protocol/eventstream/encode.go | 72 +- .../eventstream/eventstreamapi/error.go | 55 +- .../eventstreamapi/{api.go => reader.go} | 42 +- .../eventstream/eventstreamapi/shared.go | 23 + .../eventstream/eventstreamapi/signer.go | 123 +++ .../eventstreamapi/stream_writer.go | 129 +++ .../eventstream/eventstreamapi/writer.go | 109 +++ .../protocol/eventstream/header_value.go | 5 + .../private/protocol/eventstream/message.go | 2 +- .../aws-sdk-go/private/protocol/payload.go | 2 +- .../aws-sdk-go/private/protocol/protocol.go | 49 ++ .../aws-sdk-go/service/dynamodb/service.go | 2 +- .../aws/aws-sdk-go/service/ec2/api.go | 467 +++++++++-- .../aws/aws-sdk-go/service/ec2/service.go | 2 +- .../aws/aws-sdk-go/service/s3/api.go | 641 ++++++++------ .../aws/aws-sdk-go/service/s3/service.go | 3 +- .../aws/aws-sdk-go/service/sts/service.go | 2 +- .../github.com/cenkalti/backoff/.travis.yml | 10 +- vendor/github.com/cenkalti/backoff/README.md | 108 +-- vendor/github.com/cenkalti/backoff/backoff.go | 13 +- vendor/github.com/cenkalti/backoff/context.go | 63 ++ .../cenkalti/backoff/exponential.go | 9 +- vendor/github.com/cenkalti/backoff/retry.go | 46 +- vendor/github.com/cenkalti/backoff/ticker.go | 19 +- vendor/github.com/cenkalti/backoff/tries.go | 35 + .../coreos/go-systemd/journal/journal.go | 124 ++- .../github.com/go-kit/kit/log/json_logger.go | 4 +- vendor/github.com/go-logfmt/logfmt/.gitignore | 5 +- .../github.com/go-logfmt/logfmt/.travis.yml | 2 + .../github.com/go-logfmt/logfmt/CHANGELOG.md | 7 + vendor/github.com/go-logfmt/logfmt/decode.go | 6 +- vendor/github.com/go-logfmt/logfmt/fuzz.go | 126 --- vendor/github.com/go-logfmt/logfmt/go.mod | 2 +- vendor/github.com/go-logfmt/logfmt/go.sum | 2 - .../golang/protobuf/jsonpb/jsonpb.go | 12 +- .../github.com/golang/protobuf/proto/lib.go | 2 +- .../github.com/golang/protobuf/proto/text.go | 6 +- .../protoc-gen-go/generator/generator.go | 2 +- .../generator/internal/remap/remap.go | 2 +- .../protobuf/protoc-gen-go/grpc/grpc.go | 18 +- vendor/github.com/gorilla/mux/.travis.yml | 24 - .../github.com/gorilla/mux/ISSUE_TEMPLATE.md | 11 - vendor/github.com/gorilla/mux/README.md | 69 ++ vendor/github.com/gorilla/mux/doc.go | 2 +- vendor/github.com/gorilla/mux/middleware.go | 61 +- vendor/github.com/gorilla/mux/regexp.go | 37 +- vendor/github.com/kr/logfmt/.gitignore | 3 - vendor/github.com/kr/logfmt/Readme | 12 - vendor/github.com/kr/logfmt/decode.go | 184 ---- vendor/github.com/kr/logfmt/scanner.go | 149 ---- vendor/github.com/kr/logfmt/unquote.go | 149 ---- .../common/log/eventlog_formatter.go | 89 ++ .../github.com/prometheus/common/log/log.go | 368 ++++++++ .../prometheus/common/log/syslog_formatter.go | 126 +++ .../prometheus/node_exporter/LICENSE | 201 +++++ .../prometheus/node_exporter/NOTICE | 17 + .../prometheus/node_exporter/https/README.md | 24 + .../node_exporter/https/tls_config.go | 124 +++ .../node_exporter/https/web-config.yml | 10 + .../prometheus/procfs/.golangci.yml | 4 +- .../prometheus/procfs/Makefile.common | 32 +- vendor/github.com/prometheus/procfs/crypto.go | 160 ++-- .../prometheus/procfs/fixtures.ttar | 69 +- vendor/github.com/prometheus/procfs/go.mod | 1 + vendor/github.com/prometheus/procfs/go.sum | 2 + .../github.com/prometheus/procfs/loadavg.go | 62 ++ .../github.com/prometheus/procfs/mountinfo.go | 12 +- .../prometheus/procfs/net_conntrackstat.go | 153 ++++ .../prometheus/procfs/net_softnet.go | 107 +-- .../github.com/prometheus/procfs/net_udp.go | 229 +++++ .../github.com/prometheus/procfs/net_unix.go | 222 +++-- .../prometheus/procfs/proc_fdinfo.go | 32 +- .../github.com/prometheus/procfs/proc_maps.go | 208 +++++ .../prometheus/procfs/proc_status.go | 37 +- vendor/github.com/prometheus/procfs/swaps.go | 89 ++ .../weaveworks/common/server/server.go | 61 +- .../go.etcd.io/etcd/auth/range_perm_cache.go | 8 +- vendor/go.etcd.io/etcd/auth/store.go | 7 +- .../etcd/clientv3/balancer/balancer.go | 118 +-- .../etcd/clientv3/balancer/config.go | 36 - .../etcd/clientv3/balancer/connectivity.go | 58 -- .../balancer/connectivity/connectivity.go | 93 +++ .../go.etcd.io/etcd/clientv3/balancer/doc.go | 16 - .../etcd/clientv3/balancer/grpc1.7-health.go | 657 --------------- .../etcd/clientv3/balancer/picker/err.go | 11 +- .../etcd/clientv3/balancer/picker/picker.go | 67 ++ .../clientv3/balancer/picker/picker_policy.go | 49 -- .../balancer/picker/roundrobin_balanced.go | 33 +- .../balancer/resolver/endpoint/endpoint.go | 23 +- .../etcd/clientv3/balancer/utils.go | 4 +- vendor/go.etcd.io/etcd/clientv3/client.go | 171 ++-- vendor/go.etcd.io/etcd/clientv3/config.go | 4 + .../etcd/clientv3/credentials/credentials.go | 173 ++++ vendor/go.etcd.io/etcd/clientv3/doc.go | 2 +- vendor/go.etcd.io/etcd/embed/config.go | 24 +- .../go.etcd.io/etcd/embed/config_logging.go | 41 +- vendor/go.etcd.io/etcd/embed/etcd.go | 1 + vendor/go.etcd.io/etcd/embed/serve.go | 8 +- .../etcd/etcdserver/api/capability.go | 1 + .../etcd/etcdserver/api/membership/cluster.go | 6 +- .../etcd/etcdserver/api/rafthttp/http.go | 5 + .../etcd/etcdserver/api/rafthttp/metrics.go | 20 + .../api/rafthttp/snapshot_sender.go | 6 +- .../etcd/etcdserver/api/rafthttp/stream.go | 1 + .../etcd/etcdserver/api/v3rpc/grpc.go | 9 +- .../etcd/etcdserver/api/v3rpc/key.go | 2 +- .../etcdserver/api/v3rpc/rpctypes/error.go | 3 + .../etcd/etcdserver/api/v3rpc/util.go | 1 + .../etcd/etcdserver/api/v3rpc/watch.go | 2 +- vendor/go.etcd.io/etcd/etcdserver/apply.go | 78 +- .../go.etcd.io/etcd/etcdserver/apply_auth.go | 14 +- vendor/go.etcd.io/etcd/etcdserver/backend.go | 2 +- vendor/go.etcd.io/etcd/etcdserver/config.go | 1 + vendor/go.etcd.io/etcd/etcdserver/corrupt.go | 11 +- .../etcd/etcdserver/etcdserverpb/rpc.pb.go | 8 +- .../etcd/etcdserver/etcdserverpb/rpc.proto | 4 +- vendor/go.etcd.io/etcd/etcdserver/metrics.go | 9 +- vendor/go.etcd.io/etcd/etcdserver/raft.go | 68 +- vendor/go.etcd.io/etcd/etcdserver/server.go | 15 +- .../go.etcd.io/etcd/etcdserver/v3_server.go | 44 +- vendor/go.etcd.io/etcd/mvcc/kv.go | 7 +- vendor/go.etcd.io/etcd/mvcc/kv_view.go | 15 +- vendor/go.etcd.io/etcd/mvcc/kvstore.go | 72 +- .../etcd/mvcc/kvstore_compaction.go | 14 +- vendor/go.etcd.io/etcd/mvcc/kvstore_txn.go | 18 +- vendor/go.etcd.io/etcd/mvcc/metrics.go | 82 +- vendor/go.etcd.io/etcd/mvcc/metrics_txn.go | 16 +- .../go.etcd.io/etcd/mvcc/watchable_store.go | 11 +- .../etcd/mvcc/watchable_store_txn.go | 9 +- vendor/go.etcd.io/etcd/mvcc/watcher_group.go | 1 + vendor/go.etcd.io/etcd/pkg/adt/README.md | 48 ++ .../go.etcd.io/etcd/pkg/adt/interval_tree.go | 530 ++++++++++-- .../go.etcd.io/etcd/pkg/logutil/log_level.go | 70 ++ vendor/go.etcd.io/etcd/pkg/logutil/zap.go | 12 +- .../go.etcd.io/etcd/pkg/logutil/zap_raft.go | 7 +- vendor/go.etcd.io/etcd/pkg/traceutil/trace.go | 172 ++++ .../go.etcd.io/etcd/pkg/transport/listener.go | 44 +- vendor/go.etcd.io/etcd/pkg/types/set.go | 17 + .../proxy/grpcproxy/adapter/chan_stream.go | 4 +- vendor/go.etcd.io/etcd/raft/bootstrap.go | 80 ++ .../etcd/raft/confchange/confchange.go | 425 ++++++++++ .../etcd/raft/confchange/restore.go | 155 ++++ vendor/go.etcd.io/etcd/raft/log_unstable.go | 6 +- vendor/go.etcd.io/etcd/raft/logger.go | 8 +- vendor/go.etcd.io/etcd/raft/node.go | 210 +++-- .../go.etcd.io/etcd/raft/quorum/majority.go | 20 +- vendor/go.etcd.io/etcd/raft/raft.go | 286 ++++--- .../go.etcd.io/etcd/raft/raftpb/confchange.go | 170 ++++ .../go.etcd.io/etcd/raft/raftpb/confstate.go | 45 + vendor/go.etcd.io/etcd/raft/raftpb/raft.pb.go | 786 ++++++++++++++++-- vendor/go.etcd.io/etcd/raft/raftpb/raft.proto | 98 ++- vendor/go.etcd.io/etcd/raft/rawnode.go | 149 ++-- vendor/go.etcd.io/etcd/raft/status.go | 28 +- vendor/go.etcd.io/etcd/raft/storage.go | 2 + .../go.etcd.io/etcd/raft/tracker/inflights.go | 8 + .../go.etcd.io/etcd/raft/tracker/progress.go | 22 + .../go.etcd.io/etcd/raft/tracker/tracker.go | 149 ++-- vendor/go.etcd.io/etcd/raft/util.go | 115 ++- vendor/go.etcd.io/etcd/version/version.go | 2 +- vendor/go.uber.org/atomic/CHANGELOG.md | 7 +- vendor/go.uber.org/atomic/README.md | 22 +- vendor/go.uber.org/atomic/atomic.go | 9 +- vendor/go.uber.org/multierr/.gitignore | 3 + vendor/go.uber.org/multierr/.travis.yml | 14 +- vendor/go.uber.org/multierr/CHANGELOG.md | 20 + vendor/go.uber.org/multierr/Makefile | 56 +- vendor/go.uber.org/multierr/README.md | 4 +- vendor/go.uber.org/multierr/error.go | 56 +- vendor/go.uber.org/multierr/glide.lock | 19 - vendor/go.uber.org/multierr/go.mod | 12 + vendor/go.uber.org/multierr/go.sum | 45 + vendor/go.uber.org/multierr/go113.go | 52 ++ vendor/go.uber.org/multierr/tools.go | 30 + vendor/go.uber.org/tools/LICENSE | 19 + .../tools/update-license/.gitignore | 1 + .../tools/update-license/README.md | 24 + .../tools/update-license/licenses.go | 56 ++ .../go.uber.org/tools/update-license/main.go | 228 +++++ vendor/go.uber.org/zap/.gitignore | 4 + vendor/go.uber.org/zap/.readme.tmpl | 5 +- vendor/go.uber.org/zap/.travis.yml | 20 +- vendor/go.uber.org/zap/CHANGELOG.md | 25 + vendor/go.uber.org/zap/Makefile | 71 +- vendor/go.uber.org/zap/README.md | 68 +- .../zap/{check_license.sh => checklicense.sh} | 0 vendor/go.uber.org/zap/field.go | 215 +++++ vendor/go.uber.org/zap/glide.lock | 76 -- vendor/go.uber.org/zap/glide.yaml | 3 +- vendor/go.uber.org/zap/go.mod | 11 + vendor/go.uber.org/zap/go.sum | 56 ++ vendor/go.uber.org/zap/tools.go | 28 + vendor/go.uber.org/zap/zapcore/encoder.go | 35 +- vendor/go.uber.org/zap/zapcore/entry.go | 3 +- .../go.uber.org/zap/zapcore/json_encoder.go | 28 +- vendor/golang.org/x/lint/go.mod | 4 +- vendor/golang.org/x/lint/go.sum | 12 +- vendor/golang.org/x/sys/unix/mkerrors.sh | 2 +- .../x/sys/unix/syscall_linux_mips64x.go | 4 + .../golang.org/x/sys/unix/syscall_openbsd.go | 10 +- .../x/sys/unix/zerrors_linux_386.go | 46 +- .../x/sys/unix/zerrors_linux_amd64.go | 46 +- .../x/sys/unix/zerrors_linux_arm.go | 46 +- .../x/sys/unix/zerrors_linux_arm64.go | 46 +- .../x/sys/unix/zerrors_linux_mips.go | 46 +- .../x/sys/unix/zerrors_linux_mips64.go | 46 +- .../x/sys/unix/zerrors_linux_mips64le.go | 46 +- .../x/sys/unix/zerrors_linux_mipsle.go | 46 +- .../x/sys/unix/zerrors_linux_ppc64.go | 47 +- .../x/sys/unix/zerrors_linux_ppc64le.go | 47 +- .../x/sys/unix/zerrors_linux_riscv64.go | 46 +- .../x/sys/unix/zerrors_linux_s390x.go | 46 +- .../x/sys/unix/zerrors_linux_sparc64.go | 47 +- .../x/sys/unix/zsyscall_openbsd_386.go | 4 +- .../x/sys/unix/zsyscall_openbsd_amd64.go | 4 +- .../x/sys/unix/zsyscall_openbsd_arm.go | 4 +- .../x/sys/unix/zsyscall_openbsd_arm64.go | 4 +- .../x/sys/unix/zsysnum_linux_arm64.go | 1 + .../golang.org/x/sys/unix/ztypes_linux_386.go | 4 +- .../x/sys/unix/ztypes_linux_amd64.go | 4 +- .../golang.org/x/sys/unix/ztypes_linux_arm.go | 4 +- .../x/sys/unix/ztypes_linux_arm64.go | 4 +- .../x/sys/unix/ztypes_linux_mips.go | 4 +- .../x/sys/unix/ztypes_linux_mips64.go | 4 +- .../x/sys/unix/ztypes_linux_mips64le.go | 4 +- .../x/sys/unix/ztypes_linux_mipsle.go | 4 +- .../x/sys/unix/ztypes_linux_ppc64.go | 4 +- .../x/sys/unix/ztypes_linux_ppc64le.go | 4 +- .../x/sys/unix/ztypes_linux_riscv64.go | 4 +- .../x/sys/unix/ztypes_linux_s390x.go | 4 +- .../x/sys/unix/ztypes_linux_sparc64.go | 4 +- .../x/sys/windows/svc/eventlog/install.go | 80 ++ .../x/sys/windows/svc/eventlog/log.go | 70 ++ vendor/google.golang.org/grpc/.travis.yml | 4 +- .../grpc/attributes/attributes.go | 70 ++ .../grpc/balancer/balancer.go | 112 ++- .../grpc/balancer/base/balancer.go | 110 ++- .../grpc/balancer/base/base.go | 29 + .../grpc/balancer/grpclb/grpclb.go | 6 +- .../grpc/balancer/grpclb/grpclb_picker.go | 25 +- .../balancer/grpclb/grpclb_remote_balancer.go | 9 +- .../grpc/balancer/grpclb/grpclb_util.go | 9 +- .../grpc/balancer/roundrobin/roundrobin.go | 18 +- .../grpc/balancer_conn_wrappers.go | 25 +- .../grpc/balancer_v1_wrapper.go | 34 +- vendor/google.golang.org/grpc/clientconn.go | 41 +- .../alts/internal/handshaker/handshaker.go | 30 +- .../internal/proto/grpc_gcp/handshaker.pb.go | 202 +++-- .../grpc/credentials/credentials.go | 236 +----- .../grpc/credentials/{tls13.go => go12.go} | 0 .../google.golang.org/grpc/credentials/tls.go | 220 +++++ vendor/google.golang.org/grpc/dialoptions.go | 6 +- vendor/google.golang.org/grpc/go.mod | 2 +- vendor/google.golang.org/grpc/go.sum | 4 +- .../google.golang.org/grpc/health/client.go | 14 +- .../grpc/internal/binarylog/binarylog.go | 10 +- .../grpc/internal/buffer/unbounded.go | 7 + .../grpc/internal/internal.go | 2 +- .../internal/resolver/dns/dns_resolver.go | 160 ++-- .../grpc/internal/resolver/dns/go113.go | 33 + .../resolver/passthrough/passthrough.go | 4 +- .../grpc/internal/transport/handler_server.go | 8 +- .../grpc/internal/transport/http2_client.go | 17 +- .../grpc/internal/transport/http2_server.go | 114 ++- .../grpc/internal/transport/transport.go | 9 +- .../google.golang.org/grpc/picker_wrapper.go | 172 ++-- vendor/google.golang.org/grpc/pickfirst.go | 89 +- .../grpc/resolver/resolver.go | 48 +- .../grpc/resolver_conn_wrapper.go | 12 +- vendor/google.golang.org/grpc/server.go | 148 ++-- vendor/google.golang.org/grpc/stats/stats.go | 11 + vendor/google.golang.org/grpc/version.go | 2 +- vendor/google.golang.org/grpc/vet.sh | 77 +- vendor/gopkg.in/yaml.v2/scannerc.go | 47 +- vendor/gopkg.in/yaml.v2/yamlh.go | 1 + vendor/modules.txt | 47 +- 300 files changed, 11715 insertions(+), 4714 deletions(-) create mode 100644 vendor/github.com/aws/aws-sdk-go/aws/signer/v4/stream.go rename vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/{api.go => reader.go} (77%) create mode 100644 vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/shared.go create mode 100644 vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/signer.go create mode 100644 vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/stream_writer.go create mode 100644 vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/writer.go create mode 100644 vendor/github.com/aws/aws-sdk-go/private/protocol/protocol.go create mode 100644 vendor/github.com/cenkalti/backoff/context.go create mode 100644 vendor/github.com/cenkalti/backoff/tries.go delete mode 100644 vendor/github.com/go-logfmt/logfmt/fuzz.go delete mode 100644 vendor/github.com/go-logfmt/logfmt/go.sum delete mode 100644 vendor/github.com/gorilla/mux/.travis.yml delete mode 100644 vendor/github.com/gorilla/mux/ISSUE_TEMPLATE.md delete mode 100644 vendor/github.com/kr/logfmt/.gitignore delete mode 100644 vendor/github.com/kr/logfmt/Readme delete mode 100644 vendor/github.com/kr/logfmt/decode.go delete mode 100644 vendor/github.com/kr/logfmt/scanner.go delete mode 100644 vendor/github.com/kr/logfmt/unquote.go create mode 100644 vendor/github.com/prometheus/common/log/eventlog_formatter.go create mode 100644 vendor/github.com/prometheus/common/log/log.go create mode 100644 vendor/github.com/prometheus/common/log/syslog_formatter.go create mode 100644 vendor/github.com/prometheus/node_exporter/LICENSE create mode 100644 vendor/github.com/prometheus/node_exporter/NOTICE create mode 100644 vendor/github.com/prometheus/node_exporter/https/README.md create mode 100644 vendor/github.com/prometheus/node_exporter/https/tls_config.go create mode 100644 vendor/github.com/prometheus/node_exporter/https/web-config.yml create mode 100644 vendor/github.com/prometheus/procfs/loadavg.go create mode 100644 vendor/github.com/prometheus/procfs/net_conntrackstat.go create mode 100644 vendor/github.com/prometheus/procfs/net_udp.go create mode 100644 vendor/github.com/prometheus/procfs/proc_maps.go create mode 100644 vendor/github.com/prometheus/procfs/swaps.go delete mode 100644 vendor/go.etcd.io/etcd/clientv3/balancer/config.go delete mode 100644 vendor/go.etcd.io/etcd/clientv3/balancer/connectivity.go create mode 100644 vendor/go.etcd.io/etcd/clientv3/balancer/connectivity/connectivity.go delete mode 100644 vendor/go.etcd.io/etcd/clientv3/balancer/doc.go delete mode 100644 vendor/go.etcd.io/etcd/clientv3/balancer/grpc1.7-health.go delete mode 100644 vendor/go.etcd.io/etcd/clientv3/balancer/picker/picker_policy.go create mode 100644 vendor/go.etcd.io/etcd/clientv3/credentials/credentials.go create mode 100644 vendor/go.etcd.io/etcd/pkg/adt/README.md create mode 100644 vendor/go.etcd.io/etcd/pkg/logutil/log_level.go create mode 100644 vendor/go.etcd.io/etcd/pkg/traceutil/trace.go create mode 100644 vendor/go.etcd.io/etcd/raft/bootstrap.go create mode 100644 vendor/go.etcd.io/etcd/raft/confchange/confchange.go create mode 100644 vendor/go.etcd.io/etcd/raft/confchange/restore.go create mode 100644 vendor/go.etcd.io/etcd/raft/raftpb/confchange.go create mode 100644 vendor/go.etcd.io/etcd/raft/raftpb/confstate.go delete mode 100644 vendor/go.uber.org/multierr/glide.lock create mode 100644 vendor/go.uber.org/multierr/go.mod create mode 100644 vendor/go.uber.org/multierr/go.sum create mode 100644 vendor/go.uber.org/multierr/go113.go create mode 100644 vendor/go.uber.org/multierr/tools.go create mode 100644 vendor/go.uber.org/tools/LICENSE create mode 100644 vendor/go.uber.org/tools/update-license/.gitignore create mode 100644 vendor/go.uber.org/tools/update-license/README.md create mode 100644 vendor/go.uber.org/tools/update-license/licenses.go create mode 100644 vendor/go.uber.org/tools/update-license/main.go rename vendor/go.uber.org/zap/{check_license.sh => checklicense.sh} (100%) delete mode 100644 vendor/go.uber.org/zap/glide.lock create mode 100644 vendor/go.uber.org/zap/go.mod create mode 100644 vendor/go.uber.org/zap/go.sum create mode 100644 vendor/go.uber.org/zap/tools.go create mode 100644 vendor/golang.org/x/sys/windows/svc/eventlog/install.go create mode 100644 vendor/golang.org/x/sys/windows/svc/eventlog/log.go create mode 100644 vendor/google.golang.org/grpc/attributes/attributes.go rename vendor/google.golang.org/grpc/credentials/{tls13.go => go12.go} (100%) create mode 100644 vendor/google.golang.org/grpc/credentials/tls.go create mode 100644 vendor/google.golang.org/grpc/internal/resolver/dns/go113.go diff --git a/cmd/query-tee/proxy_backend.go b/cmd/query-tee/proxy_backend.go index 0c90f88c9d..d053109cde 100644 --- a/cmd/query-tee/proxy_backend.go +++ b/cmd/query-tee/proxy_backend.go @@ -5,7 +5,6 @@ import ( "io/ioutil" "net" "net/http" - "net/url" "path" "time" @@ -16,10 +15,9 @@ import ( // ProxyBackend holds the information of a single backend. type ProxyBackend struct { - name string - endpoint *url.URL - client *http.Client - timeout time.Duration + name string + clientConfig *httpclient.Config + client *http.Client // Whether this is the preferred backend from which picking up // the response and sending it back to the client. @@ -43,10 +41,9 @@ func NewProxyBackend(name string, config *httpclient.Config, preferred bool) *Pr } return &ProxyBackend{ - name: name, - endpoint: config.HTTPEndpoint.URL, - timeout: config.HTTPClientTimeout, - preferred: preferred, + name: name, + clientConfig: config, + preferred: preferred, client: &http.Client{ CheckRedirect: func(_ *http.Request, _ []*http.Request) error { return errors.New("the query-tee proxy does not follow redirects") @@ -72,19 +69,19 @@ func (b *ProxyBackend) createBackendRequest(orig *http.Request) (*http.Request, } // Replace the endpoint with the backend one. - req.URL.Scheme = b.endpoint.Scheme - req.URL.Host = b.endpoint.Host + req.URL.Scheme = b.clientConfig.HTTPEndpoint.Scheme + req.URL.Host = b.clientConfig.HTTPEndpoint.Host // Prepend the endpoint path to the request path. - req.URL.Path = path.Join(b.endpoint.Path, req.URL.Path) + req.URL.Path = path.Join(b.clientConfig.HTTPEndpoint.Path, req.URL.Path) // Replace the auth: // - If the endpoint has user and password, use it. // - If the endpoint has user only, keep it and use the request password (if any). // - If the endpoint has no user and no password, use the request auth (if any). clientUser, clientPass, clientAuth := orig.BasicAuth() - endpointUser := b.endpoint.User.Username() - endpointPass, _ := b.endpoint.User.Password() + endpointUser := b.clientConfig.HTTPEndpoint.User.Username() + endpointPass, _ := b.clientConfig.HTTPEndpoint.User.Password() if endpointUser != "" && endpointPass != "" { req.SetBasicAuth(endpointUser, endpointPass) @@ -99,7 +96,7 @@ func (b *ProxyBackend) createBackendRequest(orig *http.Request) (*http.Request, func (b *ProxyBackend) doBackendRequest(req *http.Request) (int, []byte, error) { // Honor the read timeout. - ctx, cancel := context.WithTimeout(context.Background(), b.timeout) + ctx, cancel := context.WithTimeout(context.Background(), b.clientConfig.HTTPClientTimeout) defer cancel() // Execute the request. diff --git a/go.mod b/go.mod index fb4dddeee8..cc26a234e2 100644 --- a/go.mod +++ b/go.mod @@ -11,21 +11,21 @@ require ( github.com/NYTimes/gziphandler v1.1.1 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d github.com/armon/go-metrics v0.3.0 - github.com/aws/aws-sdk-go v1.25.48 + github.com/aws/aws-sdk-go v1.27.0 github.com/blang/semver v3.5.0+incompatible github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b github.com/cespare/xxhash v1.1.0 github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb github.com/fsouza/fake-gcs-server v1.7.0 - github.com/go-kit/kit v0.9.0 + github.com/go-kit/kit v0.10.0 github.com/gocql/gocql v0.0.0-20200121121104-95d072f1b5bb github.com/gogo/protobuf v1.3.1 github.com/gogo/status v1.0.3 github.com/golang-migrate/migrate/v4 v4.7.0 - github.com/golang/protobuf v1.3.2 + github.com/golang/protobuf v1.3.3 github.com/golang/snappy v0.0.1 github.com/gomodule/redigo v2.0.0+incompatible - github.com/gorilla/mux v1.7.1 + github.com/gorilla/mux v1.7.3 github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 github.com/hashicorp/consul/api v1.3.0 github.com/hashicorp/go-cleanhttp v0.5.1 @@ -50,16 +50,16 @@ require ( github.com/stretchr/testify v1.4.0 github.com/thanos-io/thanos v0.12.1-0.20200416112106-b391ca115ed8 github.com/uber/jaeger-client-go v2.20.1+incompatible - github.com/weaveworks/common v0.0.0-20200310113808-2708ba4e60a4 + github.com/weaveworks/common v0.0.0-20200422083114-5790e7482ff8 go.etcd.io/bbolt v1.3.3 - go.etcd.io/etcd v0.0.0-20190709142735-eb7dd97135a5 - go.uber.org/atomic v1.5.0 + go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 + go.uber.org/atomic v1.5.1 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e golang.org/x/time v0.0.0-20191024005414-555d28b269f0 google.golang.org/api v0.14.0 - google.golang.org/grpc v1.25.1 - gopkg.in/yaml.v2 v2.2.7 + google.golang.org/grpc v1.26.0 + gopkg.in/yaml.v2 v2.2.8 sigs.k8s.io/yaml v1.1.0 ) @@ -72,5 +72,3 @@ replace git.apache.org/thrift.git => github.com/apache/thrift v0.0.0-20180902110 // Override reference that causes an error from Go proxy - see https://github.com/golang/go/issues/33558 replace k8s.io/client-go => k8s.io/client-go v0.0.0-20190620085101-78d2af792bab - -replace github.com/weaveworks/common => /Users/annanay/Desktop/git/go/src/github.com/weaveworks/common diff --git a/go.sum b/go.sum index 58269fac67..7012824e03 100644 --- a/go.sum +++ b/go.sum @@ -62,6 +62,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/squirrel v0.0.0-20161115235646-20f192218cf5 h1:PPfYWScYacO3Q6JMCLkyh6Ea2Q/REDTMgmiTAeiV8Jg= github.com/Masterminds/squirrel v0.0.0-20161115235646-20f192218cf5/go.mod h1:xnKTFzjGUiZtiOagBsfnvomW+nJg2usB1ZpordQWqNM= github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6nK2Q= @@ -83,6 +84,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -96,6 +99,7 @@ github.com/aliyun/aliyun-oss-go-sdk v2.0.4+incompatible/go.mod h1:T/Aws4fEfogEE9 github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/apache/thrift v0.12.0 h1:pODnxUFNcjP9UTLZGTdeh+j16A8lJbRvD3rOtrk/7bs= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= @@ -105,16 +109,22 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= github.com/aws/aws-sdk-go v1.17.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.22.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.25.48 h1:J82DYDGZHOKHdhx6hD24Tm30c2C3GchYGfN0mf9iKUk= github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.27.0 h1:0xphMHGMLBrPMfxR2AmVjZKcMEESEgWF8Kru94BNByk= +github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= +github.com/beevik/ntp v0.2.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -128,9 +138,12 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b h1:L/QXpzIa3pOvUGt1D1lA5KjYhPBAN/3iWdP7xeFS9F0= github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= +github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v0.0.0-20181003080854-62661b46c409/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff v1.0.0 h1:2XeuDgvPv/6QDyzIuxb6n36ADVocyqTLlOSpYBGYtvM= github.com/cenkalti/backoff v1.0.0/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v0.0.0-20181017004759-096ff4a8a059/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -141,11 +154,13 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk= github.com/cockroachdb/datadriven v0.0.0-20190531201743-edce55837238 h1:uNljlOxtOHrPnRoPPx+JanqjAGZpNiqAGVBfGskd/pg= github.com/cockroachdb/datadriven v0.0.0-20190531201743-edce55837238/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerd/containerd v1.2.7 h1:8lqLbl7u1j3MmiL9cJ/O275crSq7bfwUayvvatEupQk= @@ -156,10 +171,14 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d h1:t5Wuyh53qYyg9eqn4BbnlIT+vmhyww0TatL+zT3uWgI= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= +github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cortexproject/cortex v0.6.1-0.20200228110116-92ab6cbe0995/go.mod h1:3Xa3DjJxtpXqxcMGdk850lcIRb81M0fyY1MQ6udY134= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8= github.com/cznic/fileutil v0.0.0-20180108211300-6a051e75936f/go.mod h1:8S58EK26zhXSxzv7NQFpnliaOQsmDUxvoQO3rt154Vg= github.com/cznic/golex v0.0.0-20170803123110-4ab7c5e190e4/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc= @@ -208,8 +227,11 @@ github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6 github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU= github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/ema/qdisc v0.0.0-20190904071900-b82c76788043/go.mod h1:ix4kG2zvdUd8kEKSW0ZTr1XLks0epFpI4j745DXxlNE= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -221,6 +243,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/structtag v1.1.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= +github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsouza/fake-gcs-server v1.7.0 h1:Un0BXUXrRWYSmYyC1Rqm2e2WJfTPyDy/HGMz31emTi8= @@ -234,9 +258,13 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= +github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= @@ -298,6 +326,7 @@ github.com/go-openapi/validate v0.17.2/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+ github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2 h1:ky5l57HjyVRrsJfd2+Ro5Z9PjGuKbsmftwyMtk8H7js= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -305,6 +334,7 @@ github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4 h1:vF83LI8tAakwEwvWZtr github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4/go.mod h1:4Fw1eo5iaEhDUs8XyuhSVCVy52Jq3L+/3GJgYkwc+/0= github.com/gocql/gocql v0.0.0-20200121121104-95d072f1b5bb h1:gAHg4RMULuLo7Y3jhY5CHh/QuSwjeTZt4qVdJ9ytcVI= github.com/gocql/gocql v0.0.0-20200121121104-95d072f1b5bb/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY= +github.com/godbus/dbus v0.0.0-20190402143921-271e53dc4968/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/gogo/googleapis v1.1.0 h1:kFkMAZBNAn4j7K0GiZr8cRYzejq68VbheufiV3YuyFI= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -339,6 +369,8 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= @@ -397,6 +429,8 @@ github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.1 h1:Dw4jY2nghMMRsh1ol8dv1axHkDwMQK2DHerMNJsIpJU= github.com/gorilla/mux v1.7.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -446,6 +480,7 @@ github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdv github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -462,12 +497,15 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/hashicorp/serf v0.8.3/go.mod h1:UpNcs7fFbpKIyZaUuSW6EPiH+eZC7OuyFD+wc1oal+k= github.com/hashicorp/serf v0.8.5 h1:ZynDUIQiA8usmRgPdGPHFdPnb1wgGI9tK3mO9hcAJjc= github.com/hashicorp/serf v0.8.5/go.mod h1:UpNcs7fFbpKIyZaUuSW6EPiH+eZC7OuyFD+wc1oal+k= +github.com/hodgesds/perf-utils v0.0.8/go.mod h1:F6TfvsbtrF88i++hou29dTXlI2sfsJv+gRZDtmTJkAs= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb v1.7.7 h1:UvNzAPfBrKMENVbQ4mr4ccA9sW+W1Ihl0Yh1s0BiVAg= github.com/influxdata/influxdb v1.7.7/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ= github.com/jackc/pgx v3.2.0+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= github.com/jessevdk/go-flags v0.0.0-20180331124232-1c38ed7ad0cc/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -483,6 +521,9 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22 github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw= +github.com/jsimonetti/rtnetlink v0.0.0-20190830100107-3784a6c7c552/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw= +github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -536,8 +577,11 @@ github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-b github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.0 h1:fAazJekOWnfBeQYwk9jEgIWWKmBxq4ev3WfsAnezgc4= github.com/lightstep/lightstep-tracer-go v0.18.0/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= +github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lovoo/gcloud-opentracing v0.3.0 h1:nAeKG70rIsog0TelcEtt6KU0Y1s5qXtsDLnHp0urPLU= github.com/lovoo/gcloud-opentracing v0.3.0/go.mod h1:ZFqk2y38kMDDikZPAK7ynTTGuyt17nSPdS3K5e+ZTBY= +github.com/lufia/iostat v1.1.0/go.mod h1:rEPNA0xXgjHQjuI5Cy05sLlS2oRcSlWHRLrvh/AQ+Pg= +github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -555,9 +599,16 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-xmlrpc v0.0.3/go.mod h1:mqc2dz7tP5x5BKlCahN/n+hs7OSZKJkS9JsHNBRlrxA= github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mdlayher/genetlink v1.0.0/go.mod h1:0rJ0h4itni50A86M2kHcgS85ttZazNt7a8H2a2cw0Gc= +github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA= +github.com/mdlayher/netlink v0.0.0-20190828143259-340058475d09/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M= +github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M= +github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY= +github.com/mdlayher/wifi v0.0.0-20190303161829-b1436901ddee/go.mod h1:Evt/EIne46u9PtQbeTx2NTcqURpr5K4SvKtGmBuDPN8= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -600,6 +651,14 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8/go.mod h1:86wM1zFnC6/uDBfZGNwB65O+pR2OFi5q/YQaEUid1qA= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v0.0.0-20170117200651-66bb6560562f/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -625,12 +684,14 @@ github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02 h1:0R5mDLI66Qw13qN80TRz85zthQ2nf2+uDyiV23w6c3Q= github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02/go.mod h1:JNdpVEzCpXBgIiv4ds+TzhN1hrtxq6ClLrTlT9OQRSc= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9 h1:QsgXACQhd9QJhEmRumbsMQQvBtmdS0mafoVEBplWXEg= github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w= github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7lZWlQw5UXuoo= @@ -640,18 +701,25 @@ github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsq github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.1-0.20200124165624-2876d2018785 h1:Oi9nYnU9jbiUVyoRTQfMpSdGzNVmEI+/9fija3lcnjU= github.com/opentracing/opentracing-go v1.1.1-0.20200124165624-2876d2018785/go.mod h1:C+iumr2ni468+1jvcHXLCdqP9uQnoQbdX93F3aWahWU= +github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -669,6 +737,8 @@ github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQ github.com/prometheus/client_golang v1.2.0/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI= github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= +github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.5.0 h1:Ctq0iGpCmr3jeP77kbF2UxgvRwzWWz+4Bh9/vJTyg1A= github.com/prometheus/client_golang v1.5.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -677,6 +747,7 @@ github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1: github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= @@ -690,6 +761,8 @@ github.com/prometheus/common v0.8.0 h1:bLkjvFe2ZRX1DpcgZcdf7j/+MnusEps5hktST/FHA github.com/prometheus/common v0.8.0/go.mod h1:PC/OgXc+UN7B4ALwvn1yzVZmVwvhXp5JsbBv6wSv6i0= github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/node_exporter v1.0.0-rc.0.0.20200417100735-fa4edd700ebc h1:9csPynN5NJOGA4GFef/6VUOAzs7njUiEiWW+jYUamXo= +github.com/prometheus/node_exporter v1.0.0-rc.0.0.20200417100735-fa4edd700ebc/go.mod h1:FGbBv5OPKjch+jNUJmEQpMZytIdyW0NdBtWFcfSKusc= github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -703,6 +776,8 @@ github.com/prometheus/procfs v0.0.6 h1:0qbH+Yqu/cj1ViVLvEWCP6qMQ4efWUj6bQqOEA0V0 github.com/prometheus/procfs v0.0.6/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI= +github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/prometheus v0.0.0-20180315085919-58e2a31db8de/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= github.com/prometheus/prometheus v0.0.0-20190818123050-43acd0e2e93f/go.mod h1:rMTlmxGCvukf2KMu3fClMDKLLoJ5hl61MhcJ7xKakf0= github.com/prometheus/prometheus v1.8.2-0.20200107122003-4708915ac6ef/go.mod h1:7U90zPoLkWjEIQcy/rweQla82OCTUzxVHE51G3OhJbI= @@ -716,6 +791,7 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.6.0 h1:G9tHG9lebljV9mfp9SNPDL36nCDxmo3zTlAf1YgvzmI= github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190810000440-0ceca61e4d75 h1:cA+Ubq9qEVIQhIWvP2kNuSZ2CmnfBJFSRq+kO1pu2cc= @@ -737,9 +813,11 @@ github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9Nz github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 h1:bUGsEnyNbVPw06Bs80sCeARAlK8lhwqGyi6UT8ymuGk= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20180825020608-02ddb050ef6b/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd h1:ug7PpSOB5RBPK1Kg6qskGBoP3Vnj/aNYFTznWvlkGo0= github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= +github.com/siebenmann/go-kstat v0.0.0-20160321171754-d34789b79745/go.mod h1:G81aIFAMS9ECrwBYR9YxhlPjWgrItd+Kje78O6+uqm8= github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -755,6 +833,8 @@ github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIK github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/soundcloud/go-runit v0.0.0-20150630195641-06ad41a06c4a/go.mod h1:LeFCbQYJ3KJlPs/FvPz2dy1tkpxyeNESVyCNNzRXFR0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -767,6 +847,9 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= @@ -793,6 +876,11 @@ github.com/uber/jaeger-lib v1.5.1-0.20181102163054-1fc5c315e03c/go.mod h1:ComeND github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/weaveworks/common v0.0.0-20200206153930-760e36ae819a/go.mod h1:6enWAqfQBFrE8X/XdJwZr8IKgh1chStuFR0mjU/UOUw= +github.com/weaveworks/common v0.0.0-20200310113808-2708ba4e60a4/go.mod h1:6enWAqfQBFrE8X/XdJwZr8IKgh1chStuFR0mjU/UOUw= +github.com/weaveworks/common v0.0.0-20200422083114-5790e7482ff8 h1:K+Tp9/L22sx+Ez4C+cPm5dn2Ht35IuD9TVVyp0Ehr+M= +github.com/weaveworks/common v0.0.0-20200422083114-5790e7482ff8/go.mod h1:fqnqJsiQGcwBYgrCQZHgwxVEfMib76vpzKoAFTU0Vz8= github.com/weaveworks/promrus v1.2.0 h1:jOLf6pe6/vss4qGHjXmGz4oDJQA+AOCqEL3FvvZGz7M= github.com/weaveworks/promrus v1.2.0/go.mod h1:SaE82+OJ91yqjrE1rsvBWVzNZKcHYFtMUyS1+Ogs/KA= github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs= @@ -814,6 +902,8 @@ go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20190709142735-eb7dd97135a5 h1:3unozPyUjPcbSbfhBb4EgA3O1/yBYHNgRr4ZGjO9iyQ= go.etcd.io/etcd v0.0.0-20190709142735-eb7dd97135a5/go.mod h1:N0RPWo9FXJYZQI4BTkDtQylrstIigYHeR18ONnyTufk= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 h1:VcrIfasaLFkyjk6KNlXQSzO+B0fZcnECiDrKJsfxka0= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.0.4/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.0 h1:aeOqSrhl9eDRAap/3T5pCfMBEBxZ0vuXBP+RMtp2KX8= @@ -830,13 +920,22 @@ go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM= +go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/automaxprocs v1.2.0 h1:+RUihKM+nmYUoB9w0D0Ov5TJ2PpFO2FgenTxMJiZBZA= go.uber.org/automaxprocs v1.2.0/go.mod h1:YfO3fm683kQpzETxlTGZhGIVmXAhaw3gxeBADbpZtnU= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.4.0 h1:f3WCSC2KzAcBXGATIxAB1E2XuCpNU255wNKZ505qi3E= +go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0 h1:nR6NoDBgAf67s68NhaXbsojM+2gxp3S1hWkHDl27pVU= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -873,10 +972,13 @@ golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367 h1:0IiAsCRByjO2QjX7ZPkw5oU9x+n1YqRL802rjC0c3Aw= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -905,12 +1007,16 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191112182307-2180aed22343 h1:00ohfJ4K98s3m6BGUoBd8nyfp4Yl0GoIKvw5abItTjI= golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -941,6 +1047,7 @@ golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190425145619-16072639606e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -955,16 +1062,23 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190902133755-9109b7679e13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe h1:6fAMxZRR6sl1Uq8U61gxU+kPTs2tR8uOySCbBP7BN/M= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056 h1:dHtDnRWQtSx0Hjq9kvKFpBh9uPPKfQN70NZZmvssGwk= golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200217220822-9197077df867 h1:JoRuNIf+rpHl+VhScRQQvzbHed86tKkqwPMV34T8myw= +golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180805044716-cb6730876b98/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1014,6 +1128,9 @@ golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2 h1:EtTFh6h4SAKemS+CURDMTDIANuduG5zKEXShyy18bGA= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200216192241-b320d3a0f5a2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200306191617-51e69f71924f h1:bFIWQKTZ5vXyr7xMDvzbWUj5Y/WBE4a4sf35MAyZjx0= golang.org/x/tools v0.0.0-20200306191617-51e69f71924f/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1030,6 +1147,7 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.14.0 h1:uMf5uLi4eQMRrMKhCplNik4U4H8Z6C1br3zOtAa/aDE= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1058,15 +1176,19 @@ google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvx google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -1080,6 +1202,7 @@ gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/fsnotify/fsnotify.v1 v1.4.7 h1:XNNYLJHt73EyYiCZi6+xjupS9CpvmiDgjPTAjrBlQbo= gopkg.in/fsnotify/fsnotify.v1 v1.4.7/go.mod h1:Fyux9zXlo4rWoMSIzpn9fDAYjalPqJ/K1qJ27s+7ltE= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= @@ -1091,6 +1214,7 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= @@ -1101,6 +1225,8 @@ gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2 h1:XZx7nhd5GMaZpmDaEHFVafUZC7ya0fuo7cSJ3UCKYmM= gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= @@ -1148,3 +1274,4 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/pkg/configs/client/client.go b/pkg/configs/client/client.go index 5dfbcc24a7..3d1c9b89d8 100644 --- a/pkg/configs/client/client.go +++ b/pkg/configs/client/client.go @@ -88,14 +88,14 @@ func (c ConfigDBClient) GetAlerts(ctx context.Context, since userconfig.ID) (*Co return response, err } -func doRequest(endpoint string, clientConfig httpclient.Config, since userconfig.ID) (*ConfigsResponse, error) { +func doRequest(endpoint string, clientOpts httpclient.Config, since userconfig.ID) (*ConfigsResponse, error) { req, err := http.NewRequest("GET", endpoint, nil) if err != nil { return nil, err } - client := &http.Client{Timeout: clientConfig.HTTPClientTimeout} - if tlsConfig := clientConfig.GetTLSConfig(); tlsConfig != nil { + client := &http.Client{Timeout: clientOpts.HTTPClientTimeout} + if tlsConfig := clientOpts.GetTLSConfig(); tlsConfig != nil { client.Transport = &http.Transport{TLSClientConfig: tlsConfig} } diff --git a/pkg/ingester/client/client.go b/pkg/ingester/client/client.go index c28269c11b..3636dbcd64 100644 --- a/pkg/ingester/client/client.go +++ b/pkg/ingester/client/client.go @@ -34,8 +34,7 @@ type closableHealthAndIngesterClient struct { // MakeIngesterClient makes a new IngesterClient func MakeIngesterClient(addr string, cfg Config) (HealthAndIngesterClient, error) { - opts := []grpc.DialOption{grpc.WithInsecure()} - opts = append(opts, cfg.GRPCClientConfig.DialOption(grpcclient.Instrument(ingesterClientRequestDuration))...) + opts := cfg.GRPCClientConfig.DialOption(grpcclient.Instrument(ingesterClientRequestDuration)) conn, err := grpc.Dial(addr, opts...) if err != nil { return nil, err diff --git a/pkg/querier/frontend/worker.go b/pkg/querier/frontend/worker.go index d399697a52..d5be6b4870 100644 --- a/pkg/querier/frontend/worker.go +++ b/pkg/querier/frontend/worker.go @@ -223,8 +223,7 @@ func (w *worker) process(c Frontend_ProcessClient) error { } func (w *worker) connect(address string) (FrontendClient, error) { - opts := []grpc.DialOption{grpc.WithInsecure()} - opts = append(opts, w.cfg.GRPCClientConfig.DialOption([]grpc.UnaryClientInterceptor{middleware.ClientUserHeaderInterceptor}, nil)...) + opts := w.cfg.GRPCClientConfig.DialOption([]grpc.UnaryClientInterceptor{middleware.ClientUserHeaderInterceptor}, nil) conn, err := grpc.Dial(address, opts...) if err != nil { return nil, err diff --git a/pkg/querier/store_gateway_client.go b/pkg/querier/store_gateway_client.go index bfaff7c779..00c90c817c 100644 --- a/pkg/querier/store_gateway_client.go +++ b/pkg/querier/store_gateway_client.go @@ -31,8 +31,7 @@ func newStoreGatewayClientFactory(cfg grpcclient.Config, reg prometheus.Register } func dialStoreGatewayClient(cfg grpcclient.Config, addr string, requestDuration *prometheus.HistogramVec) (*storeGatewayClient, error) { - opts := []grpc.DialOption{grpc.WithInsecure()} - opts = append(opts, cfg.DialOption(grpcclient.Instrument(requestDuration))...) + opts := cfg.DialOption(grpcclient.Instrument(requestDuration)) conn, err := grpc.Dial(addr, opts...) if err != nil { return nil, errors.Wrapf(err, "failed to dial store-gateway %s", addr) diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 60b9fbde0e..3204cbf0a1 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -34,6 +34,7 @@ import ( store "github.com/cortexproject/cortex/pkg/ruler/rules" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/flagext" + "github.com/cortexproject/cortex/pkg/util/grpcclient" "github.com/cortexproject/cortex/pkg/util/services" ) @@ -59,6 +60,8 @@ var ( type Config struct { // This is used for template expansion in alerts; must be a valid URL. ExternalURL flagext.URLValue `yaml:"external_url"` + // Config parameters for the GRPC Client + GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config"` // How frequently to evaluate rules by default. EvaluationInterval time.Duration `yaml:"evaluation_interval"` // Delay the evaluation of all rules by a set interval to give a buffer @@ -95,6 +98,7 @@ type Config struct { // RegisterFlags adds the flags required to config this to the given FlagSet func (cfg *Config) RegisterFlags(f *flag.FlagSet) { + cfg.GRPCClientConfig.RegisterFlagsWithPrefix("ruler", f) cfg.StoreConfig.RegisterFlags(f) cfg.Ring.RegisterFlags(f) @@ -607,7 +611,7 @@ func (r *Ruler) getShardedRules(ctx context.Context) ([]*GroupStateDesc, error) rgs := []*GroupStateDesc{} for _, rlr := range rulers.Ingesters { - conn, err := grpc.Dial(rlr.Addr, grpc.WithInsecure()) + conn, err := grpc.Dial(rlr.Addr, r.cfg.GRPCClientConfig.DialOption(nil, nil)...) if err != nil { return nil, err } diff --git a/pkg/ruler/storage.go b/pkg/ruler/storage.go index 6bface1811..d30d035dac 100644 --- a/pkg/ruler/storage.go +++ b/pkg/ruler/storage.go @@ -12,12 +12,13 @@ import ( "github.com/cortexproject/cortex/pkg/configs/client" "github.com/cortexproject/cortex/pkg/ruler/rules" "github.com/cortexproject/cortex/pkg/ruler/rules/objectclient" + "github.com/cortexproject/cortex/pkg/util/httpclient" ) // RuleStoreConfig conigures a rule store type RuleStoreConfig struct { - Type string `yaml:"type"` - ConfigDB client.Config `yaml:"configdb"` + Type string `yaml:"type"` + ConfigDB httpclient.Config `yaml:"configdb"` // Object Storage Configs Azure azure.BlobStorageConfig `yaml:"azure"` @@ -29,7 +30,6 @@ type RuleStoreConfig struct { // RegisterFlags registers flags. func (cfg *RuleStoreConfig) RegisterFlags(f *flag.FlagSet) { - cfg.ConfigDB.RegisterFlagsWithPrefix("ruler.", f) cfg.Azure.RegisterFlagsWithPrefix("ruler.storage.", f) cfg.GCS.RegisterFlagsWithPrefix("ruler.storage.", f) cfg.S3.RegisterFlagsWithPrefix("ruler.storage.", f) diff --git a/vendor/github.com/aws/aws-sdk-go/aws/arn/arn.go b/vendor/github.com/aws/aws-sdk-go/aws/arn/arn.go index 854f939ac5..1c49674290 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/arn/arn.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/arn/arn.go @@ -75,10 +75,11 @@ func Parse(arn string) (ARN, error) { }, nil } -// IsARN returns whether the given string is an arn -// by looking for whether the string starts with arn: +// IsARN returns whether the given string is an ARN by looking for +// whether the string starts with "arn:" and contains the correct number +// of sections delimited by colons(:). func IsARN(arn string) bool { - return strings.HasPrefix(arn, arnPrefix) && strings.Count(arn, ":") > arnSections-1 + return strings.HasPrefix(arn, arnPrefix) && strings.Count(arn, ":") >= arnSections-1 } // String returns the canonical representation of the ARN diff --git a/vendor/github.com/aws/aws-sdk-go/aws/corehandlers/handlers.go b/vendor/github.com/aws/aws-sdk-go/aws/corehandlers/handlers.go index 0c60e612ea..aa902d7083 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/corehandlers/handlers.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/corehandlers/handlers.go @@ -161,7 +161,7 @@ func handleSendError(r *request.Request, err error) { } // Catch all request errors, and let the default retrier determine // if the error is retryable. - r.Error = awserr.New("RequestError", "send request failed", err) + r.Error = awserr.New(request.ErrCodeRequestError, "send request failed", err) // Override the error with a context canceled error, if that was canceled. ctx := r.Context() diff --git a/vendor/github.com/aws/aws-sdk-go/aws/credentials/stscreds/assume_role_provider.go b/vendor/github.com/aws/aws-sdk-go/aws/credentials/stscreds/assume_role_provider.go index 2e528d130d..9f37f44bcf 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/credentials/stscreds/assume_role_provider.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/credentials/stscreds/assume_role_provider.go @@ -144,6 +144,13 @@ type AssumeRoleProvider struct { // Session name, if you wish to reuse the credentials elsewhere. RoleSessionName string + // Optional, you can pass tag key-value pairs to your session. These tags are called session tags. + Tags []*sts.Tag + + // A list of keys for session tags that you want to set as transitive. + // If you set a tag key as transitive, the corresponding key and value passes to subsequent sessions in a role chain. + TransitiveTagKeys []*string + // Expiry duration of the STS credentials. Defaults to 15 minutes if not set. Duration time.Duration @@ -269,10 +276,12 @@ func (p *AssumeRoleProvider) Retrieve() (credentials.Value, error) { } jitter := time.Duration(sdkrand.SeededRand.Float64() * p.MaxJitterFrac * float64(p.Duration)) input := &sts.AssumeRoleInput{ - DurationSeconds: aws.Int64(int64((p.Duration - jitter) / time.Second)), - RoleArn: aws.String(p.RoleARN), - RoleSessionName: aws.String(p.RoleSessionName), - ExternalId: p.ExternalID, + DurationSeconds: aws.Int64(int64((p.Duration - jitter) / time.Second)), + RoleArn: aws.String(p.RoleARN), + RoleSessionName: aws.String(p.RoleSessionName), + ExternalId: p.ExternalID, + Tags: p.Tags, + TransitiveTagKeys: p.TransitiveTagKeys, } if p.Policy != nil { input.Policy = p.Policy diff --git a/vendor/github.com/aws/aws-sdk-go/aws/csm/reporter.go b/vendor/github.com/aws/aws-sdk-go/aws/csm/reporter.go index 9186587fc0..835bcd49cb 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/csm/reporter.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/csm/reporter.go @@ -89,7 +89,7 @@ func getMetricException(err awserr.Error) metricException { code := err.Code() switch code { - case "RequestError", + case request.ErrCodeRequestError, request.ErrCodeSerialization, request.CanceledErrorCode: return sdkException{ diff --git a/vendor/github.com/aws/aws-sdk-go/aws/ec2metadata/token_provider.go b/vendor/github.com/aws/aws-sdk-go/aws/ec2metadata/token_provider.go index 06f76055f3..663372a915 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/ec2metadata/token_provider.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/ec2metadata/token_provider.go @@ -62,7 +62,7 @@ func (t *tokenProvider) fetchTokenHandler(r *request.Request) { // Check if request timed out while waiting for response if e, ok := requestFailureError.OrigErr().(awserr.Error); ok { - if e.Code() == "RequestError" { + if e.Code() == request.ErrCodeRequestError { atomic.StoreUint32(&t.disabled, 1) } } diff --git a/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go b/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go index 767b2f69ea..52e593d212 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go @@ -179,17 +179,47 @@ var awsPartition = partition{ "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, - "eu-central-1": endpoint{}, - "eu-north-1": endpoint{}, - "eu-west-1": endpoint{}, - "eu-west-2": endpoint{}, - "eu-west-3": endpoint{}, - "me-south-1": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, + "ca-central-1-fips": endpoint{ + Hostname: "acm-fips.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-1-fips": endpoint{ + Hostname: "acm-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "us-east-2": endpoint{}, + "us-east-2-fips": endpoint{ + Hostname: "acm-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "us-west-1": endpoint{}, + "us-west-1-fips": endpoint{ + Hostname: "acm-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "us-west-2": endpoint{}, + "us-west-2-fips": endpoint{ + Hostname: "acm-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, }, }, "acm-pca": service{ @@ -209,12 +239,42 @@ var awsPartition = partition{ "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, - "me-south-1": endpoint{}, - "sa-east-1": endpoint{}, - "us-east-1": endpoint{}, - "us-east-2": endpoint{}, - "us-west-1": endpoint{}, - "us-west-2": endpoint{}, + "fips-ca-central-1": endpoint{ + Hostname: "acm-pca-fips.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, + "fips-us-east-1": endpoint{ + Hostname: "acm-pca-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + }, + "fips-us-east-2": endpoint{ + Hostname: "acm-pca-fips.us-east-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-2", + }, + }, + "fips-us-west-1": endpoint{ + Hostname: "acm-pca-fips.us-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-1", + }, + }, + "fips-us-west-2": endpoint{ + Hostname: "acm-pca-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + }, + "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, }, }, "api.ecr": service{ @@ -523,6 +583,7 @@ var awsPartition = partition{ "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, "me-south-1": endpoint{}, + "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, "us-west-1": endpoint{}, @@ -667,9 +728,15 @@ var awsPartition = partition{ Endpoints: endpoints{ "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, "us-west-2": endpoint{}, @@ -856,6 +923,7 @@ var awsPartition = partition{ "codecommit": service{ Endpoints: endpoints{ + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, "ap-northeast-2": endpoint{}, "ap-south-1": endpoint{}, @@ -1071,8 +1139,10 @@ var awsPartition = partition{ Endpoints: endpoints{ "ap-northeast-1": endpoint{}, + "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, "eu-central-1": endpoint{}, + "eu-west-2": endpoint{}, "us-east-1": endpoint{}, "us-west-2": endpoint{}, }, @@ -1279,6 +1349,12 @@ var awsPartition = partition{ Region: "ap-southeast-2", }, }, + "ca-central-1": endpoint{ + Hostname: "rds.ca-central-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "ca-central-1", + }, + }, "eu-central-1": endpoint{ Hostname: "rds.eu-central-1.amazonaws.com", CredentialScope: credentialScope{ @@ -1337,6 +1413,7 @@ var awsPartition = partition{ "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, @@ -1854,8 +1931,10 @@ var awsPartition = partition{ "groundstation": service{ Endpoints: endpoints{ - "us-east-2": endpoint{}, - "us-west-2": endpoint{}, + "eu-north-1": endpoint{}, + "me-south-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-2": endpoint{}, }, }, "guardduty": service{ @@ -2166,11 +2245,13 @@ var awsPartition = partition{ "ap-south-1": endpoint{}, "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, + "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, "us-west-2": endpoint{}, @@ -2179,11 +2260,20 @@ var awsPartition = partition{ "kinesisvideo": service{ Endpoints: endpoints{ + "ap-east-1": endpoint{}, "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-south-1": endpoint{}, + "ap-southeast-1": endpoint{}, "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "sa-east-1": endpoint{}, "us-east-1": endpoint{}, + "us-east-2": endpoint{}, "us-west-2": endpoint{}, }, }, @@ -2465,9 +2555,10 @@ var awsPartition = partition{ }, }, Endpoints: endpoints{ - "eu-west-1": endpoint{}, - "us-east-1": endpoint{}, - "us-west-2": endpoint{}, + "ap-southeast-2": endpoint{}, + "eu-west-1": endpoint{}, + "us-east-1": endpoint{}, + "us-west-2": endpoint{}, }, }, "monitoring": service{ @@ -2505,6 +2596,7 @@ var awsPartition = partition{ "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, @@ -2532,6 +2624,7 @@ var awsPartition = partition{ Region: "us-west-2", }, }, + "sa-east-1": endpoint{}, "us-east-1": endpoint{}, "us-east-2": endpoint{}, "us-west-1": endpoint{}, @@ -2743,6 +2836,20 @@ var awsPartition = partition{ }, }, }, + "outposts": service{ + + Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-southeast-2": endpoint{}, + "eu-central-1": endpoint{}, + "eu-west-1": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, "pinpoint": service{ Defaults: endpoint{ CredentialScope: credentialScope{ @@ -3062,6 +3169,7 @@ var awsPartition = partition{ "ap-southeast-2": endpoint{}, "ca-central-1": endpoint{}, "eu-central-1": endpoint{}, + "eu-north-1": endpoint{}, "eu-west-1": endpoint{}, "eu-west-2": endpoint{}, "eu-west-3": endpoint{}, @@ -3078,9 +3186,10 @@ var awsPartition = partition{ }, }, Endpoints: endpoints{ - "eu-west-1": endpoint{}, - "us-east-1": endpoint{}, - "us-west-2": endpoint{}, + "ap-southeast-2": endpoint{}, + "eu-west-1": endpoint{}, + "us-east-1": endpoint{}, + "us-west-2": endpoint{}, }, }, "runtime.sagemaker": service{ @@ -3157,7 +3266,8 @@ var awsPartition = partition{ SignatureVersions: []string{"s3", "s3v4"}, }, "aws-global": endpoint{ - Hostname: "s3.amazonaws.com", + Hostname: "s3.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, CredentialScope: credentialScope{ Region: "us-east-1", }, @@ -3183,7 +3293,10 @@ var awsPartition = partition{ Hostname: "s3.sa-east-1.amazonaws.com", SignatureVersions: []string{"s3", "s3v4"}, }, - "us-east-1": endpoint{}, + "us-east-1": endpoint{ + Hostname: "s3.us-east-1.amazonaws.com", + SignatureVersions: []string{"s3", "s3v4"}, + }, "us-east-2": endpoint{}, "us-west-1": endpoint{ Hostname: "s3.us-west-1.amazonaws.com", @@ -4448,6 +4561,13 @@ var awscnPartition = partition{ "cn-north-1": endpoint{}, }, }, + "health": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, "iam": service{ PartitionEndpoint: "aws-cn-global", IsRegionalized: boxedFalse, @@ -4579,6 +4699,13 @@ var awscnPartition = partition{ }, }, }, + "secretsmanager": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, "sms": service{ Endpoints: endpoints{ @@ -4701,6 +4828,13 @@ var awscnPartition = partition{ "cn-northwest-1": endpoint{}, }, }, + "xray": service{ + + Endpoints: endpoints{ + "cn-north-1": endpoint{}, + "cn-northwest-1": endpoint{}, + }, + }, }, } @@ -5588,7 +5722,6 @@ var awsisoPartition = partition{ }, "application-autoscaling": service{ Defaults: endpoint{ - Hostname: "autoscaling.us-iso-east-1.c2s.ic.gov", Protocols: []string{"http", "https"}, }, Endpoints: endpoints{ @@ -5917,7 +6050,6 @@ var awsisobPartition = partition{ Services: services{ "application-autoscaling": service{ Defaults: endpoint{ - Hostname: "autoscaling.us-isob-east-1.sc2s.sgov.gov", Protocols: []string{"http", "https"}, }, Endpoints: endpoints{ diff --git a/vendor/github.com/aws/aws-sdk-go/aws/endpoints/endpoints.go b/vendor/github.com/aws/aws-sdk-go/aws/endpoints/endpoints.go index 1f53d9cb68..ca956e5f12 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/endpoints/endpoints.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/endpoints/endpoints.go @@ -317,7 +317,7 @@ func (p Partition) EndpointFor(service, region string, opts ...func(*Options)) ( // Regions returns a map of Regions indexed by their ID. This is useful for // enumerating over the regions in a partition. func (p Partition) Regions() map[string]Region { - rs := map[string]Region{} + rs := make(map[string]Region, len(p.p.Regions)) for id, r := range p.p.Regions { rs[id] = Region{ id: id, @@ -332,7 +332,7 @@ func (p Partition) Regions() map[string]Region { // Services returns a map of Service indexed by their ID. This is useful for // enumerating over the services in a partition. func (p Partition) Services() map[string]Service { - ss := map[string]Service{} + ss := make(map[string]Service, len(p.p.Services)) for id := range p.p.Services { ss[id] = Service{ id: id, @@ -419,7 +419,7 @@ func (s Service) Regions() map[string]Region { // A region is the AWS region the service exists in. Whereas a Endpoint is // an URL that can be resolved to a instance of a service. func (s Service) Endpoints() map[string]Endpoint { - es := map[string]Endpoint{} + es := make(map[string]Endpoint, len(s.p.Services[s.id].Endpoints)) for id := range s.p.Services[s.id].Endpoints { es[id] = Endpoint{ id: id, diff --git a/vendor/github.com/aws/aws-sdk-go/aws/request/handlers.go b/vendor/github.com/aws/aws-sdk-go/aws/request/handlers.go index 185b073181..e819ab6c0e 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/request/handlers.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/request/handlers.go @@ -10,6 +10,7 @@ import ( type Handlers struct { Validate HandlerList Build HandlerList + BuildStream HandlerList Sign HandlerList Send HandlerList ValidateResponse HandlerList @@ -28,6 +29,7 @@ func (h *Handlers) Copy() Handlers { return Handlers{ Validate: h.Validate.copy(), Build: h.Build.copy(), + BuildStream: h.BuildStream.copy(), Sign: h.Sign.copy(), Send: h.Send.copy(), ValidateResponse: h.ValidateResponse.copy(), @@ -46,6 +48,7 @@ func (h *Handlers) Copy() Handlers { func (h *Handlers) Clear() { h.Validate.Clear() h.Build.Clear() + h.BuildStream.Clear() h.Send.Clear() h.Sign.Clear() h.Unmarshal.Clear() @@ -67,6 +70,9 @@ func (h *Handlers) IsEmpty() bool { if h.Build.Len() != 0 { return false } + if h.BuildStream.Len() != 0 { + return false + } if h.Send.Len() != 0 { return false } @@ -320,3 +326,18 @@ func MakeAddToUserAgentFreeFormHandler(s string) func(*Request) { AddToUserAgent(r, s) } } + +// WithSetRequestHeaders updates the operation request's HTTP header to contain +// the header key value pairs provided. If the header key already exists in the +// request's HTTP header set, the existing value(s) will be replaced. +func WithSetRequestHeaders(h map[string]string) Option { + return withRequestHeader(h).SetRequestHeaders +} + +type withRequestHeader map[string]string + +func (h withRequestHeader) SetRequestHeaders(r *Request) { + for k, v := range h { + r.HTTPRequest.Header[k] = []string{v} + } +} diff --git a/vendor/github.com/aws/aws-sdk-go/aws/request/request.go b/vendor/github.com/aws/aws-sdk-go/aws/request/request.go index 52178141da..59da73ed40 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/request/request.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/request/request.go @@ -36,6 +36,10 @@ const ( // API request that was canceled. Requests given a aws.Context may // return this error when canceled. CanceledErrorCode = "RequestCanceled" + + // ErrCodeRequestError is an error preventing the SDK from continuing to + // process the request. + ErrCodeRequestError = "RequestError" ) // A Request is the service request to be made. @@ -51,6 +55,7 @@ type Request struct { HTTPRequest *http.Request HTTPResponse *http.Response Body io.ReadSeeker + streamingBody io.ReadCloser BodyStart int64 // offset from beginning of Body that the request body starts Params interface{} Error error @@ -295,6 +300,13 @@ func (r *Request) SetReaderBody(reader io.ReadSeeker) { r.ResetBody() } +// SetStreamingBody set the reader to be used for the request that will stream +// bytes to the server. Request's Body must not be set to any reader. +func (r *Request) SetStreamingBody(reader io.ReadCloser) { + r.streamingBody = reader + r.SetReaderBody(aws.ReadSeekCloser(reader)) +} + // Presign returns the request's signed URL. Error will be returned // if the signing fails. The expire parameter is only used for presigned Amazon // S3 API requests. All other AWS services will use a fixed expiration @@ -419,6 +431,10 @@ func (r *Request) Sign() error { } func (r *Request) getNextRequestBody() (body io.ReadCloser, err error) { + if r.streamingBody != nil { + return r.streamingBody, nil + } + if r.safeBody != nil { r.safeBody.Close() } diff --git a/vendor/github.com/aws/aws-sdk-go/aws/request/retryer.go b/vendor/github.com/aws/aws-sdk-go/aws/request/retryer.go index 8015acc67e..1b61dec9c2 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/request/retryer.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/request/retryer.go @@ -75,7 +75,7 @@ func (d noOpRetryer) RetryRules(_ *Request) time.Duration { // retryableCodes is a collection of service response codes which are retry-able // without any further action. var retryableCodes = map[string]struct{}{ - "RequestError": {}, + ErrCodeRequestError: {}, "RequestTimeout": {}, ErrCodeResponseTimeout: {}, "RequestTimeoutException": {}, // Glacier's flavor of RequestTimeout @@ -83,6 +83,7 @@ var retryableCodes = map[string]struct{}{ var throttleCodes = map[string]struct{}{ "ProvisionedThroughputExceededException": {}, + "ThrottledException": {}, // SNS, XRay, ResourceGroupsTagging API "Throttling": {}, "ThrottlingException": {}, "RequestLimitExceeded": {}, @@ -176,8 +177,8 @@ func shouldRetryError(origErr error) bool { origErr := err.OrigErr() var shouldRetry bool if origErr != nil { - shouldRetry := shouldRetryError(origErr) - if err.Code() == "RequestError" && !shouldRetry { + shouldRetry = shouldRetryError(origErr) + if err.Code() == ErrCodeRequestError && !shouldRetry { return false } } diff --git a/vendor/github.com/aws/aws-sdk-go/aws/signer/v4/stream.go b/vendor/github.com/aws/aws-sdk-go/aws/signer/v4/stream.go new file mode 100644 index 0000000000..02cbd97e23 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go/aws/signer/v4/stream.go @@ -0,0 +1,63 @@ +package v4 + +import ( + "encoding/hex" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws/credentials" +) + +type credentialValueProvider interface { + Get() (credentials.Value, error) +} + +// StreamSigner implements signing of event stream encoded payloads +type StreamSigner struct { + region string + service string + + credentials credentialValueProvider + + prevSig []byte +} + +// NewStreamSigner creates a SigV4 signer used to sign Event Stream encoded messages +func NewStreamSigner(region, service string, seedSignature []byte, credentials *credentials.Credentials) *StreamSigner { + return &StreamSigner{ + region: region, + service: service, + credentials: credentials, + prevSig: seedSignature, + } +} + +// GetSignature takes an event stream encoded headers and payload and returns a signature +func (s *StreamSigner) GetSignature(headers, payload []byte, date time.Time) ([]byte, error) { + credValue, err := s.credentials.Get() + if err != nil { + return nil, err + } + + sigKey := deriveSigningKey(s.region, s.service, credValue.SecretAccessKey, date) + + keyPath := buildSigningScope(s.region, s.service, date) + + stringToSign := buildEventStreamStringToSign(headers, payload, s.prevSig, keyPath, date) + + signature := hmacSHA256(sigKey, []byte(stringToSign)) + s.prevSig = signature + + return signature, nil +} + +func buildEventStreamStringToSign(headers, payload, prevSig []byte, scope string, date time.Time) string { + return strings.Join([]string{ + "AWS4-HMAC-SHA256-PAYLOAD", + formatTime(date), + scope, + hex.EncodeToString(prevSig), + hex.EncodeToString(hashSHA256(headers)), + hex.EncodeToString(hashSHA256(payload)), + }, "\n") +} diff --git a/vendor/github.com/aws/aws-sdk-go/aws/signer/v4/v4.go b/vendor/github.com/aws/aws-sdk-go/aws/signer/v4/v4.go index 8104793aa5..03b5afb937 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/signer/v4/v4.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/signer/v4/v4.go @@ -76,9 +76,14 @@ import ( ) const ( + authorizationHeader = "Authorization" + authHeaderSignatureElem = "Signature=" + signatureQueryKey = "X-Amz-Signature" + authHeaderPrefix = "AWS4-HMAC-SHA256" timeFormat = "20060102T150405Z" shortTimeFormat = "20060102" + awsV4Request = "aws4_request" // emptyStringSHA256 is a SHA256 of an empty string emptyStringSHA256 = `e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855` @@ -87,9 +92,9 @@ const ( var ignoredHeaders = rules{ blacklist{ mapRule{ - "Authorization": struct{}{}, - "User-Agent": struct{}{}, - "X-Amzn-Trace-Id": struct{}{}, + authorizationHeader: struct{}{}, + "User-Agent": struct{}{}, + "X-Amzn-Trace-Id": struct{}{}, }, }, } @@ -229,11 +234,9 @@ type signingCtx struct { DisableURIPathEscaping bool - credValues credentials.Value - isPresign bool - formattedTime string - formattedShortTime string - unsignedPayload bool + credValues credentials.Value + isPresign bool + unsignedPayload bool bodyDigest string signedHeaders string @@ -532,39 +535,56 @@ func (ctx *signingCtx) build(disableHeaderHoisting bool) error { ctx.buildSignature() // depends on string to sign if ctx.isPresign { - ctx.Request.URL.RawQuery += "&X-Amz-Signature=" + ctx.signature + ctx.Request.URL.RawQuery += "&" + signatureQueryKey + "=" + ctx.signature } else { parts := []string{ authHeaderPrefix + " Credential=" + ctx.credValues.AccessKeyID + "/" + ctx.credentialString, "SignedHeaders=" + ctx.signedHeaders, - "Signature=" + ctx.signature, + authHeaderSignatureElem + ctx.signature, } - ctx.Request.Header.Set("Authorization", strings.Join(parts, ", ")) + ctx.Request.Header.Set(authorizationHeader, strings.Join(parts, ", ")) } return nil } -func (ctx *signingCtx) buildTime() { - ctx.formattedTime = ctx.Time.UTC().Format(timeFormat) - ctx.formattedShortTime = ctx.Time.UTC().Format(shortTimeFormat) +// GetSignedRequestSignature attempts to extract the signature of the request. +// Returning an error if the request is unsigned, or unable to extract the +// signature. +func GetSignedRequestSignature(r *http.Request) ([]byte, error) { + + if auth := r.Header.Get(authorizationHeader); len(auth) != 0 { + ps := strings.Split(auth, ", ") + for _, p := range ps { + if idx := strings.Index(p, authHeaderSignatureElem); idx >= 0 { + sig := p[len(authHeaderSignatureElem):] + if len(sig) == 0 { + return nil, fmt.Errorf("invalid request signature authorization header") + } + return hex.DecodeString(sig) + } + } + } + + if sig := r.URL.Query().Get("X-Amz-Signature"); len(sig) != 0 { + return hex.DecodeString(sig) + } + + return nil, fmt.Errorf("request not signed") +} +func (ctx *signingCtx) buildTime() { if ctx.isPresign { duration := int64(ctx.ExpireTime / time.Second) - ctx.Query.Set("X-Amz-Date", ctx.formattedTime) + ctx.Query.Set("X-Amz-Date", formatTime(ctx.Time)) ctx.Query.Set("X-Amz-Expires", strconv.FormatInt(duration, 10)) } else { - ctx.Request.Header.Set("X-Amz-Date", ctx.formattedTime) + ctx.Request.Header.Set("X-Amz-Date", formatTime(ctx.Time)) } } func (ctx *signingCtx) buildCredentialString() { - ctx.credentialString = strings.Join([]string{ - ctx.formattedShortTime, - ctx.Region, - ctx.ServiceName, - "aws4_request", - }, "/") + ctx.credentialString = buildSigningScope(ctx.Region, ctx.ServiceName, ctx.Time) if ctx.isPresign { ctx.Query.Set("X-Amz-Credential", ctx.credValues.AccessKeyID+"/"+ctx.credentialString) @@ -653,19 +673,15 @@ func (ctx *signingCtx) buildCanonicalString() { func (ctx *signingCtx) buildStringToSign() { ctx.stringToSign = strings.Join([]string{ authHeaderPrefix, - ctx.formattedTime, + formatTime(ctx.Time), ctx.credentialString, - hex.EncodeToString(makeSha256([]byte(ctx.canonicalString))), + hex.EncodeToString(hashSHA256([]byte(ctx.canonicalString))), }, "\n") } func (ctx *signingCtx) buildSignature() { - secret := ctx.credValues.SecretAccessKey - date := makeHmac([]byte("AWS4"+secret), []byte(ctx.formattedShortTime)) - region := makeHmac(date, []byte(ctx.Region)) - service := makeHmac(region, []byte(ctx.ServiceName)) - credentials := makeHmac(service, []byte("aws4_request")) - signature := makeHmac(credentials, []byte(ctx.stringToSign)) + creds := deriveSigningKey(ctx.Region, ctx.ServiceName, ctx.credValues.SecretAccessKey, ctx.Time) + signature := hmacSHA256(creds, []byte(ctx.stringToSign)) ctx.signature = hex.EncodeToString(signature) } @@ -726,13 +742,13 @@ func (ctx *signingCtx) removePresign() { ctx.Query.Del("X-Amz-SignedHeaders") } -func makeHmac(key []byte, data []byte) []byte { +func hmacSHA256(key []byte, data []byte) []byte { hash := hmac.New(sha256.New, key) hash.Write(data) return hash.Sum(nil) } -func makeSha256(data []byte) []byte { +func hashSHA256(data []byte) []byte { hash := sha256.New() hash.Write(data) return hash.Sum(nil) @@ -804,3 +820,28 @@ func stripExcessSpaces(vals []string) { vals[i] = string(buf[:m]) } } + +func buildSigningScope(region, service string, dt time.Time) string { + return strings.Join([]string{ + formatShortTime(dt), + region, + service, + awsV4Request, + }, "/") +} + +func deriveSigningKey(region, service, secretKey string, dt time.Time) []byte { + kDate := hmacSHA256([]byte("AWS4"+secretKey), []byte(formatShortTime(dt))) + kRegion := hmacSHA256(kDate, []byte(region)) + kService := hmacSHA256(kRegion, []byte(service)) + signingKey := hmacSHA256(kService, []byte(awsV4Request)) + return signingKey +} + +func formatShortTime(dt time.Time) string { + return dt.UTC().Format(shortTimeFormat) +} + +func formatTime(dt time.Time) string { + return dt.UTC().Format(timeFormat) +} diff --git a/vendor/github.com/aws/aws-sdk-go/aws/types.go b/vendor/github.com/aws/aws-sdk-go/aws/types.go index 455091540f..d542ef01bc 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/types.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/types.go @@ -2,6 +2,7 @@ package aws import ( "io" + "strings" "sync" "github.com/aws/aws-sdk-go/internal/sdkio" @@ -205,3 +206,36 @@ func (b *WriteAtBuffer) Bytes() []byte { defer b.m.Unlock() return b.buf } + +// MultiCloser is a utility to close multiple io.Closers within a single +// statement. +type MultiCloser []io.Closer + +// Close closes all of the io.Closers making up the MultiClosers. Any +// errors that occur while closing will be returned in the order they +// occur. +func (m MultiCloser) Close() error { + var errs errors + for _, c := range m { + err := c.Close() + if err != nil { + errs = append(errs, err) + } + } + if len(errs) != 0 { + return errs + } + + return nil +} + +type errors []error + +func (es errors) Error() string { + var parts []string + for _, e := range es { + parts = append(parts, e.Error()) + } + + return strings.Join(parts, "\n") +} diff --git a/vendor/github.com/aws/aws-sdk-go/aws/version.go b/vendor/github.com/aws/aws-sdk-go/aws/version.go index 437bc27ae7..acb13343bf 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/version.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/version.go @@ -5,4 +5,4 @@ package aws const SDKName = "aws-sdk-go" // SDKVersion is the version of this SDK -const SDKVersion = "1.25.48" +const SDKVersion = "1.27.0" diff --git a/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/debug.go b/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/debug.go index ecc7bf82fa..151054971a 100644 --- a/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/debug.go +++ b/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/debug.go @@ -101,7 +101,7 @@ func (hs *decodedHeaders) UnmarshalJSON(b []byte) error { } headers.Set(h.Name, value) } - (*hs) = decodedHeaders(headers) + *hs = decodedHeaders(headers) return nil } diff --git a/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/decode.go b/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/decode.go index 4b972b2d66..4743393918 100644 --- a/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/decode.go +++ b/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/decode.go @@ -21,10 +21,24 @@ type Decoder struct { // NewDecoder initializes and returns a Decoder for decoding event // stream messages from the reader provided. -func NewDecoder(r io.Reader) *Decoder { - return &Decoder{ +func NewDecoder(r io.Reader, opts ...func(*Decoder)) *Decoder { + d := &Decoder{ r: r, } + + for _, opt := range opts { + opt(d) + } + + return d +} + +// DecodeWithLogger adds a logger to be used by the decoder when decoding +// stream events. +func DecodeWithLogger(logger aws.Logger) func(*Decoder) { + return func(d *Decoder) { + d.logger = logger + } } // Decode attempts to decode a single message from the event stream reader. @@ -40,6 +54,15 @@ func (d *Decoder) Decode(payloadBuf []byte) (m Message, err error) { }() } + m, err = Decode(reader, payloadBuf) + + return m, err +} + +// Decode attempts to decode a single message from the event stream reader. +// Will return the event stream message, or error if Decode fails to read +// the message from the reader. +func Decode(reader io.Reader, payloadBuf []byte) (m Message, err error) { crc := crc32.New(crc32IEEETable) hashReader := io.TeeReader(reader, crc) @@ -72,12 +95,6 @@ func (d *Decoder) Decode(payloadBuf []byte) (m Message, err error) { return m, nil } -// UseLogger specifies the Logger that that the decoder should use to log the -// message decode to. -func (d *Decoder) UseLogger(logger aws.Logger) { - d.logger = logger -} - func logMessageDecode(logger aws.Logger, msgBuf *bytes.Buffer, msg Message, decodeErr error) { w := bytes.NewBuffer(nil) defer func() { logger.Log(w.String()) }() diff --git a/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/encode.go b/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/encode.go index 150a60981d..ffade3bc0c 100644 --- a/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/encode.go +++ b/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/encode.go @@ -3,61 +3,107 @@ package eventstream import ( "bytes" "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" "hash" "hash/crc32" "io" + + "github.com/aws/aws-sdk-go/aws" ) // Encoder provides EventStream message encoding. type Encoder struct { - w io.Writer + w io.Writer + logger aws.Logger headersBuf *bytes.Buffer } // NewEncoder initializes and returns an Encoder to encode Event Stream // messages to an io.Writer. -func NewEncoder(w io.Writer) *Encoder { - return &Encoder{ +func NewEncoder(w io.Writer, opts ...func(*Encoder)) *Encoder { + e := &Encoder{ w: w, headersBuf: bytes.NewBuffer(nil), } + + for _, opt := range opts { + opt(e) + } + + return e +} + +// EncodeWithLogger adds a logger to be used by the encode when decoding +// stream events. +func EncodeWithLogger(logger aws.Logger) func(*Encoder) { + return func(d *Encoder) { + d.logger = logger + } } // Encode encodes a single EventStream message to the io.Writer the Encoder // was created with. An error is returned if writing the message fails. -func (e *Encoder) Encode(msg Message) error { +func (e *Encoder) Encode(msg Message) (err error) { e.headersBuf.Reset() - err := encodeHeaders(e.headersBuf, msg.Headers) - if err != nil { + writer := e.w + if e.logger != nil { + encodeMsgBuf := bytes.NewBuffer(nil) + writer = io.MultiWriter(writer, encodeMsgBuf) + defer func() { + logMessageEncode(e.logger, encodeMsgBuf, msg, err) + }() + } + + if err = EncodeHeaders(e.headersBuf, msg.Headers); err != nil { return err } crc := crc32.New(crc32IEEETable) - hashWriter := io.MultiWriter(e.w, crc) + hashWriter := io.MultiWriter(writer, crc) headersLen := uint32(e.headersBuf.Len()) payloadLen := uint32(len(msg.Payload)) - if err := encodePrelude(hashWriter, crc, headersLen, payloadLen); err != nil { + if err = encodePrelude(hashWriter, crc, headersLen, payloadLen); err != nil { return err } if headersLen > 0 { - if _, err := io.Copy(hashWriter, e.headersBuf); err != nil { + if _, err = io.Copy(hashWriter, e.headersBuf); err != nil { return err } } if payloadLen > 0 { - if _, err := hashWriter.Write(msg.Payload); err != nil { + if _, err = hashWriter.Write(msg.Payload); err != nil { return err } } msgCRC := crc.Sum32() - return binary.Write(e.w, binary.BigEndian, msgCRC) + return binary.Write(writer, binary.BigEndian, msgCRC) +} + +func logMessageEncode(logger aws.Logger, msgBuf *bytes.Buffer, msg Message, encodeErr error) { + w := bytes.NewBuffer(nil) + defer func() { logger.Log(w.String()) }() + + fmt.Fprintf(w, "Message to encode:\n") + encoder := json.NewEncoder(w) + if err := encoder.Encode(msg); err != nil { + fmt.Fprintf(w, "Failed to get encoded message, %v\n", err) + } + + if encodeErr != nil { + fmt.Fprintf(w, "Encode error: %v\n", encodeErr) + return + } + + fmt.Fprintf(w, "Raw message:\n%s\n", hex.Dump(msgBuf.Bytes())) } func encodePrelude(w io.Writer, crc hash.Hash32, headersLen, payloadLen uint32) error { @@ -86,7 +132,9 @@ func encodePrelude(w io.Writer, crc hash.Hash32, headersLen, payloadLen uint32) return nil } -func encodeHeaders(w io.Writer, headers Headers) error { +// EncodeHeaders writes the header values to the writer encoded in the event +// stream format. Returns an error if a header fails to encode. +func EncodeHeaders(w io.Writer, headers Headers) error { for _, h := range headers { hn := headerName{ Len: uint8(len(h.Name)), diff --git a/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/error.go b/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/error.go index 5ea5a988b6..34c2e89d53 100644 --- a/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/error.go +++ b/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/error.go @@ -1,6 +1,9 @@ package eventstreamapi -import "fmt" +import ( + "fmt" + "sync" +) type messageError struct { code string @@ -22,3 +25,53 @@ func (e messageError) Error() string { func (e messageError) OrigErr() error { return nil } + +// OnceError wraps the behavior of recording an error +// once and signal on a channel when this has occurred. +// Signaling is done by closing of the channel. +// +// Type is safe for concurrent usage. +type OnceError struct { + mu sync.RWMutex + err error + ch chan struct{} +} + +// NewOnceError return a new OnceError +func NewOnceError() *OnceError { + return &OnceError{ + ch: make(chan struct{}, 1), + } +} + +// Err acquires a read-lock and returns an +// error if one has been set. +func (e *OnceError) Err() error { + e.mu.RLock() + err := e.err + e.mu.RUnlock() + + return err +} + +// SetError acquires a write-lock and will set +// the underlying error value if one has not been set. +func (e *OnceError) SetError(err error) { + if err == nil { + return + } + + e.mu.Lock() + if e.err == nil { + e.err = err + close(e.ch) + } + e.mu.Unlock() +} + +// ErrorSet returns a channel that will be used to signal +// that an error has been set. This channel will be closed +// when the error value has been set for OnceError. +func (e *OnceError) ErrorSet() <-chan struct{} { + return e.ch +} diff --git a/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/api.go b/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/reader.go similarity index 77% rename from vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/api.go rename to vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/reader.go index 97937c8e59..bb8ea5da16 100644 --- a/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/api.go +++ b/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/reader.go @@ -2,9 +2,7 @@ package eventstreamapi import ( "fmt" - "io" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/private/protocol" "github.com/aws/aws-sdk-go/private/protocol/eventstream" ) @@ -15,27 +13,8 @@ type Unmarshaler interface { UnmarshalEvent(protocol.PayloadUnmarshaler, eventstream.Message) error } -// EventStream headers with specific meaning to async API functionality. -const ( - MessageTypeHeader = `:message-type` // Identifies type of message. - EventMessageType = `event` - ErrorMessageType = `error` - ExceptionMessageType = `exception` - - // Message Events - EventTypeHeader = `:event-type` // Identifies message event type e.g. "Stats". - - // Message Error - ErrorCodeHeader = `:error-code` - ErrorMessageHeader = `:error-message` - - // Message Exception - ExceptionTypeHeader = `:exception-type` -) - // EventReader provides reading from the EventStream of an reader. type EventReader struct { - reader io.ReadCloser decoder *eventstream.Decoder unmarshalerForEventType func(string) (Unmarshaler, error) @@ -47,27 +26,18 @@ type EventReader struct { // NewEventReader returns a EventReader built from the reader and unmarshaler // provided. Use ReadStream method to start reading from the EventStream. func NewEventReader( - reader io.ReadCloser, + decoder *eventstream.Decoder, payloadUnmarshaler protocol.PayloadUnmarshaler, unmarshalerForEventType func(string) (Unmarshaler, error), ) *EventReader { return &EventReader{ - reader: reader, - decoder: eventstream.NewDecoder(reader), + decoder: decoder, payloadUnmarshaler: payloadUnmarshaler, unmarshalerForEventType: unmarshalerForEventType, payloadBuf: make([]byte, 10*1024), } } -// UseLogger instructs the EventReader to use the logger and log level -// specified. -func (r *EventReader) UseLogger(logger aws.Logger, logLevel aws.LogLevelType) { - if logger != nil && logLevel.Matches(aws.LogDebugWithEventStreamBody) { - r.decoder.UseLogger(logger) - } -} - // ReadEvent attempts to read a message from the EventStream and return the // unmarshaled event value that the message is for. // @@ -95,8 +65,7 @@ func (r *EventReader) ReadEvent() (event interface{}, err error) { case EventMessageType: return r.unmarshalEventMessage(msg) case ExceptionMessageType: - err = r.unmarshalEventException(msg) - return nil, err + return nil, r.unmarshalEventException(msg) case ErrorMessageType: return nil, r.unmarshalErrorMessage(msg) default: @@ -174,11 +143,6 @@ func (r *EventReader) unmarshalErrorMessage(msg eventstream.Message) (err error) return msgErr } -// Close closes the EventReader's EventStream reader. -func (r *EventReader) Close() error { - return r.reader.Close() -} - // GetHeaderString returns the value of the header as a string. If the header // is not set or the value is not a string an error will be returned. func GetHeaderString(msg eventstream.Message, headerName string) (string, error) { diff --git a/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/shared.go b/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/shared.go new file mode 100644 index 0000000000..e46b8acc20 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/shared.go @@ -0,0 +1,23 @@ +package eventstreamapi + +// EventStream headers with specific meaning to async API functionality. +const ( + ChunkSignatureHeader = `:chunk-signature` // chunk signature for message + DateHeader = `:date` // Date header for signature + + // Message header and values + MessageTypeHeader = `:message-type` // Identifies type of message. + EventMessageType = `event` + ErrorMessageType = `error` + ExceptionMessageType = `exception` + + // Message Events + EventTypeHeader = `:event-type` // Identifies message event type e.g. "Stats". + + // Message Error + ErrorCodeHeader = `:error-code` + ErrorMessageHeader = `:error-message` + + // Message Exception + ExceptionTypeHeader = `:exception-type` +) diff --git a/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/signer.go b/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/signer.go new file mode 100644 index 0000000000..3a7ba5cd57 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/signer.go @@ -0,0 +1,123 @@ +package eventstreamapi + +import ( + "bytes" + "strings" + "time" + + "github.com/aws/aws-sdk-go/private/protocol/eventstream" +) + +var timeNow = time.Now + +// StreamSigner defines an interface for the implementation of signing of event stream payloads +type StreamSigner interface { + GetSignature(headers, payload []byte, date time.Time) ([]byte, error) +} + +// SignEncoder envelopes event stream messages +// into an event stream message payload with included +// signature headers using the provided signer and encoder. +type SignEncoder struct { + signer StreamSigner + encoder Encoder + bufEncoder *BufferEncoder + + closeErr error + closed bool +} + +// NewSignEncoder returns a new SignEncoder using the provided stream signer and +// event stream encoder. +func NewSignEncoder(signer StreamSigner, encoder Encoder) *SignEncoder { + // TODO: Need to pass down logging + + return &SignEncoder{ + signer: signer, + encoder: encoder, + bufEncoder: NewBufferEncoder(), + } +} + +// Close encodes a final event stream signing envelope with an empty event stream +// payload. This final end-frame is used to mark the conclusion of the stream. +func (s *SignEncoder) Close() error { + if s.closed { + return s.closeErr + } + + if err := s.encode([]byte{}); err != nil { + if strings.Contains(err.Error(), "on closed pipe") { + return nil + } + + s.closeErr = err + s.closed = true + return s.closeErr + } + + return nil +} + +// Encode takes the provided message and add envelopes the message +// with the required signature. +func (s *SignEncoder) Encode(msg eventstream.Message) error { + payload, err := s.bufEncoder.Encode(msg) + if err != nil { + return err + } + + return s.encode(payload) +} + +func (s SignEncoder) encode(payload []byte) error { + date := timeNow() + + var msg eventstream.Message + msg.Headers.Set(DateHeader, eventstream.TimestampValue(date)) + msg.Payload = payload + + var headers bytes.Buffer + if err := eventstream.EncodeHeaders(&headers, msg.Headers); err != nil { + return err + } + + sig, err := s.signer.GetSignature(headers.Bytes(), msg.Payload, date) + if err != nil { + return err + } + + msg.Headers.Set(ChunkSignatureHeader, eventstream.BytesValue(sig)) + + return s.encoder.Encode(msg) +} + +// BufferEncoder is a utility that provides a buffered +// event stream encoder +type BufferEncoder struct { + encoder Encoder + buffer *bytes.Buffer +} + +// NewBufferEncoder returns a new BufferEncoder initialized +// with a 1024 byte buffer. +func NewBufferEncoder() *BufferEncoder { + buf := bytes.NewBuffer(make([]byte, 1024)) + return &BufferEncoder{ + encoder: eventstream.NewEncoder(buf), + buffer: buf, + } +} + +// Encode returns the encoded message as a byte slice. +// The returned byte slice will be modified on the next encode call +// and should not be held onto. +func (e *BufferEncoder) Encode(msg eventstream.Message) ([]byte, error) { + e.buffer.Reset() + + if err := e.encoder.Encode(msg); err != nil { + return nil, err + } + + return e.buffer.Bytes(), nil +} diff --git a/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/stream_writer.go b/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/stream_writer.go new file mode 100644 index 0000000000..433bb1630a --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/stream_writer.go @@ -0,0 +1,129 @@ +package eventstreamapi + +import ( + "fmt" + "io" + "sync" + + "github.com/aws/aws-sdk-go/aws" +) + +// StreamWriter provides concurrent safe writing to an event stream. +type StreamWriter struct { + eventWriter *EventWriter + stream chan eventWriteAsyncReport + + done chan struct{} + closeOnce sync.Once + err *OnceError + + streamCloser io.Closer +} + +// NewStreamWriter returns a StreamWriter for the event writer, and stream +// closer provided. +func NewStreamWriter(eventWriter *EventWriter, streamCloser io.Closer) *StreamWriter { + w := &StreamWriter{ + eventWriter: eventWriter, + streamCloser: streamCloser, + stream: make(chan eventWriteAsyncReport), + done: make(chan struct{}), + err: NewOnceError(), + } + go w.writeStream() + + return w +} + +// Close terminates the writers ability to write new events to the stream. Any +// future call to Send will fail with an error. +func (w *StreamWriter) Close() error { + w.closeOnce.Do(w.safeClose) + return w.Err() +} + +func (w *StreamWriter) safeClose() { + close(w.done) +} + +// ErrorSet returns a channel which will be closed +// if an error occurs. +func (w *StreamWriter) ErrorSet() <-chan struct{} { + return w.err.ErrorSet() +} + +// Err returns any error that occurred while attempting to write an event to the +// stream. +func (w *StreamWriter) Err() error { + return w.err.Err() +} + +// Send writes a single event to the stream returning an error if the write +// failed. +// +// Send may be called concurrently. Events will be written to the stream +// safely. +func (w *StreamWriter) Send(ctx aws.Context, event Marshaler) error { + if err := w.Err(); err != nil { + return err + } + + resultCh := make(chan error) + wrapped := eventWriteAsyncReport{ + Event: event, + Result: resultCh, + } + + select { + case w.stream <- wrapped: + case <-ctx.Done(): + return ctx.Err() + case <-w.done: + return fmt.Errorf("stream closed, unable to send event") + } + + select { + case err := <-resultCh: + return err + case <-ctx.Done(): + return ctx.Err() + case <-w.done: + return fmt.Errorf("stream closed, unable to send event") + } +} + +func (w *StreamWriter) writeStream() { + defer w.Close() + + for { + select { + case wrapper := <-w.stream: + err := w.eventWriter.WriteEvent(wrapper.Event) + wrapper.ReportResult(w.done, err) + if err != nil { + w.err.SetError(err) + return + } + + case <-w.done: + if err := w.streamCloser.Close(); err != nil { + w.err.SetError(err) + } + return + } + } +} + +type eventWriteAsyncReport struct { + Event Marshaler + Result chan<- error +} + +func (e eventWriteAsyncReport) ReportResult(cancel <-chan struct{}, err error) bool { + select { + case e.Result <- err: + return true + case <-cancel: + return false + } +} diff --git a/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/writer.go b/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/writer.go new file mode 100644 index 0000000000..10a3823dfa --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi/writer.go @@ -0,0 +1,109 @@ +package eventstreamapi + +import ( + "github.com/aws/aws-sdk-go/private/protocol" + "github.com/aws/aws-sdk-go/private/protocol/eventstream" +) + +// Marshaler provides a marshaling interface for event types to event stream +// messages. +type Marshaler interface { + MarshalEvent(protocol.PayloadMarshaler) (eventstream.Message, error) +} + +// Encoder is an stream encoder that will encode an event stream message for +// the transport. +type Encoder interface { + Encode(eventstream.Message) error +} + +// EventWriter provides a wrapper around the underlying event stream encoder +// for an io.WriteCloser. +type EventWriter struct { + encoder Encoder + payloadMarshaler protocol.PayloadMarshaler + eventTypeFor func(Marshaler) (string, error) +} + +// NewEventWriter returns a new event stream writer, that will write to the +// writer provided. Use the WriteEvent method to write an event to the stream. +func NewEventWriter(encoder Encoder, pm protocol.PayloadMarshaler, eventTypeFor func(Marshaler) (string, error), +) *EventWriter { + return &EventWriter{ + encoder: encoder, + payloadMarshaler: pm, + eventTypeFor: eventTypeFor, + } +} + +// WriteEvent writes an event to the stream. Returns an error if the event +// fails to marshal into a message, or writing to the underlying writer fails. +func (w *EventWriter) WriteEvent(event Marshaler) error { + msg, err := w.marshal(event) + if err != nil { + return err + } + + return w.encoder.Encode(msg) +} + +func (w *EventWriter) marshal(event Marshaler) (eventstream.Message, error) { + eventType, err := w.eventTypeFor(event) + if err != nil { + return eventstream.Message{}, err + } + + msg, err := event.MarshalEvent(w.payloadMarshaler) + if err != nil { + return eventstream.Message{}, err + } + + msg.Headers.Set(EventTypeHeader, eventstream.StringValue(eventType)) + return msg, nil +} + +//type EventEncoder struct { +// encoder Encoder +// ppayloadMarshaler protocol.PayloadMarshaler +// eventTypeFor func(Marshaler) (string, error) +//} +// +//func (e EventEncoder) Encode(event Marshaler) error { +// msg, err := e.marshal(event) +// if err != nil { +// return err +// } +// +// return w.encoder.Encode(msg) +//} +// +//func (e EventEncoder) marshal(event Marshaler) (eventstream.Message, error) { +// eventType, err := w.eventTypeFor(event) +// if err != nil { +// return eventstream.Message{}, err +// } +// +// msg, err := event.MarshalEvent(w.payloadMarshaler) +// if err != nil { +// return eventstream.Message{}, err +// } +// +// msg.Headers.Set(EventTypeHeader, eventstream.StringValue(eventType)) +// return msg, nil +//} +// +//func (w *EventWriter) marshal(event Marshaler) (eventstream.Message, error) { +// eventType, err := w.eventTypeFor(event) +// if err != nil { +// return eventstream.Message{}, err +// } +// +// msg, err := event.MarshalEvent(w.payloadMarshaler) +// if err != nil { +// return eventstream.Message{}, err +// } +// +// msg.Headers.Set(EventTypeHeader, eventstream.StringValue(eventType)) +// return msg, nil +//} +// diff --git a/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/header_value.go b/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/header_value.go index e3fc0766a9..9f509d8f6d 100644 --- a/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/header_value.go +++ b/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/header_value.go @@ -461,6 +461,11 @@ func (v *TimestampValue) decode(r io.Reader) error { return nil } +// MarshalJSON implements the json.Marshaler interface +func (v TimestampValue) MarshalJSON() ([]byte, error) { + return []byte(v.String()), nil +} + func timeFromEpochMilli(t int64) time.Time { secs := t / 1e3 msec := t % 1e3 diff --git a/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/message.go b/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/message.go index 2dc012a66e..25c9783cde 100644 --- a/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/message.go +++ b/vendor/github.com/aws/aws-sdk-go/private/protocol/eventstream/message.go @@ -27,7 +27,7 @@ func (m *Message) rawMessage() (rawMessage, error) { if len(m.Headers) > 0 { var headers bytes.Buffer - if err := encodeHeaders(&headers, m.Headers); err != nil { + if err := EncodeHeaders(&headers, m.Headers); err != nil { return rawMessage{}, err } raw.Headers = headers.Bytes() diff --git a/vendor/github.com/aws/aws-sdk-go/private/protocol/payload.go b/vendor/github.com/aws/aws-sdk-go/private/protocol/payload.go index e21614a125..0ea0647a57 100644 --- a/vendor/github.com/aws/aws-sdk-go/private/protocol/payload.go +++ b/vendor/github.com/aws/aws-sdk-go/private/protocol/payload.go @@ -64,7 +64,7 @@ func (h HandlerPayloadMarshal) MarshalPayload(w io.Writer, v interface{}) error metadata.ClientInfo{}, request.Handlers{}, nil, - &request.Operation{HTTPMethod: "GET"}, + &request.Operation{HTTPMethod: "PUT"}, v, nil, ) diff --git a/vendor/github.com/aws/aws-sdk-go/private/protocol/protocol.go b/vendor/github.com/aws/aws-sdk-go/private/protocol/protocol.go new file mode 100644 index 0000000000..9d521dcb95 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go/private/protocol/protocol.go @@ -0,0 +1,49 @@ +package protocol + +import ( + "fmt" + "strings" + + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/request" +) + +// RequireHTTPMinProtocol request handler is used to enforce that +// the target endpoint supports the given major and minor HTTP protocol version. +type RequireHTTPMinProtocol struct { + Major, Minor int +} + +// Handler will mark the request.Request with an error if the +// target endpoint did not connect with the required HTTP protocol +// major and minor version. +func (p RequireHTTPMinProtocol) Handler(r *request.Request) { + if r.Error != nil || r.HTTPResponse == nil { + return + } + + if !strings.HasPrefix(r.HTTPResponse.Proto, "HTTP") { + r.Error = newMinHTTPProtoError(p.Major, p.Minor, r) + } + + if r.HTTPResponse.ProtoMajor < p.Major || r.HTTPResponse.ProtoMinor < p.Minor { + r.Error = newMinHTTPProtoError(p.Major, p.Minor, r) + } +} + +// ErrCodeMinimumHTTPProtocolError error code is returned when the target endpoint +// did not match the required HTTP major and minor protocol version. +const ErrCodeMinimumHTTPProtocolError = "MinimumHTTPProtocolError" + +func newMinHTTPProtoError(major, minor int, r *request.Request) error { + return awserr.NewRequestFailure( + awserr.New("MinimumHTTPProtocolError", + fmt.Sprintf( + "operation requires minimum HTTP protocol of HTTP/%d.%d, but was %s", + major, minor, r.HTTPResponse.Proto, + ), + nil, + ), + r.HTTPResponse.StatusCode, r.RequestID, + ) +} diff --git a/vendor/github.com/aws/aws-sdk-go/service/dynamodb/service.go b/vendor/github.com/aws/aws-sdk-go/service/dynamodb/service.go index 755c870599..12201e355c 100644 --- a/vendor/github.com/aws/aws-sdk-go/service/dynamodb/service.go +++ b/vendor/github.com/aws/aws-sdk-go/service/dynamodb/service.go @@ -33,7 +33,7 @@ var initRequest func(*request.Request) const ( ServiceName = "dynamodb" // Name of service. EndpointsID = ServiceName // ID to lookup a service endpoint with. - ServiceID = "DynamoDB" // ServiceID is a unique identifer of a specific service. + ServiceID = "DynamoDB" // ServiceID is a unique identifier of a specific service. ) // New creates a new instance of the DynamoDB client with a session. diff --git a/vendor/github.com/aws/aws-sdk-go/service/ec2/api.go b/vendor/github.com/aws/aws-sdk-go/service/ec2/api.go index 2299b2c3dd..4084db9458 100644 --- a/vendor/github.com/aws/aws-sdk-go/service/ec2/api.go +++ b/vendor/github.com/aws/aws-sdk-go/service/ec2/api.go @@ -971,6 +971,9 @@ func (c *EC2) AssociateAddressRequest(input *AssociateAddressInput) (req *reques // an Elastic IP address with an instance or network interface that has an existing // Elastic IP address. // +// You cannot associate an Elastic IP address with an interface in a different +// network border group. +// // This is an idempotent operation. If you perform the operation more than once, // Amazon EC2 doesn't return an error, and you may be charged for each time // the Elastic IP address is remapped to the same instance. For more information, @@ -11763,8 +11766,10 @@ func (c *EC2) DescribeAccountAttributesRequest(input *DescribeAccountAttributesI // // * default-vpc: The ID of the default VPC for your account, or none. // -// * max-instances: The maximum number of On-Demand Instances that you can -// run. +// * max-instances: This attribute is no longer supported. The returned value +// does not reflect your actual vCPU limit for running On-Demand Instances. +// For more information, see On-Demand Instance Limits (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-on-demand-instances.html#ec2-on-demand-instances-limits) +// in the Amazon Elastic Compute Cloud User Guide. // // * vpc-max-security-groups-per-interface: The maximum number of security // groups that you can assign to a network interface. @@ -14307,7 +14312,7 @@ func (c *EC2) DescribeFleetsRequest(input *DescribeFleetsInput) (req *request.Re // DescribeFleets API operation for Amazon Elastic Compute Cloud. // -// Describes the specified EC2 Fleets or all your EC2 Fleets. +// Describes the specified EC2 Fleets or all of your EC2 Fleets. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -28695,9 +28700,10 @@ func (c *EC2) ModifyEbsDefaultKmsKeyIdRequest(input *ModifyEbsDefaultKmsKeyIdInp // for your account in this Region. // // AWS creates a unique AWS managed CMK in each Region for use with encryption -// by default. If you change the default CMK to a customer managed CMK, it is -// used instead of the AWS managed CMK. To reset the default CMK to the AWS -// managed CMK for EBS, use ResetEbsDefaultKmsKeyId. +// by default. If you change the default CMK to a symmetric customer managed +// CMK, it is used instead of the AWS managed CMK. To reset the default CMK +// to the AWS managed CMK for EBS, use ResetEbsDefaultKmsKeyId. Amazon EBS does +// not support asymmetric CMKs. // // If you delete or disable the customer managed CMK that you specified for // use with encryption by default, your instances will fail to launch. @@ -40585,6 +40591,96 @@ func (s *CapacityReservation) SetTotalInstanceCount(v int64) *CapacityReservatio return s } +// Describes the strategy for using unused Capacity Reservations for fulfilling +// On-Demand capacity. +// +// This strategy can only be used if the EC2 Fleet is of type instant. +// +// For more information about Capacity Reservations, see On-Demand Capacity +// Reservations (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-capacity-reservations.html) +// in the Amazon Elastic Compute Cloud User Guide. For examples of using Capacity +// Reservations in an EC2 Fleet, see EC2 Fleet Example Configurations (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-fleet-examples.html) +// in the Amazon Elastic Compute Cloud User Guide. +type CapacityReservationOptions struct { + _ struct{} `type:"structure"` + + // Indicates whether to use unused Capacity Reservations for fulfilling On-Demand + // capacity. + // + // If you specify use-capacity-reservations-first, the fleet uses unused Capacity + // Reservations to fulfill On-Demand capacity up to the target On-Demand capacity. + // If multiple instance pools have unused Capacity Reservations, the On-Demand + // allocation strategy (lowest-price or prioritized) is applied. If the number + // of unused Capacity Reservations is less than the On-Demand target capacity, + // the remaining On-Demand target capacity is launched according to the On-Demand + // allocation strategy (lowest-price or prioritized). + // + // If you do not specify a value, the fleet fulfils the On-Demand capacity according + // to the chosen On-Demand allocation strategy. + UsageStrategy *string `locationName:"usageStrategy" type:"string" enum:"FleetCapacityReservationUsageStrategy"` +} + +// String returns the string representation +func (s CapacityReservationOptions) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s CapacityReservationOptions) GoString() string { + return s.String() +} + +// SetUsageStrategy sets the UsageStrategy field's value. +func (s *CapacityReservationOptions) SetUsageStrategy(v string) *CapacityReservationOptions { + s.UsageStrategy = &v + return s +} + +// Describes the strategy for using unused Capacity Reservations for fulfilling +// On-Demand capacity. +// +// This strategy can only be used if the EC2 Fleet is of type instant. +// +// For more information about Capacity Reservations, see On-Demand Capacity +// Reservations (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-capacity-reservations.html) +// in the Amazon Elastic Compute Cloud User Guide. For examples of using Capacity +// Reservations in an EC2 Fleet, see EC2 Fleet Example Configurations (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-fleet-examples.html) +// in the Amazon Elastic Compute Cloud User Guide. +type CapacityReservationOptionsRequest struct { + _ struct{} `type:"structure"` + + // Indicates whether to use unused Capacity Reservations for fulfilling On-Demand + // capacity. + // + // If you specify use-capacity-reservations-first, the fleet uses unused Capacity + // Reservations to fulfill On-Demand capacity up to the target On-Demand capacity. + // If multiple instance pools have unused Capacity Reservations, the On-Demand + // allocation strategy (lowest-price or prioritized) is applied. If the number + // of unused Capacity Reservations is less than the On-Demand target capacity, + // the remaining On-Demand target capacity is launched according to the On-Demand + // allocation strategy (lowest-price or prioritized). + // + // If you do not specify a value, the fleet fulfils the On-Demand capacity according + // to the chosen On-Demand allocation strategy. + UsageStrategy *string `type:"string" enum:"FleetCapacityReservationUsageStrategy"` +} + +// String returns the string representation +func (s CapacityReservationOptionsRequest) String() string { + return awsutil.Prettify(s) +} + +// GoString returns the string representation +func (s CapacityReservationOptionsRequest) GoString() string { + return s.String() +} + +// SetUsageStrategy sets the UsageStrategy field's value. +func (s *CapacityReservationOptionsRequest) SetUsageStrategy(v string) *CapacityReservationOptionsRequest { + s.UsageStrategy = &v + return s +} + // Describes an instance's Capacity Reservation targeting option. You can specify // only one parameter at a time. If you specify CapacityReservationPreference // and CapacityReservationTarget, the request fails. @@ -42300,11 +42396,11 @@ type CopyImageInput struct { // in the Amazon Elastic Compute Cloud User Guide. Encrypted *bool `locationName:"encrypted" type:"boolean"` - // An identifier for the AWS Key Management Service (AWS KMS) customer master - // key (CMK) to use when creating the encrypted volume. This parameter is only - // required if you want to use a non-default CMK; if this parameter is not specified, - // the default CMK for EBS is used. If a KmsKeyId is specified, the Encrypted - // flag must also be set. + // An identifier for the symmetric AWS Key Management Service (AWS KMS) customer + // master key (CMK) to use when creating the encrypted volume. This parameter + // is only required if you want to use a non-default CMK; if this parameter + // is not specified, the default CMK for EBS is used. If a KmsKeyId is specified, + // the Encrypted flag must also be set. // // To specify a CMK, use its key ID, Amazon Resource Name (ARN), alias name, // or alias ARN. When using an alias name, prefix it with "alias/". For example: @@ -42323,6 +42419,8 @@ type CopyImageInput struct { // // The specified CMK must exist in the Region that the snapshot is being copied // to. + // + // Amazon EBS does not support asymmetric CMKs. KmsKeyId *string `locationName:"kmsKeyId" type:"string"` // The name of the new AMI in the destination Region. @@ -43704,12 +43802,12 @@ type CreateFleetError struct { ErrorCode *string `locationName:"errorCode" type:"string"` // The error message that describes why the instance could not be launched. - // For more information about error messages, see ee Error Codes (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html.html). + // For more information about error messages, see Error Codes (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html.html). ErrorMessage *string `locationName:"errorMessage" type:"string"` // The launch templates and overrides that were used for launching the instances. - // Any parameters that you specify in the Overrides override the same parameters - // in the launch template. + // The values that you specify in the Overrides replace the values in the launch + // template. LaunchTemplateAndOverrides *LaunchTemplateAndOverridesResponse `locationName:"launchTemplateAndOverrides" type:"structure"` // Indicates if the instance that could not be launched was a Spot Instance @@ -43754,8 +43852,8 @@ func (s *CreateFleetError) SetLifecycle(v string) *CreateFleetError { type CreateFleetInput struct { _ struct{} `type:"structure"` - // Unique, case-sensitive identifier you provide to ensure the idempotency of - // the request. For more information, see Ensuring Idempotency (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/Run_Instance_Idempotency.html). + // Unique, case-sensitive identifier that you provide to ensure the idempotency + // of the request. For more information, see Ensuring Idempotency (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/Run_Instance_Idempotency.html). ClientToken *string `type:"string"` // Checks whether you have the required permissions for the action, without @@ -43948,15 +44046,15 @@ type CreateFleetInstance struct { InstanceType *string `locationName:"instanceType" type:"string" enum:"InstanceType"` // The launch templates and overrides that were used for launching the instances. - // Any parameters that you specify in the Overrides override the same parameters - // in the launch template. + // The values that you specify in the Overrides replace the values in the launch + // template. LaunchTemplateAndOverrides *LaunchTemplateAndOverridesResponse `locationName:"launchTemplateAndOverrides" type:"structure"` // Indicates if the instance that was launched is a Spot Instance or On-Demand // Instance. Lifecycle *string `locationName:"lifecycle" type:"string" enum:"InstanceLifecycle"` - // The value is Windows for Windows instances; otherwise blank. + // The value is Windows for Windows instances. Otherwise, the value is blank. Platform *string `locationName:"platform" type:"string" enum:"PlatformValues"` } @@ -44716,6 +44814,9 @@ type CreateKeyPairOutput struct { // The name of the key pair. KeyName *string `locationName:"keyName" type:"string"` + + // The ID of the key pair. + KeyPairId *string `locationName:"keyPairId" type:"string"` } // String returns the string representation @@ -44746,6 +44847,12 @@ func (s *CreateKeyPairOutput) SetKeyName(v string) *CreateKeyPairOutput { return s } +// SetKeyPairId sets the KeyPairId field's value. +func (s *CreateKeyPairOutput) SetKeyPairId(v string) *CreateKeyPairOutput { + s.KeyPairId = &v + return s +} + type CreateLaunchTemplateInput struct { _ struct{} `type:"structure"` @@ -53338,6 +53445,8 @@ type DescribeAddressesInput struct { // * instance-id - The ID of the instance the address is associated with, // if any. // + // * network-border-group - The location from where the IP address is advertised. + // // * network-interface-id - [EC2-VPC] The ID of the network interface that // the address is associated with, if any. // @@ -55340,6 +55449,8 @@ type DescribeExportTasksInput struct { // The export task IDs. ExportTaskIds []*string `locationName:"exportTaskId" locationNameList:"ExportTaskId" type:"list"` + + Filters []*Filter `locationName:"Filter" locationNameList:"Filter" type:"list"` } // String returns the string representation @@ -55358,6 +55469,12 @@ func (s *DescribeExportTasksInput) SetExportTaskIds(v []*string) *DescribeExport return s } +// SetFilters sets the Filters field's value. +func (s *DescribeExportTasksInput) SetFilters(v []*Filter) *DescribeExportTasksInput { + s.Filters = v + return s +} + type DescribeExportTasksOutput struct { _ struct{} `type:"structure"` @@ -55606,12 +55723,12 @@ type DescribeFleetError struct { ErrorCode *string `locationName:"errorCode" type:"string"` // The error message that describes why the instance could not be launched. - // For more information about error messages, see ee Error Codes (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html.html). + // For more information about error messages, see Error Codes (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html.html). ErrorMessage *string `locationName:"errorMessage" type:"string"` // The launch templates and overrides that were used for launching the instances. - // Any parameters that you specify in the Overrides override the same parameters - // in the launch template. + // The values that you specify in the Overrides replace the values in the launch + // template. LaunchTemplateAndOverrides *LaunchTemplateAndOverridesResponse `locationName:"launchTemplateAndOverrides" type:"structure"` // Indicates if the instance that could not be launched was a Spot Instance @@ -56021,15 +56138,15 @@ type DescribeFleetsInstances struct { InstanceType *string `locationName:"instanceType" type:"string" enum:"InstanceType"` // The launch templates and overrides that were used for launching the instances. - // Any parameters that you specify in the Overrides override the same parameters - // in the launch template. + // The values that you specify in the Overrides replace the values in the launch + // template. LaunchTemplateAndOverrides *LaunchTemplateAndOverridesResponse `locationName:"launchTemplateAndOverrides" type:"structure"` // Indicates if the instance that was launched is a Spot Instance or On-Demand // Instance. Lifecycle *string `locationName:"lifecycle" type:"string" enum:"InstanceLifecycle"` - // The value is Windows for Windows instances; otherwise blank. + // The value is Windows for Windows instances. Otherwise, the value is blank. Platform *string `locationName:"platform" type:"string" enum:"PlatformValues"` } @@ -58389,8 +58506,9 @@ type DescribeInstancesInput struct { // * hypervisor - The hypervisor type of the instance (ovm | xen). // // * iam-instance-profile.arn - The instance profile associated with the - // instance. Specified as an ARN. image-id - The ID of the image used to - // launch the instance. + // instance. Specified as an ARN. + // + // * image-id - The ID of the image used to launch the instance. // // * instance-id - The ID of the instance. // @@ -58830,6 +58948,9 @@ type DescribeKeyPairsInput struct { // // Default: Describes all your key pairs. KeyNames []*string `locationName:"KeyName" locationNameList:"KeyName" type:"list"` + + // The IDs of the key pairs. + KeyPairIds []*string `locationName:"KeyPairId" locationNameList:"KeyPairId" type:"list"` } // String returns the string representation @@ -58860,6 +58981,12 @@ func (s *DescribeKeyPairsInput) SetKeyNames(v []*string) *DescribeKeyPairsInput return s } +// SetKeyPairIds sets the KeyPairIds field's value. +func (s *DescribeKeyPairsInput) SetKeyPairIds(v []*string) *DescribeKeyPairsInput { + s.KeyPairIds = v + return s +} + type DescribeKeyPairsOutput struct { _ struct{} `type:"structure"` @@ -60577,7 +60704,8 @@ type DescribeNetworkInterfacesInput struct { // The maximum number of items to return for this request. The request returns // a token that you can specify in a subsequent call to get the next set of - // results. + // results. You cannot specify this parameter and the network interface IDs + // parameter in the same request. MaxResults *int64 `min:"5" type:"integer"` // One or more network interface IDs. @@ -60695,6 +60823,9 @@ type DescribePlacementGroupsInput struct { // * strategy - The strategy of the placement group (cluster | spread | partition). Filters []*Filter `locationName:"Filter" locationNameList:"Filter" type:"list"` + // The IDs of the placement groups. + GroupIds []*string `locationName:"GroupId" locationNameList:"GroupId" type:"list"` + // The names of the placement groups. // // Default: Describes all your placement groups, or only those otherwise specified. @@ -60723,6 +60854,12 @@ func (s *DescribePlacementGroupsInput) SetFilters(v []*Filter) *DescribePlacemen return s } +// SetGroupIds sets the GroupIds field's value. +func (s *DescribePlacementGroupsInput) SetGroupIds(v []*string) *DescribePlacementGroupsInput { + s.GroupIds = v + return s +} + // SetGroupNames sets the GroupNames field's value. func (s *DescribePlacementGroupsInput) SetGroupNames(v []*string) *DescribePlacementGroupsInput { s.GroupNames = v @@ -68934,7 +69071,7 @@ type DiskInfo struct { // The number of disks with this configuration. Count *int64 `locationName:"count" type:"integer"` - // The size of the disk in GiB. + // The size of the disk in GB. SizeInGB *int64 `locationName:"sizeInGB" type:"long"` // The type of disk. @@ -69103,8 +69240,9 @@ type EbsBlockDevice struct { // size. VolumeSize *int64 `locationName:"volumeSize" type:"integer"` - // The volume type. If you set the type to io1, you must also specify the IOPS - // that the volume supports. + // The volume type. If you set the type to io1, you must also specify the Iops + // parameter. If you set the type to gp2, st1, sc1, or standard, you must omit + // the Iops parameter. // // Default: gp2 VolumeType *string `locationName:"volumeType" type:"string" enum:"VolumeType"` @@ -69395,7 +69533,10 @@ func (s *ElasticGpuHealth) SetStatus(v string) *ElasticGpuHealth { type ElasticGpuSpecification struct { _ struct{} `type:"structure"` - // The type of Elastic Graphics accelerator. + // The type of Elastic Graphics accelerator. For more information about the + // values to specify for Type, see Elastic Graphics Basics (https://docs.aws.amazon.com/AWSEC2/latest/WindowsGuide/elastic-graphics.html#elastic-graphics-basics), + // specifically the Elastic Graphics accelerator column, in the Amazon Elastic + // Compute Cloud User Guide for Windows Instances. // // Type is a required field Type *string `type:"string" required:"true"` @@ -69475,6 +69616,9 @@ type ElasticGpus struct { // The ID of the instance to which the Elastic Graphics accelerator is attached. InstanceId *string `locationName:"instanceId" type:"string"` + + // The tags assigned to the Elastic Graphics accelerator. + Tags []*Tag `locationName:"tagSet" locationNameList:"item" type:"list"` } // String returns the string representation @@ -69523,12 +69667,23 @@ func (s *ElasticGpus) SetInstanceId(v string) *ElasticGpus { return s } +// SetTags sets the Tags field's value. +func (s *ElasticGpus) SetTags(v []*Tag) *ElasticGpus { + s.Tags = v + return s +} + // Describes an elastic inference accelerator. type ElasticInferenceAccelerator struct { _ struct{} `type:"structure"` - // The type of elastic inference accelerator. The possible values are eia1.small, - // eia1.medium, and eia1.large. + // The number of elastic inference accelerators to attach to the instance. + // + // Default: 1 + Count *int64 `min:"1" type:"integer"` + + // The type of elastic inference accelerator. The possible values are eia1.medium, + // eia1.large, and eia1.xlarge. // // Type is a required field Type *string `type:"string" required:"true"` @@ -69547,6 +69702,9 @@ func (s ElasticInferenceAccelerator) GoString() string { // Validate inspects the fields of the type to determine if they are valid. func (s *ElasticInferenceAccelerator) Validate() error { invalidParams := request.ErrInvalidParams{Context: "ElasticInferenceAccelerator"} + if s.Count != nil && *s.Count < 1 { + invalidParams.Add(request.NewErrParamMinValue("Count", 1)) + } if s.Type == nil { invalidParams.Add(request.NewErrParamRequired("Type")) } @@ -69557,6 +69715,12 @@ func (s *ElasticInferenceAccelerator) Validate() error { return nil } +// SetCount sets the Count field's value. +func (s *ElasticInferenceAccelerator) SetCount(v int64) *ElasticInferenceAccelerator { + s.Count = &v + return s +} + // SetType sets the Type field's value. func (s *ElasticInferenceAccelerator) SetType(v string) *ElasticInferenceAccelerator { s.Type = &v @@ -70894,6 +71058,8 @@ type ExportTask struct { // The status message related to the export task. StatusMessage *string `locationName:"statusMessage" type:"string"` + + Tags []*Tag `locationName:"tagSet" locationNameList:"item" type:"list"` } // String returns the string representation @@ -70942,6 +71108,12 @@ func (s *ExportTask) SetStatusMessage(v string) *ExportTask { return s } +// SetTags sets the Tags field's value. +func (s *ExportTask) SetTags(v []*Tag) *ExportTask { + s.Tags = v + return s +} + // Describes the destination for an export image task. type ExportTaskS3Location struct { _ struct{} `type:"structure"` @@ -71353,8 +71525,8 @@ type FleetData struct { // is pending_termination while instances are terminating. ActivityStatus *string `locationName:"activityStatus" type:"string" enum:"FleetActivityStatus"` - // Unique, case-sensitive identifier you provide to ensure the idempotency of - // the request. For more information, see Ensuring Idempotency (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/Run_Instance_Idempotency.html). + // Unique, case-sensitive identifier that you provide to ensure the idempotency + // of the request. For more information, see Ensuring Idempotency (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/Run_Instance_Idempotency.html). // // Constraints: Maximum 64 ASCII characters ClientToken *string `locationName:"clientToken" type:"string"` @@ -71417,11 +71589,11 @@ type FleetData struct { // The type of request. Indicates whether the EC2 Fleet only requests the target // capacity, or also attempts to maintain it. If you request a certain target // capacity, EC2 Fleet only places the required requests; it does not attempt - // to replenish instances if capacity is diminished, and does not submit requests - // in alternative capacity pools if capacity is unavailable. To maintain a certain - // target capacity, EC2 Fleet places the required requests to meet this target - // capacity. It also automatically replenishes any interrupted Spot Instances. - // Default: maintain. + // to replenish instances if capacity is diminished, and it does not submit + // requests in alternative capacity pools if capacity is unavailable. To maintain + // a certain target capacity, EC2 Fleet places the required requests to meet + // this target capacity. It also automatically replenishes any interrupted Spot + // Instances. Default: maintain. Type *string `locationName:"type" type:"string" enum:"FleetType"` // The start date and time of the request, in UTC format (for example, YYYY-MM-DDTHH:MM:SSZ). @@ -75487,11 +75659,11 @@ type ImportImageInput struct { // Valid values: xen Hypervisor *string `type:"string"` - // An identifier for the AWS Key Management Service (AWS KMS) customer master - // key (CMK) to use when creating the encrypted AMI. This parameter is only - // required if you want to use a non-default CMK; if this parameter is not specified, - // the default CMK for EBS is used. If a KmsKeyId is specified, the Encrypted - // flag must also be set. + // An identifier for the symmetric AWS Key Management Service (AWS KMS) customer + // master key (CMK) to use when creating the encrypted AMI. This parameter is + // only required if you want to use a non-default CMK; if this parameter is + // not specified, the default CMK for EBS is used. If a KmsKeyId is specified, + // the Encrypted flag must also be set. // // The CMK identifier may be provided in any of the following formats: // @@ -75514,6 +75686,8 @@ type ImportImageInput struct { // will eventually report failure. // // The specified CMK must exist in the Region that the AMI is being copied to. + // + // Amazon EBS does not support asymmetric CMKs. KmsKeyId *string `type:"string"` // The ARNs of the license configurations. @@ -75653,7 +75827,7 @@ func (s *ImportImageLicenseConfigurationRequest) SetLicenseConfigurationArn(v st return s } -// The response information of license configurations. +// The response information for license configurations. type ImportImageLicenseConfigurationResponse struct { _ struct{} `type:"structure"` @@ -75698,8 +75872,8 @@ type ImportImageOutput struct { // The task ID of the import image task. ImportTaskId *string `locationName:"importTaskId" type:"string"` - // The identifier for the AWS Key Management Service (AWS KMS) customer master - // key (CMK) that was used to create the encrypted AMI. + // The identifier for the symmetric AWS Key Management Service (AWS KMS) customer + // master key (CMK) that was used to create the encrypted AMI. KmsKeyId *string `locationName:"kmsKeyId" type:"string"` // The ARNs of the license configurations. @@ -75848,7 +76022,8 @@ type ImportImageTask struct { // key (CMK) that was used to create the encrypted image. KmsKeyId *string `locationName:"kmsKeyId" type:"string"` - // The ARNs of the license configurations associated to the import image task. + // The ARNs of the license configurations that are associated with the import + // image task. LicenseSpecifications []*ImportImageLicenseConfigurationResponse `locationName:"licenseSpecifications" locationNameList:"item" type:"list"` // The license type of the virtual machine. @@ -75868,6 +76043,9 @@ type ImportImageTask struct { // A descriptive status message for the import image task. StatusMessage *string `locationName:"statusMessage" type:"string"` + + // Any tags applied to the import image task. + Tags []*Tag `locationName:"tagSet" locationNameList:"item" type:"list"` } // String returns the string representation @@ -75964,6 +76142,12 @@ func (s *ImportImageTask) SetStatusMessage(v string) *ImportImageTask { return s } +// SetTags sets the Tags field's value. +func (s *ImportImageTask) SetTags(v []*Tag) *ImportImageTask { + s.Tags = v + return s +} + type ImportInstanceInput struct { _ struct{} `type:"structure"` @@ -76447,10 +76631,10 @@ type ImportSnapshotInput struct { // in the Amazon Elastic Compute Cloud User Guide. Encrypted *bool `type:"boolean"` - // An identifier for the AWS Key Management Service (AWS KMS) customer master - // key (CMK) to use when creating the encrypted snapshot. This parameter is - // only required if you want to use a non-default CMK; if this parameter is - // not specified, the default CMK for EBS is used. If a KmsKeyId is specified, + // An identifier for the symmetric AWS Key Management Service (AWS KMS) customer + // master key (CMK) to use when creating the encrypted snapshot. This parameter + // is only required if you want to use a non-default CMK; if this parameter + // is not specified, the default CMK for EBS is used. If a KmsKeyId is specified, // the Encrypted flag must also be set. // // The CMK identifier may be provided in any of the following formats: @@ -76475,6 +76659,8 @@ type ImportSnapshotInput struct { // // The specified CMK must exist in the Region that the snapshot is being copied // to. + // + // Amazon EBS does not support asymmetric CMKs. KmsKeyId *string `type:"string"` // The name of the role to use when not using the default role, 'vmimport'. @@ -76592,6 +76778,9 @@ type ImportSnapshotTask struct { // Describes an import snapshot task. SnapshotTaskDetail *SnapshotTaskDetail `locationName:"snapshotTaskDetail" type:"structure"` + + // Any tags applied to the import snapshot task. + Tags []*Tag `locationName:"tagSet" locationNameList:"item" type:"list"` } // String returns the string representation @@ -76622,6 +76811,12 @@ func (s *ImportSnapshotTask) SetSnapshotTaskDetail(v *SnapshotTaskDetail) *Impor return s } +// SetTags sets the Tags field's value. +func (s *ImportSnapshotTask) SetTags(v []*Tag) *ImportSnapshotTask { + s.Tags = v + return s +} + type ImportVolumeInput struct { _ struct{} `type:"structure"` @@ -78738,7 +78933,7 @@ type InstanceStorageInfo struct { // Array describing the disks that are available for the instance type. Disks []*DiskInfo `locationName:"disks" locationNameList:"item" type:"list"` - // The total size of the disks, in GiB. + // The total size of the disks, in GB. TotalSizeInGB *int64 `locationName:"totalSizeInGB" type:"long"` } @@ -78826,8 +79021,8 @@ type InstanceTypeInfo struct { // Describes the processor. ProcessorInfo *ProcessorInfo `locationName:"processorInfo" type:"structure"` - // Indicates the supported root devices. - SupportedRootDevices []*string `locationName:"supportedRootDevices" locationNameList:"item" type:"list"` + // Indicates the supported root device types. + SupportedRootDeviceTypes []*string `locationName:"supportedRootDeviceTypes" locationNameList:"item" type:"list"` // Indicates whether the instance type is offered for spot or On-Demand. SupportedUsageClasses []*string `locationName:"supportedUsageClasses" locationNameList:"item" type:"list"` @@ -78960,9 +79155,9 @@ func (s *InstanceTypeInfo) SetProcessorInfo(v *ProcessorInfo) *InstanceTypeInfo return s } -// SetSupportedRootDevices sets the SupportedRootDevices field's value. -func (s *InstanceTypeInfo) SetSupportedRootDevices(v []*string) *InstanceTypeInfo { - s.SupportedRootDevices = v +// SetSupportedRootDeviceTypes sets the SupportedRootDeviceTypes field's value. +func (s *InstanceTypeInfo) SetSupportedRootDeviceTypes(v []*string) *InstanceTypeInfo { + s.SupportedRootDeviceTypes = v return s } @@ -79246,7 +79441,7 @@ type IpRange struct { // range. // // Constraints: Up to 255 characters in length. Allowed characters are a-z, - // A-Z, 0-9, spaces, and ._-:/()#,@[]+=;{}!$* + // A-Z, 0-9, spaces, and ._-:/()#,@[]+=&;{}!$* Description *string `locationName:"description" type:"string"` } @@ -79308,7 +79503,7 @@ type Ipv6Range struct { // range. // // Constraints: Up to 255 characters in length. Allowed characters are a-z, - // A-Z, 0-9, spaces, and ._-:/()#,@[]+=;{}!$* + // A-Z, 0-9, spaces, and ._-:/()#,@[]+=&;{}!$* Description *string `locationName:"description" type:"string"` } @@ -79346,6 +79541,12 @@ type KeyPairInfo struct { // The name of the key pair. KeyName *string `locationName:"keyName" type:"string"` + + // The ID of the key pair. + KeyPairId *string `locationName:"keyPairId" type:"string"` + + // Any tags applied to the key pair. + Tags []*Tag `locationName:"tagSet" locationNameList:"item" type:"list"` } // String returns the string representation @@ -79370,6 +79571,18 @@ func (s *KeyPairInfo) SetKeyName(v string) *KeyPairInfo { return s } +// SetKeyPairId sets the KeyPairId field's value. +func (s *KeyPairInfo) SetKeyPairId(v string) *KeyPairInfo { + s.KeyPairId = &v + return s +} + +// SetTags sets the Tags field's value. +func (s *KeyPairInfo) SetTags(v []*Tag) *KeyPairInfo { + s.Tags = v + return s +} + // Describes a launch permission. type LaunchPermission struct { _ struct{} `type:"structure"` @@ -80121,7 +80334,8 @@ type LaunchTemplateEbsBlockDeviceRequest struct { // it is not used in requests to create gp2, st1, sc1, or standard volumes. Iops *int64 `type:"integer"` - // The ARN of the AWS Key Management Service (AWS KMS) CMK used for encryption. + // The ARN of the symmetric AWS Key Management Service (AWS KMS) CMK used for + // encryption. KmsKeyId *string `type:"string"` // The ID of the snapshot. @@ -80193,6 +80407,11 @@ func (s *LaunchTemplateEbsBlockDeviceRequest) SetVolumeType(v string) *LaunchTem type LaunchTemplateElasticInferenceAccelerator struct { _ struct{} `type:"structure"` + // The number of elastic inference accelerators to attach to the instance. + // + // Default: 1 + Count *int64 `min:"1" type:"integer"` + // The type of elastic inference accelerator. The possible values are eia1.medium, // eia1.large, and eia1.xlarge. // @@ -80213,6 +80432,9 @@ func (s LaunchTemplateElasticInferenceAccelerator) GoString() string { // Validate inspects the fields of the type to determine if they are valid. func (s *LaunchTemplateElasticInferenceAccelerator) Validate() error { invalidParams := request.ErrInvalidParams{Context: "LaunchTemplateElasticInferenceAccelerator"} + if s.Count != nil && *s.Count < 1 { + invalidParams.Add(request.NewErrParamMinValue("Count", 1)) + } if s.Type == nil { invalidParams.Add(request.NewErrParamRequired("Type")) } @@ -80223,6 +80445,12 @@ func (s *LaunchTemplateElasticInferenceAccelerator) Validate() error { return nil } +// SetCount sets the Count field's value. +func (s *LaunchTemplateElasticInferenceAccelerator) SetCount(v int64) *LaunchTemplateElasticInferenceAccelerator { + s.Count = &v + return s +} + // SetType sets the Type field's value. func (s *LaunchTemplateElasticInferenceAccelerator) SetType(v string) *LaunchTemplateElasticInferenceAccelerator { s.Type = &v @@ -80233,6 +80461,11 @@ func (s *LaunchTemplateElasticInferenceAccelerator) SetType(v string) *LaunchTem type LaunchTemplateElasticInferenceAcceleratorResponse struct { _ struct{} `type:"structure"` + // The number of elastic inference accelerators to attach to the instance. + // + // Default: 1 + Count *int64 `locationName:"count" type:"integer"` + // The type of elastic inference accelerator. The possible values are eia1.medium, // eia1.large, and eia1.xlarge. Type *string `locationName:"type" type:"string"` @@ -80248,6 +80481,12 @@ func (s LaunchTemplateElasticInferenceAcceleratorResponse) GoString() string { return s.String() } +// SetCount sets the Count field's value. +func (s *LaunchTemplateElasticInferenceAcceleratorResponse) SetCount(v int64) *LaunchTemplateElasticInferenceAcceleratorResponse { + s.Count = &v + return s +} + // SetType sets the Type field's value. func (s *LaunchTemplateElasticInferenceAcceleratorResponse) SetType(v string) *LaunchTemplateElasticInferenceAcceleratorResponse { s.Type = &v @@ -82373,6 +82612,8 @@ type ModifyEbsDefaultKmsKeyIdInput struct { // alias, or ARN that is not valid, the action can appear to complete, but eventually // fails. // + // Amazon EBS does not support asymmetric CMKs. + // // KmsKeyId is a required field KmsKeyId *string `type:"string" required:"true"` } @@ -84877,6 +85118,8 @@ type ModifyTransitGatewayVpcAttachmentInput struct { DryRun *bool `type:"boolean"` // The new VPC attachment options. + // + // You cannot modify the IPv6 options. Options *ModifyTransitGatewayVpcAttachmentRequestOptions `type:"structure"` // The IDs of one or more subnets to remove. @@ -87704,6 +87947,10 @@ type OnDemandOptions struct { // Fleet defaults to lowest-price. AllocationStrategy *string `locationName:"allocationStrategy" type:"string" enum:"FleetOnDemandAllocationStrategy"` + // The strategy for using unused Capacity Reservations for fulfilling On-Demand + // capacity. Supported only for fleets of type instant. + CapacityReservationOptions *CapacityReservationOptions `locationName:"capacityReservationOptions" type:"structure"` + // The maximum amount per hour for On-Demand Instances that you're willing to // pay. MaxTotalPrice *string `locationName:"maxTotalPrice" type:"string"` @@ -87713,11 +87960,11 @@ type OnDemandOptions struct { MinTargetCapacity *int64 `locationName:"minTargetCapacity" type:"integer"` // Indicates that the fleet launches all On-Demand Instances into a single Availability - // Zone. + // Zone. Supported only for fleets of type instant. SingleAvailabilityZone *bool `locationName:"singleAvailabilityZone" type:"boolean"` // Indicates that the fleet uses a single instance type to launch all On-Demand - // Instances in the fleet. + // Instances in the fleet. Supported only for fleets of type instant. SingleInstanceType *bool `locationName:"singleInstanceType" type:"boolean"` } @@ -87737,6 +87984,12 @@ func (s *OnDemandOptions) SetAllocationStrategy(v string) *OnDemandOptions { return s } +// SetCapacityReservationOptions sets the CapacityReservationOptions field's value. +func (s *OnDemandOptions) SetCapacityReservationOptions(v *CapacityReservationOptions) *OnDemandOptions { + s.CapacityReservationOptions = v + return s +} + // SetMaxTotalPrice sets the MaxTotalPrice field's value. func (s *OnDemandOptions) SetMaxTotalPrice(v string) *OnDemandOptions { s.MaxTotalPrice = &v @@ -87773,6 +88026,10 @@ type OnDemandOptionsRequest struct { // Fleet defaults to lowest-price. AllocationStrategy *string `type:"string" enum:"FleetOnDemandAllocationStrategy"` + // The strategy for using unused Capacity Reservations for fulfilling On-Demand + // capacity. Supported only for fleets of type instant. + CapacityReservationOptions *CapacityReservationOptionsRequest `type:"structure"` + // The maximum amount per hour for On-Demand Instances that you're willing to // pay. MaxTotalPrice *string `type:"string"` @@ -87782,11 +88039,11 @@ type OnDemandOptionsRequest struct { MinTargetCapacity *int64 `type:"integer"` // Indicates that the fleet launches all On-Demand Instances into a single Availability - // Zone. + // Zone. Supported only for fleets of type instant. SingleAvailabilityZone *bool `type:"boolean"` // Indicates that the fleet uses a single instance type to launch all On-Demand - // Instances in the fleet. + // Instances in the fleet. Supported only for fleets of type instant. SingleInstanceType *bool `type:"boolean"` } @@ -87806,6 +88063,12 @@ func (s *OnDemandOptionsRequest) SetAllocationStrategy(v string) *OnDemandOption return s } +// SetCapacityReservationOptions sets the CapacityReservationOptions field's value. +func (s *OnDemandOptionsRequest) SetCapacityReservationOptions(v *CapacityReservationOptionsRequest) *OnDemandOptionsRequest { + s.CapacityReservationOptions = v + return s +} + // SetMaxTotalPrice sets the MaxTotalPrice field's value. func (s *OnDemandOptionsRequest) SetMaxTotalPrice(v string) *OnDemandOptionsRequest { s.MaxTotalPrice = &v @@ -88438,6 +88701,9 @@ func (s *Placement) SetTenancy(v string) *Placement { type PlacementGroup struct { _ struct{} `type:"structure"` + // The ID of the placement group. + GroupId *string `locationName:"groupId" type:"string"` + // The name of the placement group. GroupName *string `locationName:"groupName" type:"string"` @@ -88449,6 +88715,9 @@ type PlacementGroup struct { // The placement strategy. Strategy *string `locationName:"strategy" type:"string" enum:"PlacementStrategy"` + + // Any tags applied to the placement group. + Tags []*Tag `locationName:"tagSet" locationNameList:"item" type:"list"` } // String returns the string representation @@ -88461,6 +88730,12 @@ func (s PlacementGroup) GoString() string { return s.String() } +// SetGroupId sets the GroupId field's value. +func (s *PlacementGroup) SetGroupId(v string) *PlacementGroup { + s.GroupId = &v + return s +} + // SetGroupName sets the GroupName field's value. func (s *PlacementGroup) SetGroupName(v string) *PlacementGroup { s.GroupName = &v @@ -88485,6 +88760,12 @@ func (s *PlacementGroup) SetStrategy(v string) *PlacementGroup { return s } +// SetTags sets the Tags field's value. +func (s *PlacementGroup) SetTags(v []*Tag) *PlacementGroup { + s.Tags = v + return s +} + // Describes the placement group support of the instance type. type PlacementGroupInfo struct { _ struct{} `type:"structure"` @@ -88513,7 +88794,7 @@ func (s *PlacementGroupInfo) SetSupportedStrategies(v []*string) *PlacementGroup type PlacementResponse struct { _ struct{} `type:"structure"` - // The name of the placement group the instance is in. + // The name of the placement group that the instance is in. GroupName *string `locationName:"groupName" type:"string"` } @@ -98966,7 +99247,7 @@ type SpotOptions struct { // strategy. // // If the allocation strategy is diversified, EC2 Fleet launches instances from - // all the Spot Instance pools that you specify. + // all of the Spot Instance pools that you specify. // // If the allocation strategy is capacity-optimized, EC2 Fleet launches instances // from Spot Instance pools with optimal capacity for the number of instances @@ -98990,11 +99271,11 @@ type SpotOptions struct { MinTargetCapacity *int64 `locationName:"minTargetCapacity" type:"integer"` // Indicates that the fleet launches all Spot Instances into a single Availability - // Zone. + // Zone. Supported only for fleets of type instant. SingleAvailabilityZone *bool `locationName:"singleAvailabilityZone" type:"boolean"` // Indicates that the fleet uses a single instance type to launch all Spot Instances - // in the fleet. + // in the fleet. Supported only for fleets of type instant. SingleInstanceType *bool `locationName:"singleInstanceType" type:"boolean"` } @@ -99062,7 +99343,7 @@ type SpotOptionsRequest struct { // strategy. // // If the allocation strategy is diversified, EC2 Fleet launches instances from - // all the Spot Instance pools that you specify. + // all of the Spot Instance pools that you specify. // // If the allocation strategy is capacity-optimized, EC2 Fleet launches instances // from Spot Instance pools with optimal capacity for the number of instances @@ -99086,11 +99367,11 @@ type SpotOptionsRequest struct { MinTargetCapacity *int64 `type:"integer"` // Indicates that the fleet launches all Spot Instances into a single Availability - // Zone. + // Zone. Supported only for fleets of type instant. SingleAvailabilityZone *bool `type:"boolean"` // Indicates that the fleet uses a single instance type to launch all Spot Instances - // in the fleet. + // in the fleet. Supported only for fleets of type instant. SingleInstanceType *bool `type:"boolean"` } @@ -100114,9 +100395,10 @@ type TagSpecification struct { // The type of resource to tag. Currently, the resource types that support tagging // on creation are: capacity-reservation | client-vpn-endpoint | dedicated-host - // | fleet | fpga-image | instance | launch-template | snapshot | traffic-mirror-filter - // | traffic-mirror-session | traffic-mirror-target | transit-gateway | transit-gateway-attachment - // | transit-gateway-route-table | volume. + // | fleet | fpga-image | instance | key-pair | launch-template | placement-group + // | snapshot | traffic-mirror-filter | traffic-mirror-session | traffic-mirror-target + // | transit-gateway | transit-gateway-attachment | transit-gateway-route-table + // | volume. // // To tag a resource after it has been created, see CreateTags (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html). ResourceType *string `locationName:"resourceType" type:"string" enum:"ResourceType"` @@ -100154,13 +100436,13 @@ func (s *TagSpecification) SetTags(v []*Tag) *TagSpecification { // later. // // You can use the On-Demand Instance MaxTotalPrice parameter, the Spot Instance -// MaxTotalPrice, or both to ensure your fleet cost does not exceed your budget. -// If you set a maximum price per hour for the On-Demand Instances and Spot -// Instances in your request, EC2 Fleet will launch instances until it reaches -// the maximum amount you're willing to pay. When the maximum amount you're -// willing to pay is reached, the fleet stops launching instances even if it -// hasn’t met the target capacity. The MaxTotalPrice parameters are located -// in and +// MaxTotalPrice, or both to ensure that your fleet cost does not exceed your +// budget. If you set a maximum price per hour for the On-Demand Instances and +// Spot Instances in your request, EC2 Fleet will launch instances until it +// reaches the maximum amount that you're willing to pay. When the maximum amount +// you're willing to pay is reached, the fleet stops launching instances even +// if it hasn’t met the target capacity. The MaxTotalPrice parameters are +// located in and type TargetCapacitySpecification struct { _ struct{} `type:"structure"` @@ -100223,9 +100505,9 @@ func (s *TargetCapacitySpecification) SetTotalTargetCapacity(v int64) *TargetCap // MaxTotalPrice parameter, or both parameters to ensure that your fleet cost // does not exceed your budget. If you set a maximum price per hour for the // On-Demand Instances and Spot Instances in your request, EC2 Fleet will launch -// instances until it reaches the maximum amount you're willing to pay. When -// the maximum amount you're willing to pay is reached, the fleet stops launching -// instances even if it hasn’t met the target capacity. The MaxTotalPrice +// instances until it reaches the maximum amount that you're willing to pay. +// When the maximum amount you're willing to pay is reached, the fleet stops +// launching instances even if it hasn’t met the target capacity. The MaxTotalPrice // parameters are located in and . type TargetCapacitySpecificationRequest struct { _ struct{} `type:"structure"` @@ -106480,6 +106762,11 @@ const ( FleetActivityStatusFulfilled = "fulfilled" ) +const ( + // FleetCapacityReservationUsageStrategyUseCapacityReservationsFirst is a FleetCapacityReservationUsageStrategy enum value + FleetCapacityReservationUsageStrategyUseCapacityReservationsFirst = "use-capacity-reservations-first" +) + const ( // FleetEventTypeInstanceChange is a FleetEventType enum value FleetEventTypeInstanceChange = "instance-change" @@ -108126,6 +108413,9 @@ const ( // ResourceTypeInternetGateway is a ResourceType enum value ResourceTypeInternetGateway = "internet-gateway" + // ResourceTypeKeyPair is a ResourceType enum value + ResourceTypeKeyPair = "key-pair" + // ResourceTypeLaunchTemplate is a ResourceType enum value ResourceTypeLaunchTemplate = "launch-template" @@ -108138,6 +108428,9 @@ const ( // ResourceTypeNetworkInterface is a ResourceType enum value ResourceTypeNetworkInterface = "network-interface" + // ResourceTypePlacementGroup is a ResourceType enum value + ResourceTypePlacementGroup = "placement-group" + // ResourceTypeReservedInstances is a ResourceType enum value ResourceTypeReservedInstances = "reserved-instances" diff --git a/vendor/github.com/aws/aws-sdk-go/service/ec2/service.go b/vendor/github.com/aws/aws-sdk-go/service/ec2/service.go index f8c075614f..1bde2c2f5c 100644 --- a/vendor/github.com/aws/aws-sdk-go/service/ec2/service.go +++ b/vendor/github.com/aws/aws-sdk-go/service/ec2/service.go @@ -31,7 +31,7 @@ var initRequest func(*request.Request) const ( ServiceName = "ec2" // Name of service. EndpointsID = ServiceName // ID to lookup a service endpoint with. - ServiceID = "EC2" // ServiceID is a unique identifer of a specific service. + ServiceID = "EC2" // ServiceID is a unique identifier of a specific service. ) // New creates a new instance of the EC2 client with a session. diff --git a/vendor/github.com/aws/aws-sdk-go/service/s3/api.go b/vendor/github.com/aws/aws-sdk-go/service/s3/api.go index ea5dc6d73a..91d61f8a56 100644 --- a/vendor/github.com/aws/aws-sdk-go/service/s3/api.go +++ b/vendor/github.com/aws/aws-sdk-go/service/s3/api.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "sync" - "sync/atomic" "time" "github.com/aws/aws-sdk-go/aws" @@ -347,7 +346,7 @@ func (c *S3) CopyObjectRequest(input *CopyObjectInput) (req *request.Request, ou // encryption key. You can do this regardless of the form of server-side encryption // that was used to encrypt the source, or even if the source object was not // encrypted. For more information about server-side encryption, see Using Server-Side -// Encryption (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html). +// Encryption (https://docs.aws.amazon.com/AmazonS3/latest/dev/serv-side-encryption.html). // // A copy request might return an error when Amazon S3 receives the copy request // or while Amazon S3 is copying the files. If the error occurs before the copy @@ -429,13 +428,18 @@ func (c *S3) CopyObjectRequest(input *CopyObjectInput) (req *request.Request, ou // * To encrypt the target object using server-side encryption with an AWS // managed encryption key, provide the following request headers, as appropriate. // x-amz-server-side​-encryption x-amz-server-side-encryption-aws-kms-key-id -// x-amz-server-side-encryption-context If you specify x-amz-server-side-encryption:aws:kms -// but don't provide x-amz-server-side- encryption-aws-kms-key-id, Amazon -// S3 uses the AWS managed customer master key (CMK) in AWS KMS to protect -// the data. All GET and PUT requests for an object protected by AWS KMS -// fail if you don't make them with SSL or by using SigV4. For more information -// about server-side encryption with CMKs stored in AWS KMS (SSE-KMS), see -// Protecting Data Using Server-Side Encryption with CMKs stored in KMS (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html). +// x-amz-server-side-encryption-context If you specify x-amz-server-side-encryption:aws:kms, +// but don't provide x-amz-server-side-encryption-aws-kms-key-id, Amazon +// S3 uses the AWS managed CMK in AWS KMS to protect the data. If you want +// to use a customer managed AWS KMS CMK, you must provide the x-amz-server-side-encryption-aws-kms-key-id +// of the symmetric customer managed CMK. Amazon S3 only supports symmetric +// CMKs and not asymmetric CMKs. For more information, see Using Symmetric +// and Asymmetric Keys (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html) +// in the AWS Key Management Service Developer Guide. All GET and PUT requests +// for an object protected by AWS KMS fail if you don't make them with SSL +// or by using SigV4. For more information about server-side encryption with +// CMKs stored in AWS KMS (SSE-KMS), see Protecting Data Using Server-Side +// Encryption with CMKs stored in KMS (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html). // // * To encrypt the target object using server-side encryption with an encryption // key that you provide, use the following headers. x-amz-server-side​-encryption​-customer-algorithm @@ -790,7 +794,7 @@ func (c *S3) CreateMultipartUploadRequest(input *CreateMultipartUploadInput) (re // manage the keys used to encrypt data, specify the following headers in // the request. x-amz-server-side​-encryption x-amz-server-side-encryption-aws-kms-key-id // x-amz-server-side-encryption-context If you specify x-amz-server-side-encryption:aws:kms, -// but don't provide x-amz-server-side- encryption-aws-kms-key-id, Amazon +// but don't provide x-amz-server-side-encryption-aws-kms-key-id, Amazon // S3 uses the AWS managed CMK in AWS KMS to protect the data. All GET and // PUT requests for an object protected by AWS KMS fail if you don't make // them with SSL or by using SigV4. For more information about server-side @@ -2026,7 +2030,7 @@ func (c *S3) DeleteObjectTaggingRequest(input *DeleteObjectTaggingInput) (req *r // DeleteObjectTagging API operation for Amazon Simple Storage Service. // // Removes the entire tag set from the specified object. For more information -// about managing object tags, see Object Tagging (https://docs.aws.amazon.com/AmazonS3/latest/dev/Versioning.html#MultiFactorAuthenticationDelete). +// about managing object tags, see Object Tagging (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-tagging.html). // // To use this operation, you must have permission to perform the s3:DeleteObjectTagging // action. @@ -6818,9 +6822,9 @@ func (c *S3) PutBucketEncryptionRequest(input *PutBucketEncryptionInput) (req *r // This implementation of the PUT operation uses the encryption subresource // to set the default encryption state of an existing bucket. // -// This implementation of the PUT operation sets default encryption for a buckets +// This implementation of the PUT operation sets default encryption for a bucket // using server-side encryption with Amazon S3-managed keys SSE-S3 or AWS KMS -// customer master keys (CMKs) (SSE-KMS) bucket. +// customer master keys (CMKs) (SSE-KMS). // // This operation requires AWS Signature Version 4. For more information, see // Authenticating Requests (AWS Signature Version 4) (sig-v4-authenticating-requests.html). @@ -8454,19 +8458,24 @@ func (c *S3) PutObjectRequest(input *PutObjectInput) (req *request.Request, outp // manage the keys used to encrypt data, specify the following headers in // the request. x-amz-server-side​-encryption x-amz-server-side-encryption-aws-kms-key-id // x-amz-server-side-encryption-context If you specify x-amz-server-side-encryption:aws:kms, -// but don't provide x-amz-server-side- encryption-aws-kms-key-id, Amazon -// S3 uses the AWS managed CMK in AWS KMS to protect the data. All GET and -// PUT requests for an object protected by AWS KMS fail if you don't make -// them with SSL or by using SigV4. For more information about server-side -// encryption with CMKs stored in AWS KMS (SSE-KMS), see Protecting Data -// Using Server-Side Encryption with CMKs stored in AWS (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html). +// but don't provide x-amz-server-side-encryption-aws-kms-key-id, Amazon +// S3 uses the AWS managed CMK in AWS KMS to protect the data. If you want +// to use a customer managed AWS KMS CMK, you must provide the x-amz-server-side-encryption-aws-kms-key-id +// of the symmetric customer managed CMK. Amazon S3 only supports symmetric +// CMKs and not asymmetric CMKs. For more information, see Using Symmetric +// and Asymmetric Keys (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html) +// in the AWS Key Management Service Developer Guide. All GET and PUT requests +// for an object protected by AWS KMS fail if you don't make them with SSL +// or by using SigV4. For more information about server-side encryption with +// CMKs stored in AWS KMS (SSE-KMS), see Protecting Data Using Server-Side +// Encryption with CMKs stored in AWS (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html). // // * Use customer-provided encryption keys – If you want to manage your // own encryption keys, provide all the following headers in the request. // x-amz-server-side​-encryption​-customer-algorithm x-amz-server-side​-encryption​-customer-key // x-amz-server-side​-encryption​-customer-key-MD5 For more information // about server-side encryption with CMKs stored in KMS (SSE-KMS), see Protecting -// Data Using Server-Side Encryption with CMKs stored in AWS KMS (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html). +// Data Using Server-Side Encryption with CMKs stored in AWS (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html). // // Access-Control-List (ACL)-Specific Request Headers // @@ -8517,8 +8526,13 @@ func (c *S3) PutObjectRequest(input *PutObjectInput) (req *request.Request, outp // manage the keys used to encrypt data, specify the following headers in // the request. x-amz-server-side​-encryption x-amz-server-side-encryption-aws-kms-key-id // x-amz-server-side-encryption-context If you specify x-amz-server-side-encryption:aws:kms, -// but don't provide x-amz-server-side- encryption-aws-kms-key-id, Amazon -// S3 uses the default AWS KMS CMK to protect the data. All GET and PUT requests +// but don't provide x-amz-server-side-encryption-aws-kms-key-id, Amazon +// S3 uses the AWS managed CMK in AWS KMS to protect the data. If you want +// to use a customer managed AWS KMS CMK, you must provide the x-amz-server-side-encryption-aws-kms-key-id +// of the symmetric customer managed CMK. Amazon S3 only supports symmetric +// CMKs and not asymmetric CMKs. For more information, see Using Symmetric +// and Asymmetric Keys (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html) +// in the AWS Key Management Service Developer Guide. All GET and PUT requests // for an object protected by AWS KMS fail if you don't make them with SSL // or by using SigV4. For more information about server-side encryption with // CMKs stored in AWS KMS (SSE-KMS), see Protecting Data Using Server-Side @@ -9481,9 +9495,15 @@ func (c *S3) SelectObjectContentRequest(input *SelectObjectContentInput) (req *r output = &SelectObjectContentOutput{} req = c.newRequest(op, input, output) + + es := newSelectObjectContentEventStream() + req.Handlers.Unmarshal.PushBack(es.setStreamCloser) + output.EventStream = es + req.Handlers.Send.Swap(client.LogHTTPResponseHandler.Name, client.LogHTTPResponseHeaderHandler) req.Handlers.Unmarshal.Swap(restxml.UnmarshalHandler.Name, rest.UnmarshalHandler) - req.Handlers.Unmarshal.PushBack(output.runEventStreamLoop) + req.Handlers.Unmarshal.PushBack(es.runOutputStream) + req.Handlers.Unmarshal.PushBack(es.runOnStreamPartClose) return } @@ -9602,6 +9622,140 @@ func (c *S3) SelectObjectContentWithContext(ctx aws.Context, input *SelectObject return out, req.Send() } +// SelectObjectContentEventStream provides the event stream handling for the SelectObjectContent. +type SelectObjectContentEventStream struct { + + // Reader is the EventStream reader for the SelectObjectContentEventStream + // events. This value is automatically set by the SDK when the API call is made + // Use this member when unit testing your code with the SDK to mock out the + // EventStream Reader. + // + // Must not be nil. + Reader SelectObjectContentEventStreamReader + + outputReader io.ReadCloser + + // StreamCloser is the io.Closer for the EventStream connection. For HTTP + // EventStream this is the response Body. The stream will be closed when + // the Close method of the EventStream is called. + StreamCloser io.Closer + + done chan struct{} + closeOnce sync.Once + err *eventstreamapi.OnceError +} + +func newSelectObjectContentEventStream() *SelectObjectContentEventStream { + return &SelectObjectContentEventStream{ + done: make(chan struct{}), + err: eventstreamapi.NewOnceError(), + } +} + +func (es *SelectObjectContentEventStream) setStreamCloser(r *request.Request) { + es.StreamCloser = r.HTTPResponse.Body +} + +func (es *SelectObjectContentEventStream) runOnStreamPartClose(r *request.Request) { + if es.done == nil { + return + } + go es.waitStreamPartClose() + +} + +func (es *SelectObjectContentEventStream) waitStreamPartClose() { + var outputErrCh <-chan struct{} + if v, ok := es.Reader.(interface{ ErrorSet() <-chan struct{} }); ok { + outputErrCh = v.ErrorSet() + } + var outputClosedCh <-chan struct{} + if v, ok := es.Reader.(interface{ Closed() <-chan struct{} }); ok { + outputClosedCh = v.Closed() + } + + select { + case <-es.done: + case <-outputErrCh: + es.err.SetError(es.Reader.Err()) + es.Close() + case <-outputClosedCh: + if err := es.Reader.Err(); err != nil { + es.err.SetError(es.Reader.Err()) + } + es.Close() + } +} + +// Events returns a channel to read events from. +// +// These events are: +// +// * ContinuationEvent +// * EndEvent +// * ProgressEvent +// * RecordsEvent +// * StatsEvent +func (es *SelectObjectContentEventStream) Events() <-chan SelectObjectContentEventStreamEvent { + return es.Reader.Events() +} + +func (es *SelectObjectContentEventStream) runOutputStream(r *request.Request) { + var opts []func(*eventstream.Decoder) + if r.Config.Logger != nil && r.Config.LogLevel.Matches(aws.LogDebugWithEventStreamBody) { + opts = append(opts, eventstream.DecodeWithLogger(r.Config.Logger)) + } + + decoder := eventstream.NewDecoder(r.HTTPResponse.Body, opts...) + eventReader := eventstreamapi.NewEventReader(decoder, + protocol.HandlerPayloadUnmarshal{ + Unmarshalers: r.Handlers.UnmarshalStream, + }, + unmarshalerForSelectObjectContentEventStreamEvent, + ) + + es.outputReader = r.HTTPResponse.Body + es.Reader = newReadSelectObjectContentEventStream(eventReader) +} + +// Close closes the stream. This will also cause the stream to be closed. +// Close must be called when done using the stream API. Not calling Close +// may result in resource leaks. +// +// You can use the closing of the Reader's Events channel to terminate your +// application's read from the API's stream. +// +func (es *SelectObjectContentEventStream) Close() (err error) { + es.closeOnce.Do(es.safeClose) + return es.Err() +} + +func (es *SelectObjectContentEventStream) safeClose() { + if es.done != nil { + close(es.done) + } + + es.Reader.Close() + if es.outputReader != nil { + es.outputReader.Close() + } + + es.StreamCloser.Close() +} + +// Err returns any error that occurred while reading or writing EventStream +// Events from the service API's response. Returns nil if there were no errors. +func (es *SelectObjectContentEventStream) Err() error { + if err := es.err.Err(); err != nil { + return err + } + if err := es.Reader.Err(); err != nil { + return err + } + + return nil +} + const opUploadPart = "UploadPart" // UploadPartRequest generates a "aws/request.Request" representing the @@ -9967,10 +10121,10 @@ type AbortMultipartUploadInput struct { // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // For information about downloading objects from Requester Pays buckets, see - // Downloading Objects in Requestor Pays Buckets (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` @@ -11073,10 +11227,10 @@ type CompleteMultipartUploadInput struct { // The container for the multipart upload request information. MultipartUpload *CompletedMultipartUpload `locationName:"CompleteMultipartUpload" type:"structure" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // For information about downloading objects from Requester Pays buckets, see - // Downloading Objects in Requestor Pays Buckets (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` @@ -11201,7 +11355,8 @@ type CompleteMultipartUploadOutput struct { RequestCharged *string `location:"header" locationName:"x-amz-request-charged" type:"string" enum:"RequestCharged"` // If present, specifies the ID of the AWS Key Management Service (AWS KMS) - // customer master key (CMK) that was used for the object. + // symmetric customer managed customer master key (CMK) that was used for the + // object. SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` // If you specified server-side encryption either with an Amazon S3-managed @@ -11416,6 +11571,11 @@ func (s *ContinuationEvent) UnmarshalEvent( return nil } +func (s *ContinuationEvent) MarshalEvent(pm protocol.PayloadMarshaler) (msg eventstream.Message, err error) { + msg.Headers.Set(eventstreamapi.MessageTypeHeader, eventstream.StringValue(eventstreamapi.EventMessageType)) + return msg, err +} + type CopyObjectInput struct { _ struct{} `locationName:"CopyObjectRequest" type:"structure"` @@ -11513,10 +11673,10 @@ type CopyObjectInput struct { // The date and time when you want the copied object's Object Lock to expire. ObjectLockRetainUntilDate *time.Time `location:"header" locationName:"x-amz-object-lock-retain-until-date" type:"timestamp" timestampFormat:"iso8601"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // For information about downloading objects from Requester Pays buckets, see - // Downloading Objects in Requestor Pays Buckets (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` @@ -11545,7 +11705,7 @@ type CopyObjectInput struct { // requests for an object protected by AWS KMS will fail if not made via SSL // or using SigV4. For information about configuring using any of the officially // supported AWS SDKs and AWS CLI, see Specifying the Signature Version in Request - // Authentication (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version) + // Authentication (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version) // in the Amazon S3 Developer Guide. SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` @@ -11895,7 +12055,8 @@ type CopyObjectOutput struct { SSEKMSEncryptionContext *string `location:"header" locationName:"x-amz-server-side-encryption-context" type:"string" sensitive:"true"` // If present, specifies the ID of the AWS Key Management Service (AWS KMS) - // customer master key (CMK) that was used for the object. + // symmetric customer managed customer master key (CMK) that was used for the + // object. SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` // The server-side encryption algorithm used when storing this object in Amazon @@ -12275,10 +12436,10 @@ type CreateMultipartUploadInput struct { // Specifies the date and time when you want the Object Lock to expire. ObjectLockRetainUntilDate *time.Time `location:"header" locationName:"x-amz-object-lock-retain-until-date" type:"timestamp" timestampFormat:"iso8601"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // For information about downloading objects from Requester Pays buckets, see - // Downloading Objects in Requestor Pays Buckets (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` @@ -12303,11 +12464,11 @@ type CreateMultipartUploadInput struct { // encryption context key-value pairs. SSEKMSEncryptionContext *string `location:"header" locationName:"x-amz-server-side-encryption-context" type:"string" sensitive:"true"` - // Specifies the AWS KMS key ID to use for object encryption. All GET and PUT - // requests for an object protected by AWS KMS will fail if not made via SSL - // or using SigV4. For information about configuring using any of the officially - // supported AWS SDKs and AWS CLI, see Specifying the Signature Version in Request - // Authentication (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version) + // Specifies the ID of the symmetric customer managed AWS KMS CMK to use for + // object encryption. All GET and PUT requests for an object protected by AWS + // KMS will fail if not made via SSL or using SigV4. For information about configuring + // using any of the officially supported AWS SDKs and AWS CLI, see Specifying + // the Signature Version in Request Authentication (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version) // in the Amazon S3 Developer Guide. SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` @@ -12601,7 +12762,8 @@ type CreateMultipartUploadOutput struct { SSEKMSEncryptionContext *string `location:"header" locationName:"x-amz-server-side-encryption-context" type:"string" sensitive:"true"` // If present, specifies the ID of the AWS Key Management Service (AWS KMS) - // customer master key (CMK) that was used for the object. + // symmetric customer managed customer master key (CMK) that was used for the + // object. SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` // The server-side encryption algorithm used when storing this object in Amazon @@ -13805,10 +13967,10 @@ type DeleteObjectInput struct { // delete enabled. MFA *string `location:"header" locationName:"x-amz-mfa" type:"string"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // For information about downloading objects from Requester Pays buckets, see - // Downloading Objects in Requestor Pays Buckets (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` @@ -14098,10 +14260,10 @@ type DeleteObjectsInput struct { // delete enabled. MFA *string `location:"header" locationName:"x-amz-mfa" type:"string"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // For information about downloading objects from Requester Pays buckets, see - // Downloading Objects in Requestor Pays Buckets (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` } @@ -14517,8 +14679,11 @@ type Encryption struct { // the encryption context for the restore results. KMSContext *string `type:"string"` - // If the encryption type is aws:kms, this optional value specifies the AWS - // KMS key ID to use for encryption of job results. + // If the encryption type is aws:kms, this optional value specifies the ID of + // the symmetric customer managed AWS KMS CMK to use for encryption of job results. + // Amazon S3 only supports symmetric CMKs. For more information, see Using Symmetric + // and Asymmetric Keys (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html) + // in the AWS Key Management Service Developer Guide. KMSKeyId *string `type:"string" sensitive:"true"` } @@ -14568,8 +14733,12 @@ func (s *Encryption) SetKMSKeyId(v string) *Encryption { type EncryptionConfiguration struct { _ struct{} `type:"structure"` - // Specifies the AWS KMS Key ID (Key ARN or Alias ARN) for the destination bucket. - // Amazon S3 uses this key to encrypt replica objects. + // Specifies the ID (Key ARN or Alias ARN) of the customer managed customer + // master key (CMK) stored in AWS Key Management Service (KMS) for the destination + // bucket. Amazon S3 uses this key to encrypt replica objects. Amazon S3 only + // supports symmetric customer managed CMKs. For more information, see Using + // Symmetric and Asymmetric Keys (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html) + // in the AWS Key Management Service Developer Guide. ReplicaKmsKeyID *string `type:"string"` } @@ -14618,6 +14787,11 @@ func (s *EndEvent) UnmarshalEvent( return nil } +func (s *EndEvent) MarshalEvent(pm protocol.PayloadMarshaler) (msg eventstream.Message, err error) { + msg.Headers.Set(eventstreamapi.MessageTypeHeader, eventstream.StringValue(eventstreamapi.EventMessageType)) + return msg, err +} + // Container for all error elements. type Error struct { _ struct{} `type:"structure"` @@ -16854,10 +17028,10 @@ type GetObjectAclInput struct { // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // For information about downloading objects from Requester Pays buckets, see - // Downloading Objects in Requestor Pays Buckets (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` @@ -17029,10 +17203,10 @@ type GetObjectInput struct { // the HTTP Range header, see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35. Range *string `location:"header" locationName:"Range" type:"string"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // For information about downloading objects from Requester Pays buckets, see - // Downloading Objects in Requestor Pays Buckets (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` @@ -17269,10 +17443,10 @@ type GetObjectLegalHoldInput struct { // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // For information about downloading objects from Requester Pays buckets, see - // Downloading Objects in Requestor Pays Buckets (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` @@ -17562,7 +17736,8 @@ type GetObjectOutput struct { SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` // If present, specifies the ID of the AWS Key Management Service (AWS KMS) - // customer master key (CMK) that was used for the object. + // symmetric customer managed customer master key (CMK) that was used for the + // object. SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` // The server-side encryption algorithm used when storing this object in Amazon @@ -17802,10 +17977,10 @@ type GetObjectRetentionInput struct { // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // For information about downloading objects from Requester Pays buckets, see - // Downloading Objects in Requestor Pays Buckets (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` @@ -18056,10 +18231,10 @@ type GetObjectTorrentInput struct { // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // For information about downloading objects from Requester Pays buckets, see - // Downloading Objects in Requestor Pays Buckets (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` } @@ -18532,10 +18707,10 @@ type HeadObjectInput struct { // the HTTP Range header, see http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35. Range *string `location:"header" locationName:"Range" type:"string"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // For information about downloading objects from Requester Pays buckets, see - // Downloading Objects in Requestor Pays Buckets (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` @@ -18828,7 +19003,8 @@ type HeadObjectOutput struct { SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` // If present, specifies the ID of the AWS Key Management Service (AWS KMS) - // customer master key (CMK) that was used for the object. + // symmetric customer managed customer master key (CMK) that was used for the + // object. SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` // If the object is stored using server-side encryption either with an AWS KMS @@ -21676,10 +21852,10 @@ type ListPartsInput struct { // part numbers will be listed. PartNumberMarker *int64 `location:"querystring" locationName:"part-number-marker" type:"integer"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // For information about downloading objects from Requester Pays buckets, see - // Downloading Objects in Requestor Pays Buckets (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` @@ -23339,6 +23515,16 @@ func (s *ProgressEvent) UnmarshalEvent( return nil } +func (s *ProgressEvent) MarshalEvent(pm protocol.PayloadMarshaler) (msg eventstream.Message, err error) { + msg.Headers.Set(eventstreamapi.MessageTypeHeader, eventstream.StringValue(eventstreamapi.EventMessageType)) + var buf bytes.Buffer + if err = pm.MarshalPayload(&buf, s); err != nil { + return eventstream.Message{}, err + } + msg.Payload = buf.Bytes() + return msg, err +} + // The PublicAccessBlock configuration that you want to apply to this Amazon // S3 bucket. You can enable the configuration options in any combination. For // more information about when Amazon S3 considers a bucket or object public, @@ -25272,10 +25458,10 @@ type PutObjectAclInput struct { // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // For information about downloading objects from Requester Pays buckets, see - // Downloading Objects in Requestor Pays Buckets (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` @@ -25522,10 +25708,10 @@ type PutObjectInput struct { // The date and time when you want this object's Object Lock to expire. ObjectLockRetainUntilDate *time.Time `location:"header" locationName:"x-amz-object-lock-retain-until-date" type:"timestamp" timestampFormat:"iso8601"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // For information about downloading objects from Requester Pays buckets, see - // Downloading Objects in Requestor Pays Buckets (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` @@ -25552,12 +25738,14 @@ type PutObjectInput struct { // If x-amz-server-side-encryption is present and has the value of aws:kms, // this header specifies the ID of the AWS Key Management Service (AWS KMS) - // customer master key (CMK) that was used for the object. + // symmetrical customer managed customer master key (CMK) that was used for + // the object. // // If the value of x-amz-server-side-encryption is aws:kms, this header specifies - // the ID of the AWS KMS CMK that will be used for the object. If you specify - // x-amz-server-side-encryption:aws:kms, but do not providex-amz-server-side-encryption-aws-kms-key-id, - // Amazon S3 uses the AWS managed CMK in AWS to protect the data. + // the ID of the symmetric customer managed AWS KMS CMK that will be used for + // the object. If you specify x-amz-server-side-encryption:aws:kms, but do not + // providex-amz-server-side-encryption-aws-kms-key-id, Amazon S3 uses the AWS + // managed CMK in AWS to protect the data. SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` // The server-side encryption algorithm used when storing this object in Amazon @@ -25858,10 +26046,10 @@ type PutObjectLegalHoldInput struct { // specified object. LegalHold *ObjectLockLegalHold `locationName:"LegalHold" type:"structure" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // For information about downloading objects from Requester Pays buckets, see - // Downloading Objects in Requestor Pays Buckets (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` @@ -25987,10 +26175,10 @@ type PutObjectLockConfigurationInput struct { // The Object Lock configuration that you want to apply to the specified bucket. ObjectLockConfiguration *ObjectLockConfiguration `locationName:"ObjectLockConfiguration" type:"structure" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // For information about downloading objects from Requester Pays buckets, see - // Downloading Objects in Requestor Pays Buckets (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` @@ -26126,7 +26314,8 @@ type PutObjectOutput struct { // If x-amz-server-side-encryption is present and has the value of aws:kms, // this header specifies the ID of the AWS Key Management Service (AWS KMS) - // customer master key (CMK) that was used for the object. + // symmetric customer managed customer master key (CMK) that was used for the + // object. SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` // If you specified server-side encryption either with an AWS KMS customer master @@ -26228,10 +26417,10 @@ type PutObjectRetentionInput struct { // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // For information about downloading objects from Requester Pays buckets, see - // Downloading Objects in Requestor Pays Buckets (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` @@ -26762,6 +26951,13 @@ func (s *RecordsEvent) UnmarshalEvent( return nil } +func (s *RecordsEvent) MarshalEvent(pm protocol.PayloadMarshaler) (msg eventstream.Message, err error) { + msg.Headers.Set(eventstreamapi.MessageTypeHeader, eventstream.StringValue(eventstreamapi.EventMessageType)) + msg.Headers.Set(":content-type", eventstream.StringValue("application/octet-stream")) + msg.Payload = s.Payload + return msg, err +} + // Specifies how requests are redirected. In the event of an error, you can // specify a different error code to return. type Redirect struct { @@ -27426,10 +27622,10 @@ type RestoreObjectInput struct { // Key is a required field Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // For information about downloading objects from Requester Pays buckets, see - // Downloading Objects in Requestor Pays Buckets (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` @@ -27849,8 +28045,8 @@ func (s *Rule) SetTransition(v *Transition) *Rule { type SSEKMS struct { _ struct{} `locationName:"SSE-KMS" type:"structure"` - // Specifies the ID of the AWS Key Management Service (AWS KMS) customer master - // key (CMK) to use for encrypting inventory reports. + // Specifies the ID of the AWS Key Management Service (AWS KMS) symmetric customer + // managed customer master key (CMK) to use for encrypting inventory reports. // // KeyId is a required field KeyId *string `type:"string" required:"true" sensitive:"true"` @@ -27943,75 +28139,8 @@ func (s *ScanRange) SetStart(v int64) *ScanRange { return s } -// SelectObjectContentEventStream provides handling of EventStreams for -// the SelectObjectContent API. -// -// Use this type to receive SelectObjectContentEventStream events. The events -// can be read from the Events channel member. -// -// The events that can be received are: -// -// * ContinuationEvent -// * EndEvent -// * ProgressEvent -// * RecordsEvent -// * StatsEvent -type SelectObjectContentEventStream struct { - // Reader is the EventStream reader for the SelectObjectContentEventStream - // events. This value is automatically set by the SDK when the API call is made - // Use this member when unit testing your code with the SDK to mock out the - // EventStream Reader. - // - // Must not be nil. - Reader SelectObjectContentEventStreamReader - - // StreamCloser is the io.Closer for the EventStream connection. For HTTP - // EventStream this is the response Body. The stream will be closed when - // the Close method of the EventStream is called. - StreamCloser io.Closer -} - -// Close closes the EventStream. This will also cause the Events channel to be -// closed. You can use the closing of the Events channel to terminate your -// application's read from the API's EventStream. -// -// Will close the underlying EventStream reader. For EventStream over HTTP -// connection this will also close the HTTP connection. -// -// Close must be called when done using the EventStream API. Not calling Close -// may result in resource leaks. -func (es *SelectObjectContentEventStream) Close() (err error) { - es.Reader.Close() - es.StreamCloser.Close() - - return es.Err() -} - -// Err returns any error that occurred while reading EventStream Events from -// the service API's response. Returns nil if there were no errors. -func (es *SelectObjectContentEventStream) Err() error { - if err := es.Reader.Err(); err != nil { - return err - } - return nil -} - -// Events returns a channel to read EventStream Events from the -// SelectObjectContent API. -// -// These events are: -// -// * ContinuationEvent -// * EndEvent -// * ProgressEvent -// * RecordsEvent -// * StatsEvent -func (es *SelectObjectContentEventStream) Events() <-chan SelectObjectContentEventStreamEvent { - return es.Reader.Events() -} - // SelectObjectContentEventStreamEvent groups together all EventStream -// events read from the SelectObjectContent API. +// events writes for SelectObjectContentEventStream. // // These events are: // @@ -28022,11 +28151,12 @@ func (es *SelectObjectContentEventStream) Events() <-chan SelectObjectContentEve // * StatsEvent type SelectObjectContentEventStreamEvent interface { eventSelectObjectContentEventStream() + eventstreamapi.Marshaler + eventstreamapi.Unmarshaler } -// SelectObjectContentEventStreamReader provides the interface for reading EventStream -// Events from the SelectObjectContent API. The -// default implementation for this interface will be SelectObjectContentEventStream. +// SelectObjectContentEventStreamReader provides the interface for reading to the stream. The +// default implementation for this interface will be SelectObjectContentEventStreamData. // // The reader's Close method must allow multiple concurrent calls. // @@ -28041,8 +28171,7 @@ type SelectObjectContentEventStreamReader interface { // Returns a channel of events as they are read from the event stream. Events() <-chan SelectObjectContentEventStreamEvent - // Close will close the underlying event stream reader. For event stream over - // HTTP this will also close the HTTP connection. + // Close will stop the reader reading events from the stream. Close() error // Returns any error that has occurred while reading from the event stream. @@ -28052,57 +28181,44 @@ type SelectObjectContentEventStreamReader interface { type readSelectObjectContentEventStream struct { eventReader *eventstreamapi.EventReader stream chan SelectObjectContentEventStreamEvent - errVal atomic.Value + err *eventstreamapi.OnceError done chan struct{} closeOnce sync.Once } -func newReadSelectObjectContentEventStream( - reader io.ReadCloser, - unmarshalers request.HandlerList, - logger aws.Logger, - logLevel aws.LogLevelType, -) *readSelectObjectContentEventStream { +func newReadSelectObjectContentEventStream(eventReader *eventstreamapi.EventReader) *readSelectObjectContentEventStream { r := &readSelectObjectContentEventStream{ - stream: make(chan SelectObjectContentEventStreamEvent), - done: make(chan struct{}), + eventReader: eventReader, + stream: make(chan SelectObjectContentEventStreamEvent), + done: make(chan struct{}), + err: eventstreamapi.NewOnceError(), } - - r.eventReader = eventstreamapi.NewEventReader( - reader, - protocol.HandlerPayloadUnmarshal{ - Unmarshalers: unmarshalers, - }, - r.unmarshalerForEventType, - ) - r.eventReader.UseLogger(logger, logLevel) + go r.readEventStream() return r } -// Close will close the underlying event stream reader. For EventStream over -// HTTP this will also close the HTTP connection. +// Close will close the underlying event stream reader. func (r *readSelectObjectContentEventStream) Close() error { r.closeOnce.Do(r.safeClose) - return r.Err() } +func (r *readSelectObjectContentEventStream) ErrorSet() <-chan struct{} { + return r.err.ErrorSet() +} + +func (r *readSelectObjectContentEventStream) Closed() <-chan struct{} { + return r.done +} + func (r *readSelectObjectContentEventStream) safeClose() { close(r.done) - err := r.eventReader.Close() - if err != nil { - r.errVal.Store(err) - } } func (r *readSelectObjectContentEventStream) Err() error { - if v := r.errVal.Load(); v != nil { - return v.(error) - } - - return nil + return r.err.Err() } func (r *readSelectObjectContentEventStream) Events() <-chan SelectObjectContentEventStreamEvent { @@ -28110,6 +28226,7 @@ func (r *readSelectObjectContentEventStream) Events() <-chan SelectObjectContent } func (r *readSelectObjectContentEventStream) readEventStream() { + defer r.Close() defer close(r.stream) for { @@ -28124,7 +28241,7 @@ func (r *readSelectObjectContentEventStream) readEventStream() { return default: } - r.errVal.Store(err) + r.err.SetError(err) return } @@ -28136,22 +28253,16 @@ func (r *readSelectObjectContentEventStream) readEventStream() { } } -func (r *readSelectObjectContentEventStream) unmarshalerForEventType( - eventType string, -) (eventstreamapi.Unmarshaler, error) { +func unmarshalerForSelectObjectContentEventStreamEvent(eventType string) (eventstreamapi.Unmarshaler, error) { switch eventType { case "Cont": return &ContinuationEvent{}, nil - case "End": return &EndEvent{}, nil - case "Progress": return &ProgressEvent{}, nil - case "Records": return &RecordsEvent{}, nil - case "Stats": return &StatsEvent{}, nil default: @@ -28378,8 +28489,7 @@ func (s *SelectObjectContentInput) hasEndpointARN() bool { type SelectObjectContentOutput struct { _ struct{} `type:"structure" payload:"Payload"` - // Use EventStream to use the API's stream. - EventStream *SelectObjectContentEventStream `type:"structure"` + EventStream *SelectObjectContentEventStream } // String returns the string representation @@ -28392,29 +28502,17 @@ func (s SelectObjectContentOutput) GoString() string { return s.String() } -// SetEventStream sets the EventStream field's value. func (s *SelectObjectContentOutput) SetEventStream(v *SelectObjectContentEventStream) *SelectObjectContentOutput { s.EventStream = v return s } +func (s *SelectObjectContentOutput) GetEventStream() *SelectObjectContentEventStream { + return s.EventStream +} -func (s *SelectObjectContentOutput) runEventStreamLoop(r *request.Request) { - if r.Error != nil { - return - } - reader := newReadSelectObjectContentEventStream( - r.HTTPResponse.Body, - r.Handlers.UnmarshalStream, - r.Config.Logger, - r.Config.LogLevel.Value(), - ) - go reader.readEventStream() - - eventStream := &SelectObjectContentEventStream{ - StreamCloser: r.HTTPResponse.Body, - Reader: reader, - } - s.EventStream = eventStream +// GetStream returns the type to interact with the event stream. +func (s *SelectObjectContentOutput) GetStream() *SelectObjectContentEventStream { + return s.EventStream } // Describes the parameters for Select job types. @@ -28811,6 +28909,16 @@ func (s *StatsEvent) UnmarshalEvent( return nil } +func (s *StatsEvent) MarshalEvent(pm protocol.PayloadMarshaler) (msg eventstream.Message, err error) { + msg.Headers.Set(eventstreamapi.MessageTypeHeader, eventstream.StringValue(eventstreamapi.EventMessageType)) + var buf bytes.Buffer + if err = pm.MarshalPayload(&buf, s); err != nil { + return eventstream.Message{}, err + } + msg.Payload = buf.Bytes() + return msg, err +} + // Specifies data related to access patterns to be collected and made available // to analyze the tradeoffs between different storage classes for an Amazon // S3 bucket. @@ -29305,10 +29413,10 @@ type UploadPartCopyInput struct { // PartNumber is a required field PartNumber *int64 `location:"querystring" locationName:"partNumber" type:"integer" required:"true"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // For information about downloading objects from Requester Pays buckets, see - // Downloading Objects in Requestor Pays Buckets (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` @@ -29538,7 +29646,8 @@ type UploadPartCopyOutput struct { SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` // If present, specifies the ID of the AWS Key Management Service (AWS KMS) - // customer master key (CMK) that was used for the object. + // symmetric customer managed customer master key (CMK) that was used for the + // object. SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` // The server-side encryption algorithm used when storing this object in Amazon @@ -29629,10 +29738,10 @@ type UploadPartInput struct { // PartNumber is a required field PartNumber *int64 `location:"querystring" locationName:"partNumber" type:"integer" required:"true"` - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // For information about downloading objects from Requester Pays buckets, see - // Downloading Objects in Requestor Pays Buckets (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) + // Confirms that the requester knows that they will be charged for the request. + // Bucket owners need not specify this parameter in their requests. For information + // about downloading objects from requester pays buckets, see Downloading Objects + // in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) // in the Amazon S3 Developer Guide. RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` @@ -29812,7 +29921,7 @@ type UploadPartOutput struct { SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` // If present, specifies the ID of the AWS Key Management Service (AWS KMS) - // customer master key (CMK) was used for the object. + // symmetric customer managed customer master key (CMK) was used for the object. SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` // The server-side encryption algorithm used when storing this object in Amazon @@ -30461,10 +30570,10 @@ const ( RequestChargedRequester = "requester" ) -// Confirms that the requester knows that she or he will be charged for the -// request. Bucket owners need not specify this parameter in their requests. -// For information about downloading objects from Requester Pays buckets, see -// Downloading Objects in Requestor Pays Buckets (https://docs.aws.amazon.com/http:/docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) +// Confirms that the requester knows that they will be charged for the request. +// Bucket owners need not specify this parameter in their requests. For information +// about downloading objects from requester pays buckets, see Downloading Objects +// in Requestor Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html) // in the Amazon S3 Developer Guide. const ( // RequestPayerRequester is a RequestPayer enum value diff --git a/vendor/github.com/aws/aws-sdk-go/service/s3/service.go b/vendor/github.com/aws/aws-sdk-go/service/s3/service.go index ba77bb8127..b4c07b4d47 100644 --- a/vendor/github.com/aws/aws-sdk-go/service/s3/service.go +++ b/vendor/github.com/aws/aws-sdk-go/service/s3/service.go @@ -31,7 +31,7 @@ var initRequest func(*request.Request) const ( ServiceName = "s3" // Name of service. EndpointsID = ServiceName // ID to lookup a service endpoint with. - ServiceID = "S3" // ServiceID is a unique identifer of a specific service. + ServiceID = "S3" // ServiceID is a unique identifier of a specific service. ) // New creates a new instance of the S3 client with a session. @@ -78,6 +78,7 @@ func newClient(cfg aws.Config, handlers request.Handlers, partitionID, endpoint, svc.Handlers.UnmarshalMeta.PushBackNamed(restxml.UnmarshalMetaHandler) svc.Handlers.UnmarshalError.PushBackNamed(restxml.UnmarshalErrorHandler) + svc.Handlers.BuildStream.PushBackNamed(restxml.BuildHandler) svc.Handlers.UnmarshalStream.PushBackNamed(restxml.UnmarshalHandler) // Run custom client initialization if present diff --git a/vendor/github.com/aws/aws-sdk-go/service/sts/service.go b/vendor/github.com/aws/aws-sdk-go/service/sts/service.go index 586faed193..d34a685533 100644 --- a/vendor/github.com/aws/aws-sdk-go/service/sts/service.go +++ b/vendor/github.com/aws/aws-sdk-go/service/sts/service.go @@ -31,7 +31,7 @@ var initRequest func(*request.Request) const ( ServiceName = "sts" // Name of service. EndpointsID = ServiceName // ID to lookup a service endpoint with. - ServiceID = "STS" // ServiceID is a unique identifer of a specific service. + ServiceID = "STS" // ServiceID is a unique identifier of a specific service. ) // New creates a new instance of the STS client with a session. diff --git a/vendor/github.com/cenkalti/backoff/.travis.yml b/vendor/github.com/cenkalti/backoff/.travis.yml index ce9cb62333..47a6a46ec2 100644 --- a/vendor/github.com/cenkalti/backoff/.travis.yml +++ b/vendor/github.com/cenkalti/backoff/.travis.yml @@ -1,2 +1,10 @@ language: go -go: 1.3.3 +go: + - 1.7 + - 1.x + - tip +before_install: + - go get github.com/mattn/goveralls + - go get golang.org/x/tools/cmd/cover +script: + - $HOME/gopath/bin/goveralls -service=travis-ci diff --git a/vendor/github.com/cenkalti/backoff/README.md b/vendor/github.com/cenkalti/backoff/README.md index 020b8fbf33..55ebc98fc2 100644 --- a/vendor/github.com/cenkalti/backoff/README.md +++ b/vendor/github.com/cenkalti/backoff/README.md @@ -1,4 +1,4 @@ -# Exponential Backoff [![GoDoc][godoc image]][godoc] [![Build Status][travis image]][travis] +# Exponential Backoff [![GoDoc][godoc image]][godoc] [![Build Status][travis image]][travis] [![Coverage Status][coveralls image]][coveralls] This is a Go port of the exponential backoff algorithm from [Google's HTTP Client Library for Java][google-http-java-client]. @@ -7,110 +7,24 @@ is an algorithm that uses feedback to multiplicatively decrease the rate of some in order to gradually find an acceptable rate. The retries exponentially increase and stop increasing when a certain threshold is met. -## How To +## Usage -We define two functions, `Retry()` and `RetryNotify()`. -They receive an `Operation` to execute, a `BackOff` algorithm, -and an optional `Notify` error handler. +See https://godoc.org/github.com/cenkalti/backoff#pkg-examples -The operation will be executed, and will be retried on failure with delay -as given by the backoff algorithm. The backoff algorithm can also decide when to stop -retrying. -In addition, the notify error handler will be called after each failed attempt, -except for the last time, whose error should be handled by the caller. +## Contributing -```go -// An Operation is executing by Retry() or RetryNotify(). -// The operation will be retried using a backoff policy if it returns an error. -type Operation func() error - -// Notify is a notify-on-error function. It receives an operation error and -// backoff delay if the operation failed (with an error). -// -// NOTE that if the backoff policy stated to stop retrying, -// the notify function isn't called. -type Notify func(error, time.Duration) - -func Retry(Operation, BackOff) error -func RetryNotify(Operation, BackOff, Notify) -``` - -## Examples - -See more advanced examples in the [godoc][advanced example]. - -### Retry - -Simple retry helper that uses the default exponential backoff algorithm: - -```go -operation := func() error { - // An operation that might fail. - return nil // or return errors.New("some error") -} - -err := Retry(operation, NewExponentialBackOff()) -if err != nil { - // Handle error. - return err -} - -// Operation is successful. -return nil -``` - -### Ticker - -```go -operation := func() error { - // An operation that might fail - return nil // or return errors.New("some error") -} - -b := NewExponentialBackOff() -ticker := NewTicker(b) - -var err error - -// Ticks will continue to arrive when the previous operation is still running, -// so operations that take a while to fail could run in quick succession. -for range ticker.C { - if err = operation(); err != nil { - log.Println(err, "will retry...") - continue - } - - ticker.Stop() - break -} - -if err != nil { - // Operation has failed. - return err -} - -// Operation is successful. -return nil -``` - -## Getting Started - -```bash -# install -$ go get github.com/cenkalti/backoff - -# test -$ cd $GOPATH/src/github.com/cenkalti/backoff -$ go get -t ./... -$ go test -v -cover -``` +* I would like to keep this library as small as possible. +* Please don't send a PR without opening an issue and discussing it first. +* If proposed change is not a common use case, I will probably not accept it. [godoc]: https://godoc.org/github.com/cenkalti/backoff [godoc image]: https://godoc.org/github.com/cenkalti/backoff?status.png [travis]: https://travis-ci.org/cenkalti/backoff -[travis image]: https://travis-ci.org/cenkalti/backoff.png +[travis image]: https://travis-ci.org/cenkalti/backoff.png?branch=master +[coveralls]: https://coveralls.io/github/cenkalti/backoff?branch=master +[coveralls image]: https://coveralls.io/repos/github/cenkalti/backoff/badge.svg?branch=master -[google-http-java-client]: https://github.com/google/google-http-java-client +[google-http-java-client]: https://github.com/google/google-http-java-client/blob/da1aa993e90285ec18579f1553339b00e19b3ab5/google-http-client/src/main/java/com/google/api/client/util/ExponentialBackOff.java [exponential backoff wiki]: http://en.wikipedia.org/wiki/Exponential_backoff [advanced example]: https://godoc.org/github.com/cenkalti/backoff#example_ diff --git a/vendor/github.com/cenkalti/backoff/backoff.go b/vendor/github.com/cenkalti/backoff/backoff.go index 61bd6df66c..3676ee405d 100644 --- a/vendor/github.com/cenkalti/backoff/backoff.go +++ b/vendor/github.com/cenkalti/backoff/backoff.go @@ -1,6 +1,13 @@ // Package backoff implements backoff algorithms for retrying operations. // -// Also has a Retry() helper for retrying operations that may fail. +// Use Retry function for retrying operations that may fail. +// If Retry does not meet your needs, +// copy/paste the function into your project and modify as you wish. +// +// There is also Ticker type similar to time.Ticker. +// You can use it if you need to work with channels. +// +// See Examples section below for usage examples. package backoff import "time" @@ -8,7 +15,7 @@ import "time" // BackOff is a backoff policy for retrying an operation. type BackOff interface { // NextBackOff returns the duration to wait before retrying the operation, - // or backoff.Stop to indicate that no more retries should be made. + // or backoff. Stop to indicate that no more retries should be made. // // Example usage: // @@ -25,7 +32,7 @@ type BackOff interface { Reset() } -// Indicates that no more retries should be made for use in NextBackOff(). +// Stop indicates that no more retries should be made for use in NextBackOff(). const Stop time.Duration = -1 // ZeroBackOff is a fixed backoff policy whose backoff time is always zero, diff --git a/vendor/github.com/cenkalti/backoff/context.go b/vendor/github.com/cenkalti/backoff/context.go new file mode 100644 index 0000000000..7706faa2b6 --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/context.go @@ -0,0 +1,63 @@ +package backoff + +import ( + "context" + "time" +) + +// BackOffContext is a backoff policy that stops retrying after the context +// is canceled. +type BackOffContext interface { + BackOff + Context() context.Context +} + +type backOffContext struct { + BackOff + ctx context.Context +} + +// WithContext returns a BackOffContext with context ctx +// +// ctx must not be nil +func WithContext(b BackOff, ctx context.Context) BackOffContext { + if ctx == nil { + panic("nil context") + } + + if b, ok := b.(*backOffContext); ok { + return &backOffContext{ + BackOff: b.BackOff, + ctx: ctx, + } + } + + return &backOffContext{ + BackOff: b, + ctx: ctx, + } +} + +func ensureContext(b BackOff) BackOffContext { + if cb, ok := b.(BackOffContext); ok { + return cb + } + return WithContext(b, context.Background()) +} + +func (b *backOffContext) Context() context.Context { + return b.ctx +} + +func (b *backOffContext) NextBackOff() time.Duration { + select { + case <-b.ctx.Done(): + return Stop + default: + } + next := b.BackOff.NextBackOff() + if deadline, ok := b.ctx.Deadline(); ok && deadline.Sub(time.Now()) < next { + return Stop + } + return next +} diff --git a/vendor/github.com/cenkalti/backoff/exponential.go b/vendor/github.com/cenkalti/backoff/exponential.go index ae65516dc0..a031a65979 100644 --- a/vendor/github.com/cenkalti/backoff/exponential.go +++ b/vendor/github.com/cenkalti/backoff/exponential.go @@ -89,11 +89,6 @@ func NewExponentialBackOff() *ExponentialBackOff { MaxElapsedTime: DefaultMaxElapsedTime, Clock: SystemClock, } - if b.RandomizationFactor < 0 { - b.RandomizationFactor = 0 - } else if b.RandomizationFactor > 1 { - b.RandomizationFactor = 1 - } b.Reset() return b } @@ -127,7 +122,9 @@ func (b *ExponentialBackOff) NextBackOff() time.Duration { // GetElapsedTime returns the elapsed time since an ExponentialBackOff instance // is created and is reset when Reset() is called. // -// The elapsed time is computed using time.Now().UnixNano(). +// The elapsed time is computed using time.Now().UnixNano(). It is +// safe to call even while the backoff policy is used by a running +// ticker. func (b *ExponentialBackOff) GetElapsedTime() time.Duration { return b.Clock.Now().Sub(b.startTime) } diff --git a/vendor/github.com/cenkalti/backoff/retry.go b/vendor/github.com/cenkalti/backoff/retry.go index f01f2bbd07..e936a506f8 100644 --- a/vendor/github.com/cenkalti/backoff/retry.go +++ b/vendor/github.com/cenkalti/backoff/retry.go @@ -13,9 +13,11 @@ type Operation func() error // the notify function isn't called. type Notify func(error, time.Duration) -// Retry the function f until it does not return error or BackOff stops. -// f is guaranteed to be run at least once. -// It is the caller's responsibility to reset b after Retry returns. +// Retry the operation o until it does not return error or BackOff stops. +// o is guaranteed to be run at least once. +// +// If o returns a *PermanentError, the operation is not retried, and the +// wrapped error is returned. // // Retry sleeps the goroutine for the duration returned by BackOff after a // failed operation returns. @@ -26,6 +28,9 @@ func Retry(o Operation, b BackOff) error { return RetryNotify(o, b, nil) } func RetryNotify(operation Operation, b BackOff, notify Notify) error { var err error var next time.Duration + var t *time.Timer + + cb := ensureContext(b) b.Reset() for { @@ -33,7 +38,11 @@ func RetryNotify(operation Operation, b BackOff, notify Notify) error { return nil } - if next = b.NextBackOff(); next == Stop { + if permanent, ok := err.(*PermanentError); ok { + return permanent.Err + } + + if next = cb.NextBackOff(); next == Stop { return err } @@ -41,6 +50,33 @@ func RetryNotify(operation Operation, b BackOff, notify Notify) error { notify(err, next) } - time.Sleep(next) + if t == nil { + t = time.NewTimer(next) + defer t.Stop() + } else { + t.Reset(next) + } + + select { + case <-cb.Context().Done(): + return err + case <-t.C: + } + } +} + +// PermanentError signals that the operation should not be retried. +type PermanentError struct { + Err error +} + +func (e *PermanentError) Error() string { + return e.Err.Error() +} + +// Permanent wraps the given err in a *PermanentError. +func Permanent(err error) *PermanentError { + return &PermanentError{ + Err: err, } } diff --git a/vendor/github.com/cenkalti/backoff/ticker.go b/vendor/github.com/cenkalti/backoff/ticker.go index 7a5ff4ed1f..e41084b0ef 100644 --- a/vendor/github.com/cenkalti/backoff/ticker.go +++ b/vendor/github.com/cenkalti/backoff/ticker.go @@ -1,7 +1,6 @@ package backoff import ( - "runtime" "sync" "time" ) @@ -13,24 +12,27 @@ import ( type Ticker struct { C <-chan time.Time c chan time.Time - b BackOff + b BackOffContext stop chan struct{} stopOnce sync.Once } -// NewTicker returns a new Ticker containing a channel that will send the time at times -// specified by the BackOff argument. Ticker is guaranteed to tick at least once. -// The channel is closed when Stop method is called or BackOff stops. +// NewTicker returns a new Ticker containing a channel that will send +// the time at times specified by the BackOff argument. Ticker is +// guaranteed to tick at least once. The channel is closed when Stop +// method is called or BackOff stops. It is not safe to manipulate the +// provided backoff policy (notably calling NextBackOff or Reset) +// while the ticker is running. func NewTicker(b BackOff) *Ticker { c := make(chan time.Time) t := &Ticker{ C: c, c: c, - b: b, + b: ensureContext(b), stop: make(chan struct{}), } + t.b.Reset() go t.run() - runtime.SetFinalizer(t, (*Ticker).Stop) return t } @@ -42,7 +44,6 @@ func (t *Ticker) Stop() { func (t *Ticker) run() { c := t.c defer close(c) - t.b.Reset() // Ticker is guaranteed to tick at least once. afterC := t.send(time.Now()) @@ -58,6 +59,8 @@ func (t *Ticker) run() { case <-t.stop: t.c = nil // Prevent future ticks from being sent to the channel. return + case <-t.b.Context().Done(): + return } } } diff --git a/vendor/github.com/cenkalti/backoff/tries.go b/vendor/github.com/cenkalti/backoff/tries.go new file mode 100644 index 0000000000..cfeefd9b76 --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/tries.go @@ -0,0 +1,35 @@ +package backoff + +import "time" + +/* +WithMaxRetries creates a wrapper around another BackOff, which will +return Stop if NextBackOff() has been called too many times since +the last time Reset() was called + +Note: Implementation is not thread-safe. +*/ +func WithMaxRetries(b BackOff, max uint64) BackOff { + return &backOffTries{delegate: b, maxTries: max} +} + +type backOffTries struct { + delegate BackOff + maxTries uint64 + numTries uint64 +} + +func (b *backOffTries) NextBackOff() time.Duration { + if b.maxTries > 0 { + if b.maxTries <= b.numTries { + return Stop + } + b.numTries++ + } + return b.delegate.NextBackOff() +} + +func (b *backOffTries) Reset() { + b.numTries = 0 + b.delegate.Reset() +} diff --git a/vendor/github.com/coreos/go-systemd/journal/journal.go b/vendor/github.com/coreos/go-systemd/journal/journal.go index 603ad4c3b5..a0f4837a02 100644 --- a/vendor/github.com/coreos/go-systemd/journal/journal.go +++ b/vendor/github.com/coreos/go-systemd/journal/journal.go @@ -33,7 +33,10 @@ import ( "os" "strconv" "strings" + "sync" + "sync/atomic" "syscall" + "unsafe" ) // Priority of a journal message @@ -50,19 +53,35 @@ const ( PriDebug ) -var conn net.Conn +var ( + // This can be overridden at build-time: + // https://github.com/golang/go/wiki/GcToolchainTricks#including-build-information-in-the-executable + journalSocket = "/run/systemd/journal/socket" + + // unixConnPtr atomically holds the local unconnected Unix-domain socket. + // Concrete safe pointer type: *net.UnixConn + unixConnPtr unsafe.Pointer + // onceConn ensures that unixConnPtr is initialized exactly once. + onceConn sync.Once +) func init() { - var err error - conn, err = net.Dial("unixgram", "/run/systemd/journal/socket") - if err != nil { - conn = nil - } + onceConn.Do(initConn) } -// Enabled returns true if the local systemd journal is available for logging +// Enabled checks whether the local systemd journal is available for logging. func Enabled() bool { - return conn != nil + onceConn.Do(initConn) + + if (*net.UnixConn)(atomic.LoadPointer(&unixConnPtr)) == nil { + return false + } + + if _, err := net.Dial("unixgram", journalSocket); err != nil { + return false + } + + return true } // Send a message to the local systemd journal. vars is a map of journald @@ -73,8 +92,14 @@ func Enabled() bool { // (http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html) // for more details. vars may be nil. func Send(message string, priority Priority, vars map[string]string) error { + conn := (*net.UnixConn)(atomic.LoadPointer(&unixConnPtr)) if conn == nil { - return journalError("could not connect to journald socket") + return errors.New("could not initialize socket to journald") + } + + socketAddr := &net.UnixAddr{ + Name: journalSocket, + Net: "unixgram", } data := new(bytes.Buffer) @@ -84,32 +109,30 @@ func Send(message string, priority Priority, vars map[string]string) error { appendVariable(data, k, v) } - _, err := io.Copy(conn, data) - if err != nil && isSocketSpaceError(err) { - file, err := tempFd() - if err != nil { - return journalError(err.Error()) - } - defer file.Close() - _, err = io.Copy(file, data) - if err != nil { - return journalError(err.Error()) - } - - rights := syscall.UnixRights(int(file.Fd())) + _, _, err := conn.WriteMsgUnix(data.Bytes(), nil, socketAddr) + if err == nil { + return nil + } + if !isSocketSpaceError(err) { + return err + } - /* this connection should always be a UnixConn, but better safe than sorry */ - unixConn, ok := conn.(*net.UnixConn) - if !ok { - return journalError("can't send file through non-Unix connection") - } - _, _, err = unixConn.WriteMsgUnix([]byte{}, rights, nil) - if err != nil { - return journalError(err.Error()) - } - } else if err != nil { - return journalError(err.Error()) + // Large log entry, send it via tempfile and ancillary-fd. + file, err := tempFd() + if err != nil { + return err + } + defer file.Close() + _, err = io.Copy(file, data) + if err != nil { + return err + } + rights := syscall.UnixRights(int(file.Fd())) + _, _, err = conn.WriteMsgUnix([]byte{}, rights, socketAddr) + if err != nil { + return err } + return nil } @@ -120,7 +143,7 @@ func Print(priority Priority, format string, a ...interface{}) error { func appendVariable(w io.Writer, name, value string) { if err := validVarName(name); err != nil { - journalError(err.Error()) + fmt.Fprintf(os.Stderr, "variable name %s contains invalid character, ignoring\n", name) } if strings.ContainsRune(value, '\n') { /* When the value contains a newline, we write: @@ -137,9 +160,9 @@ func appendVariable(w io.Writer, name, value string) { } } -// validVarName validates a variable name to make sure it journald will accept it. +// validVarName validates a variable name to make sure journald will accept it. // The variable name must be in uppercase and consist only of characters, -// numbers and underscores, and may not begin with an underscore. (from the docs) +// numbers and underscores, and may not begin with an underscore: // https://www.freedesktop.org/software/systemd/man/sd_journal_print.html func validVarName(name string) error { if name == "" { @@ -156,20 +179,23 @@ func validVarName(name string) error { return nil } +// isSocketSpaceError checks whether the error is signaling +// an "overlarge message" condition. func isSocketSpaceError(err error) bool { opErr, ok := err.(*net.OpError) - if !ok { + if !ok || opErr == nil { return false } - sysErr, ok := opErr.Err.(syscall.Errno) - if !ok { + sysErr, ok := opErr.Err.(*os.SyscallError) + if !ok || sysErr == nil { return false } - return sysErr == syscall.EMSGSIZE || sysErr == syscall.ENOBUFS + return sysErr.Err == syscall.EMSGSIZE || sysErr.Err == syscall.ENOBUFS } +// tempFd creates a temporary, unlinked file under `/dev/shm`. func tempFd() (*os.File, error) { file, err := ioutil.TempFile("/dev/shm/", "journal.XXXXX") if err != nil { @@ -182,8 +208,18 @@ func tempFd() (*os.File, error) { return file, nil } -func journalError(s string) error { - s = "journal error: " + s - fmt.Fprintln(os.Stderr, s) - return errors.New(s) +// initConn initializes the global `unixConnPtr` socket. +// It is meant to be called exactly once, at program startup. +func initConn() { + autobind, err := net.ResolveUnixAddr("unixgram", "") + if err != nil { + return + } + + sock, err := net.ListenUnixgram("unixgram", autobind) + if err != nil { + return + } + + atomic.StorePointer(&unixConnPtr, unsafe.Pointer(sock)) } diff --git a/vendor/github.com/go-kit/kit/log/json_logger.go b/vendor/github.com/go-kit/kit/log/json_logger.go index 66094b4dd3..0cedbf8247 100644 --- a/vendor/github.com/go-kit/kit/log/json_logger.go +++ b/vendor/github.com/go-kit/kit/log/json_logger.go @@ -31,7 +31,9 @@ func (l *jsonLogger) Log(keyvals ...interface{}) error { } merge(m, k, v) } - return json.NewEncoder(l.Writer).Encode(m) + enc := json.NewEncoder(l.Writer) + enc.SetEscapeHTML(false) + return enc.Encode(m) } func merge(dst map[string]interface{}, k, v interface{}) { diff --git a/vendor/github.com/go-logfmt/logfmt/.gitignore b/vendor/github.com/go-logfmt/logfmt/.gitignore index 320e53e984..1d74e21965 100644 --- a/vendor/github.com/go-logfmt/logfmt/.gitignore +++ b/vendor/github.com/go-logfmt/logfmt/.gitignore @@ -1,4 +1 @@ -_testdata/ -_testdata2/ -logfmt-fuzz.zip -logfmt.test.exe +.vscode/ diff --git a/vendor/github.com/go-logfmt/logfmt/.travis.yml b/vendor/github.com/go-logfmt/logfmt/.travis.yml index 25976dae6d..5139309898 100644 --- a/vendor/github.com/go-logfmt/logfmt/.travis.yml +++ b/vendor/github.com/go-logfmt/logfmt/.travis.yml @@ -6,6 +6,8 @@ go: - "1.9.x" - "1.10.x" - "1.11.x" + - "1.12.x" + - "1.13.x" - "tip" before_install: diff --git a/vendor/github.com/go-logfmt/logfmt/CHANGELOG.md b/vendor/github.com/go-logfmt/logfmt/CHANGELOG.md index 3455b8ec8a..1a9a27bcf6 100644 --- a/vendor/github.com/go-logfmt/logfmt/CHANGELOG.md +++ b/vendor/github.com/go-logfmt/logfmt/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.5.0] - 2020-01-03 + +### Changed +- Remove the dependency on github.com/kr/logfmt by [@ChrisHines] +- Move fuzz code to github.com/go-logfmt/fuzzlogfmt by [@ChrisHines] + ## [0.4.0] - 2018-11-21 ### Added @@ -30,6 +36,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Decoder by [@ChrisHines] - MarshalKeyvals by [@ChrisHines] +[0.5.0]: https://github.com/go-logfmt/logfmt/compare/v0.4.0...v0.5.0 [0.4.0]: https://github.com/go-logfmt/logfmt/compare/v0.3.0...v0.4.0 [0.3.0]: https://github.com/go-logfmt/logfmt/compare/v0.2.0...v0.3.0 [0.2.0]: https://github.com/go-logfmt/logfmt/compare/v0.1.0...v0.2.0 diff --git a/vendor/github.com/go-logfmt/logfmt/decode.go b/vendor/github.com/go-logfmt/logfmt/decode.go index 04e0efff5f..2013708e48 100644 --- a/vendor/github.com/go-logfmt/logfmt/decode.go +++ b/vendor/github.com/go-logfmt/logfmt/decode.go @@ -79,7 +79,7 @@ key: dec.pos += p if dec.pos > start { dec.key = line[start:dec.pos] - if multibyte && bytes.IndexRune(dec.key, utf8.RuneError) != -1 { + if multibyte && bytes.ContainsRune(dec.key, utf8.RuneError) { dec.syntaxError(invalidKeyError) return false } @@ -97,7 +97,7 @@ key: dec.pos += p if dec.pos > start { dec.key = line[start:dec.pos] - if multibyte && bytes.IndexRune(dec.key, utf8.RuneError) != -1 { + if multibyte && bytes.ContainsRune(dec.key, utf8.RuneError) { dec.syntaxError(invalidKeyError) return false } @@ -110,7 +110,7 @@ key: dec.pos = len(line) if dec.pos > start { dec.key = line[start:dec.pos] - if multibyte && bytes.IndexRune(dec.key, utf8.RuneError) != -1 { + if multibyte && bytes.ContainsRune(dec.key, utf8.RuneError) { dec.syntaxError(invalidKeyError) return false } diff --git a/vendor/github.com/go-logfmt/logfmt/fuzz.go b/vendor/github.com/go-logfmt/logfmt/fuzz.go deleted file mode 100644 index 6553b35e83..0000000000 --- a/vendor/github.com/go-logfmt/logfmt/fuzz.go +++ /dev/null @@ -1,126 +0,0 @@ -// +build gofuzz - -package logfmt - -import ( - "bufio" - "bytes" - "fmt" - "io" - "reflect" - - kr "github.com/kr/logfmt" -) - -// Fuzz checks reserialized data matches -func Fuzz(data []byte) int { - parsed, err := parse(data) - if err != nil { - return 0 - } - var w1 bytes.Buffer - if err = write(parsed, &w1); err != nil { - panic(err) - } - parsed, err = parse(w1.Bytes()) - if err != nil { - panic(err) - } - var w2 bytes.Buffer - if err = write(parsed, &w2); err != nil { - panic(err) - } - if !bytes.Equal(w1.Bytes(), w2.Bytes()) { - panic(fmt.Sprintf("reserialized data does not match:\n%q\n%q\n", w1.Bytes(), w2.Bytes())) - } - return 1 -} - -// FuzzVsKR checks go-logfmt/logfmt against kr/logfmt -func FuzzVsKR(data []byte) int { - parsed, err := parse(data) - parsedKR, errKR := parseKR(data) - - // github.com/go-logfmt/logfmt is a stricter parser. It returns errors for - // more inputs than github.com/kr/logfmt. Ignore any inputs that have a - // stict error. - if err != nil { - return 0 - } - - // Fail if the more forgiving parser finds an error not found by the - // stricter parser. - if errKR != nil { - panic(fmt.Sprintf("unmatched error: %v", errKR)) - } - - if !reflect.DeepEqual(parsed, parsedKR) { - panic(fmt.Sprintf("parsers disagree:\n%+v\n%+v\n", parsed, parsedKR)) - } - return 1 -} - -type kv struct { - k, v []byte -} - -func parse(data []byte) ([][]kv, error) { - var got [][]kv - dec := NewDecoder(bytes.NewReader(data)) - for dec.ScanRecord() { - var kvs []kv - for dec.ScanKeyval() { - kvs = append(kvs, kv{dec.Key(), dec.Value()}) - } - got = append(got, kvs) - } - return got, dec.Err() -} - -func parseKR(data []byte) ([][]kv, error) { - var ( - s = bufio.NewScanner(bytes.NewReader(data)) - err error - h saveHandler - got [][]kv - ) - for err == nil && s.Scan() { - h.kvs = nil - err = kr.Unmarshal(s.Bytes(), &h) - got = append(got, h.kvs) - } - if err == nil { - err = s.Err() - } - return got, err -} - -type saveHandler struct { - kvs []kv -} - -func (h *saveHandler) HandleLogfmt(key, val []byte) error { - if len(key) == 0 { - key = nil - } - if len(val) == 0 { - val = nil - } - h.kvs = append(h.kvs, kv{key, val}) - return nil -} - -func write(recs [][]kv, w io.Writer) error { - enc := NewEncoder(w) - for _, rec := range recs { - for _, f := range rec { - if err := enc.EncodeKeyval(f.k, f.v); err != nil { - return err - } - } - if err := enc.EndRecord(); err != nil { - return err - } - } - return nil -} diff --git a/vendor/github.com/go-logfmt/logfmt/go.mod b/vendor/github.com/go-logfmt/logfmt/go.mod index 63d50f097e..df7192988b 100644 --- a/vendor/github.com/go-logfmt/logfmt/go.mod +++ b/vendor/github.com/go-logfmt/logfmt/go.mod @@ -1,3 +1,3 @@ module github.com/go-logfmt/logfmt -require github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 +go 1.13 diff --git a/vendor/github.com/go-logfmt/logfmt/go.sum b/vendor/github.com/go-logfmt/logfmt/go.sum deleted file mode 100644 index d0cdc16258..0000000000 --- a/vendor/github.com/go-logfmt/logfmt/go.sum +++ /dev/null @@ -1,2 +0,0 @@ -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= diff --git a/vendor/github.com/golang/protobuf/jsonpb/jsonpb.go b/vendor/github.com/golang/protobuf/jsonpb/jsonpb.go index e9cc202585..f0d66befbb 100644 --- a/vendor/github.com/golang/protobuf/jsonpb/jsonpb.go +++ b/vendor/github.com/golang/protobuf/jsonpb/jsonpb.go @@ -165,6 +165,11 @@ type wkt interface { XXX_WellKnownType() string } +var ( + wktType = reflect.TypeOf((*wkt)(nil)).Elem() + messageType = reflect.TypeOf((*proto.Message)(nil)).Elem() +) + // marshalObject writes a struct to the Writer. func (m *Marshaler) marshalObject(out *errWriter, v proto.Message, indent, typeURL string) error { if jsm, ok := v.(JSONPBMarshaler); ok { @@ -531,7 +536,8 @@ func (m *Marshaler) marshalValue(out *errWriter, prop *proto.Properties, v refle // Handle well-known types. // Most are handled up in marshalObject (because 99% are messages). - if wkt, ok := v.Interface().(wkt); ok { + if v.Type().Implements(wktType) { + wkt := v.Interface().(wkt) switch wkt.XXX_WellKnownType() { case "NullValue": out.write("null") @@ -1277,8 +1283,8 @@ func checkRequiredFields(pb proto.Message) error { } func checkRequiredFieldsInValue(v reflect.Value) error { - if pm, ok := v.Interface().(proto.Message); ok { - return checkRequiredFields(pm) + if v.Type().Implements(messageType) { + return checkRequiredFields(v.Interface().(proto.Message)) } return nil } diff --git a/vendor/github.com/golang/protobuf/proto/lib.go b/vendor/github.com/golang/protobuf/proto/lib.go index fdd328bb7f..70fbda5329 100644 --- a/vendor/github.com/golang/protobuf/proto/lib.go +++ b/vendor/github.com/golang/protobuf/proto/lib.go @@ -393,7 +393,7 @@ func (p *Buffer) Bytes() []byte { return p.buf } // than relying on this API. // // If deterministic serialization is requested, map entries will be sorted -// by keys in lexographical order. This is an implementation detail and +// by keys in lexicographical order. This is an implementation detail and // subject to change. func (p *Buffer) SetDeterministic(deterministic bool) { p.deterministic = deterministic diff --git a/vendor/github.com/golang/protobuf/proto/text.go b/vendor/github.com/golang/protobuf/proto/text.go index 1aaee725b4..d97f9b3563 100644 --- a/vendor/github.com/golang/protobuf/proto/text.go +++ b/vendor/github.com/golang/protobuf/proto/text.go @@ -456,6 +456,8 @@ func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error { return nil } +var textMarshalerType = reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem() + // writeAny writes an arbitrary field. func (tm *TextMarshaler) writeAny(w *textWriter, v reflect.Value, props *Properties) error { v = reflect.Indirect(v) @@ -519,8 +521,8 @@ func (tm *TextMarshaler) writeAny(w *textWriter, v reflect.Value, props *Propert // mutating this value. v = v.Addr() } - if etm, ok := v.Interface().(encoding.TextMarshaler); ok { - text, err := etm.MarshalText() + if v.Type().Implements(textMarshalerType) { + text, err := v.Interface().(encoding.TextMarshaler).MarshalText() if err != nil { return err } diff --git a/vendor/github.com/golang/protobuf/protoc-gen-go/generator/generator.go b/vendor/github.com/golang/protobuf/protoc-gen-go/generator/generator.go index 6f4a902b5b..81a37c7189 100644 --- a/vendor/github.com/golang/protobuf/protoc-gen-go/generator/generator.go +++ b/vendor/github.com/golang/protobuf/protoc-gen-go/generator/generator.go @@ -2062,7 +2062,7 @@ func (g *Generator) generateInternalStructFields(mc *msgCtx, topLevelFields []to } -// generateOneofFuncs adds all the utility functions for oneof, including marshalling, unmarshalling and sizer. +// generateOneofFuncs adds all the utility functions for oneof, including marshaling, unmarshaling and sizer. func (g *Generator) generateOneofFuncs(mc *msgCtx, topLevelFields []topLevelField) { ofields := []*oneofField{} for _, f := range topLevelFields { diff --git a/vendor/github.com/golang/protobuf/protoc-gen-go/generator/internal/remap/remap.go b/vendor/github.com/golang/protobuf/protoc-gen-go/generator/internal/remap/remap.go index a9b61036cc..39968eb9f9 100644 --- a/vendor/github.com/golang/protobuf/protoc-gen-go/generator/internal/remap/remap.go +++ b/vendor/github.com/golang/protobuf/protoc-gen-go/generator/internal/remap/remap.go @@ -47,7 +47,7 @@ type Location struct { } // A Map represents a mapping between token locations in an input source text -// and locations in the correspnding output text. +// and locations in the corresponding output text. type Map map[Location]Location // Find reports whether the specified span is recorded by m, and if so returns diff --git a/vendor/github.com/golang/protobuf/protoc-gen-go/grpc/grpc.go b/vendor/github.com/golang/protobuf/protoc-gen-go/grpc/grpc.go index 5d1e3f0f61..957c3f2a63 100644 --- a/vendor/github.com/golang/protobuf/protoc-gen-go/grpc/grpc.go +++ b/vendor/github.com/golang/protobuf/protoc-gen-go/grpc/grpc.go @@ -47,7 +47,7 @@ import ( // It is incremented whenever an incompatibility between the generated code and // the grpc package is introduced; the generated code references // a constant, grpc.SupportPackageIsVersionN (where N is generatedCodeVersion). -const generatedCodeVersion = 4 +const generatedCodeVersion = 6 // Paths for packages used by code generated in this file, // relative to the import_prefix of the generator.Generator. @@ -112,7 +112,7 @@ func (g *grpc) Generate(file *generator.FileDescriptor) { g.P("// Reference imports to suppress errors if they are not otherwise used.") g.P("var _ ", contextPkg, ".Context") - g.P("var _ ", grpcPkg, ".ClientConn") + g.P("var _ ", grpcPkg, ".ClientConnInterface") g.P() // Assert version compatibility. @@ -166,6 +166,10 @@ func (g *grpc) generateService(file *generator.FileDescriptor, service *pb.Servi g.P("type ", servName, "Client interface {") for i, method := range service.Method { g.gen.PrintComments(fmt.Sprintf("%s,2,%d", path, i)) // 2 means method in a service. + if method.GetOptions().GetDeprecated() { + g.P("//") + g.P(deprecationComment) + } g.P(g.generateClientSignature(servName, method)) } g.P("}") @@ -173,7 +177,7 @@ func (g *grpc) generateService(file *generator.FileDescriptor, service *pb.Servi // Client structure. g.P("type ", unexport(servName), "Client struct {") - g.P("cc *", grpcPkg, ".ClientConn") + g.P("cc ", grpcPkg, ".ClientConnInterface") g.P("}") g.P() @@ -181,7 +185,7 @@ func (g *grpc) generateService(file *generator.FileDescriptor, service *pb.Servi if deprecated { g.P(deprecationComment) } - g.P("func New", servName, "Client (cc *", grpcPkg, ".ClientConn) ", servName, "Client {") + g.P("func New", servName, "Client (cc ", grpcPkg, ".ClientConnInterface) ", servName, "Client {") g.P("return &", unexport(servName), "Client{cc}") g.P("}") g.P() @@ -213,12 +217,16 @@ func (g *grpc) generateService(file *generator.FileDescriptor, service *pb.Servi g.P("type ", serverType, " interface {") for i, method := range service.Method { g.gen.PrintComments(fmt.Sprintf("%s,2,%d", path, i)) // 2 means method in a service. + if method.GetOptions().GetDeprecated() { + g.P("//") + g.P(deprecationComment) + } g.P(g.generateServerSignature(servName, method)) } g.P("}") g.P() - // Server Unimplemented struct for forward compatability. + // Server Unimplemented struct for forward compatibility. if deprecated { g.P(deprecationComment) } diff --git a/vendor/github.com/gorilla/mux/.travis.yml b/vendor/github.com/gorilla/mux/.travis.yml deleted file mode 100644 index d003ad922f..0000000000 --- a/vendor/github.com/gorilla/mux/.travis.yml +++ /dev/null @@ -1,24 +0,0 @@ -language: go - - -matrix: - include: - - go: 1.7.x - - go: 1.8.x - - go: 1.9.x - - go: 1.10.x - - go: 1.11.x - - go: 1.x - env: LATEST=true - - go: tip - allow_failures: - - go: tip - -install: - - # Skip - -script: - - go get -t -v ./... - - diff -u <(echo -n) <(gofmt -d .) - - if [[ "$LATEST" = true ]]; then go vet .; fi - - go test -v -race ./... diff --git a/vendor/github.com/gorilla/mux/ISSUE_TEMPLATE.md b/vendor/github.com/gorilla/mux/ISSUE_TEMPLATE.md deleted file mode 100644 index 232be82e47..0000000000 --- a/vendor/github.com/gorilla/mux/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,11 +0,0 @@ -**What version of Go are you running?** (Paste the output of `go version`) - - -**What version of gorilla/mux are you at?** (Paste the output of `git rev-parse HEAD` inside `$GOPATH/src/github.com/gorilla/mux`) - - -**Describe your problem** (and what you have tried so far) - - -**Paste a minimal, runnable, reproduction of your issue below** (use backticks to format it) - diff --git a/vendor/github.com/gorilla/mux/README.md b/vendor/github.com/gorilla/mux/README.md index c661599ab2..92e422eed7 100644 --- a/vendor/github.com/gorilla/mux/README.md +++ b/vendor/github.com/gorilla/mux/README.md @@ -2,6 +2,7 @@ [![GoDoc](https://godoc.org/github.com/gorilla/mux?status.svg)](https://godoc.org/github.com/gorilla/mux) [![Build Status](https://travis-ci.org/gorilla/mux.svg?branch=master)](https://travis-ci.org/gorilla/mux) +[![CircleCI](https://circleci.com/gh/gorilla/mux.svg?style=svg)](https://circleci.com/gh/gorilla/mux) [![Sourcegraph](https://sourcegraph.com/github.com/gorilla/mux/-/badge.svg)](https://sourcegraph.com/github.com/gorilla/mux?badge) ![Gorilla Logo](http://www.gorillatoolkit.org/static/images/gorilla-icon-64.png) @@ -29,6 +30,7 @@ The name mux stands for "HTTP request multiplexer". Like the standard `http.Serv * [Walking Routes](#walking-routes) * [Graceful Shutdown](#graceful-shutdown) * [Middleware](#middleware) +* [Handling CORS Requests](#handling-cors-requests) * [Testing Handlers](#testing-handlers) * [Full Example](#full-example) @@ -491,6 +493,73 @@ r.Use(amw.Middleware) Note: The handler chain will be stopped if your middleware doesn't call `next.ServeHTTP()` with the corresponding parameters. This can be used to abort a request if the middleware writer wants to. Middlewares _should_ write to `ResponseWriter` if they _are_ going to terminate the request, and they _should not_ write to `ResponseWriter` if they _are not_ going to terminate it. +### Handling CORS Requests + +[CORSMethodMiddleware](https://godoc.org/github.com/gorilla/mux#CORSMethodMiddleware) intends to make it easier to strictly set the `Access-Control-Allow-Methods` response header. + +* You will still need to use your own CORS handler to set the other CORS headers such as `Access-Control-Allow-Origin` +* The middleware will set the `Access-Control-Allow-Methods` header to all the method matchers (e.g. `r.Methods(http.MethodGet, http.MethodPut, http.MethodOptions)` -> `Access-Control-Allow-Methods: GET,PUT,OPTIONS`) on a route +* If you do not specify any methods, then: +> _Important_: there must be an `OPTIONS` method matcher for the middleware to set the headers. + +Here is an example of using `CORSMethodMiddleware` along with a custom `OPTIONS` handler to set all the required CORS headers: + +```go +package main + +import ( + "net/http" + "github.com/gorilla/mux" +) + +func main() { + r := mux.NewRouter() + + // IMPORTANT: you must specify an OPTIONS method matcher for the middleware to set CORS headers + r.HandleFunc("/foo", fooHandler).Methods(http.MethodGet, http.MethodPut, http.MethodPatch, http.MethodOptions) + r.Use(mux.CORSMethodMiddleware(r)) + + http.ListenAndServe(":8080", r) +} + +func fooHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + if r.Method == http.MethodOptions { + return + } + + w.Write([]byte("foo")) +} +``` + +And an request to `/foo` using something like: + +```bash +curl localhost:8080/foo -v +``` + +Would look like: + +```bash +* Trying ::1... +* TCP_NODELAY set +* Connected to localhost (::1) port 8080 (#0) +> GET /foo HTTP/1.1 +> Host: localhost:8080 +> User-Agent: curl/7.59.0 +> Accept: */* +> +< HTTP/1.1 200 OK +< Access-Control-Allow-Methods: GET,PUT,PATCH,OPTIONS +< Access-Control-Allow-Origin: * +< Date: Fri, 28 Jun 2019 20:13:30 GMT +< Content-Length: 3 +< Content-Type: text/plain; charset=utf-8 +< +* Connection #0 to host localhost left intact +foo +``` + ### Testing Handlers Testing handlers in a Go web application is straightforward, and _mux_ doesn't complicate this any further. Given two files: `endpoints.go` and `endpoints_test.go`, here's how we'd test an application using _mux_. diff --git a/vendor/github.com/gorilla/mux/doc.go b/vendor/github.com/gorilla/mux/doc.go index 38957deead..bd5a38b55d 100644 --- a/vendor/github.com/gorilla/mux/doc.go +++ b/vendor/github.com/gorilla/mux/doc.go @@ -295,7 +295,7 @@ A more complex authentication middleware, which maps session token to users, cou r := mux.NewRouter() r.HandleFunc("/", handler) - amw := authenticationMiddleware{} + amw := authenticationMiddleware{tokenUsers: make(map[string]string)} amw.Populate() r.Use(amw.Middleware) diff --git a/vendor/github.com/gorilla/mux/middleware.go b/vendor/github.com/gorilla/mux/middleware.go index ceb812cee2..cf2b26dc03 100644 --- a/vendor/github.com/gorilla/mux/middleware.go +++ b/vendor/github.com/gorilla/mux/middleware.go @@ -32,37 +32,19 @@ func (r *Router) useInterface(mw middleware) { r.middlewares = append(r.middlewares, mw) } -// CORSMethodMiddleware sets the Access-Control-Allow-Methods response header -// on a request, by matching routes based only on paths. It also handles -// OPTIONS requests, by settings Access-Control-Allow-Methods, and then -// returning without calling the next http handler. +// CORSMethodMiddleware automatically sets the Access-Control-Allow-Methods response header +// on requests for routes that have an OPTIONS method matcher to all the method matchers on +// the route. Routes that do not explicitly handle OPTIONS requests will not be processed +// by the middleware. See examples for usage. func CORSMethodMiddleware(r *Router) MiddlewareFunc { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - var allMethods []string - - err := r.Walk(func(route *Route, _ *Router, _ []*Route) error { - for _, m := range route.matchers { - if _, ok := m.(*routeRegexp); ok { - if m.Match(req, &RouteMatch{}) { - methods, err := route.GetMethods() - if err != nil { - return err - } - - allMethods = append(allMethods, methods...) - } - break - } - } - return nil - }) - + allMethods, err := getAllMethodsForRoute(r, req) if err == nil { - w.Header().Set("Access-Control-Allow-Methods", strings.Join(append(allMethods, "OPTIONS"), ",")) - - if req.Method == "OPTIONS" { - return + for _, v := range allMethods { + if v == http.MethodOptions { + w.Header().Set("Access-Control-Allow-Methods", strings.Join(allMethods, ",")) + } } } @@ -70,3 +52,28 @@ func CORSMethodMiddleware(r *Router) MiddlewareFunc { }) } } + +// getAllMethodsForRoute returns all the methods from method matchers matching a given +// request. +func getAllMethodsForRoute(r *Router, req *http.Request) ([]string, error) { + var allMethods []string + + err := r.Walk(func(route *Route, _ *Router, _ []*Route) error { + for _, m := range route.matchers { + if _, ok := m.(*routeRegexp); ok { + if m.Match(req, &RouteMatch{}) { + methods, err := route.GetMethods() + if err != nil { + return err + } + + allMethods = append(allMethods, methods...) + } + break + } + } + return nil + }) + + return allMethods, err +} diff --git a/vendor/github.com/gorilla/mux/regexp.go b/vendor/github.com/gorilla/mux/regexp.go index f252886756..ac1abcd473 100644 --- a/vendor/github.com/gorilla/mux/regexp.go +++ b/vendor/github.com/gorilla/mux/regexp.go @@ -113,6 +113,13 @@ func newRouteRegexp(tpl string, typ regexpType, options routeRegexpOptions) (*ro if typ != regexpTypePrefix { pattern.WriteByte('$') } + + var wildcardHostPort bool + if typ == regexpTypeHost { + if !strings.Contains(pattern.String(), ":") { + wildcardHostPort = true + } + } reverse.WriteString(raw) if endSlash { reverse.WriteByte('/') @@ -131,13 +138,14 @@ func newRouteRegexp(tpl string, typ regexpType, options routeRegexpOptions) (*ro // Done! return &routeRegexp{ - template: template, - regexpType: typ, - options: options, - regexp: reg, - reverse: reverse.String(), - varsN: varsN, - varsR: varsR, + template: template, + regexpType: typ, + options: options, + regexp: reg, + reverse: reverse.String(), + varsN: varsN, + varsR: varsR, + wildcardHostPort: wildcardHostPort, }, nil } @@ -158,11 +166,22 @@ type routeRegexp struct { varsN []string // Variable regexps (validators). varsR []*regexp.Regexp + // Wildcard host-port (no strict port match in hostname) + wildcardHostPort bool } // Match matches the regexp against the URL host or path. func (r *routeRegexp) Match(req *http.Request, match *RouteMatch) bool { - if r.regexpType != regexpTypeHost { + if r.regexpType == regexpTypeHost { + host := getHost(req) + if r.wildcardHostPort { + // Don't be strict on the port match + if i := strings.Index(host, ":"); i != -1 { + host = host[:i] + } + } + return r.regexp.MatchString(host) + } else { if r.regexpType == regexpTypeQuery { return r.matchQueryString(req) } @@ -172,8 +191,6 @@ func (r *routeRegexp) Match(req *http.Request, match *RouteMatch) bool { } return r.regexp.MatchString(path) } - - return r.regexp.MatchString(getHost(req)) } // url builds a URL part using the given values. diff --git a/vendor/github.com/kr/logfmt/.gitignore b/vendor/github.com/kr/logfmt/.gitignore deleted file mode 100644 index 8e524f68a7..0000000000 --- a/vendor/github.com/kr/logfmt/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.test -*.swp -*.prof diff --git a/vendor/github.com/kr/logfmt/Readme b/vendor/github.com/kr/logfmt/Readme deleted file mode 100644 index 1865a11f7c..0000000000 --- a/vendor/github.com/kr/logfmt/Readme +++ /dev/null @@ -1,12 +0,0 @@ -Go package for parsing (and, eventually, generating) -log lines in the logfmt style. - -See http://godoc.org/github.com/kr/logfmt for format, and other documentation and examples. - -Copyright (C) 2013 Keith Rarick, Blake Mizerany - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/kr/logfmt/decode.go b/vendor/github.com/kr/logfmt/decode.go deleted file mode 100644 index 1397fb7467..0000000000 --- a/vendor/github.com/kr/logfmt/decode.go +++ /dev/null @@ -1,184 +0,0 @@ -// Package implements the decoding of logfmt key-value pairs. -// -// Example logfmt message: -// -// foo=bar a=14 baz="hello kitty" cool%story=bro f %^asdf -// -// Example result in JSON: -// -// { "foo": "bar", "a": 14, "baz": "hello kitty", "cool%story": "bro", "f": true, "%^asdf": true } -// -// EBNFish: -// -// ident_byte = any byte greater than ' ', excluding '=' and '"' -// string_byte = any byte excluding '"' and '\' -// garbage = !ident_byte -// ident = ident_byte, { ident byte } -// key = ident -// value = ident | '"', { string_byte | '\', '"' }, '"' -// pair = key, '=', value | key, '=' | key -// message = { garbage, pair }, garbage -package logfmt - -import ( - "reflect" - "strconv" - "strings" - "time" -) - -// Handler is the interface implemented by objects that accept logfmt -// key-value pairs. HandleLogfmt must copy the logfmt data if it -// wishes to retain the data after returning. -type Handler interface { - HandleLogfmt(key, val []byte) error -} - -// The HandlerFunc type is an adapter to allow the use of ordinary functions as -// logfmt handlers. If f is a function with the appropriate signature, -// HandlerFunc(f) is a Handler object that calls f. -type HandlerFunc func(key, val []byte) error - -func (f HandlerFunc) HandleLogfmt(key, val []byte) error { - return f(key, val) -} - -// Unmarshal parses the logfmt encoding data and stores the result in the value -// pointed to by v. If v is an Handler, HandleLogfmt will be called for each -// key-value pair. -// -// If v is not a Handler, it will pass v to NewStructHandler and use the -// returned StructHandler for decoding. -func Unmarshal(data []byte, v interface{}) (err error) { - h, ok := v.(Handler) - if !ok { - h, err = NewStructHandler(v) - if err != nil { - return err - } - } - return gotoScanner(data, h) -} - -// StructHandler unmarshals logfmt into a struct. It matches incoming keys to -// the the struct's fields (either the struct field name or its tag, preferring -// an exact match but also accepting a case-insensitive match. -// -// Field types supported by StructHandler are: -// -// all numeric types (e.g. float32, int, etc.) -// []byte -// string -// bool - true if key is present, false otherwise (the value is ignored). -// time.Duration - uses time.ParseDuration -// -// If a field is a pointer to an above type, and a matching key is not present -// in the logfmt data, the pointer will be untouched. -// -// If v is not a pointer to an Handler or struct, Unmarshal will return an -// error. -type StructHandler struct { - rv reflect.Value -} - -func NewStructHandler(v interface{}) (Handler, error) { - rv := reflect.ValueOf(v) - if rv.Kind() != reflect.Ptr || rv.IsNil() { - return nil, &InvalidUnmarshalError{reflect.TypeOf(v)} - } - return &StructHandler{rv: rv}, nil -} - -func (h *StructHandler) HandleLogfmt(key, val []byte) error { - el := h.rv.Elem() - skey := string(key) - for i := 0; i < el.NumField(); i++ { - fv := el.Field(i) - ft := el.Type().Field(i) - switch { - case ft.Name == skey: - case ft.Tag.Get("logfmt") == skey: - case strings.EqualFold(ft.Name, skey): - default: - continue - } - if fv.Kind() == reflect.Ptr { - if fv.IsNil() { - t := fv.Type().Elem() - v := reflect.New(t) - fv.Set(v) - fv = v - } - fv = fv.Elem() - } - switch fv.Interface().(type) { - case time.Duration: - d, err := time.ParseDuration(string(val)) - if err != nil { - return &UnmarshalTypeError{string(val), fv.Type()} - } - fv.Set(reflect.ValueOf(d)) - case string: - fv.SetString(string(val)) - case []byte: - b := make([]byte, len(val)) - copy(b, val) - fv.SetBytes(b) - case bool: - fv.SetBool(true) - default: - switch { - case reflect.Int <= fv.Kind() && fv.Kind() <= reflect.Int64: - v, err := strconv.ParseInt(string(val), 10, 64) - if err != nil { - return err - } - fv.SetInt(v) - case reflect.Uint32 <= fv.Kind() && fv.Kind() <= reflect.Uint64: - v, err := strconv.ParseUint(string(val), 10, 64) - if err != nil { - return err - } - fv.SetUint(v) - case reflect.Float32 <= fv.Kind() && fv.Kind() <= reflect.Float64: - v, err := strconv.ParseFloat(string(val), 10) - if err != nil { - return err - } - fv.SetFloat(v) - default: - return &UnmarshalTypeError{string(val), fv.Type()} - } - } - - } - return nil -} - -// An InvalidUnmarshalError describes an invalid argument passed to Unmarshal. -// (The argument to Unmarshal must be a non-nil pointer.) -type InvalidUnmarshalError struct { - Type reflect.Type -} - -func (e *InvalidUnmarshalError) Error() string { - if e.Type == nil { - return "logfmt: Unmarshal(nil)" - } - - if e.Type.Kind() != reflect.Ptr { - return "logfmt: Unmarshal(non-pointer " + e.Type.String() + ")" - } - return "logfmt: Unmarshal(nil " + e.Type.String() + ")" -} - -// An UnmarshalTypeError describes a logfmt value that was -// not appropriate for a value of a specific Go type. -type UnmarshalTypeError struct { - Value string // the logfmt value - Type reflect.Type // type of Go value it could not be assigned to -} - -func (e *UnmarshalTypeError) Error() string { - return "logfmt: cannot unmarshal " + e.Value + " into Go value of type " + e.Type.String() -} diff --git a/vendor/github.com/kr/logfmt/scanner.go b/vendor/github.com/kr/logfmt/scanner.go deleted file mode 100644 index 095221ff24..0000000000 --- a/vendor/github.com/kr/logfmt/scanner.go +++ /dev/null @@ -1,149 +0,0 @@ -package logfmt - -import ( - "errors" - "fmt" -) - -var ErrUnterminatedString = errors.New("logfmt: unterminated string") - -func gotoScanner(data []byte, h Handler) (err error) { - saveError := func(e error) { - if err == nil { - err = e - } - } - - var c byte - var i int - var m int - var key []byte - var val []byte - var ok bool - var esc bool - -garbage: - if i == len(data) { - return - } - - c = data[i] - switch { - case c > ' ' && c != '"' && c != '=': - key, val = nil, nil - m = i - i++ - goto key - default: - i++ - goto garbage - } - -key: - if i >= len(data) { - if m >= 0 { - key = data[m:i] - saveError(h.HandleLogfmt(key, nil)) - } - return - } - - c = data[i] - switch { - case c > ' ' && c != '"' && c != '=': - i++ - goto key - case c == '=': - key = data[m:i] - i++ - goto equal - default: - key = data[m:i] - i++ - saveError(h.HandleLogfmt(key, nil)) - goto garbage - } - -equal: - if i >= len(data) { - if m >= 0 { - i-- - key = data[m:i] - saveError(h.HandleLogfmt(key, nil)) - } - return - } - - c = data[i] - switch { - case c > ' ' && c != '"' && c != '=': - m = i - i++ - goto ivalue - case c == '"': - m = i - i++ - esc = false - goto qvalue - default: - if key != nil { - saveError(h.HandleLogfmt(key, val)) - } - i++ - goto garbage - } - -ivalue: - if i >= len(data) { - if m >= 0 { - val = data[m:i] - saveError(h.HandleLogfmt(key, val)) - } - return - } - - c = data[i] - switch { - case c > ' ' && c != '"' && c != '=': - i++ - goto ivalue - default: - val = data[m:i] - saveError(h.HandleLogfmt(key, val)) - i++ - goto garbage - } - -qvalue: - if i >= len(data) { - if m >= 0 { - saveError(ErrUnterminatedString) - } - return - } - - c = data[i] - switch c { - case '\\': - i += 2 - esc = true - goto qvalue - case '"': - i++ - val = data[m:i] - if esc { - val, ok = unquoteBytes(val) - if !ok { - saveError(fmt.Errorf("logfmt: error unquoting bytes %q", string(val))) - goto garbage - } - } else { - val = val[1 : len(val)-1] - } - saveError(h.HandleLogfmt(key, val)) - goto garbage - default: - i++ - goto qvalue - } -} diff --git a/vendor/github.com/kr/logfmt/unquote.go b/vendor/github.com/kr/logfmt/unquote.go deleted file mode 100644 index fb088a476e..0000000000 --- a/vendor/github.com/kr/logfmt/unquote.go +++ /dev/null @@ -1,149 +0,0 @@ -package logfmt - -import ( - "strconv" - "unicode" - "unicode/utf16" - "unicode/utf8" -) - -// Taken from Go's encoding/json - -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// getu4 decodes \uXXXX from the beginning of s, returning the hex value, -// or it returns -1. -func getu4(s []byte) rune { - if len(s) < 6 || s[0] != '\\' || s[1] != 'u' { - return -1 - } - r, err := strconv.ParseUint(string(s[2:6]), 16, 64) - if err != nil { - return -1 - } - return rune(r) -} - -// unquote converts a quoted JSON string literal s into an actual string t. -// The rules are different than for Go, so cannot use strconv.Unquote. -func unquote(s []byte) (t string, ok bool) { - s, ok = unquoteBytes(s) - t = string(s) - return -} - -func unquoteBytes(s []byte) (t []byte, ok bool) { - if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' { - return - } - s = s[1 : len(s)-1] - - // Check for unusual characters. If there are none, - // then no unquoting is needed, so return a slice of the - // original bytes. - r := 0 - for r < len(s) { - c := s[r] - if c == '\\' || c == '"' || c < ' ' { - break - } - if c < utf8.RuneSelf { - r++ - continue - } - rr, size := utf8.DecodeRune(s[r:]) - if rr == utf8.RuneError && size == 1 { - break - } - r += size - } - if r == len(s) { - return s, true - } - - b := make([]byte, len(s)+2*utf8.UTFMax) - w := copy(b, s[0:r]) - for r < len(s) { - // Out of room? Can only happen if s is full of - // malformed UTF-8 and we're replacing each - // byte with RuneError. - if w >= len(b)-2*utf8.UTFMax { - nb := make([]byte, (len(b)+utf8.UTFMax)*2) - copy(nb, b[0:w]) - b = nb - } - switch c := s[r]; { - case c == '\\': - r++ - if r >= len(s) { - return - } - switch s[r] { - default: - return - case '"', '\\', '/', '\'': - b[w] = s[r] - r++ - w++ - case 'b': - b[w] = '\b' - r++ - w++ - case 'f': - b[w] = '\f' - r++ - w++ - case 'n': - b[w] = '\n' - r++ - w++ - case 'r': - b[w] = '\r' - r++ - w++ - case 't': - b[w] = '\t' - r++ - w++ - case 'u': - r-- - rr := getu4(s[r:]) - if rr < 0 { - return - } - r += 6 - if utf16.IsSurrogate(rr) { - rr1 := getu4(s[r:]) - if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar { - // A valid pair; consume. - r += 6 - w += utf8.EncodeRune(b[w:], dec) - break - } - // Invalid surrogate; fall back to replacement rune. - rr = unicode.ReplacementChar - } - w += utf8.EncodeRune(b[w:], rr) - } - - // Quote, control characters are invalid. - case c == '"', c < ' ': - return - - // ASCII - case c < utf8.RuneSelf: - b[w] = c - r++ - w++ - - // Coerce to well-formed UTF-8. - default: - rr, size := utf8.DecodeRune(s[r:]) - r += size - w += utf8.EncodeRune(b[w:], rr) - } - } - return b[0:w], true -} diff --git a/vendor/github.com/prometheus/common/log/eventlog_formatter.go b/vendor/github.com/prometheus/common/log/eventlog_formatter.go new file mode 100644 index 0000000000..bcf68e6f24 --- /dev/null +++ b/vendor/github.com/prometheus/common/log/eventlog_formatter.go @@ -0,0 +1,89 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build windows + +package log + +import ( + "fmt" + "os" + + "golang.org/x/sys/windows/svc/eventlog" + + "github.com/sirupsen/logrus" +) + +func init() { + setEventlogFormatter = func(l logger, name string, debugAsInfo bool) error { + if name == "" { + return fmt.Errorf("missing name parameter") + } + + fmter, err := newEventlogger(name, debugAsInfo, l.entry.Logger.Formatter) + if err != nil { + fmt.Fprintf(os.Stderr, "error creating eventlog formatter: %v\n", err) + l.Errorf("can't connect logger to eventlog: %v", err) + return err + } + l.entry.Logger.Formatter = fmter + return nil + } +} + +type eventlogger struct { + log *eventlog.Log + debugAsInfo bool + wrap logrus.Formatter +} + +func newEventlogger(name string, debugAsInfo bool, fmter logrus.Formatter) (*eventlogger, error) { + logHandle, err := eventlog.Open(name) + if err != nil { + return nil, err + } + return &eventlogger{log: logHandle, debugAsInfo: debugAsInfo, wrap: fmter}, nil +} + +func (s *eventlogger) Format(e *logrus.Entry) ([]byte, error) { + data, err := s.wrap.Format(e) + if err != nil { + fmt.Fprintf(os.Stderr, "eventlogger: can't format entry: %v\n", err) + return data, err + } + + switch e.Level { + case logrus.PanicLevel: + fallthrough + case logrus.FatalLevel: + fallthrough + case logrus.ErrorLevel: + err = s.log.Error(102, e.Message) + case logrus.WarnLevel: + err = s.log.Warning(101, e.Message) + case logrus.InfoLevel: + err = s.log.Info(100, e.Message) + case logrus.DebugLevel: + if s.debugAsInfo { + err = s.log.Info(100, e.Message) + } + default: + err = s.log.Info(100, e.Message) + } + + if err != nil { + fmt.Fprintf(os.Stderr, "eventlogger: can't send log to eventlog: %v\n", err) + } + + return data, err +} diff --git a/vendor/github.com/prometheus/common/log/log.go b/vendor/github.com/prometheus/common/log/log.go new file mode 100644 index 0000000000..b6adbce133 --- /dev/null +++ b/vendor/github.com/prometheus/common/log/log.go @@ -0,0 +1,368 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package log implements logging via logrus. +// +// Deprecated: This package has been replaced with github.com/prometheus/common/promlog. + +package log + +import ( + "fmt" + "io" + "io/ioutil" + "log" + "net/url" + "os" + "runtime" + "strconv" + "strings" + + "github.com/sirupsen/logrus" + "gopkg.in/alecthomas/kingpin.v2" +) + +// setSyslogFormatter is nil if the target architecture does not support syslog. +var setSyslogFormatter func(logger, string, string) error + +// setEventlogFormatter is nil if the target OS does not support Eventlog (i.e., is not Windows). +var setEventlogFormatter func(logger, string, bool) error + +func setJSONFormatter() { + origLogger.Formatter = &logrus.JSONFormatter{} +} + +type loggerSettings struct { + level string + format string +} + +func (s *loggerSettings) apply(ctx *kingpin.ParseContext) error { + err := baseLogger.SetLevel(s.level) + if err != nil { + return err + } + err = baseLogger.SetFormat(s.format) + return err +} + +// AddFlags adds the flags used by this package to the Kingpin application. +// To use the default Kingpin application, call AddFlags(kingpin.CommandLine) +func AddFlags(a *kingpin.Application) { + s := loggerSettings{} + a.Flag("log.level", "Only log messages with the given severity or above. Valid levels: [debug, info, warn, error, fatal]"). + Default(origLogger.Level.String()). + StringVar(&s.level) + defaultFormat := url.URL{Scheme: "logger", Opaque: "stderr"} + a.Flag("log.format", `Set the log target and format. Example: "logger:syslog?appname=bob&local=7" or "logger:stdout?json=true"`). + Default(defaultFormat.String()). + StringVar(&s.format) + a.Action(s.apply) +} + +// Logger is the interface for loggers used in the Prometheus components. +type Logger interface { + Debug(...interface{}) + Debugln(...interface{}) + Debugf(string, ...interface{}) + + Info(...interface{}) + Infoln(...interface{}) + Infof(string, ...interface{}) + + Warn(...interface{}) + Warnln(...interface{}) + Warnf(string, ...interface{}) + + Error(...interface{}) + Errorln(...interface{}) + Errorf(string, ...interface{}) + + Fatal(...interface{}) + Fatalln(...interface{}) + Fatalf(string, ...interface{}) + + With(key string, value interface{}) Logger + + SetFormat(string) error + SetLevel(string) error +} + +type logger struct { + entry *logrus.Entry +} + +func (l logger) With(key string, value interface{}) Logger { + return logger{l.entry.WithField(key, value)} +} + +// Debug logs a message at level Debug on the standard logger. +func (l logger) Debug(args ...interface{}) { + l.sourced().Debug(args...) +} + +// Debug logs a message at level Debug on the standard logger. +func (l logger) Debugln(args ...interface{}) { + l.sourced().Debugln(args...) +} + +// Debugf logs a message at level Debug on the standard logger. +func (l logger) Debugf(format string, args ...interface{}) { + l.sourced().Debugf(format, args...) +} + +// Info logs a message at level Info on the standard logger. +func (l logger) Info(args ...interface{}) { + l.sourced().Info(args...) +} + +// Info logs a message at level Info on the standard logger. +func (l logger) Infoln(args ...interface{}) { + l.sourced().Infoln(args...) +} + +// Infof logs a message at level Info on the standard logger. +func (l logger) Infof(format string, args ...interface{}) { + l.sourced().Infof(format, args...) +} + +// Warn logs a message at level Warn on the standard logger. +func (l logger) Warn(args ...interface{}) { + l.sourced().Warn(args...) +} + +// Warn logs a message at level Warn on the standard logger. +func (l logger) Warnln(args ...interface{}) { + l.sourced().Warnln(args...) +} + +// Warnf logs a message at level Warn on the standard logger. +func (l logger) Warnf(format string, args ...interface{}) { + l.sourced().Warnf(format, args...) +} + +// Error logs a message at level Error on the standard logger. +func (l logger) Error(args ...interface{}) { + l.sourced().Error(args...) +} + +// Error logs a message at level Error on the standard logger. +func (l logger) Errorln(args ...interface{}) { + l.sourced().Errorln(args...) +} + +// Errorf logs a message at level Error on the standard logger. +func (l logger) Errorf(format string, args ...interface{}) { + l.sourced().Errorf(format, args...) +} + +// Fatal logs a message at level Fatal on the standard logger. +func (l logger) Fatal(args ...interface{}) { + l.sourced().Fatal(args...) +} + +// Fatal logs a message at level Fatal on the standard logger. +func (l logger) Fatalln(args ...interface{}) { + l.sourced().Fatalln(args...) +} + +// Fatalf logs a message at level Fatal on the standard logger. +func (l logger) Fatalf(format string, args ...interface{}) { + l.sourced().Fatalf(format, args...) +} + +func (l logger) SetLevel(level string) error { + lvl, err := logrus.ParseLevel(level) + if err != nil { + return err + } + + l.entry.Logger.Level = lvl + return nil +} + +func (l logger) SetFormat(format string) error { + u, err := url.Parse(format) + if err != nil { + return err + } + if u.Scheme != "logger" { + return fmt.Errorf("invalid scheme %s", u.Scheme) + } + jsonq := u.Query().Get("json") + if jsonq == "true" { + setJSONFormatter() + } + + switch u.Opaque { + case "syslog": + if setSyslogFormatter == nil { + return fmt.Errorf("system does not support syslog") + } + appname := u.Query().Get("appname") + facility := u.Query().Get("local") + return setSyslogFormatter(l, appname, facility) + case "eventlog": + if setEventlogFormatter == nil { + return fmt.Errorf("system does not support eventlog") + } + name := u.Query().Get("name") + debugAsInfo := false + debugAsInfoRaw := u.Query().Get("debugAsInfo") + if parsedDebugAsInfo, err := strconv.ParseBool(debugAsInfoRaw); err == nil { + debugAsInfo = parsedDebugAsInfo + } + return setEventlogFormatter(l, name, debugAsInfo) + case "stdout": + l.entry.Logger.Out = os.Stdout + case "stderr": + l.entry.Logger.Out = os.Stderr + default: + return fmt.Errorf("unsupported logger %q", u.Opaque) + } + return nil +} + +// sourced adds a source field to the logger that contains +// the file name and line where the logging happened. +func (l logger) sourced() *logrus.Entry { + _, file, line, ok := runtime.Caller(2) + if !ok { + file = "" + line = 1 + } else { + slash := strings.LastIndex(file, "/") + file = file[slash+1:] + } + return l.entry.WithField("source", fmt.Sprintf("%s:%d", file, line)) +} + +var origLogger = logrus.New() +var baseLogger = logger{entry: logrus.NewEntry(origLogger)} + +// Base returns the default Logger logging to +func Base() Logger { + return baseLogger +} + +// NewLogger returns a new Logger logging to out. +func NewLogger(w io.Writer) Logger { + l := logrus.New() + l.Out = w + return logger{entry: logrus.NewEntry(l)} +} + +// NewNopLogger returns a logger that discards all log messages. +func NewNopLogger() Logger { + l := logrus.New() + l.Out = ioutil.Discard + return logger{entry: logrus.NewEntry(l)} +} + +// With adds a field to the logger. +func With(key string, value interface{}) Logger { + return baseLogger.With(key, value) +} + +// Debug logs a message at level Debug on the standard logger. +func Debug(args ...interface{}) { + baseLogger.sourced().Debug(args...) +} + +// Debugln logs a message at level Debug on the standard logger. +func Debugln(args ...interface{}) { + baseLogger.sourced().Debugln(args...) +} + +// Debugf logs a message at level Debug on the standard logger. +func Debugf(format string, args ...interface{}) { + baseLogger.sourced().Debugf(format, args...) +} + +// Info logs a message at level Info on the standard logger. +func Info(args ...interface{}) { + baseLogger.sourced().Info(args...) +} + +// Infoln logs a message at level Info on the standard logger. +func Infoln(args ...interface{}) { + baseLogger.sourced().Infoln(args...) +} + +// Infof logs a message at level Info on the standard logger. +func Infof(format string, args ...interface{}) { + baseLogger.sourced().Infof(format, args...) +} + +// Warn logs a message at level Warn on the standard logger. +func Warn(args ...interface{}) { + baseLogger.sourced().Warn(args...) +} + +// Warnln logs a message at level Warn on the standard logger. +func Warnln(args ...interface{}) { + baseLogger.sourced().Warnln(args...) +} + +// Warnf logs a message at level Warn on the standard logger. +func Warnf(format string, args ...interface{}) { + baseLogger.sourced().Warnf(format, args...) +} + +// Error logs a message at level Error on the standard logger. +func Error(args ...interface{}) { + baseLogger.sourced().Error(args...) +} + +// Errorln logs a message at level Error on the standard logger. +func Errorln(args ...interface{}) { + baseLogger.sourced().Errorln(args...) +} + +// Errorf logs a message at level Error on the standard logger. +func Errorf(format string, args ...interface{}) { + baseLogger.sourced().Errorf(format, args...) +} + +// Fatal logs a message at level Fatal on the standard logger. +func Fatal(args ...interface{}) { + baseLogger.sourced().Fatal(args...) +} + +// Fatalln logs a message at level Fatal on the standard logger. +func Fatalln(args ...interface{}) { + baseLogger.sourced().Fatalln(args...) +} + +// Fatalf logs a message at level Fatal on the standard logger. +func Fatalf(format string, args ...interface{}) { + baseLogger.sourced().Fatalf(format, args...) +} + +// AddHook adds hook to Prometheus' original logger. +func AddHook(hook logrus.Hook) { + origLogger.Hooks.Add(hook) +} + +type errorLogWriter struct{} + +func (errorLogWriter) Write(b []byte) (int, error) { + baseLogger.sourced().Error(string(b)) + return len(b), nil +} + +// NewErrorLogger returns a log.Logger that is meant to be used +// in the ErrorLog field of an http.Server to log HTTP server errors. +func NewErrorLogger() *log.Logger { + return log.New(&errorLogWriter{}, "", 0) +} diff --git a/vendor/github.com/prometheus/common/log/syslog_formatter.go b/vendor/github.com/prometheus/common/log/syslog_formatter.go new file mode 100644 index 0000000000..f882f2f848 --- /dev/null +++ b/vendor/github.com/prometheus/common/log/syslog_formatter.go @@ -0,0 +1,126 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !windows,!nacl,!plan9 + +package log + +import ( + "fmt" + "log/syslog" + "os" + + "github.com/sirupsen/logrus" +) + +var _ logrus.Formatter = (*syslogger)(nil) + +func init() { + setSyslogFormatter = func(l logger, appname, local string) error { + if appname == "" { + return fmt.Errorf("missing appname parameter") + } + if local == "" { + return fmt.Errorf("missing local parameter") + } + + fmter, err := newSyslogger(appname, local, l.entry.Logger.Formatter) + if err != nil { + fmt.Fprintf(os.Stderr, "error creating syslog formatter: %v\n", err) + l.entry.Errorf("can't connect logger to syslog: %v", err) + return err + } + l.entry.Logger.Formatter = fmter + return nil + } +} + +var prefixTag []byte + +type syslogger struct { + wrap logrus.Formatter + out *syslog.Writer +} + +func newSyslogger(appname string, facility string, fmter logrus.Formatter) (*syslogger, error) { + priority, err := getFacility(facility) + if err != nil { + return nil, err + } + out, err := syslog.New(priority, appname) + _, isJSON := fmter.(*logrus.JSONFormatter) + if isJSON { + // add cee tag to json formatted syslogs + prefixTag = []byte("@cee:") + } + return &syslogger{ + out: out, + wrap: fmter, + }, err +} + +func getFacility(facility string) (syslog.Priority, error) { + switch facility { + case "0": + return syslog.LOG_LOCAL0, nil + case "1": + return syslog.LOG_LOCAL1, nil + case "2": + return syslog.LOG_LOCAL2, nil + case "3": + return syslog.LOG_LOCAL3, nil + case "4": + return syslog.LOG_LOCAL4, nil + case "5": + return syslog.LOG_LOCAL5, nil + case "6": + return syslog.LOG_LOCAL6, nil + case "7": + return syslog.LOG_LOCAL7, nil + } + return syslog.LOG_LOCAL0, fmt.Errorf("invalid local(%s) for syslog", facility) +} + +func (s *syslogger) Format(e *logrus.Entry) ([]byte, error) { + data, err := s.wrap.Format(e) + if err != nil { + fmt.Fprintf(os.Stderr, "syslogger: can't format entry: %v\n", err) + return data, err + } + // only append tag to data sent to syslog (line), not to what + // is returned + line := string(append(prefixTag, data...)) + + switch e.Level { + case logrus.PanicLevel: + err = s.out.Crit(line) + case logrus.FatalLevel: + err = s.out.Crit(line) + case logrus.ErrorLevel: + err = s.out.Err(line) + case logrus.WarnLevel: + err = s.out.Warning(line) + case logrus.InfoLevel: + err = s.out.Info(line) + case logrus.DebugLevel: + err = s.out.Debug(line) + default: + err = s.out.Notice(line) + } + + if err != nil { + fmt.Fprintf(os.Stderr, "syslogger: can't send log to syslog: %v\n", err) + } + + return data, err +} diff --git a/vendor/github.com/prometheus/node_exporter/LICENSE b/vendor/github.com/prometheus/node_exporter/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/vendor/github.com/prometheus/node_exporter/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/prometheus/node_exporter/NOTICE b/vendor/github.com/prometheus/node_exporter/NOTICE new file mode 100644 index 0000000000..970a9b237a --- /dev/null +++ b/vendor/github.com/prometheus/node_exporter/NOTICE @@ -0,0 +1,17 @@ +Configurable modular Prometheus exporter for various node metrics. +Copyright 2013-2015 The Prometheus Authors + +This product includes software developed at +SoundCloud Ltd. (http://soundcloud.com/). + +The following components are included in this product: + +wifi +https://github.com/mdlayher/wifi +Copyright 2016-2017 Matt Layher +Licensed under the MIT License + +netlink +https://github.com/mdlayher/netlink +Copyright 2016-2017 Matt Layher +Licensed under the MIT License diff --git a/vendor/github.com/prometheus/node_exporter/https/README.md b/vendor/github.com/prometheus/node_exporter/https/README.md new file mode 100644 index 0000000000..d9cb564336 --- /dev/null +++ b/vendor/github.com/prometheus/node_exporter/https/README.md @@ -0,0 +1,24 @@ +# HTTPS Package for Prometheus + +The `https` directory contains a Go package and a sample configuration file for running `node_exporter` with HTTPS instead of HTTP. +When running a server with TLS use the flag `--web.config` + +e.g. `./node_exporter --web.config="web-config.yml"` +If the config is kept within the https directory. + +The config file should be written in YAML format, and is reloaded on each connection to check for new certificates and/or authentication policy. + +## Sample Config +``` +tlsConfig : + # Certificate and key files for server to use to authenticate to client + tlsCertPath : + tlsKeyPath : + + # Server policy for client authentication. Maps to ClientAuth Policies + # For more detail on clientAuth options: [ClientAuthType](https://golang.org/pkg/crypto/tls/#ClientAuthType) + [ clientAuth : | default = "NoClientCert" ] + + # CA certificate for client certificate authentication to the server + [ clientCAs : ] +``` diff --git a/vendor/github.com/prometheus/node_exporter/https/tls_config.go b/vendor/github.com/prometheus/node_exporter/https/tls_config.go new file mode 100644 index 0000000000..423c267814 --- /dev/null +++ b/vendor/github.com/prometheus/node_exporter/https/tls_config.go @@ -0,0 +1,124 @@ +// Copyright 2019 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package https allows the implementation of TLS. +package https + +import ( + "crypto/tls" + "crypto/x509" + "io/ioutil" + "net/http" + + "github.com/pkg/errors" + "gopkg.in/yaml.v2" +) + +type Config struct { + TLSConfig TLSStruct `yaml:"tlsConfig"` +} + +type TLSStruct struct { + TLSCertPath string `yaml:"tlsCertPath"` + TLSKeyPath string `yaml:"tlsKeyPath"` + ClientAuth string `yaml:"clientAuth"` + ClientCAs string `yaml:"clientCAs"` +} + +func getTLSConfig(configPath string) (*tls.Config, error) { + content, err := ioutil.ReadFile(configPath) + if err != nil { + return nil, err + } + c := &Config{} + err = yaml.Unmarshal(content, c) + if err != nil { + return nil, err + } + return ConfigToTLSConfig(&c.TLSConfig) +} + +// ConfigToTLSConfig generates the golang tls.Config from the TLSStruct config. +func ConfigToTLSConfig(c *TLSStruct) (*tls.Config, error) { + cfg := &tls.Config{} + if len(c.TLSCertPath) == 0 { + return nil, errors.New("missing TLSCertPath") + } + if len(c.TLSKeyPath) == 0 { + return nil, errors.New("missing TLSKeyPath") + } + loadCert := func() (*tls.Certificate, error) { + cert, err := tls.LoadX509KeyPair(c.TLSCertPath, c.TLSKeyPath) + if err != nil { + return nil, errors.Wrap(err, "failed to load X509KeyPair") + } + return &cert, nil + } + // Confirm that certificate and key paths are valid. + if _, err := loadCert(); err != nil { + return nil, err + } + cfg.GetCertificate = func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + return loadCert() + } + + if len(c.ClientCAs) > 0 { + clientCAPool := x509.NewCertPool() + clientCAFile, err := ioutil.ReadFile(c.ClientCAs) + if err != nil { + return nil, err + } + clientCAPool.AppendCertsFromPEM(clientCAFile) + cfg.ClientCAs = clientCAPool + } + if len(c.ClientAuth) > 0 { + switch s := (c.ClientAuth); s { + case "NoClientCert": + cfg.ClientAuth = tls.NoClientCert + case "RequestClientCert": + cfg.ClientAuth = tls.RequestClientCert + case "RequireClientCert": + cfg.ClientAuth = tls.RequireAnyClientCert + case "VerifyClientCertIfGiven": + cfg.ClientAuth = tls.VerifyClientCertIfGiven + case "RequireAndVerifyClientCert": + cfg.ClientAuth = tls.RequireAndVerifyClientCert + case "": + cfg.ClientAuth = tls.NoClientCert + default: + return nil, errors.New("Invalid ClientAuth: " + s) + } + } + if len(c.ClientCAs) > 0 && cfg.ClientAuth == tls.NoClientCert { + return nil, errors.New("Client CA's have been configured without a Client Auth Policy") + } + return cfg, nil +} + +// Listen starts the server on the given address. If tlsConfigPath isn't empty the server connection will be started using TLS. +func Listen(server *http.Server, tlsConfigPath string) error { + if (tlsConfigPath) == "" { + return server.ListenAndServe() + } + var err error + server.TLSConfig, err = getTLSConfig(tlsConfigPath) + if err != nil { + return err + } + // Set the GetConfigForClient method of the HTTPS server so that the config + // and certs are reloaded on new connections. + server.TLSConfig.GetConfigForClient = func(*tls.ClientHelloInfo) (*tls.Config, error) { + return getTLSConfig(tlsConfigPath) + } + return server.ListenAndServeTLS("", "") +} diff --git a/vendor/github.com/prometheus/node_exporter/https/web-config.yml b/vendor/github.com/prometheus/node_exporter/https/web-config.yml new file mode 100644 index 0000000000..0f439dacf0 --- /dev/null +++ b/vendor/github.com/prometheus/node_exporter/https/web-config.yml @@ -0,0 +1,10 @@ +tlsConfig : + # Certificate and key files for server to use to authenticate to client + tlsCertPath : + tlsKeyPath : + + # Server policy for client authentication. Maps to ClientAuth Policies + [ clientAuth : ] + + # CA certificate for client certificate authentication to the server + [ clientCAs : ] diff --git a/vendor/github.com/prometheus/procfs/.golangci.yml b/vendor/github.com/prometheus/procfs/.golangci.yml index 7c4ce1fa84..0aa09edacb 100644 --- a/vendor/github.com/prometheus/procfs/.golangci.yml +++ b/vendor/github.com/prometheus/procfs/.golangci.yml @@ -1,4 +1,4 @@ +--- linters: enable: - - staticcheck - - govet + - golint diff --git a/vendor/github.com/prometheus/procfs/Makefile.common b/vendor/github.com/prometheus/procfs/Makefile.common index d7aea1b86f..b978dfc50d 100644 --- a/vendor/github.com/prometheus/procfs/Makefile.common +++ b/vendor/github.com/prometheus/procfs/Makefile.common @@ -69,12 +69,21 @@ else GO_BUILD_PLATFORM ?= $(GOHOSTOS)-$(GOHOSTARCH) endif -PROMU_VERSION ?= 0.4.0 +GOTEST := $(GO) test +GOTEST_DIR := +ifneq ($(CIRCLE_JOB),) +ifneq ($(shell which gotestsum),) + GOTEST_DIR := test-results + GOTEST := gotestsum --junitfile $(GOTEST_DIR)/unit-tests.xml -- +endif +endif + +PROMU_VERSION ?= 0.5.0 PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz GOLANGCI_LINT := GOLANGCI_LINT_OPTS ?= -GOLANGCI_LINT_VERSION ?= v1.16.0 +GOLANGCI_LINT_VERSION ?= v1.18.0 # golangci-lint only supports linux, darwin and windows platforms on i386/amd64. # windows isn't included here because of the path separator being different. ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin)) @@ -86,7 +95,8 @@ endif PREFIX ?= $(shell pwd) BIN_DIR ?= $(shell pwd) DOCKER_IMAGE_TAG ?= $(subst /,-,$(shell git rev-parse --abbrev-ref HEAD)) -DOCKERFILE_PATH ?= ./ +DOCKERFILE_PATH ?= ./Dockerfile +DOCKERBUILD_CONTEXT ?= ./ DOCKER_REPO ?= prom DOCKER_ARCHS ?= amd64 @@ -141,14 +151,17 @@ else endif .PHONY: common-test-short -common-test-short: +common-test-short: $(GOTEST_DIR) @echo ">> running short tests" - GO111MODULE=$(GO111MODULE) $(GO) test -short $(GOOPTS) $(pkgs) + GO111MODULE=$(GO111MODULE) $(GOTEST) -short $(GOOPTS) $(pkgs) .PHONY: common-test -common-test: +common-test: $(GOTEST_DIR) @echo ">> running all tests" - GO111MODULE=$(GO111MODULE) $(GO) test $(test-flags) $(GOOPTS) $(pkgs) + GO111MODULE=$(GO111MODULE) $(GOTEST) $(test-flags) $(GOOPTS) $(pkgs) + +$(GOTEST_DIR): + @mkdir -p $@ .PHONY: common-format common-format: @@ -200,7 +213,7 @@ endif .PHONY: common-build common-build: promu @echo ">> building binaries" - GO111MODULE=$(GO111MODULE) $(PROMU) build --prefix $(PREFIX) + GO111MODULE=$(GO111MODULE) $(PROMU) build --prefix $(PREFIX) $(PROMU_BINARIES) .PHONY: common-tarball common-tarball: promu @@ -211,9 +224,10 @@ common-tarball: promu common-docker: $(BUILD_DOCKER_ARCHS) $(BUILD_DOCKER_ARCHS): common-docker-%: docker build -t "$(DOCKER_REPO)/$(DOCKER_IMAGE_NAME)-linux-$*:$(DOCKER_IMAGE_TAG)" \ + -f $(DOCKERFILE_PATH) \ --build-arg ARCH="$*" \ --build-arg OS="linux" \ - $(DOCKERFILE_PATH) + $(DOCKERBUILD_CONTEXT) .PHONY: common-docker-publish $(PUBLISH_DOCKER_ARCHS) common-docker-publish: $(PUBLISH_DOCKER_ARCHS) diff --git a/vendor/github.com/prometheus/procfs/crypto.go b/vendor/github.com/prometheus/procfs/crypto.go index 19d4041b29..a958933757 100644 --- a/vendor/github.com/prometheus/procfs/crypto.go +++ b/vendor/github.com/prometheus/procfs/crypto.go @@ -14,10 +14,10 @@ package procfs import ( + "bufio" "bytes" "fmt" - "io/ioutil" - "strconv" + "io" "strings" "github.com/prometheus/procfs/internal/util" @@ -52,80 +52,102 @@ type Crypto struct { // structs containing the relevant info. More information available here: // https://kernel.readthedocs.io/en/sphinx-samples/crypto-API.html func (fs FS) Crypto() ([]Crypto, error) { - data, err := ioutil.ReadFile(fs.proc.Path("crypto")) + path := fs.proc.Path("crypto") + b, err := util.ReadFileNoStat(path) if err != nil { - return nil, fmt.Errorf("error parsing crypto %s: %s", fs.proc.Path("crypto"), err) + return nil, fmt.Errorf("error reading crypto %s: %s", path, err) } - crypto, err := parseCrypto(data) + + crypto, err := parseCrypto(bytes.NewReader(b)) if err != nil { - return nil, fmt.Errorf("error parsing crypto %s: %s", fs.proc.Path("crypto"), err) + return nil, fmt.Errorf("error parsing crypto %s: %s", path, err) } + return crypto, nil } -func parseCrypto(cryptoData []byte) ([]Crypto, error) { - crypto := []Crypto{} - - cryptoBlocks := bytes.Split(cryptoData, []byte("\n\n")) - - for _, block := range cryptoBlocks { - var newCryptoElem Crypto - - lines := strings.Split(string(block), "\n") - for _, line := range lines { - if strings.TrimSpace(line) == "" || line[0] == ' ' { - continue - } - fields := strings.Split(line, ":") - key := strings.TrimSpace(fields[0]) - value := strings.TrimSpace(fields[1]) - vp := util.NewValueParser(value) - - switch strings.TrimSpace(key) { - case "async": - b, err := strconv.ParseBool(value) - if err == nil { - newCryptoElem.Async = b - } - case "blocksize": - newCryptoElem.Blocksize = vp.PUInt64() - case "chunksize": - newCryptoElem.Chunksize = vp.PUInt64() - case "digestsize": - newCryptoElem.Digestsize = vp.PUInt64() - case "driver": - newCryptoElem.Driver = value - case "geniv": - newCryptoElem.Geniv = value - case "internal": - newCryptoElem.Internal = value - case "ivsize": - newCryptoElem.Ivsize = vp.PUInt64() - case "maxauthsize": - newCryptoElem.Maxauthsize = vp.PUInt64() - case "max keysize": - newCryptoElem.MaxKeysize = vp.PUInt64() - case "min keysize": - newCryptoElem.MinKeysize = vp.PUInt64() - case "module": - newCryptoElem.Module = value - case "name": - newCryptoElem.Name = value - case "priority": - newCryptoElem.Priority = vp.PInt64() - case "refcnt": - newCryptoElem.Refcnt = vp.PInt64() - case "seedsize": - newCryptoElem.Seedsize = vp.PUInt64() - case "selftest": - newCryptoElem.Selftest = value - case "type": - newCryptoElem.Type = value - case "walksize": - newCryptoElem.Walksize = vp.PUInt64() - } +// parseCrypto parses a /proc/crypto stream into Crypto elements. +func parseCrypto(r io.Reader) ([]Crypto, error) { + var out []Crypto + + s := bufio.NewScanner(r) + for s.Scan() { + text := s.Text() + switch { + case strings.HasPrefix(text, "name"): + // Each crypto element begins with its name. + out = append(out, Crypto{}) + case text == "": + continue + } + + kv := strings.Split(text, ":") + if len(kv) != 2 { + return nil, fmt.Errorf("malformed crypto line: %q", text) + } + + k := strings.TrimSpace(kv[0]) + v := strings.TrimSpace(kv[1]) + + // Parse the key/value pair into the currently focused element. + c := &out[len(out)-1] + if err := c.parseKV(k, v); err != nil { + return nil, err } - crypto = append(crypto, newCryptoElem) } - return crypto, nil + + if err := s.Err(); err != nil { + return nil, err + } + + return out, nil +} + +// parseKV parses a key/value pair into the appropriate field of c. +func (c *Crypto) parseKV(k, v string) error { + vp := util.NewValueParser(v) + + switch k { + case "async": + // Interpret literal yes as true. + c.Async = v == "yes" + case "blocksize": + c.Blocksize = vp.PUInt64() + case "chunksize": + c.Chunksize = vp.PUInt64() + case "digestsize": + c.Digestsize = vp.PUInt64() + case "driver": + c.Driver = v + case "geniv": + c.Geniv = v + case "internal": + c.Internal = v + case "ivsize": + c.Ivsize = vp.PUInt64() + case "maxauthsize": + c.Maxauthsize = vp.PUInt64() + case "max keysize": + c.MaxKeysize = vp.PUInt64() + case "min keysize": + c.MinKeysize = vp.PUInt64() + case "module": + c.Module = v + case "name": + c.Name = v + case "priority": + c.Priority = vp.PInt64() + case "refcnt": + c.Refcnt = vp.PInt64() + case "seedsize": + c.Seedsize = vp.PUInt64() + case "selftest": + c.Selftest = v + case "type": + c.Type = v + case "walksize": + c.Walksize = vp.PUInt64() + } + + return vp.Err() } diff --git a/vendor/github.com/prometheus/procfs/fixtures.ttar b/vendor/github.com/prometheus/procfs/fixtures.ttar index c50a18ace4..45a7321558 100644 --- a/vendor/github.com/prometheus/procfs/fixtures.ttar +++ b/vendor/github.com/prometheus/procfs/fixtures.ttar @@ -189,7 +189,7 @@ Ngid: 0 Pid: 26231 PPid: 1 TracerPid: 0 -Uid: 0 0 0 0 +Uid: 1000 1000 1000 0 Gid: 0 0 0 0 FDSize: 128 Groups: @@ -289,6 +289,19 @@ Max realtime priority 0 0 Max realtime timeout unlimited unlimited us Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/proc/26232/maps +Lines: 9 +55680ae1e000-55680ae20000 r--p 00000000 fd:01 47316994 /bin/cat +55680ae29000-55680ae2a000 rwxs 0000a000 fd:01 47316994 /bin/cat +55680bed6000-55680bef7000 rw-p 00000000 00:00 0 [heap] +7fdf964fc000-7fdf973f2000 r--p 00000000 fd:01 17432624 /usr/lib/locale/locale-archive +7fdf973f2000-7fdf97417000 r--p 00000000 fd:01 60571062 /lib/x86_64-linux-gnu/libc-2.29.so +7ffe9215c000-7ffe9217f000 rw-p 00000000 00:00 0 [stack] +7ffe921da000-7ffe921dd000 r--p 00000000 00:00 0 [vvar] +7ffe921dd000-7ffe921de000 r-xp 00000000 00:00 0 [vdso] +ffffffffff600000-ffffffffff601000 --xp 00000000 00:00 0 [vsyscall] +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Path: fixtures/proc/26232/root SymlinkTo: /does/not/exist # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -317,6 +330,17 @@ Lines: 8 || || Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: fixtures/proc/26234 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/proc/26234/maps +Lines: 4 +08048000-08089000 r-xp 00000000 03:01 104219 /bin/tcsh +08089000-0808c000 rw-p 00041000 03:01 104219 /bin/tcsh +0808c000-08146000 rwxp 00000000 00:00 0 +40000000-40015000 r-xp 00000000 03:01 61874 /lib/ld-2.3.2.so +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Directory: fixtures/proc/584 Mode: 755 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -554,7 +578,7 @@ power management: Mode: 444 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Path: fixtures/proc/crypto -Lines: 971 +Lines: 972 name : ccm(aes) driver : ccm_base(ctr(aes-aesni),cbcmac(aes-aesni)) module : ccm @@ -588,6 +612,7 @@ refcnt : 1 selftest : passed internal : no type : kpp +async : yes name : ecb(arc4) driver : ecb(arc4)-generic @@ -1614,6 +1639,11 @@ xpc 399724544 92823103 86219234 debug 0 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/proc/loadavg +Lines: 1 +0.02 0.04 0.05 1/497 11947 +Mode: 444 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Path: fixtures/proc/mdstat Lines: 56 Personalities : [linear] [multipath] [raid0] [raid1] [raid6] [raid5] [raid4] [raid10] @@ -1821,8 +1851,35 @@ FRAG6: inuse 0 memory 0 Mode: 444 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Path: fixtures/proc/net/softnet_stat -Lines: 1 +Lines: 2 00015c73 00020e76 F0000769 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000 +01663fb2 00000000 000109a4 00000000 00000000 00000000 00000000 00000000 00000000 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/proc/net/softnet_stat.broken +Lines: 1 +00015c73 00020e76 F0000769 00000000 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/proc/net/udp +Lines: 4 + sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode + 0: 0A000005:0016 00000000:0000 0A 00000000:00000001 00:00000000 00000000 0 0 2740 1 ffff88003d3af3c0 100 0 0 10 0 + 1: 00000000:0016 00000000:0000 0A 00000001:00000000 00:00000000 00000000 0 0 2740 1 ffff88003d3af3c0 100 0 0 10 0 + 2: 00000000:0016 00000000:0000 0A 00000001:00000001 00:00000000 00000000 0 0 2740 1 ffff88003d3af3c0 100 0 0 10 0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/proc/net/udp6 +Lines: 3 + sl local_address remote_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode ref pointer drops + 1315: 00000000000000000000000000000000:14EB 00000000000000000000000000000000:0000 07 00000000:00000000 00:00000000 00000000 981 0 21040 2 0000000013726323 0 + 6073: 000080FE00000000FFADE15609667CFE:C781 00000000000000000000000000000000:0000 07 00000000:00000000 00:00000000 00000000 1000 0 11337031 2 00000000b9256fdd 0 +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/proc/net/udp_broken +Lines: 2 + sl local_address rem_address st + 1: 00000000:0016 00000000:0000 0A Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Path: fixtures/proc/net/unix @@ -1930,6 +1987,12 @@ procs_blocked 1 softirq 5057579 250191 1481983 1647 211099 186066 0 1783454 622196 12499 508444 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: fixtures/proc/swaps +Lines: 2 +Filename Type Size Used Priority +/dev/dm-2 partition 131068 176 -2 +Mode: 444 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Directory: fixtures/proc/symlinktargets Mode: 755 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/vendor/github.com/prometheus/procfs/go.mod b/vendor/github.com/prometheus/procfs/go.mod index 0e04e5d1fd..ded48253cd 100644 --- a/vendor/github.com/prometheus/procfs/go.mod +++ b/vendor/github.com/prometheus/procfs/go.mod @@ -5,4 +5,5 @@ go 1.12 require ( github.com/google/go-cmp v0.3.1 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e + golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e ) diff --git a/vendor/github.com/prometheus/procfs/go.sum b/vendor/github.com/prometheus/procfs/go.sum index 33b824b01b..54b5f33033 100644 --- a/vendor/github.com/prometheus/procfs/go.sum +++ b/vendor/github.com/prometheus/procfs/go.sum @@ -2,3 +2,5 @@ github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e h1:LwyF2AFISC9nVbS6MgzsaQNSUsRXI49GS+YQ5KX/QH0= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/vendor/github.com/prometheus/procfs/loadavg.go b/vendor/github.com/prometheus/procfs/loadavg.go new file mode 100644 index 0000000000..00bbe14417 --- /dev/null +++ b/vendor/github.com/prometheus/procfs/loadavg.go @@ -0,0 +1,62 @@ +// Copyright 2019 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package procfs + +import ( + "fmt" + "strconv" + "strings" + + "github.com/prometheus/procfs/internal/util" +) + +// LoadAvg represents an entry in /proc/loadavg +type LoadAvg struct { + Load1 float64 + Load5 float64 + Load15 float64 +} + +// LoadAvg returns loadavg from /proc. +func (fs FS) LoadAvg() (*LoadAvg, error) { + path := fs.proc.Path("loadavg") + + data, err := util.ReadFileNoStat(path) + if err != nil { + return nil, err + } + return parseLoad(data) +} + +// Parse /proc loadavg and return 1m, 5m and 15m. +func parseLoad(loadavgBytes []byte) (*LoadAvg, error) { + loads := make([]float64, 3) + parts := strings.Fields(string(loadavgBytes)) + if len(parts) < 3 { + return nil, fmt.Errorf("malformed loadavg line: too few fields in loadavg string: %s", string(loadavgBytes)) + } + + var err error + for i, load := range parts[0:3] { + loads[i], err = strconv.ParseFloat(load, 64) + if err != nil { + return nil, fmt.Errorf("could not parse load '%s': %s", load, err) + } + } + return &LoadAvg{ + Load1: loads[0], + Load5: loads[1], + Load15: loads[2], + }, nil +} diff --git a/vendor/github.com/prometheus/procfs/mountinfo.go b/vendor/github.com/prometheus/procfs/mountinfo.go index bb01bb5a2a..9471136101 100644 --- a/vendor/github.com/prometheus/procfs/mountinfo.go +++ b/vendor/github.com/prometheus/procfs/mountinfo.go @@ -29,10 +29,10 @@ import ( // is described in the following man page. // http://man7.org/linux/man-pages/man5/proc.5.html type MountInfo struct { - // Unique Id for the mount - MountId int - // The Id of the parent mount - ParentId int + // Unique ID for the mount + MountID int + // The ID of the parent mount + ParentID int // The value of `st_dev` for the files on this FS MajorMinorVer string // The pathname of the directory in the FS that forms @@ -96,11 +96,11 @@ func parseMountInfoString(mountString string) (*MountInfo, error) { SuperOptions: mountOptionsParser(mountInfo[mountInfoLength-1]), } - mount.MountId, err = strconv.Atoi(mountInfo[0]) + mount.MountID, err = strconv.Atoi(mountInfo[0]) if err != nil { return nil, fmt.Errorf("failed to parse mount ID") } - mount.ParentId, err = strconv.Atoi(mountInfo[1]) + mount.ParentID, err = strconv.Atoi(mountInfo[1]) if err != nil { return nil, fmt.Errorf("failed to parse parent ID") } diff --git a/vendor/github.com/prometheus/procfs/net_conntrackstat.go b/vendor/github.com/prometheus/procfs/net_conntrackstat.go new file mode 100644 index 0000000000..1e27c83d50 --- /dev/null +++ b/vendor/github.com/prometheus/procfs/net_conntrackstat.go @@ -0,0 +1,153 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package procfs + +import ( + "bufio" + "bytes" + "fmt" + "io" + "strconv" + "strings" + + "github.com/prometheus/procfs/internal/util" +) + +// A ConntrackStatEntry represents one line from net/stat/nf_conntrack +// and contains netfilter conntrack statistics at one CPU core +type ConntrackStatEntry struct { + Entries uint64 + Found uint64 + Invalid uint64 + Ignore uint64 + Insert uint64 + InsertFailed uint64 + Drop uint64 + EarlyDrop uint64 + SearchRestart uint64 +} + +// Retrieves netfilter's conntrack statistics, split by CPU cores +func (fs FS) ConntrackStat() ([]ConntrackStatEntry, error) { + return readConntrackStat(fs.proc.Path("net", "stat", "nf_conntrack")) +} + +// Parses a slice of ConntrackStatEntries from the given filepath +func readConntrackStat(path string) ([]ConntrackStatEntry, error) { + // This file is small and can be read with one syscall. + b, err := util.ReadFileNoStat(path) + if err != nil { + // Do not wrap this error so the caller can detect os.IsNotExist and + // similar conditions. + return nil, err + } + + stat, err := parseConntrackStat(bytes.NewReader(b)) + if err != nil { + return nil, fmt.Errorf("failed to read conntrack stats from %q: %v", path, err) + } + + return stat, nil +} + +// Reads the contents of a conntrack statistics file and parses a slice of ConntrackStatEntries +func parseConntrackStat(r io.Reader) ([]ConntrackStatEntry, error) { + var entries []ConntrackStatEntry + + scanner := bufio.NewScanner(r) + scanner.Scan() + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + conntrackEntry, err := parseConntrackStatEntry(fields) + if err != nil { + return nil, err + } + entries = append(entries, *conntrackEntry) + } + + return entries, nil +} + +// Parses a ConntrackStatEntry from given array of fields +func parseConntrackStatEntry(fields []string) (*ConntrackStatEntry, error) { + if len(fields) != 17 { + return nil, fmt.Errorf("invalid conntrackstat entry, missing fields") + } + entry := &ConntrackStatEntry{} + + entries, err := parseConntrackStatField(fields[0]) + if err != nil { + return nil, err + } + entry.Entries = entries + + found, err := parseConntrackStatField(fields[2]) + if err != nil { + return nil, err + } + entry.Found = found + + invalid, err := parseConntrackStatField(fields[4]) + if err != nil { + return nil, err + } + entry.Invalid = invalid + + ignore, err := parseConntrackStatField(fields[5]) + if err != nil { + return nil, err + } + entry.Ignore = ignore + + insert, err := parseConntrackStatField(fields[8]) + if err != nil { + return nil, err + } + entry.Insert = insert + + insertFailed, err := parseConntrackStatField(fields[9]) + if err != nil { + return nil, err + } + entry.InsertFailed = insertFailed + + drop, err := parseConntrackStatField(fields[10]) + if err != nil { + return nil, err + } + entry.Drop = drop + + earlyDrop, err := parseConntrackStatField(fields[11]) + if err != nil { + return nil, err + } + entry.EarlyDrop = earlyDrop + + searchRestart, err := parseConntrackStatField(fields[16]) + if err != nil { + return nil, err + } + entry.SearchRestart = searchRestart + + return entry, nil +} + +// Parses a uint64 from given hex in string +func parseConntrackStatField(field string) (uint64, error) { + val, err := strconv.ParseUint(field, 16, 64) + if err != nil { + return 0, fmt.Errorf("couldn't parse \"%s\" field: %s", field, err) + } + return val, err +} diff --git a/vendor/github.com/prometheus/procfs/net_softnet.go b/vendor/github.com/prometheus/procfs/net_softnet.go index 6fcad20afc..db5debdf4a 100644 --- a/vendor/github.com/prometheus/procfs/net_softnet.go +++ b/vendor/github.com/prometheus/procfs/net_softnet.go @@ -14,78 +14,89 @@ package procfs import ( + "bufio" + "bytes" "fmt" - "io/ioutil" + "io" "strconv" "strings" + + "github.com/prometheus/procfs/internal/util" ) // For the proc file format details, -// see https://elixir.bootlin.com/linux/v4.17/source/net/core/net-procfs.c#L162 +// See: +// * Linux 2.6.23 https://elixir.bootlin.com/linux/v2.6.23/source/net/core/dev.c#L2343 +// * Linux 4.17 https://elixir.bootlin.com/linux/v4.17/source/net/core/net-procfs.c#L162 // and https://elixir.bootlin.com/linux/v4.17/source/include/linux/netdevice.h#L2810. -// SoftnetEntry contains a single row of data from /proc/net/softnet_stat -type SoftnetEntry struct { +// SoftnetStat contains a single row of data from /proc/net/softnet_stat +type SoftnetStat struct { // Number of processed packets - Processed uint + Processed uint32 // Number of dropped packets - Dropped uint + Dropped uint32 // Number of times processing packets ran out of quota - TimeSqueezed uint + TimeSqueezed uint32 } -// GatherSoftnetStats reads /proc/net/softnet_stat, parse the relevant columns, -// and then return a slice of SoftnetEntry's. -func (fs FS) GatherSoftnetStats() ([]SoftnetEntry, error) { - data, err := ioutil.ReadFile(fs.proc.Path("net/softnet_stat")) +var softNetProcFile = "net/softnet_stat" + +// NetSoftnetStat reads data from /proc/net/softnet_stat. +func (fs FS) NetSoftnetStat() ([]SoftnetStat, error) { + b, err := util.ReadFileNoStat(fs.proc.Path(softNetProcFile)) + if err != nil { + return nil, err + } + + entries, err := parseSoftnet(bytes.NewReader(b)) if err != nil { - return nil, fmt.Errorf("error reading softnet %s: %s", fs.proc.Path("net/softnet_stat"), err) + return nil, fmt.Errorf("failed to parse /proc/net/softnet_stat: %v", err) } - return parseSoftnetEntries(data) + return entries, nil } -func parseSoftnetEntries(data []byte) ([]SoftnetEntry, error) { - lines := strings.Split(string(data), "\n") - entries := make([]SoftnetEntry, 0) - var err error - const ( - expectedColumns = 11 - ) - for _, line := range lines { - columns := strings.Fields(line) +func parseSoftnet(r io.Reader) ([]SoftnetStat, error) { + const minColumns = 9 + + s := bufio.NewScanner(r) + + var stats []SoftnetStat + for s.Scan() { + columns := strings.Fields(s.Text()) width := len(columns) - if width == 0 { - continue - } - if width != expectedColumns { - return []SoftnetEntry{}, fmt.Errorf("%d columns were detected, but %d were expected", width, expectedColumns) + + if width < minColumns { + return nil, fmt.Errorf("%d columns were detected, but at least %d were expected", width, minColumns) } - var entry SoftnetEntry - if entry, err = parseSoftnetEntry(columns); err != nil { - return []SoftnetEntry{}, err + + // We only parse the first three columns at the moment. + us, err := parseHexUint32s(columns[0:3]) + if err != nil { + return nil, err } - entries = append(entries, entry) + + stats = append(stats, SoftnetStat{ + Processed: us[0], + Dropped: us[1], + TimeSqueezed: us[2], + }) } - return entries, nil + return stats, nil } -func parseSoftnetEntry(columns []string) (SoftnetEntry, error) { - var err error - var processed, dropped, timeSqueezed uint64 - if processed, err = strconv.ParseUint(columns[0], 16, 32); err != nil { - return SoftnetEntry{}, fmt.Errorf("Unable to parse column 0: %s", err) - } - if dropped, err = strconv.ParseUint(columns[1], 16, 32); err != nil { - return SoftnetEntry{}, fmt.Errorf("Unable to parse column 1: %s", err) - } - if timeSqueezed, err = strconv.ParseUint(columns[2], 16, 32); err != nil { - return SoftnetEntry{}, fmt.Errorf("Unable to parse column 2: %s", err) +func parseHexUint32s(ss []string) ([]uint32, error) { + us := make([]uint32, 0, len(ss)) + for _, s := range ss { + u, err := strconv.ParseUint(s, 16, 32) + if err != nil { + return nil, err + } + + us = append(us, uint32(u)) } - return SoftnetEntry{ - Processed: uint(processed), - Dropped: uint(dropped), - TimeSqueezed: uint(timeSqueezed), - }, nil + + return us, nil } diff --git a/vendor/github.com/prometheus/procfs/net_udp.go b/vendor/github.com/prometheus/procfs/net_udp.go new file mode 100644 index 0000000000..d017e3f18d --- /dev/null +++ b/vendor/github.com/prometheus/procfs/net_udp.go @@ -0,0 +1,229 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package procfs + +import ( + "bufio" + "encoding/hex" + "fmt" + "io" + "net" + "os" + "strconv" + "strings" +) + +const ( + // readLimit is used by io.LimitReader while reading the content of the + // /proc/net/udp{,6} files. The number of lines inside such a file is dynamic + // as each line represents a single used socket. + // In theory, the number of available sockets is 65535 (2^16 - 1) per IP. + // With e.g. 150 Byte per line and the maximum number of 65535, + // the reader needs to handle 150 Byte * 65535 =~ 10 MB for a single IP. + readLimit = 4294967296 // Byte -> 4 GiB +) + +type ( + // NetUDP represents the contents of /proc/net/udp{,6} file without the header. + NetUDP []*netUDPLine + + // NetUDPSummary provides already computed values like the total queue lengths or + // the total number of used sockets. In contrast to NetUDP it does not collect + // the parsed lines into a slice. + NetUDPSummary struct { + // TxQueueLength shows the total queue length of all parsed tx_queue lengths. + TxQueueLength uint64 + // RxQueueLength shows the total queue length of all parsed rx_queue lengths. + RxQueueLength uint64 + // UsedSockets shows the total number of parsed lines representing the + // number of used sockets. + UsedSockets uint64 + } + + // netUDPLine represents the fields parsed from a single line + // in /proc/net/udp{,6}. Fields which are not used by UDP are skipped. + // For the proc file format details, see https://linux.die.net/man/5/proc. + netUDPLine struct { + Sl uint64 + LocalAddr net.IP + LocalPort uint64 + RemAddr net.IP + RemPort uint64 + St uint64 + TxQueue uint64 + RxQueue uint64 + UID uint64 + } +) + +// NetUDP returns the IPv4 kernel/networking statistics for UDP datagrams +// read from /proc/net/udp. +func (fs FS) NetUDP() (NetUDP, error) { + return newNetUDP(fs.proc.Path("net/udp")) +} + +// NetUDP6 returns the IPv6 kernel/networking statistics for UDP datagrams +// read from /proc/net/udp6. +func (fs FS) NetUDP6() (NetUDP, error) { + return newNetUDP(fs.proc.Path("net/udp6")) +} + +// NetUDPSummary returns already computed statistics like the total queue lengths +// for UDP datagrams read from /proc/net/udp. +func (fs FS) NetUDPSummary() (*NetUDPSummary, error) { + return newNetUDPSummary(fs.proc.Path("net/udp")) +} + +// NetUDP6Summary returns already computed statistics like the total queue lengths +// for UDP datagrams read from /proc/net/udp6. +func (fs FS) NetUDP6Summary() (*NetUDPSummary, error) { + return newNetUDPSummary(fs.proc.Path("net/udp6")) +} + +// newNetUDP creates a new NetUDP{,6} from the contents of the given file. +func newNetUDP(file string) (NetUDP, error) { + f, err := os.Open(file) + if err != nil { + return nil, err + } + defer f.Close() + + netUDP := NetUDP{} + + lr := io.LimitReader(f, readLimit) + s := bufio.NewScanner(lr) + s.Scan() // skip first line with headers + for s.Scan() { + fields := strings.Fields(s.Text()) + line, err := parseNetUDPLine(fields) + if err != nil { + return nil, err + } + netUDP = append(netUDP, line) + } + if err := s.Err(); err != nil { + return nil, err + } + return netUDP, nil +} + +// newNetUDPSummary creates a new NetUDP{,6} from the contents of the given file. +func newNetUDPSummary(file string) (*NetUDPSummary, error) { + f, err := os.Open(file) + if err != nil { + return nil, err + } + defer f.Close() + + netUDPSummary := &NetUDPSummary{} + + lr := io.LimitReader(f, readLimit) + s := bufio.NewScanner(lr) + s.Scan() // skip first line with headers + for s.Scan() { + fields := strings.Fields(s.Text()) + line, err := parseNetUDPLine(fields) + if err != nil { + return nil, err + } + netUDPSummary.TxQueueLength += line.TxQueue + netUDPSummary.RxQueueLength += line.RxQueue + netUDPSummary.UsedSockets++ + } + if err := s.Err(); err != nil { + return nil, err + } + return netUDPSummary, nil +} + +// parseNetUDPLine parses a single line, represented by a list of fields. +func parseNetUDPLine(fields []string) (*netUDPLine, error) { + line := &netUDPLine{} + if len(fields) < 8 { + return nil, fmt.Errorf( + "cannot parse net udp socket line as it has less then 8 columns: %s", + strings.Join(fields, " "), + ) + } + var err error // parse error + + // sl + s := strings.Split(fields[0], ":") + if len(s) != 2 { + return nil, fmt.Errorf( + "cannot parse sl field in udp socket line: %s", fields[0]) + } + + if line.Sl, err = strconv.ParseUint(s[0], 0, 64); err != nil { + return nil, fmt.Errorf("cannot parse sl value in udp socket line: %s", err) + } + // local_address + l := strings.Split(fields[1], ":") + if len(l) != 2 { + return nil, fmt.Errorf( + "cannot parse local_address field in udp socket line: %s", fields[1]) + } + if line.LocalAddr, err = hex.DecodeString(l[0]); err != nil { + return nil, fmt.Errorf( + "cannot parse local_address value in udp socket line: %s", err) + } + if line.LocalPort, err = strconv.ParseUint(l[1], 16, 64); err != nil { + return nil, fmt.Errorf( + "cannot parse local_address port value in udp socket line: %s", err) + } + + // remote_address + r := strings.Split(fields[2], ":") + if len(r) != 2 { + return nil, fmt.Errorf( + "cannot parse rem_address field in udp socket line: %s", fields[1]) + } + if line.RemAddr, err = hex.DecodeString(r[0]); err != nil { + return nil, fmt.Errorf( + "cannot parse rem_address value in udp socket line: %s", err) + } + if line.RemPort, err = strconv.ParseUint(r[1], 16, 64); err != nil { + return nil, fmt.Errorf( + "cannot parse rem_address port value in udp socket line: %s", err) + } + + // st + if line.St, err = strconv.ParseUint(fields[3], 16, 64); err != nil { + return nil, fmt.Errorf( + "cannot parse st value in udp socket line: %s", err) + } + + // tx_queue and rx_queue + q := strings.Split(fields[4], ":") + if len(q) != 2 { + return nil, fmt.Errorf( + "cannot parse tx/rx queues in udp socket line as it has a missing colon: %s", + fields[4], + ) + } + if line.TxQueue, err = strconv.ParseUint(q[0], 16, 64); err != nil { + return nil, fmt.Errorf("cannot parse tx_queue value in udp socket line: %s", err) + } + if line.RxQueue, err = strconv.ParseUint(q[1], 16, 64); err != nil { + return nil, fmt.Errorf("cannot parse rx_queue value in udp socket line: %s", err) + } + + // uid + if line.UID, err = strconv.ParseUint(fields[7], 0, 64); err != nil { + return nil, fmt.Errorf( + "cannot parse uid value in udp socket line: %s", err) + } + + return line, nil +} diff --git a/vendor/github.com/prometheus/procfs/net_unix.go b/vendor/github.com/prometheus/procfs/net_unix.go index 93bd58f809..c55b4b18e4 100644 --- a/vendor/github.com/prometheus/procfs/net_unix.go +++ b/vendor/github.com/prometheus/procfs/net_unix.go @@ -15,7 +15,6 @@ package procfs import ( "bufio" - "errors" "fmt" "io" "os" @@ -27,25 +26,15 @@ import ( // see https://elixir.bootlin.com/linux/v4.17/source/net/unix/af_unix.c#L2815 // and https://elixir.bootlin.com/linux/latest/source/include/uapi/linux/net.h#L48. -const ( - netUnixKernelPtrIdx = iota - netUnixRefCountIdx - _ - netUnixFlagsIdx - netUnixTypeIdx - netUnixStateIdx - netUnixInodeIdx - - // Inode and Path are optional. - netUnixStaticFieldsCnt = 6 -) - +// Constants for the various /proc/net/unix enumerations. +// TODO: match against x/sys/unix or similar? const ( netUnixTypeStream = 1 netUnixTypeDgram = 2 netUnixTypeSeqpacket = 5 - netUnixFlagListen = 1 << 16 + netUnixFlagDefault = 0 + netUnixFlagListen = 1 << 16 netUnixStateUnconnected = 1 netUnixStateConnecting = 2 @@ -53,129 +42,127 @@ const ( netUnixStateDisconnected = 4 ) -var errInvalidKernelPtrFmt = errors.New("Invalid Num(the kernel table slot number) format") +// NetUNIXType is the type of the type field. +type NetUNIXType uint64 -// NetUnixType is the type of the type field. -type NetUnixType uint64 +// NetUNIXFlags is the type of the flags field. +type NetUNIXFlags uint64 -// NetUnixFlags is the type of the flags field. -type NetUnixFlags uint64 +// NetUNIXState is the type of the state field. +type NetUNIXState uint64 -// NetUnixState is the type of the state field. -type NetUnixState uint64 - -// NetUnixLine represents a line of /proc/net/unix. -type NetUnixLine struct { +// NetUNIXLine represents a line of /proc/net/unix. +type NetUNIXLine struct { KernelPtr string RefCount uint64 Protocol uint64 - Flags NetUnixFlags - Type NetUnixType - State NetUnixState + Flags NetUNIXFlags + Type NetUNIXType + State NetUNIXState Inode uint64 Path string } -// NetUnix holds the data read from /proc/net/unix. -type NetUnix struct { - Rows []*NetUnixLine +// NetUNIX holds the data read from /proc/net/unix. +type NetUNIX struct { + Rows []*NetUNIXLine } -// NewNetUnix returns data read from /proc/net/unix. -func NewNetUnix() (*NetUnix, error) { - fs, err := NewFS(DefaultMountPoint) - if err != nil { - return nil, err - } - - return fs.NewNetUnix() +// NetUNIX returns data read from /proc/net/unix. +func (fs FS) NetUNIX() (*NetUNIX, error) { + return readNetUNIX(fs.proc.Path("net/unix")) } -// NewNetUnix returns data read from /proc/net/unix. -func (fs FS) NewNetUnix() (*NetUnix, error) { - return NewNetUnixByPath(fs.proc.Path("net/unix")) -} - -// NewNetUnixByPath returns data read from /proc/net/unix by file path. -// It might returns an error with partial parsed data, if an error occur after some data parsed. -func NewNetUnixByPath(path string) (*NetUnix, error) { - f, err := os.Open(path) +// readNetUNIX reads data in /proc/net/unix format from the specified file. +func readNetUNIX(file string) (*NetUNIX, error) { + // This file could be quite large and a streaming read is desirable versus + // reading the entire contents at once. + f, err := os.Open(file) if err != nil { return nil, err } defer f.Close() - return NewNetUnixByReader(f) + + return parseNetUNIX(f) } -// NewNetUnixByReader returns data read from /proc/net/unix by a reader. -// It might returns an error with partial parsed data, if an error occur after some data parsed. -func NewNetUnixByReader(reader io.Reader) (*NetUnix, error) { - nu := &NetUnix{ - Rows: make([]*NetUnixLine, 0, 32), - } - scanner := bufio.NewScanner(reader) - // Omit the header line. - scanner.Scan() - header := scanner.Text() +// parseNetUNIX creates a NetUnix structure from the incoming stream. +func parseNetUNIX(r io.Reader) (*NetUNIX, error) { + // Begin scanning by checking for the existence of Inode. + s := bufio.NewScanner(r) + s.Scan() + // From the man page of proc(5), it does not contain an Inode field, - // but in actually it exists. - // This code works for both cases. - hasInode := strings.Contains(header, "Inode") + // but in actually it exists. This code works for both cases. + hasInode := strings.Contains(s.Text(), "Inode") - minFieldsCnt := netUnixStaticFieldsCnt + // Expect a minimum number of fields, but Inode and Path are optional: + // Num RefCount Protocol Flags Type St Inode Path + minFields := 6 if hasInode { - minFieldsCnt++ + minFields++ } - for scanner.Scan() { - line := scanner.Text() - item, err := nu.parseLine(line, hasInode, minFieldsCnt) + + var nu NetUNIX + for s.Scan() { + line := s.Text() + item, err := nu.parseLine(line, hasInode, minFields) if err != nil { - return nu, err + return nil, fmt.Errorf("failed to parse /proc/net/unix data %q: %v", line, err) } + nu.Rows = append(nu.Rows, item) } - return nu, scanner.Err() + if err := s.Err(); err != nil { + return nil, fmt.Errorf("failed to scan /proc/net/unix data: %v", err) + } + + return &nu, nil } -func (u *NetUnix) parseLine(line string, hasInode bool, minFieldsCnt int) (*NetUnixLine, error) { +func (u *NetUNIX) parseLine(line string, hasInode bool, min int) (*NetUNIXLine, error) { fields := strings.Fields(line) - fieldsLen := len(fields) - if fieldsLen < minFieldsCnt { - return nil, fmt.Errorf( - "Parse Unix domain failed: expect at least %d fields but got %d", - minFieldsCnt, fieldsLen) - } - kernelPtr, err := u.parseKernelPtr(fields[netUnixKernelPtrIdx]) - if err != nil { - return nil, fmt.Errorf("Parse Unix domain num(%s) failed: %s", fields[netUnixKernelPtrIdx], err) + + l := len(fields) + if l < min { + return nil, fmt.Errorf("expected at least %d fields but got %d", min, l) } - users, err := u.parseUsers(fields[netUnixRefCountIdx]) + + // Field offsets are as follows: + // Num RefCount Protocol Flags Type St Inode Path + + kernelPtr := strings.TrimSuffix(fields[0], ":") + + users, err := u.parseUsers(fields[1]) if err != nil { - return nil, fmt.Errorf("Parse Unix domain ref count(%s) failed: %s", fields[netUnixRefCountIdx], err) + return nil, fmt.Errorf("failed to parse ref count(%s): %v", fields[1], err) } - flags, err := u.parseFlags(fields[netUnixFlagsIdx]) + + flags, err := u.parseFlags(fields[3]) if err != nil { - return nil, fmt.Errorf("Parse Unix domain flags(%s) failed: %s", fields[netUnixFlagsIdx], err) + return nil, fmt.Errorf("failed to parse flags(%s): %v", fields[3], err) } - typ, err := u.parseType(fields[netUnixTypeIdx]) + + typ, err := u.parseType(fields[4]) if err != nil { - return nil, fmt.Errorf("Parse Unix domain type(%s) failed: %s", fields[netUnixTypeIdx], err) + return nil, fmt.Errorf("failed to parse type(%s): %v", fields[4], err) } - state, err := u.parseState(fields[netUnixStateIdx]) + + state, err := u.parseState(fields[5]) if err != nil { - return nil, fmt.Errorf("Parse Unix domain state(%s) failed: %s", fields[netUnixStateIdx], err) + return nil, fmt.Errorf("failed to parse state(%s): %v", fields[5], err) } + var inode uint64 if hasInode { - inodeStr := fields[netUnixInodeIdx] - inode, err = u.parseInode(inodeStr) + inode, err = u.parseInode(fields[6]) if err != nil { - return nil, fmt.Errorf("Parse Unix domain inode(%s) failed: %s", inodeStr, err) + return nil, fmt.Errorf("failed to parse inode(%s): %v", fields[6], err) } } - nuLine := &NetUnixLine{ + n := &NetUNIXLine{ KernelPtr: kernelPtr, RefCount: users, Type: typ, @@ -185,57 +172,56 @@ func (u *NetUnix) parseLine(line string, hasInode bool, minFieldsCnt int) (*NetU } // Path field is optional. - if fieldsLen > minFieldsCnt { - pathIdx := netUnixInodeIdx + 1 + if l > min { + // Path occurs at either index 6 or 7 depending on whether inode is + // already present. + pathIdx := 7 if !hasInode { pathIdx-- } - nuLine.Path = fields[pathIdx] - } - - return nuLine, nil -} -func (u NetUnix) parseKernelPtr(str string) (string, error) { - if !strings.HasSuffix(str, ":") { - return "", errInvalidKernelPtrFmt + n.Path = fields[pathIdx] } - return str[:len(str)-1], nil + + return n, nil } -func (u NetUnix) parseUsers(hexStr string) (uint64, error) { - return strconv.ParseUint(hexStr, 16, 32) +func (u NetUNIX) parseUsers(s string) (uint64, error) { + return strconv.ParseUint(s, 16, 32) } -func (u NetUnix) parseType(hexStr string) (NetUnixType, error) { - typ, err := strconv.ParseUint(hexStr, 16, 16) +func (u NetUNIX) parseType(s string) (NetUNIXType, error) { + typ, err := strconv.ParseUint(s, 16, 16) if err != nil { return 0, err } - return NetUnixType(typ), nil + + return NetUNIXType(typ), nil } -func (u NetUnix) parseFlags(hexStr string) (NetUnixFlags, error) { - flags, err := strconv.ParseUint(hexStr, 16, 32) +func (u NetUNIX) parseFlags(s string) (NetUNIXFlags, error) { + flags, err := strconv.ParseUint(s, 16, 32) if err != nil { return 0, err } - return NetUnixFlags(flags), nil + + return NetUNIXFlags(flags), nil } -func (u NetUnix) parseState(hexStr string) (NetUnixState, error) { - st, err := strconv.ParseInt(hexStr, 16, 8) +func (u NetUNIX) parseState(s string) (NetUNIXState, error) { + st, err := strconv.ParseInt(s, 16, 8) if err != nil { return 0, err } - return NetUnixState(st), nil + + return NetUNIXState(st), nil } -func (u NetUnix) parseInode(inodeStr string) (uint64, error) { - return strconv.ParseUint(inodeStr, 10, 64) +func (u NetUNIX) parseInode(s string) (uint64, error) { + return strconv.ParseUint(s, 10, 64) } -func (t NetUnixType) String() string { +func (t NetUNIXType) String() string { switch t { case netUnixTypeStream: return "stream" @@ -247,7 +233,7 @@ func (t NetUnixType) String() string { return "unknown" } -func (f NetUnixFlags) String() string { +func (f NetUNIXFlags) String() string { switch f { case netUnixFlagListen: return "listen" @@ -256,7 +242,7 @@ func (f NetUnixFlags) String() string { } } -func (s NetUnixState) String() string { +func (s NetUNIXState) String() string { switch s { case netUnixStateUnconnected: return "unconnected" diff --git a/vendor/github.com/prometheus/procfs/proc_fdinfo.go b/vendor/github.com/prometheus/procfs/proc_fdinfo.go index 4e7597f86b..0c9c402850 100644 --- a/vendor/github.com/prometheus/procfs/proc_fdinfo.go +++ b/vendor/github.com/prometheus/procfs/proc_fdinfo.go @@ -16,6 +16,7 @@ package procfs import ( "bufio" "bytes" + "errors" "regexp" "github.com/prometheus/procfs/internal/util" @@ -23,10 +24,11 @@ import ( // Regexp variables var ( - rPos = regexp.MustCompile(`^pos:\s+(\d+)$`) - rFlags = regexp.MustCompile(`^flags:\s+(\d+)$`) - rMntID = regexp.MustCompile(`^mnt_id:\s+(\d+)$`) - rInotify = regexp.MustCompile(`^inotify`) + rPos = regexp.MustCompile(`^pos:\s+(\d+)$`) + rFlags = regexp.MustCompile(`^flags:\s+(\d+)$`) + rMntID = regexp.MustCompile(`^mnt_id:\s+(\d+)$`) + rInotify = regexp.MustCompile(`^inotify`) + rInotifyParts = regexp.MustCompile(`^inotify\s+wd:([0-9a-f]+)\s+ino:([0-9a-f]+)\s+sdev:([0-9a-f]+)(?:\s+mask:([0-9a-f]+))?`) ) // ProcFDInfo contains represents file descriptor information. @@ -96,15 +98,21 @@ type InotifyInfo struct { // InotifyInfo constructor. Only available on kernel 3.8+. func parseInotifyInfo(line string) (*InotifyInfo, error) { - r := regexp.MustCompile(`^inotify\s+wd:([0-9a-f]+)\s+ino:([0-9a-f]+)\s+sdev:([0-9a-f]+)\s+mask:([0-9a-f]+)`) - m := r.FindStringSubmatch(line) - i := &InotifyInfo{ - WD: m[1], - Ino: m[2], - Sdev: m[3], - Mask: m[4], + m := rInotifyParts.FindStringSubmatch(line) + if len(m) >= 4 { + var mask string + if len(m) == 5 { + mask = m[4] + } + i := &InotifyInfo{ + WD: m[1], + Ino: m[2], + Sdev: m[3], + Mask: mask, + } + return i, nil } - return i, nil + return nil, errors.New("invalid inode entry: " + line) } // ProcFDInfos represents a list of ProcFDInfo structs. diff --git a/vendor/github.com/prometheus/procfs/proc_maps.go b/vendor/github.com/prometheus/procfs/proc_maps.go new file mode 100644 index 0000000000..28d5c6eb1d --- /dev/null +++ b/vendor/github.com/prometheus/procfs/proc_maps.go @@ -0,0 +1,208 @@ +// Copyright 2019 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !windows + +package procfs + +import ( + "bufio" + "fmt" + "os" + "strconv" + "strings" + + "golang.org/x/sys/unix" +) + +type ProcMapPermissions struct { + // mapping has the [R]ead flag set + Read bool + // mapping has the [W]rite flag set + Write bool + // mapping has the [X]ecutable flag set + Execute bool + // mapping has the [S]hared flag set + Shared bool + // mapping is marked as [P]rivate (copy on write) + Private bool +} + +// ProcMap contains the process memory-mappings of the process, +// read from /proc/[pid]/maps +type ProcMap struct { + // The start address of current mapping. + StartAddr uintptr + // The end address of the current mapping + EndAddr uintptr + // The permissions for this mapping + Perms *ProcMapPermissions + // The current offset into the file/fd (e.g., shared libs) + Offset int64 + // Device owner of this mapping (major:minor) in Mkdev format. + Dev uint64 + // The inode of the device above + Inode uint64 + // The file or psuedofile (or empty==anonymous) + Pathname string +} + +// parseDevice parses the device token of a line and converts it to a dev_t +// (mkdev) like structure. +func parseDevice(s string) (uint64, error) { + toks := strings.Split(s, ":") + if len(toks) < 2 { + return 0, fmt.Errorf("unexpected number of fields") + } + + major, err := strconv.ParseUint(toks[0], 16, 0) + if err != nil { + return 0, err + } + + minor, err := strconv.ParseUint(toks[1], 16, 0) + if err != nil { + return 0, err + } + + return unix.Mkdev(uint32(major), uint32(minor)), nil +} + +// parseAddress just converts a hex-string to a uintptr +func parseAddress(s string) (uintptr, error) { + a, err := strconv.ParseUint(s, 16, 0) + if err != nil { + return 0, err + } + + return uintptr(a), nil +} + +// parseAddresses parses the start-end address +func parseAddresses(s string) (uintptr, uintptr, error) { + toks := strings.Split(s, "-") + if len(toks) < 2 { + return 0, 0, fmt.Errorf("invalid address") + } + + saddr, err := parseAddress(toks[0]) + if err != nil { + return 0, 0, err + } + + eaddr, err := parseAddress(toks[1]) + if err != nil { + return 0, 0, err + } + + return saddr, eaddr, nil +} + +// parsePermissions parses a token and returns any that are set. +func parsePermissions(s string) (*ProcMapPermissions, error) { + if len(s) < 4 { + return nil, fmt.Errorf("invalid permissions token") + } + + perms := ProcMapPermissions{} + for _, ch := range s { + switch ch { + case 'r': + perms.Read = true + case 'w': + perms.Write = true + case 'x': + perms.Execute = true + case 'p': + perms.Private = true + case 's': + perms.Shared = true + } + } + + return &perms, nil +} + +// parseProcMap will attempt to parse a single line within a proc/[pid]/maps +// buffer. +func parseProcMap(text string) (*ProcMap, error) { + fields := strings.Fields(text) + if len(fields) < 5 { + return nil, fmt.Errorf("truncated procmap entry") + } + + saddr, eaddr, err := parseAddresses(fields[0]) + if err != nil { + return nil, err + } + + perms, err := parsePermissions(fields[1]) + if err != nil { + return nil, err + } + + offset, err := strconv.ParseInt(fields[2], 16, 0) + if err != nil { + return nil, err + } + + device, err := parseDevice(fields[3]) + if err != nil { + return nil, err + } + + inode, err := strconv.ParseUint(fields[4], 10, 0) + if err != nil { + return nil, err + } + + pathname := "" + + if len(fields) >= 5 { + pathname = strings.Join(fields[5:], " ") + } + + return &ProcMap{ + StartAddr: saddr, + EndAddr: eaddr, + Perms: perms, + Offset: offset, + Dev: device, + Inode: inode, + Pathname: pathname, + }, nil +} + +// ProcMaps reads from /proc/[pid]/maps to get the memory-mappings of the +// process. +func (p Proc) ProcMaps() ([]*ProcMap, error) { + file, err := os.Open(p.path("maps")) + if err != nil { + return nil, err + } + defer file.Close() + + maps := []*ProcMap{} + scan := bufio.NewScanner(file) + + for scan.Scan() { + m, err := parseProcMap(scan.Text()) + if err != nil { + return nil, err + } + + maps = append(maps, m) + } + + return maps, nil +} diff --git a/vendor/github.com/prometheus/procfs/proc_status.go b/vendor/github.com/prometheus/procfs/proc_status.go index e30c2b88f4..c58346d910 100644 --- a/vendor/github.com/prometheus/procfs/proc_status.go +++ b/vendor/github.com/prometheus/procfs/proc_status.go @@ -33,37 +33,37 @@ type ProcStatus struct { TGID int // Peak virtual memory size. - VmPeak uint64 + VmPeak uint64 // nolint:golint // Virtual memory size. - VmSize uint64 + VmSize uint64 // nolint:golint // Locked memory size. - VmLck uint64 + VmLck uint64 // nolint:golint // Pinned memory size. - VmPin uint64 + VmPin uint64 // nolint:golint // Peak resident set size. - VmHWM uint64 + VmHWM uint64 // nolint:golint // Resident set size (sum of RssAnnon RssFile and RssShmem). - VmRSS uint64 + VmRSS uint64 // nolint:golint // Size of resident anonymous memory. - RssAnon uint64 + RssAnon uint64 // nolint:golint // Size of resident file mappings. - RssFile uint64 + RssFile uint64 // nolint:golint // Size of resident shared memory. - RssShmem uint64 + RssShmem uint64 // nolint:golint // Size of data segments. - VmData uint64 + VmData uint64 // nolint:golint // Size of stack segments. - VmStk uint64 + VmStk uint64 // nolint:golint // Size of text segments. - VmExe uint64 + VmExe uint64 // nolint:golint // Shared library code size. - VmLib uint64 + VmLib uint64 // nolint:golint // Page table entries size. - VmPTE uint64 + VmPTE uint64 // nolint:golint // Size of second-level page tables. - VmPMD uint64 + VmPMD uint64 // nolint:golint // Swapped-out virtual memory size by anonymous private. - VmSwap uint64 + VmSwap uint64 // nolint:golint // Size of hugetlb memory portions HugetlbPages uint64 @@ -71,6 +71,9 @@ type ProcStatus struct { VoluntaryCtxtSwitches uint64 // Number of involuntary context switches. NonVoluntaryCtxtSwitches uint64 + + // UIDs of the process (Real, effective, saved set, and filesystem UIDs (GIDs)) + UIDs [4]string } // NewStatus returns the current status information of the process. @@ -114,6 +117,8 @@ func (s *ProcStatus) fillStatus(k string, vString string, vUint uint64, vUintByt s.TGID = int(vUint) case "Name": s.Name = vString + case "Uid": + copy(s.UIDs[:], strings.Split(vString, "\t")) case "VmPeak": s.VmPeak = vUintBytes case "VmSize": diff --git a/vendor/github.com/prometheus/procfs/swaps.go b/vendor/github.com/prometheus/procfs/swaps.go new file mode 100644 index 0000000000..15edc2212b --- /dev/null +++ b/vendor/github.com/prometheus/procfs/swaps.go @@ -0,0 +1,89 @@ +// Copyright 2019 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package procfs + +import ( + "bufio" + "bytes" + "fmt" + "strconv" + "strings" + + "github.com/prometheus/procfs/internal/util" +) + +// Swap represents an entry in /proc/swaps. +type Swap struct { + Filename string + Type string + Size int + Used int + Priority int +} + +// Swaps returns a slice of all configured swap devices on the system. +func (fs FS) Swaps() ([]*Swap, error) { + data, err := util.ReadFileNoStat(fs.proc.Path("swaps")) + if err != nil { + return nil, err + } + return parseSwaps(data) +} + +func parseSwaps(info []byte) ([]*Swap, error) { + swaps := []*Swap{} + scanner := bufio.NewScanner(bytes.NewReader(info)) + scanner.Scan() // ignore header line + for scanner.Scan() { + swapString := scanner.Text() + parsedSwap, err := parseSwapString(swapString) + if err != nil { + return nil, err + } + swaps = append(swaps, parsedSwap) + } + + err := scanner.Err() + return swaps, err +} + +func parseSwapString(swapString string) (*Swap, error) { + var err error + + swapFields := strings.Fields(swapString) + swapLength := len(swapFields) + if swapLength < 5 { + return nil, fmt.Errorf("too few fields in swap string: %s", swapString) + } + + swap := &Swap{ + Filename: swapFields[0], + Type: swapFields[1], + } + + swap.Size, err = strconv.Atoi(swapFields[2]) + if err != nil { + return nil, fmt.Errorf("invalid swap size: %s", swapFields[2]) + } + swap.Used, err = strconv.Atoi(swapFields[3]) + if err != nil { + return nil, fmt.Errorf("invalid swap used: %s", swapFields[3]) + } + swap.Priority, err = strconv.Atoi(swapFields[4]) + if err != nil { + return nil, fmt.Errorf("invalid swap priority: %s", swapFields[4]) + } + + return swap, nil +} diff --git a/vendor/github.com/weaveworks/common/server/server.go b/vendor/github.com/weaveworks/common/server/server.go index c1b584f64d..78204bcef0 100644 --- a/vendor/github.com/weaveworks/common/server/server.go +++ b/vendor/github.com/weaveworks/common/server/server.go @@ -2,10 +2,8 @@ package server import ( "crypto/tls" - "crypto/x509" "flag" "fmt" - "io/ioutil" "math" "net" "net/http" @@ -18,6 +16,7 @@ import ( "github.com/opentracing/opentracing-go" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + node_https "github.com/prometheus/node_exporter/https" "golang.org/x/net/context" "golang.org/x/net/netutil" "google.golang.org/grpc" @@ -38,13 +37,13 @@ type Config struct { HTTPListenAddress string `yaml:"http_listen_address"` HTTPListenPort int `yaml:"http_listen_port"` HTTPConnLimit int `yaml:"http_listen_conn_limit"` - HTTPCertPath string `yaml:"http_cert_path"` - HTTPKeyPath string `yaml:"http_key_path"` - HTTPCAPath string `yaml:"http_ca_path"` GRPCListenAddress string `yaml:"grpc_listen_address"` GRPCListenPort int `yaml:"grpc_listen_port"` GRPCConnLimit int `yaml:"grpc_listen_conn_limit"` + HTTPTLSConfig node_https.TLSStruct `yaml:"http_tls_config"` + GRPCTLSConfig node_https.TLSStruct `yaml:"grpc_tls_config"` + RegisterInstrumentation bool `yaml:"register_instrumentation"` ExcludeRequestInLog bool `yaml:"-"` @@ -78,9 +77,14 @@ var infinty = time.Duration(math.MaxInt64) // RegisterFlags adds the flags required to config this to the given FlagSet func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.StringVar(&cfg.HTTPListenAddress, "server.http-listen-address", "", "HTTP server listen address.") - f.StringVar(&cfg.HTTPCertPath, "server.http-cert-path", "", "HTTP server cert path.") - f.StringVar(&cfg.HTTPKeyPath, "server.http-key-path", "", "HTTP server key path.") - f.StringVar(&cfg.HTTPKeyPath, "server.http-ca-path", "", "HTTP CA path.") + f.StringVar(&cfg.HTTPTLSConfig.TLSCertPath, "server.http-tls-cert-path", "", "HTTP server cert path.") + f.StringVar(&cfg.HTTPTLSConfig.TLSKeyPath, "server.http-tls-key-path", "", "HTTP server key path.") + f.StringVar(&cfg.HTTPTLSConfig.ClientAuth, "server.http-tls-client-auth", "", "HTTP TLS Client Auth type.") + f.StringVar(&cfg.HTTPTLSConfig.ClientCAs, "server.http-tls-ca-path", "", "HTTP TLS Client CA path.") + f.StringVar(&cfg.GRPCTLSConfig.TLSCertPath, "server.grpc-tls-cert-path", "", "GRPC TLS server cert path.") + f.StringVar(&cfg.GRPCTLSConfig.TLSKeyPath, "server.grpc-tls-key-path", "", "GRPC TLS server key path.") + f.StringVar(&cfg.GRPCTLSConfig.ClientAuth, "server.grpc-tls-client-auth", "", "GRPC TLS Client Auth type.") + f.StringVar(&cfg.GRPCTLSConfig.ClientCAs, "server.grpc-tls-ca-path", "", "GRPC TLS Client CA path.") f.IntVar(&cfg.HTTPListenPort, "server.http-listen-port", 80, "HTTP server listen port.") f.IntVar(&cfg.HTTPConnLimit, "server.http-conn-limit", 0, "Maximum number of simultaneous http connections, <=0 to disable") f.StringVar(&cfg.GRPCListenAddress, "server.grpc-listen-address", "", "gRPC server listen address.") @@ -146,31 +150,20 @@ func New(cfg Config) (*Server, error) { } // Setup TLS - var cert tls.Certificate - if cfg.HTTPCertPath != "" && cfg.HTTPKeyPath != "" { - cert, err = tls.LoadX509KeyPair(cfg.HTTPCertPath, cfg.HTTPKeyPath) + var httpTLSConfig *tls.Config + if len(cfg.HTTPTLSConfig.TLSCertPath) > 0 && len(cfg.HTTPTLSConfig.TLSKeyPath) > 0 { + // Note: ConfigToTLSConfig from prometheus/node_exporter is awaiting security review. + httpTLSConfig, err = node_https.ConfigToTLSConfig(&cfg.HTTPTLSConfig) if err != nil { - log.Warnf("error loading cert %s or key %s, tls disabled", cfg.HTTPCertPath, cfg.HTTPKeyPath) + return nil, fmt.Errorf("error generating http tls config: %v", err) } } - - var caCertPool *x509.CertPool - if cfg.HTTPCAPath != "" { - caCert, err := ioutil.ReadFile(cfg.HTTPCAPath) + var grpcTLSConfig *tls.Config + if len(cfg.GRPCTLSConfig.TLSCertPath) > 0 && len(cfg.GRPCTLSConfig.TLSKeyPath) > 0 { + // Note: ConfigToTLSConfig from prometheus/node_exporter is awaiting security review. + grpcTLSConfig, err = node_https.ConfigToTLSConfig(&cfg.GRPCTLSConfig) if err != nil { - log.Warnf("error loading ca cert %s, tls disabled", cfg.HTTPCAPath) - } else { - caCertPool = x509.NewCertPool() - caCertPool.AppendCertsFromPEM(caCert) - } - } - - var tlsConfig *tls.Config - if len(cert.Certificate) > 0 && caCertPool != nil { - tlsConfig = &tls.Config{ - Certificates: []tls.Certificate{cert}, - ClientAuth: tls.RequireAndVerifyClientCert, - ClientCAs: caCertPool, + return nil, fmt.Errorf("error generating grpc tls config: %v", err) } } @@ -225,8 +218,8 @@ func New(cfg Config) (*Server, error) { grpc.MaxConcurrentStreams(uint32(cfg.GPRCServerMaxConcurrentStreams)), } grpcOptions = append(grpcOptions, cfg.GRPCOptions...) - if tlsConfig != nil { - grpcCreds := credentials.NewTLS(tlsConfig) + if grpcTLSConfig != nil { + grpcCreds := credentials.NewTLS(grpcTLSConfig) grpcOptions = append(grpcOptions, grpc.Creds(grpcCreds)) } grpcServer := grpc.NewServer(grpcOptions...) @@ -261,8 +254,8 @@ func New(cfg Config) (*Server, error) { IdleTimeout: cfg.HTTPServerIdleTimeout, Handler: middleware.Merge(httpMiddleware...).Wrap(router), } - if tlsConfig != nil { - httpServer.TLSConfig = tlsConfig + if httpTLSConfig != nil { + httpServer.TLSConfig = httpTLSConfig } return &Server{ @@ -302,7 +295,7 @@ func (s *Server) Run() error { if s.HTTPServer.TLSConfig == nil { err = s.HTTPServer.Serve(s.httpListener) } else { - err = s.HTTPServer.ServeTLS(s.httpListener, s.cfg.HTTPCertPath, s.cfg.HTTPKeyPath) + err = s.HTTPServer.ServeTLS(s.httpListener, s.cfg.HTTPTLSConfig.TLSCertPath, s.cfg.HTTPTLSConfig.TLSKeyPath) } if err == http.ErrServerClosed { err = nil diff --git a/vendor/go.etcd.io/etcd/auth/range_perm_cache.go b/vendor/go.etcd.io/etcd/auth/range_perm_cache.go index 8de2d175c2..7b6c182409 100644 --- a/vendor/go.etcd.io/etcd/auth/range_perm_cache.go +++ b/vendor/go.etcd.io/etcd/auth/range_perm_cache.go @@ -28,8 +28,8 @@ func getMergedPerms(lg *zap.Logger, tx backend.BatchTx, userName string) *unifie return nil } - readPerms := &adt.IntervalTree{} - writePerms := &adt.IntervalTree{} + readPerms := adt.NewIntervalTree() + writePerms := adt.NewIntervalTree() for _, roleName := range user.Roles { role := getRole(tx, roleName) @@ -148,6 +148,6 @@ func (as *authStore) invalidateCachedPerm(userName string) { } type unifiedRangePermissions struct { - readPerms *adt.IntervalTree - writePerms *adt.IntervalTree + readPerms adt.IntervalTree + writePerms adt.IntervalTree } diff --git a/vendor/go.etcd.io/etcd/auth/store.go b/vendor/go.etcd.io/etcd/auth/store.go index c90f477460..52122554a1 100644 --- a/vendor/go.etcd.io/etcd/auth/store.go +++ b/vendor/go.etcd.io/etcd/auth/store.go @@ -57,6 +57,7 @@ var ( ErrUserNotFound = errors.New("auth: user not found") ErrRoleAlreadyExist = errors.New("auth: role already exists") ErrRoleNotFound = errors.New("auth: role not found") + ErrRoleEmpty = errors.New("auth: role name is empty") ErrAuthFailed = errors.New("auth: authentication failed, invalid user ID or password") ErrPermissionDenied = errors.New("auth: permission denied") ErrRoleNotGranted = errors.New("auth: role is not granted to the user") @@ -387,7 +388,7 @@ func (as *authStore) UserAdd(r *pb.AuthUserAddRequest) (*pb.AuthUserAddResponse, var hashed []byte var err error - if !r.Options.NoPassword { + if r.Options != nil && !r.Options.NoPassword { hashed, err = bcrypt.GenerateFromPassword([]byte(r.Password), as.bcryptCost) if err != nil { if as.lg != nil { @@ -796,6 +797,10 @@ func (as *authStore) RoleDelete(r *pb.AuthRoleDeleteRequest) (*pb.AuthRoleDelete } func (as *authStore) RoleAdd(r *pb.AuthRoleAddRequest) (*pb.AuthRoleAddResponse, error) { + if len(r.Name) == 0 { + return nil, ErrRoleEmpty + } + tx := as.be.BatchTx() tx.Lock() defer tx.Unlock() diff --git a/vendor/go.etcd.io/etcd/clientv3/balancer/balancer.go b/vendor/go.etcd.io/etcd/clientv3/balancer/balancer.go index c39702ec47..d02a7eec7c 100644 --- a/vendor/go.etcd.io/etcd/clientv3/balancer/balancer.go +++ b/vendor/go.etcd.io/etcd/clientv3/balancer/balancer.go @@ -12,24 +12,45 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package balancer implements client balancer. package balancer import ( - "fmt" "strconv" "sync" "time" + "go.etcd.io/etcd/clientv3/balancer/connectivity" "go.etcd.io/etcd/clientv3/balancer/picker" "go.uber.org/zap" "google.golang.org/grpc/balancer" - "google.golang.org/grpc/connectivity" + grpcconnectivity "google.golang.org/grpc/connectivity" "google.golang.org/grpc/resolver" _ "google.golang.org/grpc/resolver/dns" // register DNS resolver _ "google.golang.org/grpc/resolver/passthrough" // register passthrough resolver ) +// Config defines balancer configurations. +type Config struct { + // Policy configures balancer policy. + Policy picker.Policy + + // Picker implements gRPC picker. + // Leave empty if "Policy" field is not custom. + // TODO: currently custom policy is not supported. + // Picker picker.Picker + + // Name defines an additional name for balancer. + // Useful for balancer testing to avoid register conflicts. + // If empty, defaults to policy name. + Name string + + // Logger configures balancer logging. + // If nil, logs are discarded. + Logger *zap.Logger +} + // RegisterBuilder creates and registers a builder. Since this function calls balancer.Register, it // must be invoked at initialization time. func RegisterBuilder(cfg Config) { @@ -59,16 +80,13 @@ func (b *builder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) balan addrToSc: make(map[resolver.Address]balancer.SubConn), scToAddr: make(map[balancer.SubConn]resolver.Address), - scToSt: make(map[balancer.SubConn]connectivity.State), + scToSt: make(map[balancer.SubConn]grpcconnectivity.State), - currentConn: nil, - csEvltr: &connectivityStateEvaluator{}, + currentConn: nil, + connectivityRecorder: connectivity.New(b.cfg.Logger), // initialize picker always returns "ErrNoSubConnAvailable" - Picker: picker.NewErr(balancer.ErrNoSubConnAvailable), - } - if bb.lg == nil { - bb.lg = zap.NewNop() + picker: picker.NewErr(balancer.ErrNoSubConnAvailable), } // TODO: support multiple connections @@ -112,13 +130,12 @@ type baseBalancer struct { addrToSc map[resolver.Address]balancer.SubConn scToAddr map[balancer.SubConn]resolver.Address - scToSt map[balancer.SubConn]connectivity.State + scToSt map[balancer.SubConn]grpcconnectivity.State - currentConn balancer.ClientConn - currentState connectivity.State - csEvltr *connectivityStateEvaluator + currentConn balancer.ClientConn + connectivityRecorder connectivity.Recorder - picker.Picker + picker picker.Picker } // HandleResolvedAddrs implements "grpc/balancer.Balancer" interface. @@ -128,7 +145,11 @@ func (bb *baseBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) bb.lg.Warn("HandleResolvedAddrs called with error", zap.String("balancer-id", bb.id), zap.Error(err)) return } - bb.lg.Info("resolved", zap.String("balancer-id", bb.id), zap.Strings("addresses", addrsToStrings(addrs))) + bb.lg.Info("resolved", + zap.String("picker", bb.picker.String()), + zap.String("balancer-id", bb.id), + zap.Strings("addresses", addrsToStrings(addrs)), + ) bb.mu.Lock() defer bb.mu.Unlock() @@ -139,12 +160,13 @@ func (bb *baseBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) if _, ok := bb.addrToSc[addr]; !ok { sc, err := bb.currentConn.NewSubConn([]resolver.Address{addr}, balancer.NewSubConnOptions{}) if err != nil { - bb.lg.Warn("NewSubConn failed", zap.String("balancer-id", bb.id), zap.Error(err), zap.String("address", addr.Addr)) + bb.lg.Warn("NewSubConn failed", zap.String("picker", bb.picker.String()), zap.String("balancer-id", bb.id), zap.Error(err), zap.String("address", addr.Addr)) continue } + bb.lg.Info("created subconn", zap.String("address", addr.Addr)) bb.addrToSc[addr] = sc bb.scToAddr[sc] = addr - bb.scToSt[sc] = connectivity.Idle + bb.scToSt[sc] = grpcconnectivity.Idle sc.Connect() } } @@ -157,6 +179,7 @@ func (bb *baseBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) bb.lg.Info( "removed subconn", + zap.String("picker", bb.picker.String()), zap.String("balancer-id", bb.id), zap.String("address", addr.Addr), zap.String("subconn", scToString(sc)), @@ -171,7 +194,7 @@ func (bb *baseBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) } // HandleSubConnStateChange implements "grpc/balancer.Balancer" interface. -func (bb *baseBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) { +func (bb *baseBalancer) HandleSubConnStateChange(sc balancer.SubConn, s grpcconnectivity.State) { bb.mu.Lock() defer bb.mu.Unlock() @@ -179,8 +202,10 @@ func (bb *baseBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connecti if !ok { bb.lg.Warn( "state change for an unknown subconn", + zap.String("picker", bb.picker.String()), zap.String("balancer-id", bb.id), zap.String("subconn", scToString(sc)), + zap.Int("subconn-size", len(bb.scToAddr)), zap.String("state", s.String()), ) return @@ -188,9 +213,11 @@ func (bb *baseBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connecti bb.lg.Info( "state changed", + zap.String("picker", bb.picker.String()), zap.String("balancer-id", bb.id), - zap.Bool("connected", s == connectivity.Ready), + zap.Bool("connected", s == grpcconnectivity.Ready), zap.String("subconn", scToString(sc)), + zap.Int("subconn-size", len(bb.scToAddr)), zap.String("address", bb.scToAddr[sc].Addr), zap.String("old-state", old.String()), zap.String("new-state", s.String()), @@ -198,68 +225,63 @@ func (bb *baseBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connecti bb.scToSt[sc] = s switch s { - case connectivity.Idle: + case grpcconnectivity.Idle: sc.Connect() - case connectivity.Shutdown: + case grpcconnectivity.Shutdown: // When an address was removed by resolver, b called RemoveSubConn but // kept the sc's state in scToSt. Remove state for this sc here. delete(bb.scToAddr, sc) delete(bb.scToSt, sc) } - oldAggrState := bb.currentState - bb.currentState = bb.csEvltr.recordTransition(old, s) + oldAggrState := bb.connectivityRecorder.GetCurrentState() + bb.connectivityRecorder.RecordTransition(old, s) - // Regenerate picker when one of the following happens: + // Update balancer picker when one of the following happens: // - this sc became ready from not-ready // - this sc became not-ready from ready // - the aggregated state of balancer became TransientFailure from non-TransientFailure // - the aggregated state of balancer became non-TransientFailure from TransientFailure - if (s == connectivity.Ready) != (old == connectivity.Ready) || - (bb.currentState == connectivity.TransientFailure) != (oldAggrState == connectivity.TransientFailure) { - bb.regeneratePicker() + if (s == grpcconnectivity.Ready) != (old == grpcconnectivity.Ready) || + (bb.connectivityRecorder.GetCurrentState() == grpcconnectivity.TransientFailure) != (oldAggrState == grpcconnectivity.TransientFailure) { + bb.updatePicker() } - bb.currentConn.UpdateBalancerState(bb.currentState, bb.Picker) + bb.currentConn.UpdateBalancerState(bb.connectivityRecorder.GetCurrentState(), bb.picker) } -func (bb *baseBalancer) regeneratePicker() { - if bb.currentState == connectivity.TransientFailure { +func (bb *baseBalancer) updatePicker() { + if bb.connectivityRecorder.GetCurrentState() == grpcconnectivity.TransientFailure { + bb.picker = picker.NewErr(balancer.ErrTransientFailure) bb.lg.Info( - "generated transient error picker", + "updated picker to transient error picker", + zap.String("picker", bb.picker.String()), zap.String("balancer-id", bb.id), zap.String("policy", bb.policy.String()), ) - bb.Picker = picker.NewErr(balancer.ErrTransientFailure) return } // only pass ready subconns to picker - scs := make([]balancer.SubConn, 0) - addrToSc := make(map[resolver.Address]balancer.SubConn) scToAddr := make(map[balancer.SubConn]resolver.Address) for addr, sc := range bb.addrToSc { - if st, ok := bb.scToSt[sc]; ok && st == connectivity.Ready { - scs = append(scs, sc) - addrToSc[addr] = sc + if st, ok := bb.scToSt[sc]; ok && st == grpcconnectivity.Ready { scToAddr[sc] = addr } } - switch bb.policy { - case picker.RoundrobinBalanced: - bb.Picker = picker.NewRoundrobinBalanced(bb.lg, scs, addrToSc, scToAddr) - - default: - panic(fmt.Errorf("invalid balancer picker policy (%d)", bb.policy)) - } - + bb.picker = picker.New(picker.Config{ + Policy: bb.policy, + Logger: bb.lg, + SubConnToResolverAddress: scToAddr, + }) bb.lg.Info( - "generated picker", + "updated picker", + zap.String("picker", bb.picker.String()), zap.String("balancer-id", bb.id), zap.String("policy", bb.policy.String()), - zap.Strings("subconn-ready", scsToStrings(addrToSc)), - zap.Int("subconn-size", len(addrToSc)), + zap.Strings("subconn-ready", scsToStrings(scToAddr)), + zap.Int("subconn-size", len(scToAddr)), ) } diff --git a/vendor/go.etcd.io/etcd/clientv3/balancer/config.go b/vendor/go.etcd.io/etcd/clientv3/balancer/config.go deleted file mode 100644 index 0339a84d08..0000000000 --- a/vendor/go.etcd.io/etcd/clientv3/balancer/config.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2018 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package balancer - -import ( - "go.etcd.io/etcd/clientv3/balancer/picker" - - "go.uber.org/zap" -) - -// Config defines balancer configurations. -type Config struct { - // Policy configures balancer policy. - Policy picker.Policy - - // Name defines an additional name for balancer. - // Useful for balancer testing to avoid register conflicts. - // If empty, defaults to policy name. - Name string - - // Logger configures balancer logging. - // If nil, logs are discarded. - Logger *zap.Logger -} diff --git a/vendor/go.etcd.io/etcd/clientv3/balancer/connectivity.go b/vendor/go.etcd.io/etcd/clientv3/balancer/connectivity.go deleted file mode 100644 index 6cdeb3fa3a..0000000000 --- a/vendor/go.etcd.io/etcd/clientv3/balancer/connectivity.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2018 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package balancer - -import "google.golang.org/grpc/connectivity" - -// connectivityStateEvaluator gets updated by addrConns when their -// states transition, based on which it evaluates the state of -// ClientConn. -type connectivityStateEvaluator struct { - numReady uint64 // Number of addrConns in ready state. - numConnecting uint64 // Number of addrConns in connecting state. - numTransientFailure uint64 // Number of addrConns in transientFailure. -} - -// recordTransition records state change happening in every subConn and based on -// that it evaluates what aggregated state should be. -// It can only transition between Ready, Connecting and TransientFailure. Other states, -// Idle and Shutdown are transitioned into by ClientConn; in the beginning of the connection -// before any subConn is created ClientConn is in idle state. In the end when ClientConn -// closes it is in Shutdown state. -// -// recordTransition should only be called synchronously from the same goroutine. -func (cse *connectivityStateEvaluator) recordTransition(oldState, newState connectivity.State) connectivity.State { - // Update counters. - for idx, state := range []connectivity.State{oldState, newState} { - updateVal := 2*uint64(idx) - 1 // -1 for oldState and +1 for new. - switch state { - case connectivity.Ready: - cse.numReady += updateVal - case connectivity.Connecting: - cse.numConnecting += updateVal - case connectivity.TransientFailure: - cse.numTransientFailure += updateVal - } - } - - // Evaluate. - if cse.numReady > 0 { - return connectivity.Ready - } - if cse.numConnecting > 0 { - return connectivity.Connecting - } - return connectivity.TransientFailure -} diff --git a/vendor/go.etcd.io/etcd/clientv3/balancer/connectivity/connectivity.go b/vendor/go.etcd.io/etcd/clientv3/balancer/connectivity/connectivity.go new file mode 100644 index 0000000000..4c4ad363a7 --- /dev/null +++ b/vendor/go.etcd.io/etcd/clientv3/balancer/connectivity/connectivity.go @@ -0,0 +1,93 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package connectivity implements client connectivity operations. +package connectivity + +import ( + "sync" + + "go.uber.org/zap" + "google.golang.org/grpc/connectivity" +) + +// Recorder records gRPC connectivity. +type Recorder interface { + GetCurrentState() connectivity.State + RecordTransition(oldState, newState connectivity.State) +} + +// New returns a new Recorder. +func New(lg *zap.Logger) Recorder { + return &recorder{lg: lg} +} + +// recorder takes the connectivity states of multiple SubConns +// and returns one aggregated connectivity state. +// ref. https://github.com/grpc/grpc-go/blob/master/balancer/balancer.go +type recorder struct { + lg *zap.Logger + + mu sync.RWMutex + + cur connectivity.State + + numReady uint64 // Number of addrConns in ready state. + numConnecting uint64 // Number of addrConns in connecting state. + numTransientFailure uint64 // Number of addrConns in transientFailure. +} + +func (rc *recorder) GetCurrentState() (state connectivity.State) { + rc.mu.RLock() + defer rc.mu.RUnlock() + return rc.cur +} + +// RecordTransition records state change happening in subConn and based on that +// it evaluates what aggregated state should be. +// +// - If at least one SubConn in Ready, the aggregated state is Ready; +// - Else if at least one SubConn in Connecting, the aggregated state is Connecting; +// - Else the aggregated state is TransientFailure. +// +// Idle and Shutdown are not considered. +// +// ref. https://github.com/grpc/grpc-go/blob/master/balancer/balancer.go +func (rc *recorder) RecordTransition(oldState, newState connectivity.State) { + rc.mu.Lock() + defer rc.mu.Unlock() + + for idx, state := range []connectivity.State{oldState, newState} { + updateVal := 2*uint64(idx) - 1 // -1 for oldState and +1 for new. + switch state { + case connectivity.Ready: + rc.numReady += updateVal + case connectivity.Connecting: + rc.numConnecting += updateVal + case connectivity.TransientFailure: + rc.numTransientFailure += updateVal + default: + rc.lg.Warn("connectivity recorder received unknown state", zap.String("connectivity-state", state.String())) + } + } + + switch { // must be exclusive, no overlap + case rc.numReady > 0: + rc.cur = connectivity.Ready + case rc.numConnecting > 0: + rc.cur = connectivity.Connecting + default: + rc.cur = connectivity.TransientFailure + } +} diff --git a/vendor/go.etcd.io/etcd/clientv3/balancer/doc.go b/vendor/go.etcd.io/etcd/clientv3/balancer/doc.go deleted file mode 100644 index 45af5e9d10..0000000000 --- a/vendor/go.etcd.io/etcd/clientv3/balancer/doc.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2018 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package balancer implements client balancer. -package balancer diff --git a/vendor/go.etcd.io/etcd/clientv3/balancer/grpc1.7-health.go b/vendor/go.etcd.io/etcd/clientv3/balancer/grpc1.7-health.go deleted file mode 100644 index 2153767354..0000000000 --- a/vendor/go.etcd.io/etcd/clientv3/balancer/grpc1.7-health.go +++ /dev/null @@ -1,657 +0,0 @@ -// Copyright 2018 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package balancer - -import ( - "context" - "errors" - "io/ioutil" - "net/url" - "strings" - "sync" - "time" - - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/grpclog" - healthpb "google.golang.org/grpc/health/grpc_health_v1" - "google.golang.org/grpc/status" -) - -// TODO: replace with something better -var lg = grpclog.NewLoggerV2(ioutil.Discard, ioutil.Discard, ioutil.Discard) - -const ( - minHealthRetryDuration = 3 * time.Second - unknownService = "unknown service grpc.health.v1.Health" -) - -// ErrNoAddrAvailable is returned by Get() when the balancer does not have -// any active connection to endpoints at the time. -// This error is returned only when opts.BlockingWait is true. -var ErrNoAddrAvailable = status.Error(codes.Unavailable, "there is no address available") - -type NotifyMsg int - -const ( - NotifyReset NotifyMsg = iota - NotifyNext -) - -// GRPC17Health does the bare minimum to expose multiple eps -// to the grpc reconnection code path -type GRPC17Health struct { - // addrs are the client's endpoint addresses for grpc - addrs []grpc.Address - - // eps holds the raw endpoints from the client - eps []string - - // notifyCh notifies grpc of the set of addresses for connecting - notifyCh chan []grpc.Address - - // readyc closes once the first connection is up - readyc chan struct{} - readyOnce sync.Once - - // healthCheck checks an endpoint's health. - healthCheck func(ep string) (bool, error) - healthCheckTimeout time.Duration - - unhealthyMu sync.RWMutex - unhealthyHostPorts map[string]time.Time - - // mu protects all fields below. - mu sync.RWMutex - - // upc closes when pinAddr transitions from empty to non-empty or the balancer closes. - upc chan struct{} - - // downc closes when grpc calls down() on pinAddr - downc chan struct{} - - // stopc is closed to signal updateNotifyLoop should stop. - stopc chan struct{} - stopOnce sync.Once - wg sync.WaitGroup - - // donec closes when all goroutines are exited - donec chan struct{} - - // updateAddrsC notifies updateNotifyLoop to update addrs. - updateAddrsC chan NotifyMsg - - // grpc issues TLS cert checks using the string passed into dial so - // that string must be the host. To recover the full scheme://host URL, - // have a map from hosts to the original endpoint. - hostPort2ep map[string]string - - // pinAddr is the currently pinned address; set to the empty string on - // initialization and shutdown. - pinAddr string - - closed bool -} - -// DialFunc defines gRPC dial function. -type DialFunc func(ep string, dopts ...grpc.DialOption) (*grpc.ClientConn, error) - -// NewGRPC17Health returns a new health balancer with gRPC v1.7. -func NewGRPC17Health( - eps []string, - timeout time.Duration, - dialFunc DialFunc, -) *GRPC17Health { - notifyCh := make(chan []grpc.Address) - addrs := eps2addrs(eps) - hb := &GRPC17Health{ - addrs: addrs, - eps: eps, - notifyCh: notifyCh, - readyc: make(chan struct{}), - healthCheck: func(ep string) (bool, error) { return grpcHealthCheck(ep, dialFunc) }, - unhealthyHostPorts: make(map[string]time.Time), - upc: make(chan struct{}), - stopc: make(chan struct{}), - downc: make(chan struct{}), - donec: make(chan struct{}), - updateAddrsC: make(chan NotifyMsg), - hostPort2ep: getHostPort2ep(eps), - } - if timeout < minHealthRetryDuration { - timeout = minHealthRetryDuration - } - hb.healthCheckTimeout = timeout - - close(hb.downc) - go hb.updateNotifyLoop() - hb.wg.Add(1) - go func() { - defer hb.wg.Done() - hb.updateUnhealthy() - }() - return hb -} - -func (b *GRPC17Health) Start(target string, config grpc.BalancerConfig) error { return nil } - -func (b *GRPC17Health) ConnectNotify() <-chan struct{} { - b.mu.Lock() - defer b.mu.Unlock() - return b.upc -} - -func (b *GRPC17Health) UpdateAddrsC() chan NotifyMsg { return b.updateAddrsC } -func (b *GRPC17Health) StopC() chan struct{} { return b.stopc } - -func (b *GRPC17Health) Ready() <-chan struct{} { return b.readyc } - -func (b *GRPC17Health) Endpoint(hostPort string) string { - b.mu.RLock() - defer b.mu.RUnlock() - return b.hostPort2ep[hostPort] -} - -func (b *GRPC17Health) Pinned() string { - b.mu.RLock() - defer b.mu.RUnlock() - return b.pinAddr -} - -func (b *GRPC17Health) HostPortError(hostPort string, err error) { - if b.Endpoint(hostPort) == "" { - lg.Infof("clientv3/balancer: %q is stale (skip marking as unhealthy on %q)", hostPort, err.Error()) - return - } - - b.unhealthyMu.Lock() - b.unhealthyHostPorts[hostPort] = time.Now() - b.unhealthyMu.Unlock() - lg.Infof("clientv3/balancer: %q is marked unhealthy (%q)", hostPort, err.Error()) -} - -func (b *GRPC17Health) removeUnhealthy(hostPort, msg string) { - if b.Endpoint(hostPort) == "" { - lg.Infof("clientv3/balancer: %q was not in unhealthy (%q)", hostPort, msg) - return - } - - b.unhealthyMu.Lock() - delete(b.unhealthyHostPorts, hostPort) - b.unhealthyMu.Unlock() - lg.Infof("clientv3/balancer: %q is removed from unhealthy (%q)", hostPort, msg) -} - -func (b *GRPC17Health) countUnhealthy() (count int) { - b.unhealthyMu.RLock() - count = len(b.unhealthyHostPorts) - b.unhealthyMu.RUnlock() - return count -} - -func (b *GRPC17Health) isUnhealthy(hostPort string) (unhealthy bool) { - b.unhealthyMu.RLock() - _, unhealthy = b.unhealthyHostPorts[hostPort] - b.unhealthyMu.RUnlock() - return unhealthy -} - -func (b *GRPC17Health) cleanupUnhealthy() { - b.unhealthyMu.Lock() - for k, v := range b.unhealthyHostPorts { - if time.Since(v) > b.healthCheckTimeout { - delete(b.unhealthyHostPorts, k) - lg.Infof("clientv3/balancer: removed %q from unhealthy after %v", k, b.healthCheckTimeout) - } - } - b.unhealthyMu.Unlock() -} - -func (b *GRPC17Health) liveAddrs() ([]grpc.Address, map[string]struct{}) { - unhealthyCnt := b.countUnhealthy() - - b.mu.RLock() - defer b.mu.RUnlock() - - hbAddrs := b.addrs - if len(b.addrs) == 1 || unhealthyCnt == 0 || unhealthyCnt == len(b.addrs) { - liveHostPorts := make(map[string]struct{}, len(b.hostPort2ep)) - for k := range b.hostPort2ep { - liveHostPorts[k] = struct{}{} - } - return hbAddrs, liveHostPorts - } - - addrs := make([]grpc.Address, 0, len(b.addrs)-unhealthyCnt) - liveHostPorts := make(map[string]struct{}, len(addrs)) - for _, addr := range b.addrs { - if !b.isUnhealthy(addr.Addr) { - addrs = append(addrs, addr) - liveHostPorts[addr.Addr] = struct{}{} - } - } - return addrs, liveHostPorts -} - -func (b *GRPC17Health) updateUnhealthy() { - for { - select { - case <-time.After(b.healthCheckTimeout): - b.cleanupUnhealthy() - pinned := b.Pinned() - if pinned == "" || b.isUnhealthy(pinned) { - select { - case b.updateAddrsC <- NotifyNext: - case <-b.stopc: - return - } - } - case <-b.stopc: - return - } - } -} - -// NeedUpdate returns true if all connections are down or -// addresses do not include current pinned address. -func (b *GRPC17Health) NeedUpdate() bool { - // updating notifyCh can trigger new connections, - // need update addrs if all connections are down - // or addrs does not include pinAddr. - b.mu.RLock() - update := !hasAddr(b.addrs, b.pinAddr) - b.mu.RUnlock() - return update -} - -func (b *GRPC17Health) UpdateAddrs(eps ...string) { - np := getHostPort2ep(eps) - - b.mu.Lock() - defer b.mu.Unlock() - - match := len(np) == len(b.hostPort2ep) - if match { - for k, v := range np { - if b.hostPort2ep[k] != v { - match = false - break - } - } - } - if match { - // same endpoints, so no need to update address - return - } - - b.hostPort2ep = np - b.addrs, b.eps = eps2addrs(eps), eps - - b.unhealthyMu.Lock() - b.unhealthyHostPorts = make(map[string]time.Time) - b.unhealthyMu.Unlock() -} - -func (b *GRPC17Health) Next() { - b.mu.RLock() - downc := b.downc - b.mu.RUnlock() - select { - case b.updateAddrsC <- NotifyNext: - case <-b.stopc: - } - // wait until disconnect so new RPCs are not issued on old connection - select { - case <-downc: - case <-b.stopc: - } -} - -func (b *GRPC17Health) updateNotifyLoop() { - defer close(b.donec) - - for { - b.mu.RLock() - upc, downc, addr := b.upc, b.downc, b.pinAddr - b.mu.RUnlock() - // downc or upc should be closed - select { - case <-downc: - downc = nil - default: - } - select { - case <-upc: - upc = nil - default: - } - switch { - case downc == nil && upc == nil: - // stale - select { - case <-b.stopc: - return - default: - } - case downc == nil: - b.notifyAddrs(NotifyReset) - select { - case <-upc: - case msg := <-b.updateAddrsC: - b.notifyAddrs(msg) - case <-b.stopc: - return - } - case upc == nil: - select { - // close connections that are not the pinned address - case b.notifyCh <- []grpc.Address{{Addr: addr}}: - case <-downc: - case <-b.stopc: - return - } - select { - case <-downc: - b.notifyAddrs(NotifyReset) - case msg := <-b.updateAddrsC: - b.notifyAddrs(msg) - case <-b.stopc: - return - } - } - } -} - -func (b *GRPC17Health) notifyAddrs(msg NotifyMsg) { - if msg == NotifyNext { - select { - case b.notifyCh <- []grpc.Address{}: - case <-b.stopc: - return - } - } - b.mu.RLock() - pinAddr := b.pinAddr - downc := b.downc - b.mu.RUnlock() - addrs, hostPorts := b.liveAddrs() - - var waitDown bool - if pinAddr != "" { - _, ok := hostPorts[pinAddr] - waitDown = !ok - } - - select { - case b.notifyCh <- addrs: - if waitDown { - select { - case <-downc: - case <-b.stopc: - } - } - case <-b.stopc: - } -} - -func (b *GRPC17Health) Up(addr grpc.Address) func(error) { - if !b.mayPin(addr) { - return func(err error) {} - } - - b.mu.Lock() - defer b.mu.Unlock() - - // gRPC might call Up after it called Close. We add this check - // to "fix" it up at application layer. Otherwise, will panic - // if b.upc is already closed. - if b.closed { - return func(err error) {} - } - - // gRPC might call Up on a stale address. - // Prevent updating pinAddr with a stale address. - if !hasAddr(b.addrs, addr.Addr) { - return func(err error) {} - } - - if b.pinAddr != "" { - lg.Infof("clientv3/balancer: %q is up but not pinned (already pinned %q)", addr.Addr, b.pinAddr) - return func(err error) {} - } - - // notify waiting Get()s and pin first connected address - close(b.upc) - b.downc = make(chan struct{}) - b.pinAddr = addr.Addr - lg.Infof("clientv3/balancer: pin %q", addr.Addr) - - // notify client that a connection is up - b.readyOnce.Do(func() { close(b.readyc) }) - - return func(err error) { - // If connected to a black hole endpoint or a killed server, the gRPC ping - // timeout will induce a network I/O error, and retrying until success; - // finding healthy endpoint on retry could take several timeouts and redials. - // To avoid wasting retries, gray-list unhealthy endpoints. - b.HostPortError(addr.Addr, err) - - b.mu.Lock() - b.upc = make(chan struct{}) - close(b.downc) - b.pinAddr = "" - b.mu.Unlock() - lg.Infof("clientv3/balancer: unpin %q (%q)", addr.Addr, err.Error()) - } -} - -func (b *GRPC17Health) mayPin(addr grpc.Address) bool { - if b.Endpoint(addr.Addr) == "" { // stale host:port - return false - } - - b.unhealthyMu.RLock() - unhealthyCnt := len(b.unhealthyHostPorts) - failedTime, bad := b.unhealthyHostPorts[addr.Addr] - b.unhealthyMu.RUnlock() - - b.mu.RLock() - skip := len(b.addrs) == 1 || unhealthyCnt == 0 || len(b.addrs) == unhealthyCnt - b.mu.RUnlock() - if skip || !bad { - return true - } - - // prevent isolated member's endpoint from being infinitely retried, as follows: - // 1. keepalive pings detects GoAway with http2.ErrCodeEnhanceYourCalm - // 2. balancer 'Up' unpins with grpc: failed with network I/O error - // 3. grpc-healthcheck still SERVING, thus retry to pin - // instead, return before grpc-healthcheck if failed within healthcheck timeout - if elapsed := time.Since(failedTime); elapsed < b.healthCheckTimeout { - lg.Infof("clientv3/balancer: %q is up but not pinned (failed %v ago, require minimum %v after failure)", addr.Addr, elapsed, b.healthCheckTimeout) - return false - } - - if ok, _ := b.healthCheck(addr.Addr); ok { - b.removeUnhealthy(addr.Addr, "health check success") - return true - } - - b.HostPortError(addr.Addr, errors.New("health check failed")) - return false -} - -func (b *GRPC17Health) Get(ctx context.Context, opts grpc.BalancerGetOptions) (grpc.Address, func(), error) { - var ( - addr string - closed bool - ) - - // If opts.BlockingWait is false (for fail-fast RPCs), it should return - // an address it has notified via Notify immediately instead of blocking. - if !opts.BlockingWait { - b.mu.RLock() - closed = b.closed - addr = b.pinAddr - b.mu.RUnlock() - if closed { - return grpc.Address{Addr: ""}, nil, grpc.ErrClientConnClosing - } - if addr == "" { - return grpc.Address{Addr: ""}, nil, ErrNoAddrAvailable - } - return grpc.Address{Addr: addr}, func() {}, nil - } - - for { - b.mu.RLock() - ch := b.upc - b.mu.RUnlock() - select { - case <-ch: - case <-b.donec: - return grpc.Address{Addr: ""}, nil, grpc.ErrClientConnClosing - case <-ctx.Done(): - return grpc.Address{Addr: ""}, nil, ctx.Err() - } - b.mu.RLock() - closed = b.closed - addr = b.pinAddr - b.mu.RUnlock() - // Close() which sets b.closed = true can be called before Get(), Get() must exit if balancer is closed. - if closed { - return grpc.Address{Addr: ""}, nil, grpc.ErrClientConnClosing - } - if addr != "" { - break - } - } - return grpc.Address{Addr: addr}, func() {}, nil -} - -func (b *GRPC17Health) Notify() <-chan []grpc.Address { return b.notifyCh } - -func (b *GRPC17Health) Close() error { - b.mu.Lock() - // In case gRPC calls close twice. TODO: remove the checking - // when we are sure that gRPC wont call close twice. - if b.closed { - b.mu.Unlock() - <-b.donec - return nil - } - b.closed = true - b.stopOnce.Do(func() { close(b.stopc) }) - b.pinAddr = "" - - // In the case of following scenario: - // 1. upc is not closed; no pinned address - // 2. client issues an RPC, calling invoke(), which calls Get(), enters for loop, blocks - // 3. client.conn.Close() calls balancer.Close(); closed = true - // 4. for loop in Get() never exits since ctx is the context passed in by the client and may not be canceled - // we must close upc so Get() exits from blocking on upc - select { - case <-b.upc: - default: - // terminate all waiting Get()s - close(b.upc) - } - - b.mu.Unlock() - b.wg.Wait() - - // wait for updateNotifyLoop to finish - <-b.donec - close(b.notifyCh) - - return nil -} - -func grpcHealthCheck(ep string, dialFunc func(ep string, dopts ...grpc.DialOption) (*grpc.ClientConn, error)) (bool, error) { - conn, err := dialFunc(ep) - if err != nil { - return false, err - } - defer conn.Close() - cli := healthpb.NewHealthClient(conn) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - resp, err := cli.Check(ctx, &healthpb.HealthCheckRequest{}) - cancel() - if err != nil { - if s, ok := status.FromError(err); ok && s.Code() == codes.Unavailable { - if s.Message() == unknownService { // etcd < v3.3.0 - return true, nil - } - } - return false, err - } - return resp.Status == healthpb.HealthCheckResponse_SERVING, nil -} - -func hasAddr(addrs []grpc.Address, targetAddr string) bool { - for _, addr := range addrs { - if targetAddr == addr.Addr { - return true - } - } - return false -} - -func getHost(ep string) string { - url, uerr := url.Parse(ep) - if uerr != nil || !strings.Contains(ep, "://") { - return ep - } - return url.Host -} - -func eps2addrs(eps []string) []grpc.Address { - addrs := make([]grpc.Address, len(eps)) - for i := range eps { - addrs[i].Addr = getHost(eps[i]) - } - return addrs -} - -func getHostPort2ep(eps []string) map[string]string { - hm := make(map[string]string, len(eps)) - for i := range eps { - _, host, _ := parseEndpoint(eps[i]) - hm[host] = eps[i] - } - return hm -} - -func parseEndpoint(endpoint string) (proto string, host string, scheme string) { - proto = "tcp" - host = endpoint - url, uerr := url.Parse(endpoint) - if uerr != nil || !strings.Contains(endpoint, "://") { - return proto, host, scheme - } - scheme = url.Scheme - - // strip scheme:// prefix since grpc dials by host - host = url.Host - switch url.Scheme { - case "http", "https": - case "unix", "unixs": - proto = "unix" - host = url.Host + url.Path - default: - proto, host = "", "" - } - return proto, host, scheme -} diff --git a/vendor/go.etcd.io/etcd/clientv3/balancer/picker/err.go b/vendor/go.etcd.io/etcd/clientv3/balancer/picker/err.go index c70ce158b6..9e043789c8 100644 --- a/vendor/go.etcd.io/etcd/clientv3/balancer/picker/err.go +++ b/vendor/go.etcd.io/etcd/clientv3/balancer/picker/err.go @@ -22,13 +22,18 @@ import ( // NewErr returns a picker that always returns err on "Pick". func NewErr(err error) Picker { - return &errPicker{err: err} + return &errPicker{p: Error, err: err} } type errPicker struct { + p Policy err error } -func (p *errPicker) Pick(context.Context, balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) { - return nil, nil, p.err +func (ep *errPicker) String() string { + return ep.p.String() +} + +func (ep *errPicker) Pick(context.Context, balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) { + return nil, nil, ep.err } diff --git a/vendor/go.etcd.io/etcd/clientv3/balancer/picker/picker.go b/vendor/go.etcd.io/etcd/clientv3/balancer/picker/picker.go index 7ea761bdb5..bd1a5d25e8 100644 --- a/vendor/go.etcd.io/etcd/clientv3/balancer/picker/picker.go +++ b/vendor/go.etcd.io/etcd/clientv3/balancer/picker/picker.go @@ -15,10 +15,77 @@ package picker import ( + "fmt" + + "go.uber.org/zap" "google.golang.org/grpc/balancer" + "google.golang.org/grpc/resolver" ) // Picker defines balancer Picker methods. type Picker interface { balancer.Picker + String() string +} + +// Config defines picker configuration. +type Config struct { + // Policy specifies etcd clientv3's built in balancer policy. + Policy Policy + + // Logger defines picker logging object. + Logger *zap.Logger + + // SubConnToResolverAddress maps each gRPC sub-connection to an address. + // Basically, it is a list of addresses that the Picker can pick from. + SubConnToResolverAddress map[balancer.SubConn]resolver.Address +} + +// Policy defines balancer picker policy. +type Policy uint8 + +const ( + // Error is error picker policy. + Error Policy = iota + + // RoundrobinBalanced balances loads over multiple endpoints + // and implements failover in roundrobin fashion. + RoundrobinBalanced + + // Custom defines custom balancer picker. + // TODO: custom picker is not supported yet. + Custom +) + +func (p Policy) String() string { + switch p { + case Error: + return "picker-error" + + case RoundrobinBalanced: + return "picker-roundrobin-balanced" + + case Custom: + panic("'custom' picker policy is not supported yet") + + default: + panic(fmt.Errorf("invalid balancer picker policy (%d)", p)) + } +} + +// New creates a new Picker. +func New(cfg Config) Picker { + switch cfg.Policy { + case Error: + panic("'error' picker policy is not supported here; use 'picker.NewErr'") + + case RoundrobinBalanced: + return newRoundrobinBalanced(cfg) + + case Custom: + panic("'custom' picker policy is not supported yet") + + default: + panic(fmt.Errorf("invalid balancer picker policy (%d)", cfg.Policy)) + } } diff --git a/vendor/go.etcd.io/etcd/clientv3/balancer/picker/picker_policy.go b/vendor/go.etcd.io/etcd/clientv3/balancer/picker/picker_policy.go deleted file mode 100644 index 7bca39cb1e..0000000000 --- a/vendor/go.etcd.io/etcd/clientv3/balancer/picker/picker_policy.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2018 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package picker - -import "fmt" - -// Policy defines balancer picker policy. -type Policy uint8 - -const ( - // TODO: custom picker is not supported yet. - // custom defines custom balancer picker. - custom Policy = iota - - // RoundrobinBalanced balance loads over multiple endpoints - // and implements failover in roundrobin fashion. - RoundrobinBalanced Policy = iota - - // TODO: only send loads to pinned address "RoundrobinFailover" - // just like how 3.3 client works - // - // TODO: prioritize leader - // TODO: health-check - // TODO: weighted roundrobin - // TODO: power of two random choice -) - -func (p Policy) String() string { - switch p { - case custom: - panic("'custom' picker policy is not supported yet") - case RoundrobinBalanced: - return "etcd-client-roundrobin-balanced" - default: - panic(fmt.Errorf("invalid balancer picker policy (%d)", p)) - } -} diff --git a/vendor/go.etcd.io/etcd/clientv3/balancer/picker/roundrobin_balanced.go b/vendor/go.etcd.io/etcd/clientv3/balancer/picker/roundrobin_balanced.go index b043d572dd..1b8b285737 100644 --- a/vendor/go.etcd.io/etcd/clientv3/balancer/picker/roundrobin_balanced.go +++ b/vendor/go.etcd.io/etcd/clientv3/balancer/picker/roundrobin_balanced.go @@ -24,32 +24,33 @@ import ( "google.golang.org/grpc/resolver" ) -// NewRoundrobinBalanced returns a new roundrobin balanced picker. -func NewRoundrobinBalanced( - lg *zap.Logger, - scs []balancer.SubConn, - addrToSc map[resolver.Address]balancer.SubConn, - scToAddr map[balancer.SubConn]resolver.Address, -) Picker { +// newRoundrobinBalanced returns a new roundrobin balanced picker. +func newRoundrobinBalanced(cfg Config) Picker { + scs := make([]balancer.SubConn, 0, len(cfg.SubConnToResolverAddress)) + for sc := range cfg.SubConnToResolverAddress { + scs = append(scs, sc) + } return &rrBalanced{ - lg: lg, + p: RoundrobinBalanced, + lg: cfg.Logger, scs: scs, - addrToSc: addrToSc, - scToAddr: scToAddr, + scToAddr: cfg.SubConnToResolverAddress, } } type rrBalanced struct { - lg *zap.Logger + p Policy - mu sync.RWMutex - next int - scs []balancer.SubConn + lg *zap.Logger - addrToSc map[resolver.Address]balancer.SubConn + mu sync.RWMutex + next int + scs []balancer.SubConn scToAddr map[balancer.SubConn]resolver.Address } +func (rb *rrBalanced) String() string { return rb.p.String() } + // Pick is called for every client request. func (rb *rrBalanced) Pick(ctx context.Context, opts balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) { rb.mu.RLock() @@ -68,6 +69,7 @@ func (rb *rrBalanced) Pick(ctx context.Context, opts balancer.PickOptions) (bala rb.lg.Debug( "picked", + zap.String("picker", rb.p.String()), zap.String("address", picked), zap.Int("subconn-index", cur), zap.Int("subconn-size", n), @@ -77,6 +79,7 @@ func (rb *rrBalanced) Pick(ctx context.Context, opts balancer.PickOptions) (bala // TODO: error handling? fss := []zapcore.Field{ zap.Error(info.Err), + zap.String("picker", rb.p.String()), zap.String("address", picked), zap.Bool("success", info.Err == nil), zap.Bool("bytes-sent", info.BytesSent), diff --git a/vendor/go.etcd.io/etcd/clientv3/balancer/resolver/endpoint/endpoint.go b/vendor/go.etcd.io/etcd/clientv3/balancer/resolver/endpoint/endpoint.go index 1f32039e37..864b5df642 100644 --- a/vendor/go.etcd.io/etcd/clientv3/balancer/resolver/endpoint/endpoint.go +++ b/vendor/go.etcd.io/etcd/clientv3/balancer/resolver/endpoint/endpoint.go @@ -16,7 +16,9 @@ package endpoint import ( + "context" "fmt" + "net" "net/url" "strings" "sync" @@ -228,13 +230,18 @@ func ParseTarget(target string) (string, string, error) { return parts[0], parts[1], nil } -// ParseHostPort splits a ":" string into the host and port parts. -// The port part is optional. -func ParseHostPort(hostPort string) (host string, port string) { - parts := strings.SplitN(hostPort, ":", 2) - host = parts[0] - if len(parts) > 1 { - port = parts[1] +// Dialer dials a endpoint using net.Dialer. +// Context cancelation and timeout are supported. +func Dialer(ctx context.Context, dialEp string) (net.Conn, error) { + proto, host, _ := ParseEndpoint(dialEp) + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + dialer := &net.Dialer{} + if deadline, ok := ctx.Deadline(); ok { + dialer.Deadline = deadline } - return host, port + return dialer.DialContext(ctx, proto, host) } diff --git a/vendor/go.etcd.io/etcd/clientv3/balancer/utils.go b/vendor/go.etcd.io/etcd/clientv3/balancer/utils.go index a11faeb7e6..48eb875074 100644 --- a/vendor/go.etcd.io/etcd/clientv3/balancer/utils.go +++ b/vendor/go.etcd.io/etcd/clientv3/balancer/utils.go @@ -29,9 +29,9 @@ func scToString(sc balancer.SubConn) string { return fmt.Sprintf("%p", sc) } -func scsToStrings(scs map[resolver.Address]balancer.SubConn) (ss []string) { +func scsToStrings(scs map[balancer.SubConn]resolver.Address) (ss []string) { ss = make([]string, 0, len(scs)) - for a, sc := range scs { + for sc, a := range scs { ss = append(ss, fmt.Sprintf("%s (%s)", a.Addr, scToString(sc))) } sort.Strings(ss) diff --git a/vendor/go.etcd.io/etcd/clientv3/client.go b/vendor/go.etcd.io/etcd/clientv3/client.go index b91cbf958e..215e054798 100644 --- a/vendor/go.etcd.io/etcd/clientv3/client.go +++ b/vendor/go.etcd.io/etcd/clientv3/client.go @@ -16,7 +16,6 @@ package clientv3 import ( "context" - "crypto/tls" "errors" "fmt" "net" @@ -30,12 +29,13 @@ import ( "go.etcd.io/etcd/clientv3/balancer" "go.etcd.io/etcd/clientv3/balancer/picker" "go.etcd.io/etcd/clientv3/balancer/resolver/endpoint" + "go.etcd.io/etcd/clientv3/credentials" "go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes" "go.etcd.io/etcd/pkg/logutil" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials" + grpccredentials "google.golang.org/grpc/credentials" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" @@ -51,12 +51,17 @@ var ( func init() { lg := zap.NewNop() if os.Getenv("ETCD_CLIENT_DEBUG") != "" { + lcfg := logutil.DefaultZapLoggerConfig + lcfg.Level = zap.NewAtomicLevelAt(zap.DebugLevel) + var err error - lg, err = zap.NewProductionConfig().Build() // info level logging + lg, err = lcfg.Build() // info level logging if err != nil { panic(err) } } + + // TODO: support custom balancer balancer.RegisterBuilder(balancer.Config{ Policy: picker.RoundrobinBalanced, Name: roundRobinBalancerName, @@ -76,7 +81,7 @@ type Client struct { conn *grpc.ClientConn cfg Config - creds *credentials.TransportCredentials + creds grpccredentials.TransportCredentials resolverGroup *endpoint.ResolverGroup mu *sync.RWMutex @@ -86,9 +91,8 @@ type Client struct { // Username is a user name for authentication. Username string // Password is a password for authentication. - Password string - // tokenCred is an instance of WithPerRPCCredentials()'s argument - tokenCred *authTokenCredential + Password string + authTokenBundle credentials.Bundle callOpts []grpc.CallOption @@ -125,8 +129,12 @@ func NewFromURLs(urls []string) (*Client, error) { // Close shuts down the client's etcd connections. func (c *Client) Close() error { c.cancel() - c.Watcher.Close() - c.Lease.Close() + if c.Watcher != nil { + c.Watcher.Close() + } + if c.Lease != nil { + c.Lease.Close() + } if c.resolverGroup != nil { c.resolverGroup.Close() } @@ -193,24 +201,7 @@ func (c *Client) autoSync() { } } -type authTokenCredential struct { - token string - tokenMu *sync.RWMutex -} - -func (cred authTokenCredential) RequireTransportSecurity() bool { - return false -} - -func (cred authTokenCredential) GetRequestMetadata(ctx context.Context, s ...string) (map[string]string, error) { - cred.tokenMu.RLock() - defer cred.tokenMu.RUnlock() - return map[string]string{ - rpctypes.TokenFieldNameGRPC: cred.token, - }, nil -} - -func (c *Client) processCreds(scheme string) (creds *credentials.TransportCredentials) { +func (c *Client) processCreds(scheme string) (creds grpccredentials.TransportCredentials) { creds = c.creds switch scheme { case "unix": @@ -220,9 +211,7 @@ func (c *Client) processCreds(scheme string) (creds *credentials.TransportCreden if creds != nil { break } - tlsconfig := &tls.Config{} - emptyCreds := credentials.NewTLS(tlsconfig) - creds = &emptyCreds + creds = credentials.NewBundle(credentials.Config{}).TransportCredentials() default: creds = nil } @@ -230,7 +219,7 @@ func (c *Client) processCreds(scheme string) (creds *credentials.TransportCreden } // dialSetupOpts gives the dial opts prior to any authentication. -func (c *Client) dialSetupOpts(creds *credentials.TransportCredentials, dopts ...grpc.DialOption) (opts []grpc.DialOption, err error) { +func (c *Client) dialSetupOpts(creds grpccredentials.TransportCredentials, dopts ...grpc.DialOption) (opts []grpc.DialOption, err error) { if c.cfg.DialKeepAliveTime > 0 { params := keepalive.ClientParameters{ Time: c.cfg.DialKeepAliveTime, @@ -241,24 +230,17 @@ func (c *Client) dialSetupOpts(creds *credentials.TransportCredentials, dopts .. } opts = append(opts, dopts...) - // Provide a net dialer that supports cancelation and timeout. - f := func(dialEp string, t time.Duration) (net.Conn, error) { - proto, host, _ := endpoint.ParseEndpoint(dialEp) - select { - case <-c.ctx.Done(): - return nil, c.ctx.Err() - default: - } - dialer := &net.Dialer{Timeout: t} - return dialer.DialContext(c.ctx, proto, host) - } - opts = append(opts, grpc.WithDialer(f)) - + dialer := endpoint.Dialer if creds != nil { - opts = append(opts, grpc.WithTransportCredentials(*creds)) + opts = append(opts, grpc.WithTransportCredentials(creds)) + // gRPC load balancer workaround. See credentials.transportCredential for details. + if credsDialer, ok := creds.(TransportCredentialsWithDialer); ok { + dialer = credsDialer.Dialer + } } else { opts = append(opts, grpc.WithInsecure()) } + opts = append(opts, grpc.WithContextDialer(dialer)) // Interceptor retry and backoff. // TODO: Replace all of clientv3/retry.go with interceptor based retry, or with @@ -277,7 +259,10 @@ func (c *Client) dialSetupOpts(creds *credentials.TransportCredentials, dopts .. // Dial connects to a single endpoint using the client's config. func (c *Client) Dial(ep string) (*grpc.ClientConn, error) { - creds := c.directDialCreds(ep) + creds, err := c.directDialCreds(ep) + if err != nil { + return nil, err + } // Use the grpc passthrough resolver to directly dial a single endpoint. // This resolver passes through the 'unix' and 'unixs' endpoints schemes used // by etcd without modification, allowing us to directly dial endpoints and @@ -289,8 +274,8 @@ func (c *Client) getToken(ctx context.Context) error { var err error // return last error in a case of fail var auth *authenticator - for i := 0; i < len(c.cfg.Endpoints); i++ { - ep := c.cfg.Endpoints[i] + eps := c.Endpoints() + for _, ep := range eps { // use dial options without dopts to avoid reusing the client balancer var dOpts []grpc.DialOption _, host, _ := endpoint.ParseEndpoint(ep) @@ -318,10 +303,7 @@ func (c *Client) getToken(ctx context.Context) error { continue } - c.tokenCred.tokenMu.Lock() - c.tokenCred.token = resp.Token - c.tokenCred.tokenMu.Unlock() - + c.authTokenBundle.UpdateAuthToken(resp.Token) return nil } @@ -338,16 +320,14 @@ func (c *Client) dialWithBalancer(ep string, dopts ...grpc.DialOption) (*grpc.Cl } // dial configures and dials any grpc balancer target. -func (c *Client) dial(target string, creds *credentials.TransportCredentials, dopts ...grpc.DialOption) (*grpc.ClientConn, error) { +func (c *Client) dial(target string, creds grpccredentials.TransportCredentials, dopts ...grpc.DialOption) (*grpc.ClientConn, error) { opts, err := c.dialSetupOpts(creds, dopts...) if err != nil { return nil, fmt.Errorf("failed to configure dialer: %v", err) } if c.Username != "" && c.Password != "" { - c.tokenCred = &authTokenCredential{ - tokenMu: &sync.RWMutex{}, - } + c.authTokenBundle = credentials.NewBundle(credentials.Config{}) ctx, cancel := c.ctx, func() {} if c.cfg.DialTimeout > 0 { @@ -364,7 +344,7 @@ func (c *Client) dial(target string, creds *credentials.TransportCredentials, do return nil, err } } else { - opts = append(opts, grpc.WithPerRPCCredentials(c.tokenCred)) + opts = append(opts, grpc.WithPerRPCCredentials(c.authTokenBundle.PerRPCCredentials())) } cancel() } @@ -385,26 +365,30 @@ func (c *Client) dial(target string, creds *credentials.TransportCredentials, do return conn, nil } -func (c *Client) directDialCreds(ep string) *credentials.TransportCredentials { - _, hostPort, scheme := endpoint.ParseEndpoint(ep) +func (c *Client) directDialCreds(ep string) (grpccredentials.TransportCredentials, error) { + _, host, scheme := endpoint.ParseEndpoint(ep) creds := c.creds if len(scheme) != 0 { creds = c.processCreds(scheme) if creds != nil { - c := *creds - clone := c.Clone() + clone := creds.Clone() // Set the server name must to the endpoint hostname without port since grpc // otherwise attempts to check if x509 cert is valid for the full endpoint // including the scheme and port, which fails. - host, _ := endpoint.ParseHostPort(hostPort) - clone.OverrideServerName(host) - creds = &clone + overrideServerName, _, err := net.SplitHostPort(host) + if err != nil { + // Either the host didn't have a port or the host could not be parsed. Either way, continue with the + // original host string. + overrideServerName = host + } + clone.OverrideServerName(overrideServerName) + creds = clone } } - return creds + return creds, nil } -func (c *Client) dialWithBalancerCreds(ep string) *credentials.TransportCredentials { +func (c *Client) dialWithBalancerCreds(ep string) grpccredentials.TransportCredentials { _, _, scheme := endpoint.ParseEndpoint(ep) creds := c.creds if len(scheme) != 0 { @@ -424,10 +408,9 @@ func newClient(cfg *Config) (*Client, error) { if cfg == nil { cfg = &Config{} } - var creds *credentials.TransportCredentials + var creds grpccredentials.TransportCredentials if cfg.TLS != nil { - c := credentials.NewTLS(cfg.TLS) - creds = &c + creds = credentials.NewBundle(credentials.Config{TLSConfig: cfg.TLS}).TransportCredentials() } // use a temporary skeleton client to bootstrap first connection @@ -541,13 +524,17 @@ func (c *Client) roundRobinQuorumBackoff(waitBetween time.Duration, jitterFracti func (c *Client) checkVersion() (err error) { var wg sync.WaitGroup - errc := make(chan error, len(c.cfg.Endpoints)) + + eps := c.Endpoints() + errc := make(chan error, len(eps)) ctx, cancel := context.WithCancel(c.ctx) if c.cfg.DialTimeout > 0 { - ctx, cancel = context.WithTimeout(ctx, c.cfg.DialTimeout) + cancel() + ctx, cancel = context.WithTimeout(c.ctx, c.cfg.DialTimeout) } - wg.Add(len(c.cfg.Endpoints)) - for _, ep := range c.cfg.Endpoints { + + wg.Add(len(eps)) + for _, ep := range eps { // if cluster is current, any endpoint gives a recent version go func(e string) { defer wg.Done() @@ -559,8 +546,15 @@ func (c *Client) checkVersion() (err error) { vs := strings.Split(resp.Version, ".") maj, min := 0, 0 if len(vs) >= 2 { - maj, _ = strconv.Atoi(vs[0]) - min, rerr = strconv.Atoi(vs[1]) + var serr error + if maj, serr = strconv.Atoi(vs[0]); serr != nil { + errc <- serr + return + } + if min, serr = strconv.Atoi(vs[1]); serr != nil { + errc <- serr + return + } } if maj < 3 || (maj == 3 && min < 2) { rerr = ErrOldCluster @@ -569,7 +563,7 @@ func (c *Client) checkVersion() (err error) { }(ep) } // wait for success - for i := 0; i < len(c.cfg.Endpoints); i++ { + for range eps { if err = <-errc; err == nil { break } @@ -609,10 +603,13 @@ func isUnavailableErr(ctx context.Context, err error) bool { if err == nil { return false } - ev, _ := status.FromError(err) - // Unavailable codes mean the system will be right back. - // (e.g., can't connect, lost leader) - return ev.Code() == codes.Unavailable + ev, ok := status.FromError(err) + if ok { + // Unavailable codes mean the system will be right back. + // (e.g., can't connect, lost leader) + return ev.Code() == codes.Unavailable + } + return false } func toErr(ctx context.Context, err error) error { @@ -632,9 +629,6 @@ func toErr(ctx context.Context, err error) error { if ctx.Err() != nil { err = ctx.Err() } - case codes.Unavailable: - case codes.FailedPrecondition: - err = grpc.ErrClientConnClosing } } return err @@ -654,16 +648,25 @@ func IsConnCanceled(err error) bool { if err == nil { return false } - // >= gRPC v1.10.x + + // >= gRPC v1.23.x s, ok := status.FromError(err) if ok { // connection is canceled or server has already closed the connection return s.Code() == codes.Canceled || s.Message() == "transport is closing" } + // >= gRPC v1.10.x if err == context.Canceled { return true } + // <= gRPC v1.7.x returns 'errors.New("grpc: the client connection is closing")' return strings.Contains(err.Error(), "grpc: the client connection is closing") } + +// TransportCredentialsWithDialer is for a gRPC load balancer workaround. See credentials.transportCredential for details. +type TransportCredentialsWithDialer interface { + grpccredentials.TransportCredentials + Dialer(ctx context.Context, dialEp string) (net.Conn, error) +} diff --git a/vendor/go.etcd.io/etcd/clientv3/config.go b/vendor/go.etcd.io/etcd/clientv3/config.go index bd0376880f..11d447d575 100644 --- a/vendor/go.etcd.io/etcd/clientv3/config.go +++ b/vendor/go.etcd.io/etcd/clientv3/config.go @@ -68,6 +68,8 @@ type Config struct { RejectOldCluster bool `json:"reject-old-cluster"` // DialOptions is a list of dial options for the grpc client (e.g., for interceptors). + // For example, pass "grpc.WithBlock()" to block until the underlying connection is up. + // Without this, Dial returns immediately and connecting the server happens in background. DialOptions []grpc.DialOption // Context is the default client context; it can be used to cancel grpc dial out and @@ -81,4 +83,6 @@ type Config struct { // PermitWithoutStream when set will allow client to send keepalive pings to server without any active streams(RPCs). PermitWithoutStream bool `json:"permit-without-stream"` + + // TODO: support custom balancer picker } diff --git a/vendor/go.etcd.io/etcd/clientv3/credentials/credentials.go b/vendor/go.etcd.io/etcd/clientv3/credentials/credentials.go new file mode 100644 index 0000000000..63389c08bf --- /dev/null +++ b/vendor/go.etcd.io/etcd/clientv3/credentials/credentials.go @@ -0,0 +1,173 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package credentials implements gRPC credential interface with etcd specific logic. +// e.g., client handshake with custom authority parameter +package credentials + +import ( + "context" + "crypto/tls" + "net" + "sync" + + "go.etcd.io/etcd/clientv3/balancer/resolver/endpoint" + "go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes" + grpccredentials "google.golang.org/grpc/credentials" +) + +// Config defines gRPC credential configuration. +type Config struct { + TLSConfig *tls.Config +} + +// Bundle defines gRPC credential interface. +type Bundle interface { + grpccredentials.Bundle + UpdateAuthToken(token string) +} + +// NewBundle constructs a new gRPC credential bundle. +func NewBundle(cfg Config) Bundle { + return &bundle{ + tc: newTransportCredential(cfg.TLSConfig), + rc: newPerRPCCredential(), + } +} + +// bundle implements "grpccredentials.Bundle" interface. +type bundle struct { + tc *transportCredential + rc *perRPCCredential +} + +func (b *bundle) TransportCredentials() grpccredentials.TransportCredentials { + return b.tc +} + +func (b *bundle) PerRPCCredentials() grpccredentials.PerRPCCredentials { + return b.rc +} + +func (b *bundle) NewWithMode(mode string) (grpccredentials.Bundle, error) { + // no-op + return nil, nil +} + +// transportCredential implements "grpccredentials.TransportCredentials" interface. +// transportCredential wraps TransportCredentials to track which +// addresses are dialed for which endpoints, and then sets the authority when checking the endpoint's cert to the +// hostname or IP of the dialed endpoint. +// This is a workaround of a gRPC load balancer issue. gRPC uses the dialed target's service name as the authority when +// checking all endpoint certs, which does not work for etcd servers using their hostname or IP as the Subject Alternative Name +// in their TLS certs. +// To enable, include both WithTransportCredentials(creds) and WithContextDialer(creds.Dialer) +// when dialing. +type transportCredential struct { + gtc grpccredentials.TransportCredentials + mu sync.Mutex + // addrToEndpoint maps from the connection addresses that are dialed to the hostname or IP of the + // endpoint provided to the dialer when dialing + addrToEndpoint map[string]string +} + +func newTransportCredential(cfg *tls.Config) *transportCredential { + return &transportCredential{ + gtc: grpccredentials.NewTLS(cfg), + addrToEndpoint: map[string]string{}, + } +} + +func (tc *transportCredential) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, grpccredentials.AuthInfo, error) { + // Set the authority when checking the endpoint's cert to the hostname or IP of the dialed endpoint + tc.mu.Lock() + dialEp, ok := tc.addrToEndpoint[rawConn.RemoteAddr().String()] + tc.mu.Unlock() + if ok { + _, host, _ := endpoint.ParseEndpoint(dialEp) + authority = host + } + return tc.gtc.ClientHandshake(ctx, authority, rawConn) +} + +// return true if given string is an IP. +func isIP(ep string) bool { + return net.ParseIP(ep) != nil +} + +func (tc *transportCredential) ServerHandshake(rawConn net.Conn) (net.Conn, grpccredentials.AuthInfo, error) { + return tc.gtc.ServerHandshake(rawConn) +} + +func (tc *transportCredential) Info() grpccredentials.ProtocolInfo { + return tc.gtc.Info() +} + +func (tc *transportCredential) Clone() grpccredentials.TransportCredentials { + copy := map[string]string{} + tc.mu.Lock() + for k, v := range tc.addrToEndpoint { + copy[k] = v + } + tc.mu.Unlock() + return &transportCredential{ + gtc: tc.gtc.Clone(), + addrToEndpoint: copy, + } +} + +func (tc *transportCredential) OverrideServerName(serverNameOverride string) error { + return tc.gtc.OverrideServerName(serverNameOverride) +} + +func (tc *transportCredential) Dialer(ctx context.Context, dialEp string) (net.Conn, error) { + // Keep track of which addresses are dialed for which endpoints + conn, err := endpoint.Dialer(ctx, dialEp) + if conn != nil { + tc.mu.Lock() + tc.addrToEndpoint[conn.RemoteAddr().String()] = dialEp + tc.mu.Unlock() + } + return conn, err +} + +// perRPCCredential implements "grpccredentials.PerRPCCredentials" interface. +type perRPCCredential struct { + authToken string + authTokenMu sync.RWMutex +} + +func newPerRPCCredential() *perRPCCredential { return &perRPCCredential{} } + +func (rc *perRPCCredential) RequireTransportSecurity() bool { return false } + +func (rc *perRPCCredential) GetRequestMetadata(ctx context.Context, s ...string) (map[string]string, error) { + rc.authTokenMu.RLock() + authToken := rc.authToken + rc.authTokenMu.RUnlock() + return map[string]string{rpctypes.TokenFieldNameGRPC: authToken}, nil +} + +func (b *bundle) UpdateAuthToken(token string) { + if b.rc == nil { + return + } + b.rc.UpdateAuthToken(token) +} + +func (rc *perRPCCredential) UpdateAuthToken(token string) { + rc.authTokenMu.Lock() + rc.authToken = token + rc.authTokenMu.Unlock() +} diff --git a/vendor/go.etcd.io/etcd/clientv3/doc.go b/vendor/go.etcd.io/etcd/clientv3/doc.go index 01a3f5961a..913cd28255 100644 --- a/vendor/go.etcd.io/etcd/clientv3/doc.go +++ b/vendor/go.etcd.io/etcd/clientv3/doc.go @@ -90,7 +90,7 @@ // // with etcd clientv3 <= v3.3 // if err == context.Canceled { // // grpc balancer calls 'Get' with an inflight client.Close -// } else if err == grpc.ErrClientConnClosing { +// } else if err == grpc.ErrClientConnClosing { // <= gRCP v1.7.x // // grpc balancer calls 'Get' after client.Close. // } // // with etcd clientv3 >= v3.4 diff --git a/vendor/go.etcd.io/etcd/embed/config.go b/vendor/go.etcd.io/etcd/embed/config.go index 616e46cce9..2f64d927f2 100644 --- a/vendor/go.etcd.io/etcd/embed/config.go +++ b/vendor/go.etcd.io/etcd/embed/config.go @@ -30,6 +30,7 @@ import ( "go.etcd.io/etcd/etcdserver" "go.etcd.io/etcd/etcdserver/api/v3compactor" "go.etcd.io/etcd/pkg/flags" + "go.etcd.io/etcd/pkg/logutil" "go.etcd.io/etcd/pkg/netutil" "go.etcd.io/etcd/pkg/srv" "go.etcd.io/etcd/pkg/tlsutil" @@ -69,9 +70,8 @@ const ( // It's enabled by default. DefaultStrictReconfigCheck = true // DefaultEnableV2 is the default value for "--enable-v2" flag. - // v2 is enabled by default. - // TODO: disable v2 when deprecated. - DefaultEnableV2 = true + // v2 API is disabled by default. + DefaultEnableV2 = false // maxElectionMs specifies the maximum value of election timeout. // More details are listed in ../Documentation/tuning.md#time-parameters. @@ -280,6 +280,7 @@ type Config struct { ExperimentalBackendFreelistType string `json:"experimental-backend-bbolt-freelist-type"` // ExperimentalEnableLeaseCheckpoint enables primary lessor to persist lease remainingTTL to prevent indefinite auto-renewal of long lived leases. ExperimentalEnableLeaseCheckpoint bool `json:"experimental-enable-lease-checkpoint"` + ExperimentalCompactionBatchLimit int `json:"experimental-compaction-batch-limit"` // ForceNewCluster starts a new cluster even if previously started; unsafe. ForceNewCluster bool `json:"force-new-cluster"` @@ -292,11 +293,8 @@ type Config struct { // Logger is logger options: "zap", "capnslog". // WARN: "capnslog" is being deprecated in v3.5. Logger string `json:"logger"` - - // DeprecatedLogOutput is to be deprecated in v3.5. - // Just here for safe migration in v3.4. - DeprecatedLogOutput []string `json:"log-output"` - + // LogLevel configures log level. Only supports debug, info, warn, error, panic, or fatal. Default 'info'. + LogLevel string `json:"log-level"` // LogOutputs is either: // - "default" as os.Stderr, // - "stderr" as os.Stderr, @@ -305,9 +303,6 @@ type Config struct { // It can be multiple when "Logger" is zap. LogOutputs []string `json:"log-outputs"` - // Debug is true, to enable debug level logging. - Debug bool `json:"debug"` - // ZapLoggerBuilder is used to build the zap logger. ZapLoggerBuilder func(*Config) error @@ -330,6 +325,12 @@ type Config struct { // TO BE DEPRECATED + // DeprecatedLogOutput is to be deprecated in v3.5. + // Just here for safe migration in v3.4. + DeprecatedLogOutput []string `json:"log-output"` + // Debug is true, to enable debug level logging. + // WARNING: to be deprecated in 3.5. Use "--log-level=debug" instead. + Debug bool `json:"debug"` // LogPkgLevels is being deprecated in v3.5. // Only valid if "logger" option is "capnslog". // WARN: DO NOT USE THIS! @@ -416,6 +417,7 @@ func NewConfig() *Config { DeprecatedLogOutput: []string{DefaultLogOutput}, LogOutputs: []string{DefaultLogOutput}, Debug: false, + LogLevel: logutil.DefaultLogLevel, LogPkgLevels: "", } cfg.InitialCluster = cfg.InitialClusterFromName(cfg.Name) diff --git a/vendor/go.etcd.io/etcd/embed/config_logging.go b/vendor/go.etcd.io/etcd/embed/config_logging.go index bddaacabe5..e42103cb18 100644 --- a/vendor/go.etcd.io/etcd/embed/config_logging.go +++ b/vendor/go.etcd.io/etcd/embed/config_logging.go @@ -69,11 +69,25 @@ func (cfg *Config) setupLogging() error { return fmt.Errorf("'--log-output=%q' and '--log-outputs=%q' are incompatible; only set --log-outputs", cfg.DeprecatedLogOutput, cfg.LogOutputs) } if !reflect.DeepEqual(cfg.DeprecatedLogOutput, []string{DefaultLogOutput}) { - fmt.Fprintf(os.Stderr, "Deprecated '--log-output' flag is set to %q\n", cfg.DeprecatedLogOutput) + fmt.Fprintf(os.Stderr, "[WARNING] Deprecated '--log-output' flag is set to %q\n", cfg.DeprecatedLogOutput) fmt.Fprintln(os.Stderr, "Please use '--log-outputs' flag") } } + // TODO: remove after deprecating log related flags in v3.5 + if cfg.Debug { + fmt.Fprintf(os.Stderr, "[WARNING] Deprecated '--debug' flag is set to %v (use '--log-level=debug' instead\n", cfg.Debug) + } + if cfg.Debug && cfg.LogLevel != "debug" { + fmt.Fprintf(os.Stderr, "[WARNING] Deprecated '--debug' flag is set to %v with inconsistent '--log-level=%s' flag\n", cfg.Debug, cfg.LogLevel) + } + if cfg.Logger == "capnslog" { + fmt.Fprintf(os.Stderr, "[WARNING] Deprecated '--logger=%s' flag is set; use '--logger=zap' flag instead\n", cfg.Logger) + } + if cfg.LogPkgLevels != "" { + fmt.Fprintf(os.Stderr, "[WARNING] Deprecated '--log-package-levels=%s' flag is set; use '--logger=zap' flag instead\n", cfg.LogPkgLevels) + } + switch cfg.Logger { case "capnslog": // TODO: deprecate this in v3.5 cfg.ClientTLSInfo.HandshakeFailure = logTLSHandshakeFailure @@ -85,7 +99,7 @@ func (cfg *Config) setupLogging() error { // enable info, warning, error grpclog.SetLoggerV2(grpclog.NewLoggerV2(os.Stderr, os.Stderr, os.Stderr)) } else { - capnslog.SetGlobalLogLevel(capnslog.INFO) + capnslog.SetGlobalLogLevel(logutil.ConvertToCapnslogLogLevel(cfg.LogLevel)) // only discard info grpclog.SetLoggerV2(grpclog.NewLoggerV2(ioutil.Discard, os.Stderr, os.Stderr)) } @@ -156,10 +170,15 @@ func (cfg *Config) setupLogging() error { } if !isJournal { - copied := logutil.AddOutputPaths(logutil.DefaultZapLoggerConfig, outputPaths, errOutputPaths) - - if cfg.Debug { - copied.Level = zap.NewAtomicLevelAt(zap.DebugLevel) + copied := logutil.DefaultZapLoggerConfig + copied.OutputPaths = outputPaths + copied.ErrorOutputPaths = errOutputPaths + copied = logutil.MergeOutputPaths(copied) + copied.Level = zap.NewAtomicLevelAt(logutil.ConvertToZapLevel(cfg.LogLevel)) + if cfg.Debug || cfg.LogLevel == "debug" { + // enable tracing even when "--debug --log-level info" + // in order to keep backward compatibility with <= v3.3 + // TODO: remove "Debug" check in v3.5 grpc.EnableTracing = true } if cfg.ZapLoggerBuilder == nil { @@ -201,9 +220,11 @@ func (cfg *Config) setupLogging() error { return lerr } - lvl := zap.NewAtomicLevelAt(zap.InfoLevel) - if cfg.Debug { - lvl = zap.NewAtomicLevelAt(zap.DebugLevel) + lvl := zap.NewAtomicLevelAt(logutil.ConvertToZapLevel(cfg.LogLevel)) + if cfg.Debug || cfg.LogLevel == "debug" { + // enable tracing even when "--debug --log-level info" + // in order to keep backward compatibility with <= v3.3 + // TODO: remove "Debug" check in v3.5 grpc.EnableTracing = true } @@ -242,7 +263,7 @@ func (cfg *Config) setupLogging() error { serverName := state.ServerName if len(state.PeerCertificates) > 0 { cert := state.PeerCertificates[0] - ips := make([]string, 0, len(cert.IPAddresses)) + ips := make([]string, len(cert.IPAddresses)) for i := range cert.IPAddresses { ips[i] = cert.IPAddresses[i].String() } diff --git a/vendor/go.etcd.io/etcd/embed/etcd.go b/vendor/go.etcd.io/etcd/embed/etcd.go index 8fa48f41d1..ac7dbc987f 100644 --- a/vendor/go.etcd.io/etcd/embed/etcd.go +++ b/vendor/go.etcd.io/etcd/embed/etcd.go @@ -205,6 +205,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { ForceNewCluster: cfg.ForceNewCluster, EnableGRPCGateway: cfg.EnableGRPCGateway, EnableLeaseCheckpoint: cfg.ExperimentalEnableLeaseCheckpoint, + CompactionBatchLimit: cfg.ExperimentalCompactionBatchLimit, } print(e.cfg.logger, *cfg, srvcfg, memberInitialized) if e.Server, err = etcdserver.NewServer(srvcfg); err != nil { diff --git a/vendor/go.etcd.io/etcd/embed/serve.go b/vendor/go.etcd.io/etcd/embed/serve.go index 8e6e7b769a..a3b20c46c3 100644 --- a/vendor/go.etcd.io/etcd/embed/serve.go +++ b/vendor/go.etcd.io/etcd/embed/serve.go @@ -23,6 +23,7 @@ import ( "net/http" "strings" + "go.etcd.io/etcd/clientv3/credentials" "go.etcd.io/etcd/etcdserver" "go.etcd.io/etcd/etcdserver/api/v3client" "go.etcd.io/etcd/etcdserver/api/v3election" @@ -43,7 +44,6 @@ import ( "go.uber.org/zap" "golang.org/x/net/trace" "google.golang.org/grpc" - "google.golang.org/grpc/credentials" ) type serveCtx struct { @@ -163,8 +163,8 @@ func (sctx *serveCtx) serve( dtls := tlscfg.Clone() // trust local server dtls.InsecureSkipVerify = true - creds := credentials.NewTLS(dtls) - opts := []grpc.DialOption{grpc.WithTransportCredentials(creds)} + bundle := credentials.NewBundle(credentials.Config{TLSConfig: dtls}) + opts := []grpc.DialOption{grpc.WithTransportCredentials(bundle.TransportCredentials())} gwmux, err = sctx.registerGateway(opts) if err != nil { return err @@ -189,7 +189,7 @@ func (sctx *serveCtx) serve( sctx.serversC <- &servers{secure: true, grpc: gs, http: srv} if sctx.lg != nil { sctx.lg.Info( - "serving client traffic insecurely", + "serving client traffic securely", zap.String("address", sctx.l.Addr().String()), ) } else { diff --git a/vendor/go.etcd.io/etcd/etcdserver/api/capability.go b/vendor/go.etcd.io/etcd/etcdserver/api/capability.go index ec0d600e88..8b13f47429 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/api/capability.go +++ b/vendor/go.etcd.io/etcd/etcdserver/api/capability.go @@ -40,6 +40,7 @@ var ( "3.1.0": {AuthCapability: true, V3rpcCapability: true}, "3.2.0": {AuthCapability: true, V3rpcCapability: true}, "3.3.0": {AuthCapability: true, V3rpcCapability: true}, + "3.4.0": {AuthCapability: true, V3rpcCapability: true}, } enableMapMu sync.RWMutex diff --git a/vendor/go.etcd.io/etcd/etcdserver/api/membership/cluster.go b/vendor/go.etcd.io/etcd/etcdserver/api/membership/cluster.go index 81f515d2f3..b1a011b503 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/api/membership/cluster.go +++ b/vendor/go.etcd.io/etcd/etcdserver/api/membership/cluster.go @@ -565,6 +565,7 @@ func (c *RaftCluster) SetVersion(ver *semver.Version, onSet func(*zap.Logger, *s plog.Noticef("set the initial cluster version to %v", version.Cluster(ver.String())) } } + oldVer := c.version c.version = ver mustDetectDowngrade(c.lg, c.version) if c.v2store != nil { @@ -573,7 +574,10 @@ func (c *RaftCluster) SetVersion(ver *semver.Version, onSet func(*zap.Logger, *s if c.be != nil { mustSaveClusterVersionToBackend(c.be, ver) } - ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": ver.String()}).Set(1) + if oldVer != nil { + ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(oldVer.String())}).Set(0) + } + ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(ver.String())}).Set(1) onSet(c.lg, ver) } diff --git a/vendor/go.etcd.io/etcd/etcdserver/api/rafthttp/http.go b/vendor/go.etcd.io/etcd/etcdserver/api/rafthttp/http.go index 18e9c53f24..d0e0c81e20 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/api/rafthttp/http.go +++ b/vendor/go.etcd.io/etcd/etcdserver/api/rafthttp/http.go @@ -258,6 +258,11 @@ func (h *snapshotHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + snapshotReceiveInflights.WithLabelValues(from).Inc() + defer func() { + snapshotReceiveInflights.WithLabelValues(from).Dec() + }() + if h.lg != nil { h.lg.Info( "receiving database snapshot", diff --git a/vendor/go.etcd.io/etcd/etcdserver/api/rafthttp/metrics.go b/vendor/go.etcd.io/etcd/etcdserver/api/rafthttp/metrics.go index ce51248d88..02fff84be7 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/api/rafthttp/metrics.go +++ b/vendor/go.etcd.io/etcd/etcdserver/api/rafthttp/metrics.go @@ -80,6 +80,15 @@ var ( []string{"To"}, ) + snapshotSendInflights = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "etcd", + Subsystem: "network", + Name: "snapshot_send_inflights_total", + Help: "Total number of inflight snapshot sends", + }, + []string{"To"}, + ) + snapshotSendFailures = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "network", @@ -111,6 +120,15 @@ var ( []string{"From"}, ) + snapshotReceiveInflights = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "etcd", + Subsystem: "network", + Name: "snapshot_receive_inflights_total", + Help: "Total number of inflight snapshot receives", + }, + []string{"From"}, + ) + snapshotReceiveFailures = prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: "etcd", Subsystem: "network", @@ -156,9 +174,11 @@ func init() { prometheus.MustRegister(recvFailures) prometheus.MustRegister(snapshotSend) + prometheus.MustRegister(snapshotSendInflights) prometheus.MustRegister(snapshotSendFailures) prometheus.MustRegister(snapshotSendSeconds) prometheus.MustRegister(snapshotReceive) + prometheus.MustRegister(snapshotReceiveInflights) prometheus.MustRegister(snapshotReceiveFailures) prometheus.MustRegister(snapshotReceiveSeconds) diff --git a/vendor/go.etcd.io/etcd/etcdserver/api/rafthttp/snapshot_sender.go b/vendor/go.etcd.io/etcd/etcdserver/api/rafthttp/snapshot_sender.go index 85abaeaa4d..62efb0cdc3 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/api/rafthttp/snapshot_sender.go +++ b/vendor/go.etcd.io/etcd/etcdserver/api/rafthttp/snapshot_sender.go @@ -90,6 +90,11 @@ func (s *snapshotSender) send(merged snap.Message) { plog.Infof("start to send database snapshot [index: %d, to %s]...", m.Snapshot.Metadata.Index, types.ID(m.To)) } + snapshotSendInflights.WithLabelValues(to).Inc() + defer func() { + snapshotSendInflights.WithLabelValues(to).Dec() + }() + err := s.post(req) defer merged.CloseWithError(err) if err != nil { @@ -139,7 +144,6 @@ func (s *snapshotSender) send(merged snap.Message) { } sentBytes.WithLabelValues(to).Add(float64(merged.TotalSize)) - snapshotSend.WithLabelValues(to).Inc() snapshotSendSeconds.WithLabelValues(to).Observe(time.Since(start).Seconds()) } diff --git a/vendor/go.etcd.io/etcd/etcdserver/api/rafthttp/stream.go b/vendor/go.etcd.io/etcd/etcdserver/api/rafthttp/stream.go index dcb2223ca5..cf7d8ccf62 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/api/rafthttp/stream.go +++ b/vendor/go.etcd.io/etcd/etcdserver/api/rafthttp/stream.go @@ -57,6 +57,7 @@ var ( "3.1.0": {streamTypeMsgAppV2, streamTypeMessage}, "3.2.0": {streamTypeMsgAppV2, streamTypeMessage}, "3.3.0": {streamTypeMsgAppV2, streamTypeMessage}, + "3.4.0": {streamTypeMsgAppV2, streamTypeMessage}, } ) diff --git a/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/grpc.go b/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/grpc.go index 957c818957..3332016617 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/grpc.go +++ b/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/grpc.go @@ -21,10 +21,10 @@ import ( "go.etcd.io/etcd/etcdserver" pb "go.etcd.io/etcd/etcdserver/etcdserverpb" - "github.com/grpc-ecosystem/go-grpc-middleware" - "github.com/grpc-ecosystem/go-grpc-prometheus" + grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + "go.etcd.io/etcd/clientv3/credentials" "google.golang.org/grpc" - "google.golang.org/grpc/credentials" "google.golang.org/grpc/health" healthpb "google.golang.org/grpc/health/grpc_health_v1" ) @@ -39,7 +39,8 @@ func Server(s *etcdserver.EtcdServer, tls *tls.Config, gopts ...grpc.ServerOptio var opts []grpc.ServerOption opts = append(opts, grpc.CustomCodec(&codec{})) if tls != nil { - opts = append(opts, grpc.Creds(credentials.NewTLS(tls))) + bundle := credentials.NewBundle(credentials.Config{TLSConfig: tls}) + opts = append(opts, grpc.Creds(bundle.TransportCredentials())) } opts = append(opts, grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer( newLogUnaryInterceptor(s), diff --git a/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/key.go b/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/key.go index fdb002e0dd..ff59bac346 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/key.go +++ b/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/key.go @@ -179,7 +179,7 @@ func checkTxnRequest(r *pb.TxnRequest, maxTxnOps int) error { // there is an overlap, returns an error. If no overlap, return put and delete // sets for recursive evaluation. func checkIntervals(reqs []*pb.RequestOp) (map[string]struct{}, adt.IntervalTree, error) { - var dels adt.IntervalTree + dels := adt.NewIntervalTree() // collect deletes from this level; build first to check lower level overlapped puts for _, req := range reqs { diff --git a/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes/error.go b/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes/error.go index 3bbc26b8fd..e6a281460d 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes/error.go +++ b/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes/error.go @@ -54,6 +54,7 @@ var ( ErrGRPCUserNotFound = status.New(codes.FailedPrecondition, "etcdserver: user name not found").Err() ErrGRPCRoleAlreadyExist = status.New(codes.FailedPrecondition, "etcdserver: role name already exists").Err() ErrGRPCRoleNotFound = status.New(codes.FailedPrecondition, "etcdserver: role name not found").Err() + ErrGRPCRoleEmpty = status.New(codes.InvalidArgument, "etcdserver: role name is empty").Err() ErrGRPCAuthFailed = status.New(codes.InvalidArgument, "etcdserver: authentication failed, invalid user ID or password").Err() ErrGRPCPermissionDenied = status.New(codes.PermissionDenied, "etcdserver: permission denied").Err() ErrGRPCRoleNotGranted = status.New(codes.FailedPrecondition, "etcdserver: role is not granted to the user").Err() @@ -110,6 +111,7 @@ var ( ErrorDesc(ErrGRPCUserNotFound): ErrGRPCUserNotFound, ErrorDesc(ErrGRPCRoleAlreadyExist): ErrGRPCRoleAlreadyExist, ErrorDesc(ErrGRPCRoleNotFound): ErrGRPCRoleNotFound, + ErrorDesc(ErrGRPCRoleEmpty): ErrGRPCRoleEmpty, ErrorDesc(ErrGRPCAuthFailed): ErrGRPCAuthFailed, ErrorDesc(ErrGRPCPermissionDenied): ErrGRPCPermissionDenied, ErrorDesc(ErrGRPCRoleNotGranted): ErrGRPCRoleNotGranted, @@ -168,6 +170,7 @@ var ( ErrUserNotFound = Error(ErrGRPCUserNotFound) ErrRoleAlreadyExist = Error(ErrGRPCRoleAlreadyExist) ErrRoleNotFound = Error(ErrGRPCRoleNotFound) + ErrRoleEmpty = Error(ErrGRPCRoleEmpty) ErrAuthFailed = Error(ErrGRPCAuthFailed) ErrPermissionDenied = Error(ErrGRPCPermissionDenied) ErrRoleNotGranted = Error(ErrGRPCRoleNotGranted) diff --git a/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/util.go b/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/util.go index 748632b5e2..281ddc7a0d 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/util.go +++ b/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/util.go @@ -69,6 +69,7 @@ var toGRPCErrorMap = map[error]error{ auth.ErrUserNotFound: rpctypes.ErrGRPCUserNotFound, auth.ErrRoleAlreadyExist: rpctypes.ErrGRPCRoleAlreadyExist, auth.ErrRoleNotFound: rpctypes.ErrGRPCRoleNotFound, + auth.ErrRoleEmpty: rpctypes.ErrGRPCRoleEmpty, auth.ErrAuthFailed: rpctypes.ErrGRPCAuthFailed, auth.ErrPermissionDenied: rpctypes.ErrGRPCPermissionDenied, auth.ErrRoleNotGranted: rpctypes.ErrGRPCRoleNotGranted, diff --git a/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/watch.go b/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/watch.go index 7848cdd04a..f41cb6c056 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/watch.go +++ b/vendor/go.etcd.io/etcd/etcdserver/api/v3rpc/watch.go @@ -179,7 +179,7 @@ func (ws *watchServer) Watch(stream pb.Watch_WatchServer) (err error) { } } else { if sws.lg != nil { - sws.lg.Warn("failed to receive watch request from gRPC stream", zap.Error(err)) + sws.lg.Warn("failed to receive watch request from gRPC stream", zap.Error(rerr)) } else { plog.Warningf("failed to receive watch request from gRPC stream (%q)", rerr.Error()) } diff --git a/vendor/go.etcd.io/etcd/etcdserver/apply.go b/vendor/go.etcd.io/etcd/etcdserver/apply.go index 1f06ad0dd6..822b5e3220 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/apply.go +++ b/vendor/go.etcd.io/etcd/etcdserver/apply.go @@ -26,6 +26,7 @@ import ( "go.etcd.io/etcd/lease" "go.etcd.io/etcd/mvcc" "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/pkg/traceutil" "go.etcd.io/etcd/pkg/types" "github.com/gogo/protobuf/proto" @@ -43,17 +44,18 @@ type applyResult struct { // to being logically reflected by the node. Currently only used for // Compaction requests. physc <-chan struct{} + trace *traceutil.Trace } // applierV3 is the interface for processing V3 raft messages type applierV3 interface { Apply(r *pb.InternalRaftRequest) *applyResult - Put(txn mvcc.TxnWrite, p *pb.PutRequest) (*pb.PutResponse, error) - Range(txn mvcc.TxnRead, r *pb.RangeRequest) (*pb.RangeResponse, error) + Put(txn mvcc.TxnWrite, p *pb.PutRequest) (*pb.PutResponse, *traceutil.Trace, error) + Range(ctx context.Context, txn mvcc.TxnRead, r *pb.RangeRequest) (*pb.RangeResponse, error) DeleteRange(txn mvcc.TxnWrite, dr *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, error) Txn(rt *pb.TxnRequest) (*pb.TxnResponse, error) - Compaction(compaction *pb.CompactionRequest) (*pb.CompactionResponse, <-chan struct{}, error) + Compaction(compaction *pb.CompactionRequest) (*pb.CompactionResponse, <-chan struct{}, *traceutil.Trace, error) LeaseGrant(lc *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) LeaseRevoke(lc *pb.LeaseRevokeRequest) (*pb.LeaseRevokeResponse, error) @@ -119,15 +121,15 @@ func (a *applierV3backend) Apply(r *pb.InternalRaftRequest) *applyResult { // call into a.s.applyV3.F instead of a.F so upper appliers can check individual calls switch { case r.Range != nil: - ar.resp, ar.err = a.s.applyV3.Range(nil, r.Range) + ar.resp, ar.err = a.s.applyV3.Range(context.TODO(), nil, r.Range) case r.Put != nil: - ar.resp, ar.err = a.s.applyV3.Put(nil, r.Put) + ar.resp, ar.trace, ar.err = a.s.applyV3.Put(nil, r.Put) case r.DeleteRange != nil: ar.resp, ar.err = a.s.applyV3.DeleteRange(nil, r.DeleteRange) case r.Txn != nil: ar.resp, ar.err = a.s.applyV3.Txn(r.Txn) case r.Compaction != nil: - ar.resp, ar.physc, ar.err = a.s.applyV3.Compaction(r.Compaction) + ar.resp, ar.physc, ar.trace, ar.err = a.s.applyV3.Compaction(r.Compaction) case r.LeaseGrant != nil: ar.resp, ar.err = a.s.applyV3.LeaseGrant(r.LeaseGrant) case r.LeaseRevoke != nil: @@ -174,32 +176,39 @@ func (a *applierV3backend) Apply(r *pb.InternalRaftRequest) *applyResult { return ar } -func (a *applierV3backend) Put(txn mvcc.TxnWrite, p *pb.PutRequest) (resp *pb.PutResponse, err error) { +func (a *applierV3backend) Put(txn mvcc.TxnWrite, p *pb.PutRequest) (resp *pb.PutResponse, trace *traceutil.Trace, err error) { resp = &pb.PutResponse{} resp.Header = &pb.ResponseHeader{} - + trace = traceutil.New("put", + a.s.getLogger(), + traceutil.Field{Key: "key", Value: string(p.Key)}, + traceutil.Field{Key: "req_size", Value: proto.Size(p)}, + ) val, leaseID := p.Value, lease.LeaseID(p.Lease) if txn == nil { if leaseID != lease.NoLease { if l := a.s.lessor.Lookup(leaseID); l == nil { - return nil, lease.ErrLeaseNotFound + return nil, nil, lease.ErrLeaseNotFound } } - txn = a.s.KV().Write() + txn = a.s.KV().Write(trace) defer txn.End() } var rr *mvcc.RangeResult if p.IgnoreValue || p.IgnoreLease || p.PrevKv { + trace.DisableStep() rr, err = txn.Range(p.Key, nil, mvcc.RangeOptions{}) if err != nil { - return nil, err + return nil, nil, err } + trace.EnableStep() + trace.Step("get previous kv pair") } if p.IgnoreValue || p.IgnoreLease { if rr == nil || len(rr.KVs) == 0 { // ignore_{lease,value} flag expects previous key-value pair - return nil, ErrKeyNotFound + return nil, nil, ErrKeyNotFound } } if p.IgnoreValue { @@ -215,7 +224,8 @@ func (a *applierV3backend) Put(txn mvcc.TxnWrite, p *pb.PutRequest) (resp *pb.Pu } resp.Header.Revision = txn.Put(p.Key, val, leaseID) - return resp, nil + trace.AddField(traceutil.Field{Key: "response_revision", Value: resp.Header.Revision}) + return resp, trace, nil } func (a *applierV3backend) DeleteRange(txn mvcc.TxnWrite, dr *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, error) { @@ -224,7 +234,7 @@ func (a *applierV3backend) DeleteRange(txn mvcc.TxnWrite, dr *pb.DeleteRangeRequ end := mkGteRange(dr.RangeEnd) if txn == nil { - txn = a.s.kv.Write() + txn = a.s.kv.Write(traceutil.TODO()) defer txn.End() } @@ -245,12 +255,14 @@ func (a *applierV3backend) DeleteRange(txn mvcc.TxnWrite, dr *pb.DeleteRangeRequ return resp, nil } -func (a *applierV3backend) Range(txn mvcc.TxnRead, r *pb.RangeRequest) (*pb.RangeResponse, error) { +func (a *applierV3backend) Range(ctx context.Context, txn mvcc.TxnRead, r *pb.RangeRequest) (*pb.RangeResponse, error) { + trace := traceutil.Get(ctx) + resp := &pb.RangeResponse{} resp.Header = &pb.ResponseHeader{} if txn == nil { - txn = a.s.kv.Read() + txn = a.s.kv.Read(trace) defer txn.End() } @@ -327,7 +339,7 @@ func (a *applierV3backend) Range(txn mvcc.TxnRead, r *pb.RangeRequest) (*pb.Rang rr.KVs = rr.KVs[:r.Limit] resp.More = true } - + trace.Step("filter and sort the key-value pairs") resp.Header.Revision = rr.Rev resp.Count = int64(rr.Count) resp.Kvs = make([]*mvccpb.KeyValue, len(rr.KVs)) @@ -337,12 +349,13 @@ func (a *applierV3backend) Range(txn mvcc.TxnRead, r *pb.RangeRequest) (*pb.Rang } resp.Kvs[i] = &rr.KVs[i] } + trace.Step("assemble the response") return resp, nil } func (a *applierV3backend) Txn(rt *pb.TxnRequest) (*pb.TxnResponse, error) { isWrite := !isTxnReadonly(rt) - txn := mvcc.NewReadOnlyTxnWrite(a.s.KV().Read()) + txn := mvcc.NewReadOnlyTxnWrite(a.s.KV().Read(traceutil.TODO())) txnPath := compareToPath(txn, rt) if isWrite { @@ -364,7 +377,7 @@ func (a *applierV3backend) Txn(rt *pb.TxnRequest) (*pb.TxnResponse, error) { // be the revision of the write txn. if isWrite { txn.End() - txn = a.s.KV().Write() + txn = a.s.KV().Write(traceutil.TODO()) } a.applyTxn(txn, rt, txnPath, txnResp) rev := txn.Rev() @@ -516,7 +529,7 @@ func (a *applierV3backend) applyTxn(txn mvcc.TxnWrite, rt *pb.TxnRequest, txnPat respi := tresp.Responses[i].Response switch tv := req.Request.(type) { case *pb.RequestOp_RequestRange: - resp, err := a.Range(txn, tv.RequestRange) + resp, err := a.Range(context.TODO(), txn, tv.RequestRange) if err != nil { if lg != nil { lg.Panic("unexpected error during txn", zap.Error(err)) @@ -526,7 +539,7 @@ func (a *applierV3backend) applyTxn(txn mvcc.TxnWrite, rt *pb.TxnRequest, txnPat } respi.(*pb.ResponseOp_ResponseRange).ResponseRange = resp case *pb.RequestOp_RequestPut: - resp, err := a.Put(txn, tv.RequestPut) + resp, _, err := a.Put(txn, tv.RequestPut) if err != nil { if lg != nil { lg.Panic("unexpected error during txn", zap.Error(err)) @@ -557,17 +570,22 @@ func (a *applierV3backend) applyTxn(txn mvcc.TxnWrite, rt *pb.TxnRequest, txnPat return txns } -func (a *applierV3backend) Compaction(compaction *pb.CompactionRequest) (*pb.CompactionResponse, <-chan struct{}, error) { +func (a *applierV3backend) Compaction(compaction *pb.CompactionRequest) (*pb.CompactionResponse, <-chan struct{}, *traceutil.Trace, error) { resp := &pb.CompactionResponse{} resp.Header = &pb.ResponseHeader{} - ch, err := a.s.KV().Compact(compaction.Revision) + trace := traceutil.New("compact", + a.s.getLogger(), + traceutil.Field{Key: "revision", Value: compaction.Revision}, + ) + + ch, err := a.s.KV().Compact(trace, compaction.Revision) if err != nil { - return nil, ch, err + return nil, ch, nil, err } // get the current revision. which key to get is not important. rr, _ := a.s.KV().Range([]byte("compaction"), nil, mvcc.RangeOptions{}) resp.Header.Revision = rr.Rev - return resp, ch, err + return resp, ch, trace, err } func (a *applierV3backend) LeaseGrant(lc *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) { @@ -674,8 +692,8 @@ type applierV3Capped struct { // with Puts so that the number of keys in the store is capped. func newApplierV3Capped(base applierV3) applierV3 { return &applierV3Capped{applierV3: base} } -func (a *applierV3Capped) Put(txn mvcc.TxnWrite, p *pb.PutRequest) (*pb.PutResponse, error) { - return nil, ErrNoSpace +func (a *applierV3Capped) Put(txn mvcc.TxnWrite, p *pb.PutRequest) (*pb.PutResponse, *traceutil.Trace, error) { + return nil, nil, ErrNoSpace } func (a *applierV3Capped) Txn(r *pb.TxnRequest) (*pb.TxnResponse, error) { @@ -824,13 +842,13 @@ func newQuotaApplierV3(s *EtcdServer, app applierV3) applierV3 { return "aApplierV3{app, NewBackendQuota(s, "v3-applier")} } -func (a *quotaApplierV3) Put(txn mvcc.TxnWrite, p *pb.PutRequest) (*pb.PutResponse, error) { +func (a *quotaApplierV3) Put(txn mvcc.TxnWrite, p *pb.PutRequest) (*pb.PutResponse, *traceutil.Trace, error) { ok := a.q.Available(p) - resp, err := a.applierV3.Put(txn, p) + resp, trace, err := a.applierV3.Put(txn, p) if err == nil && !ok { err = ErrNoSpace } - return resp, err + return resp, trace, err } func (a *quotaApplierV3) Txn(rt *pb.TxnRequest) (*pb.TxnResponse, error) { diff --git a/vendor/go.etcd.io/etcd/etcdserver/apply_auth.go b/vendor/go.etcd.io/etcd/etcdserver/apply_auth.go index 4b094ad5d8..269af4758c 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/apply_auth.go +++ b/vendor/go.etcd.io/etcd/etcdserver/apply_auth.go @@ -15,12 +15,14 @@ package etcdserver import ( + "context" "sync" "go.etcd.io/etcd/auth" pb "go.etcd.io/etcd/etcdserver/etcdserverpb" "go.etcd.io/etcd/lease" "go.etcd.io/etcd/mvcc" + "go.etcd.io/etcd/pkg/traceutil" ) type authApplierV3 struct { @@ -61,9 +63,9 @@ func (aa *authApplierV3) Apply(r *pb.InternalRaftRequest) *applyResult { return ret } -func (aa *authApplierV3) Put(txn mvcc.TxnWrite, r *pb.PutRequest) (*pb.PutResponse, error) { +func (aa *authApplierV3) Put(txn mvcc.TxnWrite, r *pb.PutRequest) (*pb.PutResponse, *traceutil.Trace, error) { if err := aa.as.IsPutPermitted(&aa.authInfo, r.Key); err != nil { - return nil, err + return nil, nil, err } if err := aa.checkLeasePuts(lease.LeaseID(r.Lease)); err != nil { @@ -71,23 +73,23 @@ func (aa *authApplierV3) Put(txn mvcc.TxnWrite, r *pb.PutRequest) (*pb.PutRespon // be written by this user. It means the user cannot revoke the // lease so attaching the lease to the newly written key should // be forbidden. - return nil, err + return nil, nil, err } if r.PrevKv { err := aa.as.IsRangePermitted(&aa.authInfo, r.Key, nil) if err != nil { - return nil, err + return nil, nil, err } } return aa.applierV3.Put(txn, r) } -func (aa *authApplierV3) Range(txn mvcc.TxnRead, r *pb.RangeRequest) (*pb.RangeResponse, error) { +func (aa *authApplierV3) Range(ctx context.Context, txn mvcc.TxnRead, r *pb.RangeRequest) (*pb.RangeResponse, error) { if err := aa.as.IsRangePermitted(&aa.authInfo, r.Key, r.RangeEnd); err != nil { return nil, err } - return aa.applierV3.Range(txn, r) + return aa.applierV3.Range(ctx, txn, r) } func (aa *authApplierV3) DeleteRange(txn mvcc.TxnWrite, r *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, error) { diff --git a/vendor/go.etcd.io/etcd/etcdserver/backend.go b/vendor/go.etcd.io/etcd/etcdserver/backend.go index 7fd8d17b56..01ba192568 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/backend.go +++ b/vendor/go.etcd.io/etcd/etcdserver/backend.go @@ -102,7 +102,7 @@ func openBackend(cfg ServerConfig) backend.Backend { // case, replace the db with the snapshot db sent by the leader. func recoverSnapshotBackend(cfg ServerConfig, oldbe backend.Backend, snapshot raftpb.Snapshot) (backend.Backend, error) { var cIndex consistentIndex - kv := mvcc.New(cfg.Logger, oldbe, &lease.FakeLessor{}, &cIndex) + kv := mvcc.New(cfg.Logger, oldbe, &lease.FakeLessor{}, &cIndex, mvcc.StoreConfig{CompactionBatchLimit: cfg.CompactionBatchLimit}) defer kv.Close() if snapshot.Metadata.Index <= kv.ConsistentIndex() { return oldbe, nil diff --git a/vendor/go.etcd.io/etcd/etcdserver/config.go b/vendor/go.etcd.io/etcd/etcdserver/config.go index 9597c6cb8b..88cd721c32 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/config.go +++ b/vendor/go.etcd.io/etcd/etcdserver/config.go @@ -112,6 +112,7 @@ type ServerConfig struct { AutoCompactionRetention time.Duration AutoCompactionMode string + CompactionBatchLimit int QuotaBackendBytes int64 MaxTxnOps uint diff --git a/vendor/go.etcd.io/etcd/etcdserver/corrupt.go b/vendor/go.etcd.io/etcd/etcdserver/corrupt.go index 32678a7c51..2351eef445 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/corrupt.go +++ b/vendor/go.etcd.io/etcd/etcdserver/corrupt.go @@ -23,6 +23,7 @@ import ( "go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes" pb "go.etcd.io/etcd/etcdserver/etcdserverpb" "go.etcd.io/etcd/mvcc" + "go.etcd.io/etcd/pkg/traceutil" "go.etcd.io/etcd/pkg/types" "go.uber.org/zap" @@ -382,11 +383,11 @@ type applierV3Corrupt struct { func newApplierV3Corrupt(a applierV3) *applierV3Corrupt { return &applierV3Corrupt{a} } -func (a *applierV3Corrupt) Put(txn mvcc.TxnWrite, p *pb.PutRequest) (*pb.PutResponse, error) { - return nil, ErrCorrupt +func (a *applierV3Corrupt) Put(txn mvcc.TxnWrite, p *pb.PutRequest) (*pb.PutResponse, *traceutil.Trace, error) { + return nil, nil, ErrCorrupt } -func (a *applierV3Corrupt) Range(txn mvcc.TxnRead, p *pb.RangeRequest) (*pb.RangeResponse, error) { +func (a *applierV3Corrupt) Range(ctx context.Context, txn mvcc.TxnRead, p *pb.RangeRequest) (*pb.RangeResponse, error) { return nil, ErrCorrupt } @@ -398,8 +399,8 @@ func (a *applierV3Corrupt) Txn(rt *pb.TxnRequest) (*pb.TxnResponse, error) { return nil, ErrCorrupt } -func (a *applierV3Corrupt) Compaction(compaction *pb.CompactionRequest) (*pb.CompactionResponse, <-chan struct{}, error) { - return nil, nil, ErrCorrupt +func (a *applierV3Corrupt) Compaction(compaction *pb.CompactionRequest) (*pb.CompactionResponse, <-chan struct{}, *traceutil.Trace, error) { + return nil, nil, nil, ErrCorrupt } func (a *applierV3Corrupt) LeaseGrant(lc *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) { diff --git a/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/rpc.pb.go b/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/rpc.pb.go index 73efc3040f..199ee6244d 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/rpc.pb.go +++ b/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/rpc.pb.go @@ -4537,7 +4537,7 @@ type AuthClient interface { AuthDisable(ctx context.Context, in *AuthDisableRequest, opts ...grpc.CallOption) (*AuthDisableResponse, error) // Authenticate processes an authenticate request. Authenticate(ctx context.Context, in *AuthenticateRequest, opts ...grpc.CallOption) (*AuthenticateResponse, error) - // UserAdd adds a new user. + // UserAdd adds a new user. User name cannot be empty. UserAdd(ctx context.Context, in *AuthUserAddRequest, opts ...grpc.CallOption) (*AuthUserAddResponse, error) // UserGet gets detailed user information. UserGet(ctx context.Context, in *AuthUserGetRequest, opts ...grpc.CallOption) (*AuthUserGetResponse, error) @@ -4551,7 +4551,7 @@ type AuthClient interface { UserGrantRole(ctx context.Context, in *AuthUserGrantRoleRequest, opts ...grpc.CallOption) (*AuthUserGrantRoleResponse, error) // UserRevokeRole revokes a role of specified user. UserRevokeRole(ctx context.Context, in *AuthUserRevokeRoleRequest, opts ...grpc.CallOption) (*AuthUserRevokeRoleResponse, error) - // RoleAdd adds a new role. + // RoleAdd adds a new role. Role name cannot be empty. RoleAdd(ctx context.Context, in *AuthRoleAddRequest, opts ...grpc.CallOption) (*AuthRoleAddResponse, error) // RoleGet gets detailed role information. RoleGet(ctx context.Context, in *AuthRoleGetRequest, opts ...grpc.CallOption) (*AuthRoleGetResponse, error) @@ -4726,7 +4726,7 @@ type AuthServer interface { AuthDisable(context.Context, *AuthDisableRequest) (*AuthDisableResponse, error) // Authenticate processes an authenticate request. Authenticate(context.Context, *AuthenticateRequest) (*AuthenticateResponse, error) - // UserAdd adds a new user. + // UserAdd adds a new user. User name cannot be empty. UserAdd(context.Context, *AuthUserAddRequest) (*AuthUserAddResponse, error) // UserGet gets detailed user information. UserGet(context.Context, *AuthUserGetRequest) (*AuthUserGetResponse, error) @@ -4740,7 +4740,7 @@ type AuthServer interface { UserGrantRole(context.Context, *AuthUserGrantRoleRequest) (*AuthUserGrantRoleResponse, error) // UserRevokeRole revokes a role of specified user. UserRevokeRole(context.Context, *AuthUserRevokeRoleRequest) (*AuthUserRevokeRoleResponse, error) - // RoleAdd adds a new role. + // RoleAdd adds a new role. Role name cannot be empty. RoleAdd(context.Context, *AuthRoleAddRequest) (*AuthRoleAddResponse, error) // RoleGet gets detailed role information. RoleGet(context.Context, *AuthRoleGetRequest) (*AuthRoleGetResponse, error) diff --git a/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/rpc.proto b/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/rpc.proto index 565f8fae50..423eabada4 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/rpc.proto +++ b/vendor/go.etcd.io/etcd/etcdserver/etcdserverpb/rpc.proto @@ -264,7 +264,7 @@ service Auth { }; } - // UserAdd adds a new user. + // UserAdd adds a new user. User name cannot be empty. rpc UserAdd(AuthUserAddRequest) returns (AuthUserAddResponse) { option (google.api.http) = { post: "/v3/auth/user/add" @@ -320,7 +320,7 @@ service Auth { }; } - // RoleAdd adds a new role. + // RoleAdd adds a new role. Role name cannot be empty. rpc RoleAdd(AuthRoleAddRequest) returns (AuthRoleAddResponse) { option (google.api.http) = { post: "/v3/auth/role/add" diff --git a/vendor/go.etcd.io/etcd/etcdserver/metrics.go b/vendor/go.etcd.io/etcd/etcdserver/metrics.go index a868331fb4..e0c0cde855 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/metrics.go +++ b/vendor/go.etcd.io/etcd/etcdserver/metrics.go @@ -76,6 +76,12 @@ var ( Name: "slow_apply_total", Help: "The total number of slow apply requests (likely overloaded from slow disk).", }) + applySnapshotInProgress = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "etcd", + Subsystem: "server", + Name: "snapshot_apply_in_progress_total", + Help: "1 if the server is applying the incoming snapshot. 0 if none.", + }) proposalsCommitted = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: "etcd", Subsystem: "server", @@ -153,6 +159,7 @@ func init() { prometheus.MustRegister(leaderChanges) prometheus.MustRegister(heartbeatSendFailures) prometheus.MustRegister(slowApplies) + prometheus.MustRegister(applySnapshotInProgress) prometheus.MustRegister(proposalsCommitted) prometheus.MustRegister(proposalsApplied) prometheus.MustRegister(proposalsPending) @@ -200,7 +207,7 @@ func monitorFileDescriptor(lg *zap.Logger, done <-chan struct{}) { } if used >= limit/5*4 { if lg != nil { - lg.Warn("80%% of file descriptors are used", zap.Uint64("used", used), zap.Uint64("limit", limit)) + lg.Warn("80% of file descriptors are used", zap.Uint64("used", used), zap.Uint64("limit", limit)) } else { plog.Warningf("80%% of the file descriptor limit is used [used = %d, limit = %d]", used, limit) } diff --git a/vendor/go.etcd.io/etcd/etcdserver/raft.go b/vendor/go.etcd.io/etcd/etcdserver/raft.go index 6f42c3504d..c0fe979053 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/raft.go +++ b/vendor/go.etcd.io/etcd/etcdserver/raft.go @@ -58,13 +58,6 @@ var ( ) func init() { - lcfg := logutil.DefaultZapLoggerConfig - lg, err := logutil.NewRaftLogger(&lcfg) - if err != nil { - log.Fatalf("cannot create raft logger %v", err) - } - raft.SetLogger(lg) - expvar.Publish("raft.status", expvar.Func(func() interface{} { raftStatusMu.Lock() defer raftStatusMu.Unlock() @@ -124,6 +117,18 @@ type raftNodeConfig struct { } func newRaftNode(cfg raftNodeConfig) *raftNode { + var lg raft.Logger + if cfg.lg != nil { + lg = logutil.NewRaftLoggerZap(cfg.lg) + } else { + lcfg := logutil.DefaultZapLoggerConfig + var err error + lg, err = logutil.NewRaftLogger(&lcfg) + if err != nil { + log.Fatalf("cannot create raft logger %v", err) + } + } + raft.SetLogger(lg) r := &raftNode{ lg: cfg.lg, tickMu: new(sync.Mutex), @@ -479,7 +484,11 @@ func startNode(cfg ServerConfig, cl *membership.RaftCluster, ids []types.ID) (id } } - n = raft.StartNode(c, peers) + if len(peers) == 0 { + n = raft.RestartNode(c) + } else { + n = raft.StartNode(c, peers) + } raftStatusMu.Lock() raftStatus = n.Status raftStatusMu.Unlock() @@ -643,7 +652,7 @@ func restartAsStandaloneNode(cfg ServerConfig, snapshot *raftpb.Snapshot) (types func getIDs(lg *zap.Logger, snap *raftpb.Snapshot, ents []raftpb.Entry) []uint64 { ids := make(map[uint64]bool) if snap != nil { - for _, id := range snap.Metadata.ConfState.Nodes { + for _, id := range snap.Metadata.ConfState.Voters { ids[id] = true } } @@ -682,27 +691,18 @@ func getIDs(lg *zap.Logger, snap *raftpb.Snapshot, ents []raftpb.Entry) []uint64 // If `self` is not inside the given ids, it creates a Raft entry to add a // default member with the given `self`. func createConfigChangeEnts(lg *zap.Logger, ids []uint64, self uint64, term, index uint64) []raftpb.Entry { - ents := make([]raftpb.Entry, 0) - next := index + 1 found := false for _, id := range ids { if id == self { found = true - continue - } - cc := &raftpb.ConfChange{ - Type: raftpb.ConfChangeRemoveNode, - NodeID: id, - } - e := raftpb.Entry{ - Type: raftpb.EntryConfChange, - Data: pbutil.MustMarshal(cc), - Term: term, - Index: next, } - ents = append(ents, e) - next++ } + + var ents []raftpb.Entry + next := index + 1 + + // NB: always add self first, then remove other nodes. Raft will panic if the + // set of voters ever becomes empty. if !found { m := membership.Member{ ID: types.ID(self), @@ -728,6 +728,26 @@ func createConfigChangeEnts(lg *zap.Logger, ids []uint64, self uint64, term, ind Index: next, } ents = append(ents, e) + next++ + } + + for _, id := range ids { + if id == self { + continue + } + cc := &raftpb.ConfChange{ + Type: raftpb.ConfChangeRemoveNode, + NodeID: id, + } + e := raftpb.Entry{ + Type: raftpb.EntryConfChange, + Data: pbutil.MustMarshal(cc), + Term: term, + Index: next, + } + ents = append(ents, e) + next++ } + return ents } diff --git a/vendor/go.etcd.io/etcd/etcdserver/server.go b/vendor/go.etcd.io/etcd/etcdserver/server.go index 236696e811..976acd684c 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/server.go +++ b/vendor/go.etcd.io/etcd/etcdserver/server.go @@ -50,6 +50,7 @@ import ( "go.etcd.io/etcd/pkg/pbutil" "go.etcd.io/etcd/pkg/runtime" "go.etcd.io/etcd/pkg/schedule" + "go.etcd.io/etcd/pkg/traceutil" "go.etcd.io/etcd/pkg/types" "go.etcd.io/etcd/pkg/wait" "go.etcd.io/etcd/raft" @@ -539,7 +540,7 @@ func NewServer(cfg ServerConfig) (srv *EtcdServer, err error) { CheckpointInterval: cfg.LeaseCheckpointInterval, ExpiredLeasesRetryInterval: srv.Cfg.ReqTimeout(), }) - srv.kv = mvcc.New(srv.getLogger(), srv.be, srv.lessor, &srv.consistIndex) + srv.kv = mvcc.New(srv.getLogger(), srv.be, srv.lessor, &srv.consistIndex, mvcc.StoreConfig{CompactionBatchLimit: cfg.CompactionBatchLimit}) if beExist { kvindex := srv.kv.ConsistentIndex() // TODO: remove kvindex != 0 checking when we do not expect users to upgrade @@ -785,7 +786,7 @@ func (s *EtcdServer) start() { } else { plog.Infof("starting server... [version: %v, cluster version: %v]", version.Version, version.Cluster(s.ClusterVersion().String())) } - membership.ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": s.ClusterVersion().String()}).Set(1) + membership.ClusterVersionMetrics.With(prometheus.Labels{"cluster_version": version.Cluster(s.ClusterVersion().String())}).Set(1) } else { if lg != nil { lg.Info( @@ -1113,6 +1114,7 @@ func (s *EtcdServer) applySnapshot(ep *etcdProgress, apply *apply) { if raft.IsEmptySnap(apply.snapshot) { return } + applySnapshotInProgress.Inc() lg := s.getLogger() if lg != nil { @@ -1138,6 +1140,7 @@ func (s *EtcdServer) applySnapshot(ep *etcdProgress, apply *apply) { } else { plog.Infof("finished applying incoming snapshot at index %d", ep.snapi) } + applySnapshotInProgress.Dec() }() if apply.snapshot.Metadata.Index <= ep.appliedi { @@ -1176,7 +1179,7 @@ func (s *EtcdServer) applySnapshot(ep *etcdProgress, apply *apply) { plog.Info("recovering lessor...") } - s.lessor.Recover(newbe, func() lease.TxnDelete { return s.kv.Write() }) + s.lessor.Recover(newbe, func() lease.TxnDelete { return s.kv.Write(traceutil.TODO()) }) if lg != nil { lg.Info("restored lease store") @@ -1980,6 +1983,12 @@ func (s *EtcdServer) sync(timeout time.Duration) { // static clientURLs of the server. // The function keeps attempting to register until it succeeds, // or its server is stopped. +// +// Use v2 store to encode member attributes, and apply through Raft +// but does not go through v2 API endpoint, which means even with v2 +// client handler disabled (e.g. --enable-v2=false), cluster can still +// process publish requests through rafthttp +// TODO: Deprecate v2 store func (s *EtcdServer) publish(timeout time.Duration) { b, err := json.Marshal(s.attributes) if err != nil { diff --git a/vendor/go.etcd.io/etcd/etcdserver/v3_server.go b/vendor/go.etcd.io/etcd/etcdserver/v3_server.go index b2084618b8..bfe08ea35c 100644 --- a/vendor/go.etcd.io/etcd/etcdserver/v3_server.go +++ b/vendor/go.etcd.io/etcd/etcdserver/v3_server.go @@ -26,6 +26,7 @@ import ( "go.etcd.io/etcd/lease" "go.etcd.io/etcd/lease/leasehttp" "go.etcd.io/etcd/mvcc" + "go.etcd.io/etcd/pkg/traceutil" "go.etcd.io/etcd/raft" "github.com/gogo/protobuf/proto" @@ -38,6 +39,7 @@ const ( // However, if the committed entries are very heavy to apply, the gap might grow. // We should stop accepting new proposals if the gap growing to a certain point. maxGapBetweenApplyAndCommitIndex = 5000 + traceThreshold = 100 * time.Millisecond ) type RaftKV interface { @@ -85,14 +87,29 @@ type Authenticator interface { } func (s *EtcdServer) Range(ctx context.Context, r *pb.RangeRequest) (*pb.RangeResponse, error) { + trace := traceutil.New("range", + s.getLogger(), + traceutil.Field{Key: "range_begin", Value: string(r.Key)}, + traceutil.Field{Key: "range_end", Value: string(r.RangeEnd)}, + ) + ctx = context.WithValue(ctx, traceutil.TraceKey, trace) + var resp *pb.RangeResponse var err error defer func(start time.Time) { warnOfExpensiveReadOnlyRangeRequest(s.getLogger(), start, r, resp, err) + if resp != nil { + trace.AddField( + traceutil.Field{Key: "response_count", Value: len(resp.Kvs)}, + traceutil.Field{Key: "response_revision", Value: resp.Header.Revision}, + ) + } + trace.LogIfLong(traceThreshold) }(time.Now()) if !r.Serializable { err = s.linearizableReadNotify(ctx) + trace.Step("agreement among raft nodes before linearized reading") if err != nil { return nil, err } @@ -101,7 +118,7 @@ func (s *EtcdServer) Range(ctx context.Context, r *pb.RangeRequest) (*pb.RangeRe return s.authStore.IsRangePermitted(ai, r.Key, r.RangeEnd) } - get := func() { resp, err = s.applyV3Base.Range(nil, r) } + get := func() { resp, err = s.applyV3Base.Range(ctx, nil, r) } if serr := s.doSerialize(ctx, chk, get); serr != nil { err = serr return nil, err @@ -110,6 +127,7 @@ func (s *EtcdServer) Range(ctx context.Context, r *pb.RangeRequest) (*pb.RangeRe } func (s *EtcdServer) Put(ctx context.Context, r *pb.PutRequest) (*pb.PutResponse, error) { + ctx = context.WithValue(ctx, traceutil.StartTimeKey, time.Now()) resp, err := s.raftRequest(ctx, pb.InternalRaftRequest{Put: r}) if err != nil { return nil, err @@ -186,7 +204,18 @@ func isTxnReadonly(r *pb.TxnRequest) bool { } func (s *EtcdServer) Compact(ctx context.Context, r *pb.CompactionRequest) (*pb.CompactionResponse, error) { + startTime := time.Now() result, err := s.processInternalRaftRequestOnce(ctx, pb.InternalRaftRequest{Compaction: r}) + trace := traceutil.TODO() + if result != nil && result.trace != nil { + trace = result.trace + defer func() { + trace.LogIfLong(traceThreshold) + }() + applyStart := result.trace.GetStartTime() + result.trace.SetStartTime(startTime) + trace.InsertStep(0, applyStart, "process raft request") + } if r.Physical && result != nil && result.physc != nil { <-result.physc // The compaction is done deleting keys; the hash is now settled @@ -195,6 +224,7 @@ func (s *EtcdServer) Compact(ctx context.Context, r *pb.CompactionRequest) (*pb. // if the compaction resumes. Force the finished compaction to // commit so it won't resume following a crash. s.be.ForceCommit() + trace.Step("physically apply compaction") } if err != nil { return nil, err @@ -210,6 +240,7 @@ func (s *EtcdServer) Compact(ctx context.Context, r *pb.CompactionRequest) (*pb. resp.Header = &pb.ResponseHeader{} } resp.Header.Revision = s.kv.Rev() + trace.AddField(traceutil.Field{Key: "response_revision", Value: resp.Header.Revision}) return resp, nil } @@ -533,6 +564,15 @@ func (s *EtcdServer) raftRequestOnce(ctx context.Context, r pb.InternalRaftReque if result.err != nil { return nil, result.err } + if startTime, ok := ctx.Value(traceutil.StartTimeKey).(time.Time); ok && result.trace != nil { + applyStart := result.trace.GetStartTime() + // The trace object is created in apply. Here reset the start time to trace + // the raft request time by the difference between the request start time + // and apply start time + result.trace.SetStartTime(startTime) + result.trace.InsertStep(0, applyStart, "process raft request") + result.trace.LogIfLong(traceThreshold) + } return result.resp, nil } @@ -547,6 +587,7 @@ func (s *EtcdServer) raftRequest(ctx context.Context, r pb.InternalRaftRequest) // doSerialize handles the auth logic, with permissions checked by "chk", for a serialized request "get". Returns a non-nil error on authentication failure. func (s *EtcdServer) doSerialize(ctx context.Context, chk func(*auth.AuthInfo) error, get func()) error { + trace := traceutil.Get(ctx) ai, err := s.AuthInfoFromCtx(ctx) if err != nil { return err @@ -558,6 +599,7 @@ func (s *EtcdServer) doSerialize(ctx context.Context, chk func(*auth.AuthInfo) e if err = chk(ai); err != nil { return err } + trace.Step("get authentication metadata") // fetch response for serialized request get() // check for stale token revision in case the auth store was updated while diff --git a/vendor/go.etcd.io/etcd/mvcc/kv.go b/vendor/go.etcd.io/etcd/mvcc/kv.go index 8e898a5ad3..c057f92611 100644 --- a/vendor/go.etcd.io/etcd/mvcc/kv.go +++ b/vendor/go.etcd.io/etcd/mvcc/kv.go @@ -18,6 +18,7 @@ import ( "go.etcd.io/etcd/lease" "go.etcd.io/etcd/mvcc/backend" "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/pkg/traceutil" ) type RangeOptions struct { @@ -102,10 +103,10 @@ type KV interface { WriteView // Read creates a read transaction. - Read() TxnRead + Read(trace *traceutil.Trace) TxnRead // Write creates a write transaction. - Write() TxnWrite + Write(trace *traceutil.Trace) TxnWrite // Hash computes the hash of the KV's backend. Hash() (hash uint32, revision int64, err error) @@ -114,7 +115,7 @@ type KV interface { HashByRev(rev int64) (hash uint32, revision int64, compactRev int64, err error) // Compact frees all superseded keys with revisions less than rev. - Compact(rev int64) (<-chan struct{}, error) + Compact(trace *traceutil.Trace, rev int64) (<-chan struct{}, error) // Commit commits outstanding txns into the underlying backend. Commit() diff --git a/vendor/go.etcd.io/etcd/mvcc/kv_view.go b/vendor/go.etcd.io/etcd/mvcc/kv_view.go index bd2e77729f..d4f0ca6880 100644 --- a/vendor/go.etcd.io/etcd/mvcc/kv_view.go +++ b/vendor/go.etcd.io/etcd/mvcc/kv_view.go @@ -14,24 +14,27 @@ package mvcc -import "go.etcd.io/etcd/lease" +import ( + "go.etcd.io/etcd/lease" + "go.etcd.io/etcd/pkg/traceutil" +) type readView struct{ kv KV } func (rv *readView) FirstRev() int64 { - tr := rv.kv.Read() + tr := rv.kv.Read(traceutil.TODO()) defer tr.End() return tr.FirstRev() } func (rv *readView) Rev() int64 { - tr := rv.kv.Read() + tr := rv.kv.Read(traceutil.TODO()) defer tr.End() return tr.Rev() } func (rv *readView) Range(key, end []byte, ro RangeOptions) (r *RangeResult, err error) { - tr := rv.kv.Read() + tr := rv.kv.Read(traceutil.TODO()) defer tr.End() return tr.Range(key, end, ro) } @@ -39,13 +42,13 @@ func (rv *readView) Range(key, end []byte, ro RangeOptions) (r *RangeResult, err type writeView struct{ kv KV } func (wv *writeView) DeleteRange(key, end []byte) (n, rev int64) { - tw := wv.kv.Write() + tw := wv.kv.Write(traceutil.TODO()) defer tw.End() return tw.DeleteRange(key, end) } func (wv *writeView) Put(key, value []byte, lease lease.LeaseID) (rev int64) { - tw := wv.kv.Write() + tw := wv.kv.Write(traceutil.TODO()) defer tw.End() return tw.Put(key, value, lease) } diff --git a/vendor/go.etcd.io/etcd/mvcc/kvstore.go b/vendor/go.etcd.io/etcd/mvcc/kvstore.go index bc6b895ed7..ed05bc2882 100644 --- a/vendor/go.etcd.io/etcd/mvcc/kvstore.go +++ b/vendor/go.etcd.io/etcd/mvcc/kvstore.go @@ -29,6 +29,7 @@ import ( "go.etcd.io/etcd/mvcc/backend" "go.etcd.io/etcd/mvcc/mvccpb" "go.etcd.io/etcd/pkg/schedule" + "go.etcd.io/etcd/pkg/traceutil" "github.com/coreos/pkg/capnslog" "go.uber.org/zap" @@ -60,6 +61,7 @@ const ( ) var restoreChunkKeys = 10000 // non-const for testing +var defaultCompactBatchLimit = 1000 // ConsistentIndexGetter is an interface that wraps the Get method. // Consistent index is the offset of an entry in a consistent replicated log. @@ -68,6 +70,10 @@ type ConsistentIndexGetter interface { ConsistentIndex() uint64 } +type StoreConfig struct { + CompactionBatchLimit int +} + type store struct { ReadView WriteView @@ -76,6 +82,8 @@ type store struct { // through atomics so must be 64-bit aligned. consistentIndex uint64 + cfg StoreConfig + // mu read locks for txns and write locks for non-txn store changes. mu sync.RWMutex @@ -108,8 +116,12 @@ type store struct { // NewStore returns a new store. It is useful to create a store inside // mvcc pkg. It should only be used for testing externally. -func NewStore(lg *zap.Logger, b backend.Backend, le lease.Lessor, ig ConsistentIndexGetter) *store { +func NewStore(lg *zap.Logger, b backend.Backend, le lease.Lessor, ig ConsistentIndexGetter, cfg StoreConfig) *store { + if cfg.CompactionBatchLimit == 0 { + cfg.CompactionBatchLimit = defaultCompactBatchLimit + } s := &store{ + cfg: cfg, b: b, ig: ig, kvindex: newTreeIndex(lg), @@ -129,7 +141,7 @@ func NewStore(lg *zap.Logger, b backend.Backend, le lease.Lessor, ig ConsistentI s.ReadView = &readView{s} s.WriteView = &writeView{s} if s.le != nil { - s.le.SetRangeDeleter(func() lease.TxnDelete { return s.Write() }) + s.le.SetRangeDeleter(func() lease.TxnDelete { return s.Write(traceutil.TODO()) }) } tx := s.b.BatchTx() @@ -258,9 +270,10 @@ func (s *store) updateCompactRev(rev int64) (<-chan struct{}, error) { return nil, nil } -func (s *store) compact(rev int64) (<-chan struct{}, error) { +func (s *store) compact(trace *traceutil.Trace, rev int64) (<-chan struct{}, error) { start := time.Now() keep := s.kvindex.Compact(rev) + trace.Step("compact in-memory index tree") ch := make(chan struct{}) var j = func(ctx context.Context) { if ctx.Err() != nil { @@ -277,6 +290,7 @@ func (s *store) compact(rev int64) (<-chan struct{}, error) { s.fifoSched.Schedule(j) indexCompactionPauseMs.Observe(float64(time.Since(start) / time.Millisecond)) + trace.Step("schedule compaction") return ch, nil } @@ -286,21 +300,21 @@ func (s *store) compactLockfree(rev int64) (<-chan struct{}, error) { return ch, err } - return s.compact(rev) + return s.compact(traceutil.TODO(), rev) } -func (s *store) Compact(rev int64) (<-chan struct{}, error) { +func (s *store) Compact(trace *traceutil.Trace, rev int64) (<-chan struct{}, error) { s.mu.Lock() ch, err := s.updateCompactRev(rev) - + trace.Step("check and update compact revision") if err != nil { s.mu.Unlock() return ch, err } s.mu.Unlock() - return s.compact(rev) + return s.compact(trace, rev) } // DefaultIgnores is a map of keys to ignore in hash checking. @@ -344,19 +358,7 @@ func (s *store) Restore(b backend.Backend) error { } func (s *store) restore() error { - b := s.b - reportDbTotalSizeInBytesMu.Lock() - reportDbTotalSizeInBytes = func() float64 { return float64(b.Size()) } - reportDbTotalSizeInBytesMu.Unlock() - reportDbTotalSizeInBytesDebuggingMu.Lock() - reportDbTotalSizeInBytesDebugging = func() float64 { return float64(b.Size()) } - reportDbTotalSizeInBytesDebuggingMu.Unlock() - reportDbTotalSizeInUseInBytesMu.Lock() - reportDbTotalSizeInUseInBytes = func() float64 { return float64(b.SizeInUse()) } - reportDbTotalSizeInUseInBytesMu.Unlock() - reportDbOpenReadTxNMu.Lock() - reportDbOpenReadTxN = func() float64 { return float64(b.OpenReadTxN()) } - reportDbOpenReadTxNMu.Unlock() + s.setupMetricsReporter() min, max := newRevBytes(), newRevBytes() revToBytes(revision{main: 1}, min) @@ -568,6 +570,36 @@ func (s *store) ConsistentIndex() uint64 { return v } +func (s *store) setupMetricsReporter() { + b := s.b + reportDbTotalSizeInBytesMu.Lock() + reportDbTotalSizeInBytes = func() float64 { return float64(b.Size()) } + reportDbTotalSizeInBytesMu.Unlock() + reportDbTotalSizeInBytesDebugMu.Lock() + reportDbTotalSizeInBytesDebug = func() float64 { return float64(b.Size()) } + reportDbTotalSizeInBytesDebugMu.Unlock() + reportDbTotalSizeInUseInBytesMu.Lock() + reportDbTotalSizeInUseInBytes = func() float64 { return float64(b.SizeInUse()) } + reportDbTotalSizeInUseInBytesMu.Unlock() + reportDbOpenReadTxNMu.Lock() + reportDbOpenReadTxN = func() float64 { return float64(b.OpenReadTxN()) } + reportDbOpenReadTxNMu.Unlock() + reportCurrentRevMu.Lock() + reportCurrentRev = func() float64 { + s.revMu.RLock() + defer s.revMu.RUnlock() + return float64(s.currentRev) + } + reportCurrentRevMu.Unlock() + reportCompactRevMu.Lock() + reportCompactRev = func() float64 { + s.revMu.RLock() + defer s.revMu.RUnlock() + return float64(s.compactMainRev) + } + reportCompactRevMu.Unlock() +} + // appendMarkTombstone appends tombstone mark to normal revision bytes. func appendMarkTombstone(lg *zap.Logger, b []byte) []byte { if len(b) != revBytesLen { diff --git a/vendor/go.etcd.io/etcd/mvcc/kvstore_compaction.go b/vendor/go.etcd.io/etcd/mvcc/kvstore_compaction.go index b1fcac8a9a..2adb498543 100644 --- a/vendor/go.etcd.io/etcd/mvcc/kvstore_compaction.go +++ b/vendor/go.etcd.io/etcd/mvcc/kvstore_compaction.go @@ -30,25 +30,23 @@ func (s *store) scheduleCompaction(compactMainRev int64, keep map[revision]struc end := make([]byte, 8) binary.BigEndian.PutUint64(end, uint64(compactMainRev+1)) - batchsize := int64(10000) last := make([]byte, 8+1+8) for { var rev revision start := time.Now() + tx := s.b.BatchTx() tx.Lock() - - keys, _ := tx.UnsafeRange(keyBucketName, last, end, batchsize) + keys, _ := tx.UnsafeRange(keyBucketName, last, end, int64(s.cfg.CompactionBatchLimit)) for _, key := range keys { rev = bytesToRev(key) if _, ok := keep[rev]; !ok { tx.UnsafeDelete(keyBucketName, key) - keyCompactions++ } } - if len(keys) < int(batchsize) { + if len(keys) < s.cfg.CompactionBatchLimit { rbytes := make([]byte, 8+1+8) revToBytes(revision{main: compactMainRev}, rbytes) tx.UnsafePut(metaBucketName, finishedCompactKeyName, rbytes) @@ -60,7 +58,7 @@ func (s *store) scheduleCompaction(compactMainRev int64, keep map[revision]struc zap.Duration("took", time.Since(totalStart)), ) } else { - plog.Printf("finished scheduled compaction at %d (took %v)", compactMainRev, time.Since(totalStart)) + plog.Infof("finished scheduled compaction at %d (took %v)", compactMainRev, time.Since(totalStart)) } return true } @@ -68,10 +66,12 @@ func (s *store) scheduleCompaction(compactMainRev int64, keep map[revision]struc // update last revToBytes(revision{main: rev.main, sub: rev.sub + 1}, last) tx.Unlock() + // Immediately commit the compaction deletes instead of letting them accumulate in the write buffer + s.b.ForceCommit() dbCompactionPauseMs.Observe(float64(time.Since(start) / time.Millisecond)) select { - case <-time.After(100 * time.Millisecond): + case <-time.After(10 * time.Millisecond): case <-s.stopc: return false } diff --git a/vendor/go.etcd.io/etcd/mvcc/kvstore_txn.go b/vendor/go.etcd.io/etcd/mvcc/kvstore_txn.go index 9698254644..716a6d82ff 100644 --- a/vendor/go.etcd.io/etcd/mvcc/kvstore_txn.go +++ b/vendor/go.etcd.io/etcd/mvcc/kvstore_txn.go @@ -18,6 +18,7 @@ import ( "go.etcd.io/etcd/lease" "go.etcd.io/etcd/mvcc/backend" "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/pkg/traceutil" "go.uber.org/zap" ) @@ -27,9 +28,11 @@ type storeTxnRead struct { firstRev int64 rev int64 + + trace *traceutil.Trace } -func (s *store) Read() TxnRead { +func (s *store) Read(trace *traceutil.Trace) TxnRead { s.mu.RLock() s.revMu.RLock() // backend holds b.readTx.RLock() only when creating the concurrentReadTx. After @@ -38,7 +41,7 @@ func (s *store) Read() TxnRead { tx.RLock() // RLock is no-op. concurrentReadTx does not need to be locked after it is created. firstRev, rev := s.compactMainRev, s.currentRev s.revMu.RUnlock() - return newMetricsTxnRead(&storeTxnRead{s, tx, firstRev, rev}) + return newMetricsTxnRead(&storeTxnRead{s, tx, firstRev, rev, trace}) } func (tr *storeTxnRead) FirstRev() int64 { return tr.firstRev } @@ -61,12 +64,12 @@ type storeTxnWrite struct { changes []mvccpb.KeyValue } -func (s *store) Write() TxnWrite { +func (s *store) Write(trace *traceutil.Trace) TxnWrite { s.mu.RLock() tx := s.b.BatchTx() tx.Lock() tw := &storeTxnWrite{ - storeTxnRead: storeTxnRead{s, tx, 0, 0}, + storeTxnRead: storeTxnRead{s, tx, 0, 0, trace}, tx: tx, beginRev: s.currentRev, changes: make([]mvccpb.KeyValue, 0, 4), @@ -124,6 +127,7 @@ func (tr *storeTxnRead) rangeKeys(key, end []byte, curRev int64, ro RangeOptions } revpairs := tr.s.kvindex.Revisions(key, end, rev) + tr.trace.Step("range keys from in-memory index tree") if len(revpairs) == 0 { return &RangeResult{KVs: nil, Count: 0, Rev: curRev}, nil } @@ -163,6 +167,7 @@ func (tr *storeTxnRead) rangeKeys(key, end []byte, curRev int64, ro RangeOptions } } } + tr.trace.Step("range keys from bolt db") return &RangeResult{KVs: kvs, Count: len(revpairs), Rev: curRev}, nil } @@ -178,7 +183,7 @@ func (tw *storeTxnWrite) put(key, value []byte, leaseID lease.LeaseID) { c = created.main oldLease = tw.s.le.GetLease(lease.LeaseItem{Key: string(key)}) } - + tw.trace.Step("get key's previous created_revision and leaseID") ibytes := newRevBytes() idxRev := revision{main: rev, sub: int64(len(tw.changes))} revToBytes(idxRev, ibytes) @@ -205,9 +210,11 @@ func (tw *storeTxnWrite) put(key, value []byte, leaseID lease.LeaseID) { } } + tw.trace.Step("marshal mvccpb.KeyValue") tw.tx.UnsafeSeqPut(keyBucketName, ibytes, d) tw.s.kvindex.Put(key, idxRev) tw.changes = append(tw.changes, kv) + tw.trace.Step("store kv pair into bolt db") if oldLease != lease.NoLease { if tw.s.le == nil { @@ -234,6 +241,7 @@ func (tw *storeTxnWrite) put(key, value []byte, leaseID lease.LeaseID) { panic("unexpected error from lease Attach") } } + tw.trace.Step("attach lease to kv pair") } func (tw *storeTxnWrite) deleteRange(key, end []byte) int64 { diff --git a/vendor/go.etcd.io/etcd/mvcc/metrics.go b/vendor/go.etcd.io/etcd/mvcc/metrics.go index 4f3c49aefa..7526ee4b59 100644 --- a/vendor/go.etcd.io/etcd/mvcc/metrics.go +++ b/vendor/go.etcd.io/etcd/mvcc/metrics.go @@ -22,6 +22,13 @@ import ( var ( rangeCounter = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "etcd", + Subsystem: "mvcc", + Name: "range_total", + Help: "Total number of ranges seen by this member.", + }) + rangeCounterDebug = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "etcd_debugging", Subsystem: "mvcc", @@ -30,6 +37,14 @@ var ( }) putCounter = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "etcd", + Subsystem: "mvcc", + Name: "put_total", + Help: "Total number of puts seen by this member.", + }) + // TODO: remove in 3.5 release + putCounterDebug = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "etcd_debugging", Subsystem: "mvcc", @@ -38,6 +53,14 @@ var ( }) deleteCounter = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "etcd", + Subsystem: "mvcc", + Name: "delete_total", + Help: "Total number of deletes seen by this member.", + }) + // TODO: remove in 3.5 release + deleteCounterDebug = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "etcd_debugging", Subsystem: "mvcc", @@ -46,6 +69,13 @@ var ( }) txnCounter = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: "etcd", + Subsystem: "mvcc", + Name: "txn_total", + Help: "Total number of txns seen by this member.", + }) + txnCounterDebug = prometheus.NewCounter( prometheus.CounterOpts{ Namespace: "etcd_debugging", Subsystem: "mvcc", @@ -162,21 +192,21 @@ var ( reportDbTotalSizeInBytes = func() float64 { return 0 } // TODO: remove this in v3.5 - dbTotalSizeDebugging = prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + dbTotalSizeDebug = prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Namespace: "etcd_debugging", Subsystem: "mvcc", Name: "db_total_size_in_bytes", Help: "Total size of the underlying database physically allocated in bytes.", }, func() float64 { - reportDbTotalSizeInBytesDebuggingMu.RLock() - defer reportDbTotalSizeInBytesDebuggingMu.RUnlock() - return reportDbTotalSizeInBytesDebugging() + reportDbTotalSizeInBytesDebugMu.RLock() + defer reportDbTotalSizeInBytesDebugMu.RUnlock() + return reportDbTotalSizeInBytesDebug() }, ) // overridden by mvcc initialization - reportDbTotalSizeInBytesDebuggingMu sync.RWMutex - reportDbTotalSizeInBytesDebugging = func() float64 { return 0 } + reportDbTotalSizeInBytesDebugMu sync.RWMutex + reportDbTotalSizeInBytesDebug = func() float64 { return 0 } dbTotalSizeInUse = prometheus.NewGaugeFunc(prometheus.GaugeOpts{ Namespace: "etcd", @@ -234,13 +264,49 @@ var ( // highest bucket start of 0.01 sec * 2^14 == 163.84 sec Buckets: prometheus.ExponentialBuckets(.01, 2, 15), }) + + currentRev = prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Namespace: "etcd_debugging", + Subsystem: "mvcc", + Name: "current_revision", + Help: "The current revision of store.", + }, + func() float64 { + reportCurrentRevMu.RLock() + defer reportCurrentRevMu.RUnlock() + return reportCurrentRev() + }, + ) + // overridden by mvcc initialization + reportCurrentRevMu sync.RWMutex + reportCurrentRev = func() float64 { return 0 } + + compactRev = prometheus.NewGaugeFunc(prometheus.GaugeOpts{ + Namespace: "etcd_debugging", + Subsystem: "mvcc", + Name: "compact_revision", + Help: "The revision of the last compaction in store.", + }, + func() float64 { + reportCompactRevMu.RLock() + defer reportCompactRevMu.RUnlock() + return reportCompactRev() + }, + ) + // overridden by mvcc initialization + reportCompactRevMu sync.RWMutex + reportCompactRev = func() float64 { return 0 } ) func init() { prometheus.MustRegister(rangeCounter) + prometheus.MustRegister(rangeCounterDebug) prometheus.MustRegister(putCounter) + prometheus.MustRegister(putCounterDebug) prometheus.MustRegister(deleteCounter) + prometheus.MustRegister(deleteCounterDebug) prometheus.MustRegister(txnCounter) + prometheus.MustRegister(txnCounterDebug) prometheus.MustRegister(keysGauge) prometheus.MustRegister(watchStreamGauge) prometheus.MustRegister(watcherGauge) @@ -252,11 +318,13 @@ func init() { prometheus.MustRegister(dbCompactionTotalMs) prometheus.MustRegister(dbCompactionKeysCounter) prometheus.MustRegister(dbTotalSize) - prometheus.MustRegister(dbTotalSizeDebugging) + prometheus.MustRegister(dbTotalSizeDebug) prometheus.MustRegister(dbTotalSizeInUse) prometheus.MustRegister(dbOpenReadTxN) prometheus.MustRegister(hashSec) prometheus.MustRegister(hashRevSec) + prometheus.MustRegister(currentRev) + prometheus.MustRegister(compactRev) } // ReportEventReceived reports that an event is received. diff --git a/vendor/go.etcd.io/etcd/mvcc/metrics_txn.go b/vendor/go.etcd.io/etcd/mvcc/metrics_txn.go index b4a29bc952..64b629c785 100644 --- a/vendor/go.etcd.io/etcd/mvcc/metrics_txn.go +++ b/vendor/go.etcd.io/etcd/mvcc/metrics_txn.go @@ -50,8 +50,18 @@ func (tw *metricsTxnWrite) End() { defer tw.TxnWrite.End() if sum := tw.ranges + tw.puts + tw.deletes; sum > 1 { txnCounter.Inc() + txnCounterDebug.Inc() // TODO: remove in 3.5 release } - rangeCounter.Add(float64(tw.ranges)) - putCounter.Add(float64(tw.puts)) - deleteCounter.Add(float64(tw.deletes)) + + ranges := float64(tw.ranges) + rangeCounter.Add(ranges) + rangeCounterDebug.Add(ranges) // TODO: remove in 3.5 release + + puts := float64(tw.puts) + putCounter.Add(puts) + putCounterDebug.Add(puts) // TODO: remove in 3.5 release + + deletes := float64(tw.deletes) + deleteCounter.Add(deletes) + deleteCounterDebug.Add(deletes) // TODO: remove in 3.5 release } diff --git a/vendor/go.etcd.io/etcd/mvcc/watchable_store.go b/vendor/go.etcd.io/etcd/mvcc/watchable_store.go index 46a9af5ed7..a51e5aa529 100644 --- a/vendor/go.etcd.io/etcd/mvcc/watchable_store.go +++ b/vendor/go.etcd.io/etcd/mvcc/watchable_store.go @@ -21,6 +21,7 @@ import ( "go.etcd.io/etcd/lease" "go.etcd.io/etcd/mvcc/backend" "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/pkg/traceutil" "go.uber.org/zap" ) @@ -68,13 +69,13 @@ type watchableStore struct { // cancel operations. type cancelFunc func() -func New(lg *zap.Logger, b backend.Backend, le lease.Lessor, ig ConsistentIndexGetter) ConsistentWatchableKV { - return newWatchableStore(lg, b, le, ig) +func New(lg *zap.Logger, b backend.Backend, le lease.Lessor, ig ConsistentIndexGetter, cfg StoreConfig) ConsistentWatchableKV { + return newWatchableStore(lg, b, le, ig, cfg) } -func newWatchableStore(lg *zap.Logger, b backend.Backend, le lease.Lessor, ig ConsistentIndexGetter) *watchableStore { +func newWatchableStore(lg *zap.Logger, b backend.Backend, le lease.Lessor, ig ConsistentIndexGetter, cfg StoreConfig) *watchableStore { s := &watchableStore{ - store: NewStore(lg, b, le, ig), + store: NewStore(lg, b, le, ig, cfg), victimc: make(chan struct{}, 1), unsynced: newWatcherGroup(), synced: newWatcherGroup(), @@ -84,7 +85,7 @@ func newWatchableStore(lg *zap.Logger, b backend.Backend, le lease.Lessor, ig Co s.store.WriteView = &writeView{s} if s.le != nil { // use this store as the deleter so revokes trigger watch events - s.le.SetRangeDeleter(func() lease.TxnDelete { return s.Write() }) + s.le.SetRangeDeleter(func() lease.TxnDelete { return s.Write(traceutil.TODO()) }) } s.wg.Add(2) go s.syncWatchersLoop() diff --git a/vendor/go.etcd.io/etcd/mvcc/watchable_store_txn.go b/vendor/go.etcd.io/etcd/mvcc/watchable_store_txn.go index 3bcfa4d756..70b12983d9 100644 --- a/vendor/go.etcd.io/etcd/mvcc/watchable_store_txn.go +++ b/vendor/go.etcd.io/etcd/mvcc/watchable_store_txn.go @@ -14,7 +14,10 @@ package mvcc -import "go.etcd.io/etcd/mvcc/mvccpb" +import ( + "go.etcd.io/etcd/mvcc/mvccpb" + "go.etcd.io/etcd/pkg/traceutil" +) func (tw *watchableStoreTxnWrite) End() { changes := tw.Changes() @@ -48,4 +51,6 @@ type watchableStoreTxnWrite struct { s *watchableStore } -func (s *watchableStore) Write() TxnWrite { return &watchableStoreTxnWrite{s.store.Write(), s} } +func (s *watchableStore) Write(trace *traceutil.Trace) TxnWrite { + return &watchableStoreTxnWrite{s.store.Write(trace), s} +} diff --git a/vendor/go.etcd.io/etcd/mvcc/watcher_group.go b/vendor/go.etcd.io/etcd/mvcc/watcher_group.go index 07029335a8..151f0de718 100644 --- a/vendor/go.etcd.io/etcd/mvcc/watcher_group.go +++ b/vendor/go.etcd.io/etcd/mvcc/watcher_group.go @@ -156,6 +156,7 @@ type watcherGroup struct { func newWatcherGroup() watcherGroup { return watcherGroup{ keyWatchers: make(watcherSetByKey), + ranges: adt.NewIntervalTree(), watchers: make(watcherSet), } } diff --git a/vendor/go.etcd.io/etcd/pkg/adt/README.md b/vendor/go.etcd.io/etcd/pkg/adt/README.md new file mode 100644 index 0000000000..a2089cd4b9 --- /dev/null +++ b/vendor/go.etcd.io/etcd/pkg/adt/README.md @@ -0,0 +1,48 @@ + +## Red-Black Tree + +*"Introduction to Algorithms" (Cormen et al, 3rd ed.), Chapter 13* + +1. Every node is either red or black. +2. The root is black. +3. Every leaf (NIL) is black. +4. If a node is red, then both its children are black. +5. For each node, all simple paths from the node to descendant leaves contain the +same number of black nodes. + +For example, + +```go +import ( + "fmt" + + "go.etcd.io/etcd/pkg/adt" +) + +func main() { + ivt := adt.NewIntervalTree() + ivt.Insert(NewInt64Interval(510, 511), 0) + ivt.Insert(NewInt64Interval(82, 83), 0) + ivt.Insert(NewInt64Interval(830, 831), 0) + ... +``` + +After inserting the values `510`, `82`, `830`, `11`, `383`, `647`, `899`, `261`, `410`, `514`, `815`, `888`, `972`, `238`, `292`, `953`. + +![red-black-tree-01-insertion.png](img/red-black-tree-01-insertion.png) + +Deleting the node `514` should not trigger any rebalancing: + +![red-black-tree-02-delete-514.png](img/red-black-tree-02-delete-514.png) + +Deleting the node `11` triggers multiple rotates for rebalancing: + +![red-black-tree-03-delete-11.png](img/red-black-tree-03-delete-11.png) +![red-black-tree-04-delete-11.png](img/red-black-tree-04-delete-11.png) +![red-black-tree-05-delete-11.png](img/red-black-tree-05-delete-11.png) +![red-black-tree-06-delete-11.png](img/red-black-tree-06-delete-11.png) +![red-black-tree-07-delete-11.png](img/red-black-tree-07-delete-11.png) +![red-black-tree-08-delete-11.png](img/red-black-tree-08-delete-11.png) +![red-black-tree-09-delete-11.png](img/red-black-tree-09-delete-11.png) + +Try yourself at https://www.cs.usfca.edu/~galles/visualization/RedBlack.html. diff --git a/vendor/go.etcd.io/etcd/pkg/adt/interval_tree.go b/vendor/go.etcd.io/etcd/pkg/adt/interval_tree.go index 342eab75bd..2e5b2ddb88 100644 --- a/vendor/go.etcd.io/etcd/pkg/adt/interval_tree.go +++ b/vendor/go.etcd.io/etcd/pkg/adt/interval_tree.go @@ -16,7 +16,9 @@ package adt import ( "bytes" + "fmt" "math" + "strings" ) // Comparable is an interface for trichotomic comparisons. @@ -35,6 +37,17 @@ const ( red ) +func (c rbcolor) String() string { + switch c { + case black: + return "black" + case red: + return "black" + default: + panic(fmt.Errorf("unknown color %d", c)) + } +} + // Interval implements a Comparable interval [begin, end) // TODO: support different sorts of intervals: (a,b), [a,b], (a, b] type Interval struct { @@ -74,39 +87,39 @@ type intervalNode struct { c rbcolor } -func (x *intervalNode) color() rbcolor { - if x == nil { +func (x *intervalNode) color(sentinel *intervalNode) rbcolor { + if x == sentinel { return black } return x.c } -func (x *intervalNode) height() int { - if x == nil { +func (x *intervalNode) height(sentinel *intervalNode) int { + if x == sentinel { return 0 } - ld := x.left.height() - rd := x.right.height() + ld := x.left.height(sentinel) + rd := x.right.height(sentinel) if ld < rd { return rd + 1 } return ld + 1 } -func (x *intervalNode) min() *intervalNode { - for x.left != nil { +func (x *intervalNode) min(sentinel *intervalNode) *intervalNode { + for x.left != sentinel { x = x.left } return x } // successor is the next in-order node in the tree -func (x *intervalNode) successor() *intervalNode { - if x.right != nil { - return x.right.min() +func (x *intervalNode) successor(sentinel *intervalNode) *intervalNode { + if x.right != sentinel { + return x.right.min(sentinel) } y := x.parent - for y != nil && x == y.right { + for y != sentinel && x == y.right { x = y y = y.parent } @@ -114,14 +127,14 @@ func (x *intervalNode) successor() *intervalNode { } // updateMax updates the maximum values for a node and its ancestors -func (x *intervalNode) updateMax() { - for x != nil { +func (x *intervalNode) updateMax(sentinel *intervalNode) { + for x != sentinel { oldmax := x.max max := x.iv.Ivl.End - if x.left != nil && x.left.max.Compare(max) > 0 { + if x.left != sentinel && x.left.max.Compare(max) > 0 { max = x.left.max } - if x.right != nil && x.right.max.Compare(max) > 0 { + if x.right != sentinel && x.right.max.Compare(max) > 0 { max = x.right.max } if oldmax.Compare(max) == 0 { @@ -135,66 +148,151 @@ func (x *intervalNode) updateMax() { type nodeVisitor func(n *intervalNode) bool // visit will call a node visitor on each node that overlaps the given interval -func (x *intervalNode) visit(iv *Interval, nv nodeVisitor) bool { - if x == nil { +func (x *intervalNode) visit(iv *Interval, sentinel *intervalNode, nv nodeVisitor) bool { + if x == sentinel { return true } v := iv.Compare(&x.iv.Ivl) switch { case v < 0: - if !x.left.visit(iv, nv) { + if !x.left.visit(iv, sentinel, nv) { return false } case v > 0: maxiv := Interval{x.iv.Ivl.Begin, x.max} if maxiv.Compare(iv) == 0 { - if !x.left.visit(iv, nv) || !x.right.visit(iv, nv) { + if !x.left.visit(iv, sentinel, nv) || !x.right.visit(iv, sentinel, nv) { return false } } default: - if !x.left.visit(iv, nv) || !nv(x) || !x.right.visit(iv, nv) { + if !x.left.visit(iv, sentinel, nv) || !nv(x) || !x.right.visit(iv, sentinel, nv) { return false } } return true } +// IntervalValue represents a range tree node that contains a range and a value. type IntervalValue struct { Ivl Interval Val interface{} } // IntervalTree represents a (mostly) textbook implementation of the -// "Introduction to Algorithms" (Cormen et al, 2nd ed.) chapter 13 red-black tree +// "Introduction to Algorithms" (Cormen et al, 3rd ed.) chapter 13 red-black tree // and chapter 14.3 interval tree with search supporting "stabbing queries". -type IntervalTree struct { +type IntervalTree interface { + // Insert adds a node with the given interval into the tree. + Insert(ivl Interval, val interface{}) + // Delete removes the node with the given interval from the tree, returning + // true if a node is in fact removed. + Delete(ivl Interval) bool + // Len gives the number of elements in the tree. + Len() int + // Height is the number of levels in the tree; one node has height 1. + Height() int + // MaxHeight is the expected maximum tree height given the number of nodes. + MaxHeight() int + // Visit calls a visitor function on every tree node intersecting the given interval. + // It will visit each interval [x, y) in ascending order sorted on x. + Visit(ivl Interval, ivv IntervalVisitor) + // Find gets the IntervalValue for the node matching the given interval + Find(ivl Interval) *IntervalValue + // Intersects returns true if there is some tree node intersecting the given interval. + Intersects(iv Interval) bool + // Contains returns true if the interval tree's keys cover the entire given interval. + Contains(ivl Interval) bool + // Stab returns a slice with all elements in the tree intersecting the interval. + Stab(iv Interval) []*IntervalValue + // Union merges a given interval tree into the receiver. + Union(inIvt IntervalTree, ivl Interval) +} + +// NewIntervalTree returns a new interval tree. +func NewIntervalTree() IntervalTree { + sentinel := &intervalNode{ + iv: IntervalValue{}, + max: nil, + left: nil, + right: nil, + parent: nil, + c: black, + } + return &intervalTree{ + root: sentinel, + count: 0, + sentinel: sentinel, + } +} + +type intervalTree struct { root *intervalNode count int + + // red-black NIL node + // use 'sentinel' as a dummy object to simplify boundary conditions + // use the sentinel to treat a nil child of a node x as an ordinary node whose parent is x + // use one shared sentinel to represent all nil leaves and the root's parent + sentinel *intervalNode } +// TODO: make this consistent with textbook implementation +// +// "Introduction to Algorithms" (Cormen et al, 3rd ed.), chapter 13.4, p324 +// +// 0. RB-DELETE(T, z) +// 1. +// 2. y = z +// 3. y-original-color = y.color +// 4. +// 5. if z.left == T.nil +// 6. x = z.right +// 7. RB-TRANSPLANT(T, z, z.right) +// 8. else if z.right == T.nil +// 9. x = z.left +// 10. RB-TRANSPLANT(T, z, z.left) +// 11. else +// 12. y = TREE-MINIMUM(z.right) +// 13. y-original-color = y.color +// 14. x = y.right +// 15. if y.p == z +// 16. x.p = y +// 17. else +// 18. RB-TRANSPLANT(T, y, y.right) +// 19. y.right = z.right +// 20. y.right.p = y +// 21. RB-TRANSPLANT(T, z, y) +// 22. y.left = z.left +// 23. y.left.p = y +// 24. y.color = z.color +// 25. +// 26. if y-original-color == BLACK +// 27. RB-DELETE-FIXUP(T, x) + // Delete removes the node with the given interval from the tree, returning // true if a node is in fact removed. -func (ivt *IntervalTree) Delete(ivl Interval) bool { +func (ivt *intervalTree) Delete(ivl Interval) bool { z := ivt.find(ivl) - if z == nil { + if z == ivt.sentinel { return false } y := z - if z.left != nil && z.right != nil { - y = z.successor() + if z.left != ivt.sentinel && z.right != ivt.sentinel { + y = z.successor(ivt.sentinel) } - x := y.left - if x == nil { + x := ivt.sentinel + if y.left != ivt.sentinel { + x = y.left + } else if y.right != ivt.sentinel { x = y.right } - if x != nil { - x.parent = y.parent - } - if y.parent == nil { + x.parent = y.parent + + if y.parent == ivt.sentinel { ivt.root = x } else { if y == y.parent.left { @@ -202,14 +300,14 @@ func (ivt *IntervalTree) Delete(ivl Interval) bool { } else { y.parent.right = x } - y.parent.updateMax() + y.parent.updateMax(ivt.sentinel) } if y != z { z.iv = y.iv - z.updateMax() + z.updateMax(ivt.sentinel) } - if y.color() == black && x != nil { + if y.color(ivt.sentinel) == black { ivt.deleteFixup(x) } @@ -217,11 +315,55 @@ func (ivt *IntervalTree) Delete(ivl Interval) bool { return true } -func (ivt *IntervalTree) deleteFixup(x *intervalNode) { - for x != ivt.root && x.color() == black && x.parent != nil { - if x == x.parent.left { +// "Introduction to Algorithms" (Cormen et al, 3rd ed.), chapter 13.4, p326 +// +// 0. RB-DELETE-FIXUP(T, z) +// 1. +// 2. while x ≠ T.root and x.color == BLACK +// 3. if x == x.p.left +// 4. w = x.p.right +// 5. if w.color == RED +// 6. w.color = BLACK +// 7. x.p.color = RED +// 8. LEFT-ROTATE(T, x, p) +// 9. if w.left.color == BLACK and w.right.color == BLACK +// 10. w.color = RED +// 11. x = x.p +// 12. else if w.right.color == BLACK +// 13. w.left.color = BLACK +// 14. w.color = RED +// 15. RIGHT-ROTATE(T, w) +// 16. w = w.p.right +// 17. w.color = x.p.color +// 18. x.p.color = BLACK +// 19. LEFT-ROTATE(T, w.p) +// 20. x = T.root +// 21. else +// 22. w = x.p.left +// 23. if w.color == RED +// 24. w.color = BLACK +// 25. x.p.color = RED +// 26. RIGHT-ROTATE(T, x, p) +// 27. if w.right.color == BLACK and w.left.color == BLACK +// 28. w.color = RED +// 29. x = x.p +// 30. else if w.left.color == BLACK +// 31. w.right.color = BLACK +// 32. w.color = RED +// 33. LEFT-ROTATE(T, w) +// 34. w = w.p.left +// 35. w.color = x.p.color +// 36. x.p.color = BLACK +// 37. RIGHT-ROTATE(T, w.p) +// 38. x = T.root +// 39. +// 40. x.color = BLACK +// +func (ivt *intervalTree) deleteFixup(x *intervalNode) { + for x != ivt.root && x.color(ivt.sentinel) == black { + if x == x.parent.left { // line 3-20 w := x.parent.right - if w.color() == red { + if w.color(ivt.sentinel) == red { w.c = black x.parent.c = red ivt.rotateLeft(x.parent) @@ -230,26 +372,26 @@ func (ivt *IntervalTree) deleteFixup(x *intervalNode) { if w == nil { break } - if w.left.color() == black && w.right.color() == black { + if w.left.color(ivt.sentinel) == black && w.right.color(ivt.sentinel) == black { w.c = red x = x.parent } else { - if w.right.color() == black { + if w.right.color(ivt.sentinel) == black { w.left.c = black w.c = red ivt.rotateRight(w) w = x.parent.right } - w.c = x.parent.color() + w.c = x.parent.color(ivt.sentinel) x.parent.c = black w.right.c = black ivt.rotateLeft(x.parent) x = ivt.root } - } else { + } else { // line 22-38 // same as above but with left and right exchanged w := x.parent.left - if w.color() == red { + if w.color(ivt.sentinel) == red { w.c = black x.parent.c = red ivt.rotateRight(x.parent) @@ -258,17 +400,17 @@ func (ivt *IntervalTree) deleteFixup(x *intervalNode) { if w == nil { break } - if w.left.color() == black && w.right.color() == black { + if w.left.color(ivt.sentinel) == black && w.right.color(ivt.sentinel) == black { w.c = red x = x.parent } else { - if w.left.color() == black { + if w.left.color(ivt.sentinel) == black { w.right.c = black w.c = red ivt.rotateLeft(w) w = x.parent.left } - w.c = x.parent.color() + w.c = x.parent.color(ivt.sentinel) x.parent.c = black w.left.c = black ivt.rotateRight(x.parent) @@ -276,17 +418,60 @@ func (ivt *IntervalTree) deleteFixup(x *intervalNode) { } } } + if x != nil { x.c = black } } +func (ivt *intervalTree) createIntervalNode(ivl Interval, val interface{}) *intervalNode { + return &intervalNode{ + iv: IntervalValue{ivl, val}, + max: ivl.End, + c: red, + left: ivt.sentinel, + right: ivt.sentinel, + parent: ivt.sentinel, + } +} + +// TODO: make this consistent with textbook implementation +// +// "Introduction to Algorithms" (Cormen et al, 3rd ed.), chapter 13.3, p315 +// +// 0. RB-INSERT(T, z) +// 1. +// 2. y = T.nil +// 3. x = T.root +// 4. +// 5. while x ≠ T.nil +// 6. y = x +// 7. if z.key < x.key +// 8. x = x.left +// 9. else +// 10. x = x.right +// 11. +// 12. z.p = y +// 13. +// 14. if y == T.nil +// 15. T.root = z +// 16. else if z.key < y.key +// 17. y.left = z +// 18. else +// 19. y.right = z +// 20. +// 21. z.left = T.nil +// 22. z.right = T.nil +// 23. z.color = RED +// 24. +// 25. RB-INSERT-FIXUP(T, z) + // Insert adds a node with the given interval into the tree. -func (ivt *IntervalTree) Insert(ivl Interval, val interface{}) { - var y *intervalNode - z := &intervalNode{iv: IntervalValue{ivl, val}, max: ivl.End, c: red} +func (ivt *intervalTree) Insert(ivl Interval, val interface{}) { + y := ivt.sentinel + z := ivt.createIntervalNode(ivl, val) x := ivt.root - for x != nil { + for x != ivt.sentinel { y = x if z.iv.Ivl.Begin.Compare(x.iv.Ivl.Begin) < 0 { x = x.left @@ -296,7 +481,7 @@ func (ivt *IntervalTree) Insert(ivl Interval, val interface{}) { } z.parent = y - if y == nil { + if y == ivt.sentinel { ivt.root = z } else { if z.iv.Ivl.Begin.Compare(y.iv.Ivl.Begin) < 0 { @@ -304,18 +489,54 @@ func (ivt *IntervalTree) Insert(ivl Interval, val interface{}) { } else { y.right = z } - y.updateMax() + y.updateMax(ivt.sentinel) } z.c = red + ivt.insertFixup(z) ivt.count++ } -func (ivt *IntervalTree) insertFixup(z *intervalNode) { - for z.parent != nil && z.parent.parent != nil && z.parent.color() == red { - if z.parent == z.parent.parent.left { +// "Introduction to Algorithms" (Cormen et al, 3rd ed.), chapter 13.3, p316 +// +// 0. RB-INSERT-FIXUP(T, z) +// 1. +// 2. while z.p.color == RED +// 3. if z.p == z.p.p.left +// 4. y = z.p.p.right +// 5. if y.color == RED +// 6. z.p.color = BLACK +// 7. y.color = BLACK +// 8. z.p.p.color = RED +// 9. z = z.p.p +// 10. else if z == z.p.right +// 11. z = z.p +// 12. LEFT-ROTATE(T, z) +// 13. z.p.color = BLACK +// 14. z.p.p.color = RED +// 15. RIGHT-ROTATE(T, z.p.p) +// 16. else +// 17. y = z.p.p.left +// 18. if y.color == RED +// 19. z.p.color = BLACK +// 20. y.color = BLACK +// 21. z.p.p.color = RED +// 22. z = z.p.p +// 23. else if z == z.p.right +// 24. z = z.p +// 25. RIGHT-ROTATE(T, z) +// 26. z.p.color = BLACK +// 27. z.p.p.color = RED +// 28. LEFT-ROTATE(T, z.p.p) +// 29. +// 30. T.root.color = BLACK +// +func (ivt *intervalTree) insertFixup(z *intervalNode) { + for z.parent.color(ivt.sentinel) == red { + if z.parent == z.parent.parent.left { // line 3-15 + y := z.parent.parent.right - if y.color() == red { + if y.color(ivt.sentinel) == red { y.c = black z.parent.c = black z.parent.parent.c = red @@ -329,10 +550,10 @@ func (ivt *IntervalTree) insertFixup(z *intervalNode) { z.parent.parent.c = red ivt.rotateRight(z.parent.parent) } - } else { + } else { // line 16-28 // same as then with left/right exchanged y := z.parent.parent.left - if y.color() == red { + if y.color(ivt.sentinel) == red { y.c = black z.parent.c = black z.parent.parent.c = red @@ -348,42 +569,109 @@ func (ivt *IntervalTree) insertFixup(z *intervalNode) { } } } + + // line 30 ivt.root.c = black } // rotateLeft moves x so it is left of its right child -func (ivt *IntervalTree) rotateLeft(x *intervalNode) { +// +// "Introduction to Algorithms" (Cormen et al, 3rd ed.), chapter 13.2, p313 +// +// 0. LEFT-ROTATE(T, x) +// 1. +// 2. y = x.right +// 3. x.right = y.left +// 4. +// 5. if y.left ≠ T.nil +// 6. y.left.p = x +// 7. +// 8. y.p = x.p +// 9. +// 10. if x.p == T.nil +// 11. T.root = y +// 12. else if x == x.p.left +// 13. x.p.left = y +// 14. else +// 15. x.p.right = y +// 16. +// 17. y.left = x +// 18. x.p = y +// +func (ivt *intervalTree) rotateLeft(x *intervalNode) { + // rotateLeft x must have right child + if x.right == ivt.sentinel { + return + } + + // line 2-3 y := x.right x.right = y.left - if y.left != nil { + + // line 5-6 + if y.left != ivt.sentinel { y.left.parent = x } - x.updateMax() + x.updateMax(ivt.sentinel) + + // line 10-15, 18 ivt.replaceParent(x, y) + + // line 17 y.left = x - y.updateMax() + y.updateMax(ivt.sentinel) } -// rotateLeft moves x so it is right of its left child -func (ivt *IntervalTree) rotateRight(x *intervalNode) { - if x == nil { +// rotateRight moves x so it is right of its left child +// +// 0. RIGHT-ROTATE(T, x) +// 1. +// 2. y = x.left +// 3. x.left = y.right +// 4. +// 5. if y.right ≠ T.nil +// 6. y.right.p = x +// 7. +// 8. y.p = x.p +// 9. +// 10. if x.p == T.nil +// 11. T.root = y +// 12. else if x == x.p.right +// 13. x.p.right = y +// 14. else +// 15. x.p.left = y +// 16. +// 17. y.right = x +// 18. x.p = y +// +func (ivt *intervalTree) rotateRight(x *intervalNode) { + // rotateRight x must have left child + if x.left == ivt.sentinel { return } + + // line 2-3 y := x.left x.left = y.right - if y.right != nil { + + // line 5-6 + if y.right != ivt.sentinel { y.right.parent = x } - x.updateMax() + x.updateMax(ivt.sentinel) + + // line 10-15, 18 ivt.replaceParent(x, y) + + // line 17 y.right = x - y.updateMax() + y.updateMax(ivt.sentinel) } // replaceParent replaces x's parent with y -func (ivt *IntervalTree) replaceParent(x *intervalNode, y *intervalNode) { +func (ivt *intervalTree) replaceParent(x *intervalNode, y *intervalNode) { y.parent = x.parent - if x.parent == nil { + if x.parent == ivt.sentinel { ivt.root = y } else { if x == x.parent.left { @@ -391,19 +679,19 @@ func (ivt *IntervalTree) replaceParent(x *intervalNode, y *intervalNode) { } else { x.parent.right = y } - x.parent.updateMax() + x.parent.updateMax(ivt.sentinel) } x.parent = y } // Len gives the number of elements in the tree -func (ivt *IntervalTree) Len() int { return ivt.count } +func (ivt *intervalTree) Len() int { return ivt.count } // Height is the number of levels in the tree; one node has height 1. -func (ivt *IntervalTree) Height() int { return ivt.root.height() } +func (ivt *intervalTree) Height() int { return ivt.root.height(ivt.sentinel) } // MaxHeight is the expected maximum tree height given the number of nodes -func (ivt *IntervalTree) MaxHeight() int { +func (ivt *intervalTree) MaxHeight() int { return int((2 * math.Log2(float64(ivt.Len()+1))) + 0.5) } @@ -412,12 +700,13 @@ type IntervalVisitor func(n *IntervalValue) bool // Visit calls a visitor function on every tree node intersecting the given interval. // It will visit each interval [x, y) in ascending order sorted on x. -func (ivt *IntervalTree) Visit(ivl Interval, ivv IntervalVisitor) { - ivt.root.visit(&ivl, func(n *intervalNode) bool { return ivv(&n.iv) }) +func (ivt *intervalTree) Visit(ivl Interval, ivv IntervalVisitor) { + ivt.root.visit(&ivl, ivt.sentinel, func(n *intervalNode) bool { return ivv(&n.iv) }) } // find the exact node for a given interval -func (ivt *IntervalTree) find(ivl Interval) (ret *intervalNode) { +func (ivt *intervalTree) find(ivl Interval) *intervalNode { + ret := ivt.sentinel f := func(n *intervalNode) bool { if n.iv.Ivl != ivl { return true @@ -425,34 +714,34 @@ func (ivt *IntervalTree) find(ivl Interval) (ret *intervalNode) { ret = n return false } - ivt.root.visit(&ivl, f) + ivt.root.visit(&ivl, ivt.sentinel, f) return ret } // Find gets the IntervalValue for the node matching the given interval -func (ivt *IntervalTree) Find(ivl Interval) (ret *IntervalValue) { +func (ivt *intervalTree) Find(ivl Interval) (ret *IntervalValue) { n := ivt.find(ivl) - if n == nil { + if n == ivt.sentinel { return nil } return &n.iv } // Intersects returns true if there is some tree node intersecting the given interval. -func (ivt *IntervalTree) Intersects(iv Interval) bool { +func (ivt *intervalTree) Intersects(iv Interval) bool { x := ivt.root - for x != nil && iv.Compare(&x.iv.Ivl) != 0 { - if x.left != nil && x.left.max.Compare(iv.Begin) > 0 { + for x != ivt.sentinel && iv.Compare(&x.iv.Ivl) != 0 { + if x.left != ivt.sentinel && x.left.max.Compare(iv.Begin) > 0 { x = x.left } else { x = x.right } } - return x != nil + return x != ivt.sentinel } // Contains returns true if the interval tree's keys cover the entire given interval. -func (ivt *IntervalTree) Contains(ivl Interval) bool { +func (ivt *intervalTree) Contains(ivl Interval) bool { var maxEnd, minBegin Comparable isContiguous := true @@ -476,7 +765,7 @@ func (ivt *IntervalTree) Contains(ivl Interval) bool { } // Stab returns a slice with all elements in the tree intersecting the interval. -func (ivt *IntervalTree) Stab(iv Interval) (ivs []*IntervalValue) { +func (ivt *intervalTree) Stab(iv Interval) (ivs []*IntervalValue) { if ivt.count == 0 { return nil } @@ -486,7 +775,7 @@ func (ivt *IntervalTree) Stab(iv Interval) (ivs []*IntervalValue) { } // Union merges a given interval tree into the receiver. -func (ivt *IntervalTree) Union(inIvt IntervalTree, ivl Interval) { +func (ivt *intervalTree) Union(inIvt IntervalTree, ivl Interval) { f := func(n *IntervalValue) bool { ivt.Insert(n.Ivl, n.Val) return true @@ -494,6 +783,63 @@ func (ivt *IntervalTree) Union(inIvt IntervalTree, ivl Interval) { inIvt.Visit(ivl, f) } +type visitedInterval struct { + root Interval + left Interval + right Interval + color rbcolor + depth int +} + +func (vi visitedInterval) String() string { + bd := new(strings.Builder) + bd.WriteString(fmt.Sprintf("root [%v,%v,%v], left [%v,%v], right [%v,%v], depth %d", + vi.root.Begin, vi.root.End, vi.color, + vi.left.Begin, vi.left.End, + vi.right.Begin, vi.right.End, + vi.depth, + )) + return bd.String() +} + +// visitLevel traverses tree in level order. +// used for testing +func (ivt *intervalTree) visitLevel() []visitedInterval { + if ivt.root == ivt.sentinel { + return nil + } + + rs := make([]visitedInterval, 0, ivt.Len()) + + type pair struct { + node *intervalNode + depth int + } + queue := []pair{{ivt.root, 0}} + for len(queue) > 0 { + f := queue[0] + queue = queue[1:] + + vi := visitedInterval{ + root: f.node.iv.Ivl, + color: f.node.color(ivt.sentinel), + depth: f.depth, + } + if f.node.left != ivt.sentinel { + vi.left = f.node.left.iv.Ivl + queue = append(queue, pair{f.node.left, f.depth + 1}) + } + if f.node.right != ivt.sentinel { + vi.right = f.node.right.iv.Ivl + queue = append(queue, pair{f.node.right, f.depth + 1}) + } + + rs = append(rs, vi) + } + + return rs +} + type StringComparable string func (s StringComparable) Compare(c Comparable) int { @@ -543,6 +889,7 @@ func (s StringAffineComparable) Compare(c Comparable) int { func NewStringAffineInterval(begin, end string) Interval { return Interval{StringAffineComparable(begin), StringAffineComparable(end)} } + func NewStringAffinePoint(s string) Interval { return NewStringAffineInterval(s, s+"\x00") } @@ -551,6 +898,10 @@ func NewInt64Interval(a int64, b int64) Interval { return Interval{Int64Comparable(a), Int64Comparable(b)} } +func newInt64EmptyInterval() Interval { + return Interval{Begin: nil, End: nil} +} + func NewInt64Point(a int64) Interval { return Interval{Int64Comparable(a), Int64Comparable(a + 1)} } @@ -591,6 +942,7 @@ func (b BytesAffineComparable) Compare(c Comparable) int { func NewBytesAffineInterval(begin, end []byte) Interval { return Interval{BytesAffineComparable(begin), BytesAffineComparable(end)} } + func NewBytesAffinePoint(b []byte) Interval { be := make([]byte, len(b)+1) copy(be, b) diff --git a/vendor/go.etcd.io/etcd/pkg/logutil/log_level.go b/vendor/go.etcd.io/etcd/pkg/logutil/log_level.go new file mode 100644 index 0000000000..d57e173944 --- /dev/null +++ b/vendor/go.etcd.io/etcd/pkg/logutil/log_level.go @@ -0,0 +1,70 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logutil + +import ( + "fmt" + + "github.com/coreos/pkg/capnslog" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +var DefaultLogLevel = "info" + +// ConvertToZapLevel converts log level string to zapcore.Level. +func ConvertToZapLevel(lvl string) zapcore.Level { + switch lvl { + case "debug": + return zap.DebugLevel + case "info": + return zap.InfoLevel + case "warn": + return zap.WarnLevel + case "error": + return zap.ErrorLevel + case "dpanic": + return zap.DPanicLevel + case "panic": + return zap.PanicLevel + case "fatal": + return zap.FatalLevel + default: + panic(fmt.Sprintf("unknown level %q", lvl)) + } +} + +// ConvertToCapnslogLogLevel convert log level string to capnslog.LogLevel. +// TODO: deprecate this in 3.5 +func ConvertToCapnslogLogLevel(lvl string) capnslog.LogLevel { + switch lvl { + case "debug": + return capnslog.DEBUG + case "info": + return capnslog.INFO + case "warn": + return capnslog.WARNING + case "error": + return capnslog.ERROR + case "dpanic": + return capnslog.CRITICAL + case "panic": + return capnslog.CRITICAL + case "fatal": + return capnslog.CRITICAL + default: + panic(fmt.Sprintf("unknown level %q", lvl)) + } +} diff --git a/vendor/go.etcd.io/etcd/pkg/logutil/zap.go b/vendor/go.etcd.io/etcd/pkg/logutil/zap.go index 313d914c10..8fc6e03b77 100644 --- a/vendor/go.etcd.io/etcd/pkg/logutil/zap.go +++ b/vendor/go.etcd.io/etcd/pkg/logutil/zap.go @@ -23,7 +23,7 @@ import ( // DefaultZapLoggerConfig defines default zap logger configuration. var DefaultZapLoggerConfig = zap.Config{ - Level: zap.NewAtomicLevelAt(zap.InfoLevel), + Level: zap.NewAtomicLevelAt(ConvertToZapLevel(DefaultLogLevel)), Development: false, Sampling: &zap.SamplingConfig{ @@ -53,15 +53,12 @@ var DefaultZapLoggerConfig = zap.Config{ ErrorOutputPaths: []string{"stderr"}, } -// AddOutputPaths adds output paths to the existing output paths, resolving conflicts. -func AddOutputPaths(cfg zap.Config, outputPaths, errorOutputPaths []string) zap.Config { +// MergeOutputPaths merges logging output paths, resolving conflicts. +func MergeOutputPaths(cfg zap.Config) zap.Config { outputs := make(map[string]struct{}) for _, v := range cfg.OutputPaths { outputs[v] = struct{}{} } - for _, v := range outputPaths { - outputs[v] = struct{}{} - } outputSlice := make([]string, 0) if _, ok := outputs["/dev/null"]; ok { // "/dev/null" to discard all @@ -78,9 +75,6 @@ func AddOutputPaths(cfg zap.Config, outputPaths, errorOutputPaths []string) zap. for _, v := range cfg.ErrorOutputPaths { errOutputs[v] = struct{}{} } - for _, v := range errorOutputPaths { - errOutputs[v] = struct{}{} - } errOutputSlice := make([]string, 0) if _, ok := errOutputs["/dev/null"]; ok { // "/dev/null" to discard all diff --git a/vendor/go.etcd.io/etcd/pkg/logutil/zap_raft.go b/vendor/go.etcd.io/etcd/pkg/logutil/zap_raft.go index e92cba04cb..f016b3054e 100644 --- a/vendor/go.etcd.io/etcd/pkg/logutil/zap_raft.go +++ b/vendor/go.etcd.io/etcd/pkg/logutil/zap_raft.go @@ -23,7 +23,7 @@ import ( "go.uber.org/zap/zapcore" ) -// NewRaftLogger converts "*zap.Logger" to "raft.Logger". +// NewRaftLogger builds "raft.Logger" from "*zap.Config". func NewRaftLogger(lcfg *zap.Config) (raft.Logger, error) { if lcfg == nil { return nil, errors.New("nil zap.Config") @@ -35,6 +35,11 @@ func NewRaftLogger(lcfg *zap.Config) (raft.Logger, error) { return &zapRaftLogger{lg: lg, sugar: lg.Sugar()}, nil } +// NewRaftLoggerZap converts "*zap.Logger" to "raft.Logger". +func NewRaftLoggerZap(lg *zap.Logger) raft.Logger { + return &zapRaftLogger{lg: lg, sugar: lg.Sugar()} +} + // NewRaftLoggerFromZapCore creates "raft.Logger" from "zap.Core" // and "zapcore.WriteSyncer". func NewRaftLoggerFromZapCore(cr zapcore.Core, syncer zapcore.WriteSyncer) raft.Logger { diff --git a/vendor/go.etcd.io/etcd/pkg/traceutil/trace.go b/vendor/go.etcd.io/etcd/pkg/traceutil/trace.go new file mode 100644 index 0000000000..2d247dd9ac --- /dev/null +++ b/vendor/go.etcd.io/etcd/pkg/traceutil/trace.go @@ -0,0 +1,172 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package traceutil implements tracing utilities using "context". +package traceutil + +import ( + "bytes" + "context" + "fmt" + "math/rand" + "time" + + "go.uber.org/zap" +) + +const ( + TraceKey = "trace" + StartTimeKey = "startTime" +) + +// Field is a kv pair to record additional details of the trace. +type Field struct { + Key string + Value interface{} +} + +func (f *Field) format() string { + return fmt.Sprintf("%s:%v; ", f.Key, f.Value) +} + +func writeFields(fields []Field) string { + if len(fields) == 0 { + return "" + } + var buf bytes.Buffer + buf.WriteString("{") + for _, f := range fields { + buf.WriteString(f.format()) + } + buf.WriteString("}") + return buf.String() +} + +type Trace struct { + operation string + lg *zap.Logger + fields []Field + startTime time.Time + steps []step + stepDisabled bool +} + +type step struct { + time time.Time + msg string + fields []Field +} + +func New(op string, lg *zap.Logger, fields ...Field) *Trace { + return &Trace{operation: op, lg: lg, startTime: time.Now(), fields: fields} +} + +// TODO returns a non-nil, empty Trace +func TODO() *Trace { + return &Trace{} +} + +func Get(ctx context.Context) *Trace { + if trace, ok := ctx.Value(TraceKey).(*Trace); ok && trace != nil { + return trace + } + return TODO() +} + +func (t *Trace) GetStartTime() time.Time { + return t.startTime +} + +func (t *Trace) SetStartTime(time time.Time) { + t.startTime = time +} + +func (t *Trace) InsertStep(at int, time time.Time, msg string, fields ...Field) { + newStep := step{time, msg, fields} + if at < len(t.steps) { + t.steps = append(t.steps[:at+1], t.steps[at:]...) + t.steps[at] = newStep + } else { + t.steps = append(t.steps, newStep) + } +} + +// Step adds step to trace +func (t *Trace) Step(msg string, fields ...Field) { + if !t.stepDisabled { + t.steps = append(t.steps, step{time: time.Now(), msg: msg, fields: fields}) + } +} + +// DisableStep sets the flag to prevent the trace from adding steps +func (t *Trace) DisableStep() { + t.stepDisabled = true +} + +// EnableStep re-enable the trace to add steps +func (t *Trace) EnableStep() { + t.stepDisabled = false +} + +func (t *Trace) AddField(fields ...Field) { + for _, f := range fields { + t.fields = append(t.fields, f) + } +} + +// Log dumps all steps in the Trace +func (t *Trace) Log() { + t.LogWithStepThreshold(0) +} + +// LogIfLong dumps logs if the duration is longer than threshold +func (t *Trace) LogIfLong(threshold time.Duration) { + if time.Since(t.startTime) > threshold { + stepThreshold := threshold / time.Duration(len(t.steps)+1) + t.LogWithStepThreshold(stepThreshold) + } +} + +// LogWithStepThreshold only dumps step whose duration is longer than step threshold +func (t *Trace) LogWithStepThreshold(threshold time.Duration) { + msg, fs := t.logInfo(threshold) + if t.lg != nil { + t.lg.Info(msg, fs...) + } +} + +func (t *Trace) logInfo(threshold time.Duration) (string, []zap.Field) { + endTime := time.Now() + totalDuration := endTime.Sub(t.startTime) + traceNum := rand.Int31() + msg := fmt.Sprintf("trace[%d] %s", traceNum, t.operation) + + var steps []string + lastStepTime := t.startTime + for _, step := range t.steps { + stepDuration := step.time.Sub(lastStepTime) + if stepDuration > threshold { + steps = append(steps, fmt.Sprintf("trace[%d] '%v' %s (duration: %v)", + traceNum, step.msg, writeFields(step.fields), stepDuration)) + } + lastStepTime = step.time + } + + fs := []zap.Field{zap.String("detail", writeFields(t.fields)), + zap.Duration("duration", totalDuration), + zap.Time("start", t.startTime), + zap.Time("end", endTime), + zap.Strings("steps", steps)} + return msg, fs +} diff --git a/vendor/go.etcd.io/etcd/pkg/transport/listener.go b/vendor/go.etcd.io/etcd/pkg/transport/listener.go index 0c593e8e2b..80e35bda59 100644 --- a/vendor/go.etcd.io/etcd/pkg/transport/listener.go +++ b/vendor/go.etcd.io/etcd/pkg/transport/listener.go @@ -56,16 +56,20 @@ func wrapTLS(scheme string, tlsinfo *TLSInfo, l net.Listener) (net.Listener, err if scheme != "https" && scheme != "unixs" { return l, nil } + if tlsinfo != nil && tlsinfo.SkipClientSANVerify { + return NewTLSListener(l, tlsinfo) + } return newTLSListener(l, tlsinfo, checkSAN) } type TLSInfo struct { - CertFile string - KeyFile string - TrustedCAFile string - ClientCertAuth bool - CRLFile string - InsecureSkipVerify bool + CertFile string + KeyFile string + TrustedCAFile string + ClientCertAuth bool + CRLFile string + InsecureSkipVerify bool + SkipClientSANVerify bool // ServerName ensures the cert matches the given host in case of discovery / virtual hosting ServerName string @@ -88,6 +92,10 @@ type TLSInfo struct { // AllowedCN is a CN which must be provided by a client. AllowedCN string + // AllowedHostname is an IP address or hostname that must match the TLS + // certificate provided by a client. + AllowedHostname string + // Logger logs TLS errors. // If nil, all logs are discarded. Logger *zap.Logger @@ -105,7 +113,7 @@ func (info TLSInfo) Empty() bool { return info.CertFile == "" && info.KeyFile == "" } -func SelfCert(lg *zap.Logger, dirpath string, hosts []string) (info TLSInfo, err error) { +func SelfCert(lg *zap.Logger, dirpath string, hosts []string, additionalUsages ...x509.ExtKeyUsage) (info TLSInfo, err error) { if err = os.MkdirAll(dirpath, 0700); err != nil { return } @@ -141,7 +149,7 @@ func SelfCert(lg *zap.Logger, dirpath string, hosts []string) (info TLSInfo, err NotAfter: time.Now().Add(365 * (24 * time.Hour)), KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + ExtKeyUsage: append([]x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, additionalUsages...), BasicConstraintsValid: true, } @@ -256,16 +264,32 @@ func (info TLSInfo) baseConfig() (*tls.Config, error) { cfg.CipherSuites = info.CipherSuites } + // Client certificates may be verified by either an exact match on the CN, + // or a more general check of the CN and SANs. + var verifyCertificate func(*x509.Certificate) bool if info.AllowedCN != "" { + if info.AllowedHostname != "" { + return nil, fmt.Errorf("AllowedCN and AllowedHostname are mutually exclusive (cn=%q, hostname=%q)", info.AllowedCN, info.AllowedHostname) + } + verifyCertificate = func(cert *x509.Certificate) bool { + return info.AllowedCN == cert.Subject.CommonName + } + } + if info.AllowedHostname != "" { + verifyCertificate = func(cert *x509.Certificate) bool { + return cert.VerifyHostname(info.AllowedHostname) == nil + } + } + if verifyCertificate != nil { cfg.VerifyPeerCertificate = func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { for _, chains := range verifiedChains { if len(chains) != 0 { - if info.AllowedCN == chains[0].Subject.CommonName { + if verifyCertificate(chains[0]) { return nil } } } - return errors.New("CommonName authentication failed") + return errors.New("client certificate authentication failed") } } diff --git a/vendor/go.etcd.io/etcd/pkg/types/set.go b/vendor/go.etcd.io/etcd/pkg/types/set.go index c111b0c0c0..e7a3cdc9ab 100644 --- a/vendor/go.etcd.io/etcd/pkg/types/set.go +++ b/vendor/go.etcd.io/etcd/pkg/types/set.go @@ -148,6 +148,14 @@ func (ts *tsafeSet) Contains(value string) (exists bool) { func (ts *tsafeSet) Equals(other Set) bool { ts.m.RLock() defer ts.m.RUnlock() + + // If ts and other represent the same variable, avoid calling + // ts.us.Equals(other), to avoid double RLock bug + if _other, ok := other.(*tsafeSet); ok { + if _other == ts { + return true + } + } return ts.us.Equals(other) } @@ -173,6 +181,15 @@ func (ts *tsafeSet) Copy() Set { func (ts *tsafeSet) Sub(other Set) Set { ts.m.RLock() defer ts.m.RUnlock() + + // If ts and other represent the same variable, avoid calling + // ts.us.Sub(other), to avoid double RLock bug + if _other, ok := other.(*tsafeSet); ok { + if _other == ts { + usResult := NewUnsafeSet() + return &tsafeSet{usResult, sync.RWMutex{}} + } + } usResult := ts.us.Sub(other).(*unsafeSet) return &tsafeSet{usResult, sync.RWMutex{}} } diff --git a/vendor/go.etcd.io/etcd/proxy/grpcproxy/adapter/chan_stream.go b/vendor/go.etcd.io/etcd/proxy/grpcproxy/adapter/chan_stream.go index 82e3411931..1af514b1fd 100644 --- a/vendor/go.etcd.io/etcd/proxy/grpcproxy/adapter/chan_stream.go +++ b/vendor/go.etcd.io/etcd/proxy/grpcproxy/adapter/chan_stream.go @@ -18,7 +18,9 @@ import ( "context" "google.golang.org/grpc" + "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" ) // chanServerStream implements grpc.ServerStream with a chanStream @@ -120,7 +122,7 @@ func (s *chanStream) RecvMsg(m interface{}) error { select { case msg, ok := <-s.recvc: if !ok { - return grpc.ErrClientConnClosing + return status.Error(codes.Canceled, "the client connection is closing") } if err, ok := msg.(error); ok { return err diff --git a/vendor/go.etcd.io/etcd/raft/bootstrap.go b/vendor/go.etcd.io/etcd/raft/bootstrap.go new file mode 100644 index 0000000000..bd82b2041a --- /dev/null +++ b/vendor/go.etcd.io/etcd/raft/bootstrap.go @@ -0,0 +1,80 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raft + +import ( + "errors" + + pb "go.etcd.io/etcd/raft/raftpb" +) + +// Bootstrap initializes the RawNode for first use by appending configuration +// changes for the supplied peers. This method returns an error if the Storage +// is nonempty. +// +// It is recommended that instead of calling this method, applications bootstrap +// their state manually by setting up a Storage that has a first index > 1 and +// which stores the desired ConfState as its InitialState. +func (rn *RawNode) Bootstrap(peers []Peer) error { + if len(peers) == 0 { + return errors.New("must provide at least one peer to Bootstrap") + } + lastIndex, err := rn.raft.raftLog.storage.LastIndex() + if err != nil { + return err + } + + if lastIndex != 0 { + return errors.New("can't bootstrap a nonempty Storage") + } + + // We've faked out initial entries above, but nothing has been + // persisted. Start with an empty HardState (thus the first Ready will + // emit a HardState update for the app to persist). + rn.prevHardSt = emptyState + + // TODO(tbg): remove StartNode and give the application the right tools to + // bootstrap the initial membership in a cleaner way. + rn.raft.becomeFollower(1, None) + ents := make([]pb.Entry, len(peers)) + for i, peer := range peers { + cc := pb.ConfChange{Type: pb.ConfChangeAddNode, NodeID: peer.ID, Context: peer.Context} + data, err := cc.Marshal() + if err != nil { + return err + } + + ents[i] = pb.Entry{Type: pb.EntryConfChange, Term: 1, Index: uint64(i + 1), Data: data} + } + rn.raft.raftLog.append(ents...) + + // Now apply them, mainly so that the application can call Campaign + // immediately after StartNode in tests. Note that these nodes will + // be added to raft twice: here and when the application's Ready + // loop calls ApplyConfChange. The calls to addNode must come after + // all calls to raftLog.append so progress.next is set after these + // bootstrapping entries (it is an error if we try to append these + // entries since they have already been committed). + // We do not set raftLog.applied so the application will be able + // to observe all conf changes via Ready.CommittedEntries. + // + // TODO(bdarnell): These entries are still unstable; do we need to preserve + // the invariant that committed < unstable? + rn.raft.raftLog.committed = uint64(len(ents)) + for _, peer := range peers { + rn.raft.applyConfChange(pb.ConfChange{NodeID: peer.ID, Type: pb.ConfChangeAddNode}.AsV2()) + } + return nil +} diff --git a/vendor/go.etcd.io/etcd/raft/confchange/confchange.go b/vendor/go.etcd.io/etcd/raft/confchange/confchange.go new file mode 100644 index 0000000000..a0dc486df4 --- /dev/null +++ b/vendor/go.etcd.io/etcd/raft/confchange/confchange.go @@ -0,0 +1,425 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package confchange + +import ( + "errors" + "fmt" + "strings" + + "go.etcd.io/etcd/raft/quorum" + pb "go.etcd.io/etcd/raft/raftpb" + "go.etcd.io/etcd/raft/tracker" +) + +// Changer facilitates configuration changes. It exposes methods to handle +// simple and joint consensus while performing the proper validation that allows +// refusing invalid configuration changes before they affect the active +// configuration. +type Changer struct { + Tracker tracker.ProgressTracker + LastIndex uint64 +} + +// EnterJoint verifies that the outgoing (=right) majority config of the joint +// config is empty and initializes it with a copy of the incoming (=left) +// majority config. That is, it transitions from +// +// (1 2 3)&&() +// to +// (1 2 3)&&(1 2 3). +// +// The supplied changes are then applied to the incoming majority config, +// resulting in a joint configuration that in terms of the Raft thesis[1] +// (Section 4.3) corresponds to `C_{new,old}`. +// +// [1]: https://github.com/ongardie/dissertation/blob/master/online-trim.pdf +func (c Changer) EnterJoint(autoLeave bool, ccs ...pb.ConfChangeSingle) (tracker.Config, tracker.ProgressMap, error) { + cfg, prs, err := c.checkAndCopy() + if err != nil { + return c.err(err) + } + if joint(cfg) { + err := errors.New("config is already joint") + return c.err(err) + } + if len(incoming(cfg.Voters)) == 0 { + // We allow adding nodes to an empty config for convenience (testing and + // bootstrap), but you can't enter a joint state. + err := errors.New("can't make a zero-voter config joint") + return c.err(err) + } + // Clear the outgoing config. + *outgoingPtr(&cfg.Voters) = quorum.MajorityConfig{} + // Copy incoming to outgoing. + for id := range incoming(cfg.Voters) { + outgoing(cfg.Voters)[id] = struct{}{} + } + + if err := c.apply(&cfg, prs, ccs...); err != nil { + return c.err(err) + } + cfg.AutoLeave = autoLeave + return checkAndReturn(cfg, prs) +} + +// LeaveJoint transitions out of a joint configuration. It is an error to call +// this method if the configuration is not joint, i.e. if the outgoing majority +// config Voters[1] is empty. +// +// The outgoing majority config of the joint configuration will be removed, +// that is, the incoming config is promoted as the sole decision maker. In the +// notation of the Raft thesis[1] (Section 4.3), this method transitions from +// `C_{new,old}` into `C_new`. +// +// At the same time, any staged learners (LearnersNext) the addition of which +// was held back by an overlapping voter in the former outgoing config will be +// inserted into Learners. +// +// [1]: https://github.com/ongardie/dissertation/blob/master/online-trim.pdf +func (c Changer) LeaveJoint() (tracker.Config, tracker.ProgressMap, error) { + cfg, prs, err := c.checkAndCopy() + if err != nil { + return c.err(err) + } + if !joint(cfg) { + err := errors.New("can't leave a non-joint config") + return c.err(err) + } + if len(outgoing(cfg.Voters)) == 0 { + err := fmt.Errorf("configuration is not joint: %v", cfg) + return c.err(err) + } + for id := range cfg.LearnersNext { + nilAwareAdd(&cfg.Learners, id) + prs[id].IsLearner = true + } + cfg.LearnersNext = nil + + for id := range outgoing(cfg.Voters) { + _, isVoter := incoming(cfg.Voters)[id] + _, isLearner := cfg.Learners[id] + + if !isVoter && !isLearner { + delete(prs, id) + } + } + *outgoingPtr(&cfg.Voters) = nil + cfg.AutoLeave = false + + return checkAndReturn(cfg, prs) +} + +// Simple carries out a series of configuration changes that (in aggregate) +// mutates the incoming majority config Voters[0] by at most one. This method +// will return an error if that is not the case, if the resulting quorum is +// zero, or if the configuration is in a joint state (i.e. if there is an +// outgoing configuration). +func (c Changer) Simple(ccs ...pb.ConfChangeSingle) (tracker.Config, tracker.ProgressMap, error) { + cfg, prs, err := c.checkAndCopy() + if err != nil { + return c.err(err) + } + if joint(cfg) { + err := errors.New("can't apply simple config change in joint config") + return c.err(err) + } + if err := c.apply(&cfg, prs, ccs...); err != nil { + return c.err(err) + } + if n := symdiff(incoming(c.Tracker.Voters), incoming(cfg.Voters)); n > 1 { + return tracker.Config{}, nil, errors.New("more than one voter changed without entering joint config") + } + if err := checkInvariants(cfg, prs); err != nil { + return tracker.Config{}, tracker.ProgressMap{}, nil + } + + return checkAndReturn(cfg, prs) +} + +// apply a change to the configuration. By convention, changes to voters are +// always made to the incoming majority config Voters[0]. Voters[1] is either +// empty or preserves the outgoing majority configuration while in a joint state. +func (c Changer) apply(cfg *tracker.Config, prs tracker.ProgressMap, ccs ...pb.ConfChangeSingle) error { + for _, cc := range ccs { + if cc.NodeID == 0 { + // etcd replaces the NodeID with zero if it decides (downstream of + // raft) to not apply a change, so we have to have explicit code + // here to ignore these. + continue + } + switch cc.Type { + case pb.ConfChangeAddNode: + c.makeVoter(cfg, prs, cc.NodeID) + case pb.ConfChangeAddLearnerNode: + c.makeLearner(cfg, prs, cc.NodeID) + case pb.ConfChangeRemoveNode: + c.remove(cfg, prs, cc.NodeID) + case pb.ConfChangeUpdateNode: + default: + return fmt.Errorf("unexpected conf type %d", cc.Type) + } + } + if len(incoming(cfg.Voters)) == 0 { + return errors.New("removed all voters") + } + return nil +} + +// makeVoter adds or promotes the given ID to be a voter in the incoming +// majority config. +func (c Changer) makeVoter(cfg *tracker.Config, prs tracker.ProgressMap, id uint64) { + pr := prs[id] + if pr == nil { + c.initProgress(cfg, prs, id, false /* isLearner */) + return + } + + pr.IsLearner = false + nilAwareDelete(&cfg.Learners, id) + nilAwareDelete(&cfg.LearnersNext, id) + incoming(cfg.Voters)[id] = struct{}{} + return +} + +// makeLearner makes the given ID a learner or stages it to be a learner once +// an active joint configuration is exited. +// +// The former happens when the peer is not a part of the outgoing config, in +// which case we either add a new learner or demote a voter in the incoming +// config. +// +// The latter case occurs when the configuration is joint and the peer is a +// voter in the outgoing config. In that case, we do not want to add the peer +// as a learner because then we'd have to track a peer as a voter and learner +// simultaneously. Instead, we add the learner to LearnersNext, so that it will +// be added to Learners the moment the outgoing config is removed by +// LeaveJoint(). +func (c Changer) makeLearner(cfg *tracker.Config, prs tracker.ProgressMap, id uint64) { + pr := prs[id] + if pr == nil { + c.initProgress(cfg, prs, id, true /* isLearner */) + return + } + if pr.IsLearner { + return + } + // Remove any existing voter in the incoming config... + c.remove(cfg, prs, id) + // ... but save the Progress. + prs[id] = pr + // Use LearnersNext if we can't add the learner to Learners directly, i.e. + // if the peer is still tracked as a voter in the outgoing config. It will + // be turned into a learner in LeaveJoint(). + // + // Otherwise, add a regular learner right away. + if _, onRight := outgoing(cfg.Voters)[id]; onRight { + nilAwareAdd(&cfg.LearnersNext, id) + } else { + pr.IsLearner = true + nilAwareAdd(&cfg.Learners, id) + } +} + +// remove this peer as a voter or learner from the incoming config. +func (c Changer) remove(cfg *tracker.Config, prs tracker.ProgressMap, id uint64) { + if _, ok := prs[id]; !ok { + return + } + + delete(incoming(cfg.Voters), id) + nilAwareDelete(&cfg.Learners, id) + nilAwareDelete(&cfg.LearnersNext, id) + + // If the peer is still a voter in the outgoing config, keep the Progress. + if _, onRight := outgoing(cfg.Voters)[id]; !onRight { + delete(prs, id) + } +} + +// initProgress initializes a new progress for the given node or learner. +func (c Changer) initProgress(cfg *tracker.Config, prs tracker.ProgressMap, id uint64, isLearner bool) { + if !isLearner { + incoming(cfg.Voters)[id] = struct{}{} + } else { + nilAwareAdd(&cfg.Learners, id) + } + prs[id] = &tracker.Progress{ + // Initializing the Progress with the last index means that the follower + // can be probed (with the last index). + // + // TODO(tbg): seems awfully optimistic. Using the first index would be + // better. The general expectation here is that the follower has no log + // at all (and will thus likely need a snapshot), though the app may + // have applied a snapshot out of band before adding the replica (thus + // making the first index the better choice). + Next: c.LastIndex, + Match: 0, + Inflights: tracker.NewInflights(c.Tracker.MaxInflight), + IsLearner: isLearner, + // When a node is first added, we should mark it as recently active. + // Otherwise, CheckQuorum may cause us to step down if it is invoked + // before the added node has had a chance to communicate with us. + RecentActive: true, + } +} + +// checkInvariants makes sure that the config and progress are compatible with +// each other. This is used to check both what the Changer is initialized with, +// as well as what it returns. +func checkInvariants(cfg tracker.Config, prs tracker.ProgressMap) error { + // NB: intentionally allow the empty config. In production we'll never see a + // non-empty config (we prevent it from being created) but we will need to + // be able to *create* an initial config, for example during bootstrap (or + // during tests). Instead of having to hand-code this, we allow + // transitioning from an empty config into any other legal and non-empty + // config. + for _, ids := range []map[uint64]struct{}{ + cfg.Voters.IDs(), + cfg.Learners, + cfg.LearnersNext, + } { + for id := range ids { + if _, ok := prs[id]; !ok { + return fmt.Errorf("no progress for %d", id) + } + } + } + + // Any staged learner was staged because it could not be directly added due + // to a conflicting voter in the outgoing config. + for id := range cfg.LearnersNext { + if _, ok := outgoing(cfg.Voters)[id]; !ok { + return fmt.Errorf("%d is in LearnersNext, but not Voters[1]", id) + } + if prs[id].IsLearner { + return fmt.Errorf("%d is in LearnersNext, but is already marked as learner", id) + } + } + // Conversely Learners and Voters doesn't intersect at all. + for id := range cfg.Learners { + if _, ok := outgoing(cfg.Voters)[id]; ok { + return fmt.Errorf("%d is in Learners and Voters[1]", id) + } + if _, ok := incoming(cfg.Voters)[id]; ok { + return fmt.Errorf("%d is in Learners and Voters[0]", id) + } + if !prs[id].IsLearner { + return fmt.Errorf("%d is in Learners, but is not marked as learner", id) + } + } + + if !joint(cfg) { + // We enforce that empty maps are nil instead of zero. + if outgoing(cfg.Voters) != nil { + return fmt.Errorf("Voters[1] must be nil when not joint") + } + if cfg.LearnersNext != nil { + return fmt.Errorf("LearnersNext must be nil when not joint") + } + if cfg.AutoLeave { + return fmt.Errorf("AutoLeave must be false when not joint") + } + } + + return nil +} + +// checkAndCopy copies the tracker's config and progress map (deeply enough for +// the purposes of the Changer) and returns those copies. It returns an error +// if checkInvariants does. +func (c Changer) checkAndCopy() (tracker.Config, tracker.ProgressMap, error) { + cfg := c.Tracker.Config.Clone() + prs := tracker.ProgressMap{} + + for id, pr := range c.Tracker.Progress { + // A shallow copy is enough because we only mutate the Learner field. + ppr := *pr + prs[id] = &ppr + } + return checkAndReturn(cfg, prs) +} + +// checkAndReturn calls checkInvariants on the input and returns either the +// resulting error or the input. +func checkAndReturn(cfg tracker.Config, prs tracker.ProgressMap) (tracker.Config, tracker.ProgressMap, error) { + if err := checkInvariants(cfg, prs); err != nil { + return tracker.Config{}, tracker.ProgressMap{}, err + } + return cfg, prs, nil +} + +// err returns zero values and an error. +func (c Changer) err(err error) (tracker.Config, tracker.ProgressMap, error) { + return tracker.Config{}, nil, err +} + +// nilAwareAdd populates a map entry, creating the map if necessary. +func nilAwareAdd(m *map[uint64]struct{}, id uint64) { + if *m == nil { + *m = map[uint64]struct{}{} + } + (*m)[id] = struct{}{} +} + +// nilAwareDelete deletes from a map, nil'ing the map itself if it is empty after. +func nilAwareDelete(m *map[uint64]struct{}, id uint64) { + if *m == nil { + return + } + delete(*m, id) + if len(*m) == 0 { + *m = nil + } +} + +// symdiff returns the count of the symmetric difference between the sets of +// uint64s, i.e. len( (l - r) \union (r - l)). +func symdiff(l, r map[uint64]struct{}) int { + var n int + pairs := [][2]quorum.MajorityConfig{ + {l, r}, // count elems in l but not in r + {r, l}, // count elems in r but not in l + } + for _, p := range pairs { + for id := range p[0] { + if _, ok := p[1][id]; !ok { + n++ + } + } + } + return n +} + +func joint(cfg tracker.Config) bool { + return len(outgoing(cfg.Voters)) > 0 +} + +func incoming(voters quorum.JointConfig) quorum.MajorityConfig { return voters[0] } +func outgoing(voters quorum.JointConfig) quorum.MajorityConfig { return voters[1] } +func outgoingPtr(voters *quorum.JointConfig) *quorum.MajorityConfig { return &voters[1] } + +// Describe prints the type and NodeID of the configuration changes as a +// space-delimited string. +func Describe(ccs ...pb.ConfChangeSingle) string { + var buf strings.Builder + for _, cc := range ccs { + if buf.Len() > 0 { + buf.WriteByte(' ') + } + fmt.Fprintf(&buf, "%s(%d)", cc.Type, cc.NodeID) + } + return buf.String() +} diff --git a/vendor/go.etcd.io/etcd/raft/confchange/restore.go b/vendor/go.etcd.io/etcd/raft/confchange/restore.go new file mode 100644 index 0000000000..724068da00 --- /dev/null +++ b/vendor/go.etcd.io/etcd/raft/confchange/restore.go @@ -0,0 +1,155 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package confchange + +import ( + pb "go.etcd.io/etcd/raft/raftpb" + "go.etcd.io/etcd/raft/tracker" +) + +// toConfChangeSingle translates a conf state into 1) a slice of operations creating +// first the config that will become the outgoing one, and then the incoming one, and +// b) another slice that, when applied to the config resulted from 1), represents the +// ConfState. +func toConfChangeSingle(cs pb.ConfState) (out []pb.ConfChangeSingle, in []pb.ConfChangeSingle) { + // Example to follow along this code: + // voters=(1 2 3) learners=(5) outgoing=(1 2 4 6) learners_next=(4) + // + // This means that before entering the joint config, the configuration + // had voters (1 2 4) and perhaps some learners that are already gone. + // The new set of voters is (1 2 3), i.e. (1 2) were kept around, and (4 6) + // are no longer voters; however 4 is poised to become a learner upon leaving + // the joint state. + // We can't tell whether 5 was a learner before entering the joint config, + // but it doesn't matter (we'll pretend that it wasn't). + // + // The code below will construct + // outgoing = add 1; add 2; add 4; add 6 + // incoming = remove 1; remove 2; remove 4; remove 6 + // add 1; add 2; add 3; + // add-learner 5; + // add-learner 4; + // + // So, when starting with an empty config, after applying 'outgoing' we have + // + // quorum=(1 2 4 6) + // + // From which we enter a joint state via 'incoming' + // + // quorum=(1 2 3)&&(1 2 4 6) learners=(5) learners_next=(4) + // + // as desired. + + for _, id := range cs.VotersOutgoing { + // If there are outgoing voters, first add them one by one so that the + // (non-joint) config has them all. + out = append(out, pb.ConfChangeSingle{ + Type: pb.ConfChangeAddNode, + NodeID: id, + }) + + } + + // We're done constructing the outgoing slice, now on to the incoming one + // (which will apply on top of the config created by the outgoing slice). + + // First, we'll remove all of the outgoing voters. + for _, id := range cs.VotersOutgoing { + in = append(in, pb.ConfChangeSingle{ + Type: pb.ConfChangeRemoveNode, + NodeID: id, + }) + } + // Then we'll add the incoming voters and learners. + for _, id := range cs.Voters { + in = append(in, pb.ConfChangeSingle{ + Type: pb.ConfChangeAddNode, + NodeID: id, + }) + } + for _, id := range cs.Learners { + in = append(in, pb.ConfChangeSingle{ + Type: pb.ConfChangeAddLearnerNode, + NodeID: id, + }) + } + // Same for LearnersNext; these are nodes we want to be learners but which + // are currently voters in the outgoing config. + for _, id := range cs.LearnersNext { + in = append(in, pb.ConfChangeSingle{ + Type: pb.ConfChangeAddLearnerNode, + NodeID: id, + }) + } + return out, in +} + +func chain(chg Changer, ops ...func(Changer) (tracker.Config, tracker.ProgressMap, error)) (tracker.Config, tracker.ProgressMap, error) { + for _, op := range ops { + cfg, prs, err := op(chg) + if err != nil { + return tracker.Config{}, nil, err + } + chg.Tracker.Config = cfg + chg.Tracker.Progress = prs + } + return chg.Tracker.Config, chg.Tracker.Progress, nil +} + +// Restore takes a Changer (which must represent an empty configuration), and +// runs a sequence of changes enacting the configuration described in the +// ConfState. +// +// TODO(tbg) it's silly that this takes a Changer. Unravel this by making sure +// the Changer only needs a ProgressMap (not a whole Tracker) at which point +// this can just take LastIndex and MaxInflight directly instead and cook up +// the results from that alone. +func Restore(chg Changer, cs pb.ConfState) (tracker.Config, tracker.ProgressMap, error) { + outgoing, incoming := toConfChangeSingle(cs) + + var ops []func(Changer) (tracker.Config, tracker.ProgressMap, error) + + if len(outgoing) == 0 { + // No outgoing config, so just apply the incoming changes one by one. + for _, cc := range incoming { + cc := cc // loop-local copy + ops = append(ops, func(chg Changer) (tracker.Config, tracker.ProgressMap, error) { + return chg.Simple(cc) + }) + } + } else { + // The ConfState describes a joint configuration. + // + // First, apply all of the changes of the outgoing config one by one, so + // that it temporarily becomes the incoming active config. For example, + // if the config is (1 2 3)&(2 3 4), this will establish (2 3 4)&(). + for _, cc := range outgoing { + cc := cc // loop-local copy + ops = append(ops, func(chg Changer) (tracker.Config, tracker.ProgressMap, error) { + return chg.Simple(cc) + }) + } + // Now enter the joint state, which rotates the above additions into the + // outgoing config, and adds the incoming config in. Continuing the + // example above, we'd get (1 2 3)&(2 3 4), i.e. the incoming operations + // would be removing 2,3,4 and then adding in 1,2,3 while transitioning + // into a joint state. + ops = append(ops, func(chg Changer) (tracker.Config, tracker.ProgressMap, error) { + return chg.EnterJoint(cs.AutoLeave, incoming...) + }) + } + + return chain(chg, ops...) +} diff --git a/vendor/go.etcd.io/etcd/raft/log_unstable.go b/vendor/go.etcd.io/etcd/raft/log_unstable.go index 1005bf65cc..1bff5a7bdc 100644 --- a/vendor/go.etcd.io/etcd/raft/log_unstable.go +++ b/vendor/go.etcd.io/etcd/raft/log_unstable.go @@ -55,10 +55,7 @@ func (u *unstable) maybeLastIndex() (uint64, bool) { // is any. func (u *unstable) maybeTerm(i uint64) (uint64, bool) { if i < u.offset { - if u.snapshot == nil { - return 0, false - } - if u.snapshot.Metadata.Index == i { + if u.snapshot != nil && u.snapshot.Metadata.Index == i { return u.snapshot.Metadata.Term, true } return 0, false @@ -71,6 +68,7 @@ func (u *unstable) maybeTerm(i uint64) (uint64, bool) { if i > last { return 0, false } + return u.entries[i-u.offset].Term, true } diff --git a/vendor/go.etcd.io/etcd/raft/logger.go b/vendor/go.etcd.io/etcd/raft/logger.go index 426a77d344..6d89629650 100644 --- a/vendor/go.etcd.io/etcd/raft/logger.go +++ b/vendor/go.etcd.io/etcd/raft/logger.go @@ -19,6 +19,7 @@ import ( "io/ioutil" "log" "os" + "sync" ) type Logger interface { @@ -41,11 +42,16 @@ type Logger interface { Panicf(format string, v ...interface{}) } -func SetLogger(l Logger) { raftLogger = l } +func SetLogger(l Logger) { + raftLoggerMu.Lock() + raftLogger = l + raftLoggerMu.Unlock() +} var ( defaultLogger = &DefaultLogger{Logger: log.New(os.Stderr, "raft", log.LstdFlags)} discardLogger = &DefaultLogger{Logger: log.New(ioutil.Discard, "", 0)} + raftLoggerMu sync.Mutex raftLogger = Logger(defaultLogger) ) diff --git a/vendor/go.etcd.io/etcd/raft/node.go b/vendor/go.etcd.io/etcd/raft/node.go index 4a3b2f1dfd..ab6185b99e 100644 --- a/vendor/go.etcd.io/etcd/raft/node.go +++ b/vendor/go.etcd.io/etcd/raft/node.go @@ -132,10 +132,20 @@ type Node interface { // Propose proposes that data be appended to the log. Note that proposals can be lost without // notice, therefore it is user's job to ensure proposal retries. Propose(ctx context.Context, data []byte) error - // ProposeConfChange proposes config change. - // At most one ConfChange can be in the process of going through consensus. - // Application needs to call ApplyConfChange when applying EntryConfChange type entry. - ProposeConfChange(ctx context.Context, cc pb.ConfChange) error + // ProposeConfChange proposes a configuration change. Like any proposal, the + // configuration change may be dropped with or without an error being + // returned. In particular, configuration changes are dropped unless the + // leader has certainty that there is no prior unapplied configuration + // change in its log. + // + // The method accepts either a pb.ConfChange (deprecated) or pb.ConfChangeV2 + // message. The latter allows arbitrary configuration changes via joint + // consensus, notably including replacing a voter. Passing a ConfChangeV2 + // message is only allowed if all Nodes participating in the cluster run a + // version of this library aware of the V2 API. See pb.ConfChangeV2 for + // usage details and semantics. + ProposeConfChange(ctx context.Context, cc pb.ConfChangeI) error + // Step advances the state machine using the given message. ctx.Err() will be returned, if any. Step(ctx context.Context, msg pb.Message) error @@ -156,11 +166,13 @@ type Node interface { // a long time to apply the snapshot data. To continue receiving Ready without blocking raft // progress, it can call Advance before finishing applying the last ready. Advance() - // ApplyConfChange applies config change to the local node. - // Returns an opaque ConfState protobuf which must be recorded - // in snapshots. Will never return nil; it returns a pointer only - // to match MemoryStorage.Compact. - ApplyConfChange(cc pb.ConfChange) *pb.ConfState + // ApplyConfChange applies a config change (previously passed to + // ProposeConfChange) to the node. This must be called whenever a config + // change is observed in Ready.CommittedEntries. + // + // Returns an opaque non-nil ConfState protobuf which must be recorded in + // snapshots. + ApplyConfChange(cc pb.ConfChangeI) *pb.ConfState // TransferLeadership attempts to transfer leadership to the given transferee. TransferLeadership(ctx context.Context, lead, transferee uint64) @@ -197,52 +209,21 @@ type Peer struct { // StartNode returns a new Node given configuration and a list of raft peers. // It appends a ConfChangeAddNode entry for each given peer to the initial log. +// +// Peers must not be zero length; call RestartNode in that case. func StartNode(c *Config, peers []Peer) Node { - r := newRaft(c) - // become the follower at term 1 and apply initial configuration - // entries of term 1 - r.becomeFollower(1, None) - for _, peer := range peers { - cc := pb.ConfChange{Type: pb.ConfChangeAddNode, NodeID: peer.ID, Context: peer.Context} - d, err := cc.Marshal() - if err != nil { - panic("unexpected marshal error") - } - // TODO(tbg): this should append the ConfChange for the own node first - // and also call applyConfChange below for that node first. Otherwise - // we have a Raft group (for a little while) that doesn't have itself - // in its config, which is bad. - // This whole way of setting things up is rickety. The app should just - // populate the initial ConfState appropriately and then all of this - // goes away. - e := pb.Entry{ - Type: pb.EntryConfChange, - Term: 1, - Index: r.raftLog.lastIndex() + 1, - Data: d, - } - r.raftLog.append(e) + if len(peers) == 0 { + panic("no peers given; use RestartNode instead") } - // Mark these initial entries as committed. - // TODO(bdarnell): These entries are still unstable; do we need to preserve - // the invariant that committed < unstable? - r.raftLog.committed = r.raftLog.lastIndex() - // Now apply them, mainly so that the application can call Campaign - // immediately after StartNode in tests. Note that these nodes will - // be added to raft twice: here and when the application's Ready - // loop calls ApplyConfChange. The calls to addNode must come after - // all calls to raftLog.append so progress.next is set after these - // bootstrapping entries (it is an error if we try to append these - // entries since they have already been committed). - // We do not set raftLog.applied so the application will be able - // to observe all conf changes via Ready.CommittedEntries. - for _, peer := range peers { - r.applyConfChange(pb.ConfChange{NodeID: peer.ID, Type: pb.ConfChangeAddNode}) + rn, err := NewRawNode(c) + if err != nil { + panic(err) } + rn.Bootstrap(peers) + + n := newNode(rn) - n := newNode() - n.logger = c.Logger - go n.run(r) + go n.run() return &n } @@ -251,11 +232,12 @@ func StartNode(c *Config, peers []Peer) Node { // If the caller has an existing state machine, pass in the last log index that // has been applied to it; otherwise use zero. func RestartNode(c *Config) Node { - r := newRaft(c) - - n := newNode() - n.logger = c.Logger - go n.run(r) + rn, err := NewRawNode(c) + if err != nil { + panic(err) + } + n := newNode(rn) + go n.run() return &n } @@ -268,7 +250,7 @@ type msgWithResult struct { type node struct { propc chan msgWithResult recvc chan pb.Message - confc chan pb.ConfChange + confc chan pb.ConfChangeV2 confstatec chan pb.ConfState readyc chan Ready advancec chan struct{} @@ -277,14 +259,14 @@ type node struct { stop chan struct{} status chan chan Status - logger Logger + rn *RawNode } -func newNode() node { +func newNode(rn *RawNode) node { return node{ propc: make(chan msgWithResult), recvc: make(chan pb.Message), - confc: make(chan pb.ConfChange), + confc: make(chan pb.ConfChangeV2), confstatec: make(chan pb.ConfState), readyc: make(chan Ready), advancec: make(chan struct{}), @@ -295,6 +277,7 @@ func newNode() node { done: make(chan struct{}), stop: make(chan struct{}), status: make(chan chan Status), + rn: rn, } } @@ -310,30 +293,30 @@ func (n *node) Stop() { <-n.done } -func (n *node) run(r *raft) { +func (n *node) run() { var propc chan msgWithResult var readyc chan Ready var advancec chan struct{} - var prevLastUnstablei, prevLastUnstablet uint64 - var havePrevLastUnstablei bool - var prevSnapi uint64 - var applyingToI uint64 var rd Ready + r := n.rn.raft + lead := None - prevSoftSt := r.softState() - prevHardSt := emptyState for { if advancec != nil { readyc = nil - } else { - rd = newReady(r, prevSoftSt, prevHardSt) - if rd.containsUpdates() { - readyc = n.readyc - } else { - readyc = nil - } + } else if n.rn.HasReady() { + // Populate a Ready. Note that this Ready is not guaranteed to + // actually be handled. We will arm readyc, but there's no guarantee + // that we will actually send on it. It's possible that we will + // service another channel instead, loop around, and then populate + // the Ready again. We could instead force the previous Ready to be + // handled first, but it's generally good to emit larger Readys plus + // it simplifies testing (by emitting less frequently and more + // predictably). + rd = n.rn.readyWithoutAccept() + readyc = n.readyc } if lead != r.lead { @@ -369,11 +352,27 @@ func (n *node) run(r *raft) { r.Step(m) } case cc := <-n.confc: + _, okBefore := r.prs.Progress[r.id] cs := r.applyConfChange(cc) - if _, ok := r.prs.Progress[r.id]; !ok { - // block incoming proposal when local node is - // removed - if cc.NodeID == r.id { + // If the node was removed, block incoming proposals. Note that we + // only do this if the node was in the config before. Nodes may be + // a member of the group without knowing this (when they're catching + // up on the log and don't have the latest config) and we don't want + // to block the proposal channel in that case. + // + // NB: propc is reset when the leader changes, which, if we learn + // about it, sort of implies that we got readded, maybe? This isn't + // very sound and likely has bugs. + if _, okAfter := r.prs.Progress[r.id]; okBefore && !okAfter { + var found bool + for _, sl := range [][]uint64{cs.Voters, cs.VotersOutgoing} { + for _, id := range sl { + if id == r.id { + found = true + } + } + } + if !found { propc = nil } } @@ -382,40 +381,13 @@ func (n *node) run(r *raft) { case <-n.done: } case <-n.tickc: - r.tick() + n.rn.Tick() case readyc <- rd: - if rd.SoftState != nil { - prevSoftSt = rd.SoftState - } - if len(rd.Entries) > 0 { - prevLastUnstablei = rd.Entries[len(rd.Entries)-1].Index - prevLastUnstablet = rd.Entries[len(rd.Entries)-1].Term - havePrevLastUnstablei = true - } - if !IsEmptyHardState(rd.HardState) { - prevHardSt = rd.HardState - } - if !IsEmptySnap(rd.Snapshot) { - prevSnapi = rd.Snapshot.Metadata.Index - } - if index := rd.appliedCursor(); index != 0 { - applyingToI = index - } - - r.msgs = nil - r.readStates = nil - r.reduceUncommittedSize(rd.CommittedEntries) + n.rn.acceptReady(rd) advancec = n.advancec case <-advancec: - if applyingToI != 0 { - r.raftLog.appliedTo(applyingToI) - applyingToI = 0 - } - if havePrevLastUnstablei { - r.raftLog.stableTo(prevLastUnstablei, prevLastUnstablet) - havePrevLastUnstablei = false - } - r.raftLog.stableSnapTo(prevSnapi) + n.rn.Advance(rd) + rd = Ready{} advancec = nil case c := <-n.status: c <- getStatus(r) @@ -433,7 +405,7 @@ func (n *node) Tick() { case n.tickc <- struct{}{}: case <-n.done: default: - n.logger.Warningf("A tick missed to fire. Node blocks too long!") + n.rn.raft.logger.Warningf("%x (leader %v) A tick missed to fire. Node blocks too long!", n.rn.raft.id, n.rn.raft.id == n.rn.raft.lead) } } @@ -452,12 +424,20 @@ func (n *node) Step(ctx context.Context, m pb.Message) error { return n.step(ctx, m) } -func (n *node) ProposeConfChange(ctx context.Context, cc pb.ConfChange) error { - data, err := cc.Marshal() +func confChangeToMsg(c pb.ConfChangeI) (pb.Message, error) { + typ, data, err := pb.MarshalConfChange(c) + if err != nil { + return pb.Message{}, err + } + return pb.Message{Type: pb.MsgProp, Entries: []pb.Entry{{Type: typ, Data: data}}}, nil +} + +func (n *node) ProposeConfChange(ctx context.Context, cc pb.ConfChangeI) error { + msg, err := confChangeToMsg(cc) if err != nil { return err } - return n.Step(ctx, pb.Message{Type: pb.MsgProp, Entries: []pb.Entry{{Type: pb.EntryConfChange, Data: data}}}) + return n.Step(ctx, msg) } func (n *node) step(ctx context.Context, m pb.Message) error { @@ -518,10 +498,10 @@ func (n *node) Advance() { } } -func (n *node) ApplyConfChange(cc pb.ConfChange) *pb.ConfState { +func (n *node) ApplyConfChange(cc pb.ConfChangeI) *pb.ConfState { var cs pb.ConfState select { - case n.confc <- cc: + case n.confc <- cc.AsV2(): case <-n.done: } select { diff --git a/vendor/go.etcd.io/etcd/raft/quorum/majority.go b/vendor/go.etcd.io/etcd/raft/quorum/majority.go index 5eba503444..8858a36b63 100644 --- a/vendor/go.etcd.io/etcd/raft/quorum/majority.go +++ b/vendor/go.etcd.io/etcd/raft/quorum/majority.go @@ -102,9 +102,17 @@ func (c MajorityConfig) Describe(l AckedIndexer) string { return buf.String() } -type uint64Slice []uint64 +// Slice returns the MajorityConfig as a sorted slice. +func (c MajorityConfig) Slice() []uint64 { + var sl []uint64 + for id := range c { + sl = append(sl, id) + } + sort.Slice(sl, func(i, j int) bool { return sl[i] < sl[j] }) + return sl +} -func insertionSort(sl uint64Slice) { +func insertionSort(sl []uint64) { a, b := 0, len(sl) for i := a + 1; i < b; i++ { for j := i; j > a && sl[j] < sl[j-1]; j-- { @@ -131,12 +139,12 @@ func (c MajorityConfig) CommittedIndex(l AckedIndexer) Index { // performance is a lesser concern (additionally the performance // implications of an allocation here are far from drastic). var stk [7]uint64 - srt := uint64Slice(stk[:]) - - if cap(srt) < n { + var srt []uint64 + if len(stk) >= n { + srt = stk[:n] + } else { srt = make([]uint64, n) } - srt = srt[:n] { // Fill the slice with the indexes observed. Any unused slots will be diff --git a/vendor/go.etcd.io/etcd/raft/raft.go b/vendor/go.etcd.io/etcd/raft/raft.go index a42bb4e63d..d3c3f42574 100644 --- a/vendor/go.etcd.io/etcd/raft/raft.go +++ b/vendor/go.etcd.io/etcd/raft/raft.go @@ -20,10 +20,12 @@ import ( "fmt" "math" "math/rand" + "sort" "strings" "sync" "time" + "go.etcd.io/etcd/raft/confchange" "go.etcd.io/etcd/raft/quorum" pb "go.etcd.io/etcd/raft/raftpb" "go.etcd.io/etcd/raft/tracker" @@ -262,7 +264,8 @@ type raft struct { maxMsgSize uint64 maxUncommittedSize uint64 - prs tracker.ProgressTracker + // TODO(tbg): rename to trk. + prs tracker.ProgressTracker state StateType @@ -326,18 +329,18 @@ func newRaft(c *Config) *raft { if err != nil { panic(err) // TODO(bdarnell) } - peers := c.peers - learners := c.learners - if len(cs.Nodes) > 0 || len(cs.Learners) > 0 { - if len(peers) > 0 || len(learners) > 0 { + + if len(c.peers) > 0 || len(c.learners) > 0 { + if len(cs.Voters) > 0 || len(cs.Learners) > 0 { // TODO(bdarnell): the peers argument is always nil except in // tests; the argument should be removed and these tests should be // updated to specify their nodes through a snapshot. - panic("cannot specify both newRaft(peers, learners) and ConfState.(Nodes, Learners)") + panic("cannot specify both newRaft(peers, learners) and ConfState.(Voters, Learners)") } - peers = cs.Nodes - learners = cs.Learners + cs.Voters = c.peers + cs.Learners = c.learners } + r := &raft{ id: c.ID, lead: None, @@ -354,20 +357,17 @@ func newRaft(c *Config) *raft { readOnly: newReadOnly(c.ReadOnlyOption), disableProposalForwarding: c.DisableProposalForwarding, } - for _, p := range peers { - // Add node to active config. - r.prs.InitProgress(p, 0 /* match */, 1 /* next */, false /* isLearner */) - } - for _, p := range learners { - // Add learner to active config. - r.prs.InitProgress(p, 0 /* match */, 1 /* next */, true /* isLearner */) - if r.id == p { - r.isLearner = true - } + cfg, prs, err := confchange.Restore(confchange.Changer{ + Tracker: r.prs, + LastIndex: raftlog.lastIndex(), + }, cs) + if err != nil { + panic(err) } + assertConfStatesEquivalent(r.logger, cs, r.switchToConfig(cfg, prs)) - if !isHardStateEqual(hs, emptyState) { + if !IsEmptyHardState(hs) { r.loadState(hs) } if c.Applied > 0 { @@ -530,7 +530,6 @@ func (r *raft) bcastAppend() { if id == r.id { return } - r.sendAppend(id) }) } @@ -554,6 +553,46 @@ func (r *raft) bcastHeartbeatWithCtx(ctx []byte) { }) } +func (r *raft) advance(rd Ready) { + // If entries were applied (or a snapshot), update our cursor for + // the next Ready. Note that if the current HardState contains a + // new Commit index, this does not mean that we're also applying + // all of the new entries due to commit pagination by size. + if index := rd.appliedCursor(); index > 0 { + r.raftLog.appliedTo(index) + if r.prs.Config.AutoLeave && index >= r.pendingConfIndex && r.state == StateLeader { + // If the current (and most recent, at least for this leader's term) + // configuration should be auto-left, initiate that now. + ccdata, err := (&pb.ConfChangeV2{}).Marshal() + if err != nil { + panic(err) + } + ent := pb.Entry{ + Type: pb.EntryConfChangeV2, + Data: ccdata, + } + if !r.appendEntry(ent) { + // If we could not append the entry, bump the pending conf index + // so that we'll try again later. + // + // TODO(tbg): test this case. + r.pendingConfIndex = r.raftLog.lastIndex() + } else { + r.logger.Infof("initiating automatic transition out of joint configuration %s", r.prs.Config) + } + } + } + r.reduceUncommittedSize(rd.CommittedEntries) + + if len(rd.Entries) > 0 { + e := rd.Entries[len(rd.Entries)-1] + r.raftLog.stableTo(e.Index, e.Term) + } + if !IsEmptySnap(rd.Snapshot) { + r.raftLog.stableSnapTo(rd.Snapshot.Metadata.Index) + } +} + // maybeCommit attempts to advance the commit index. Returns true if // the commit index changed (in which case the caller should call // r.bcastAppend). @@ -756,7 +795,16 @@ func (r *raft) campaign(t CampaignType) { } return } - for id := range r.prs.Voters.IDs() { + var ids []uint64 + { + idMap := r.prs.Voters.IDs() + ids = make([]uint64, 0, len(idMap)) + for id := range idMap { + ids = append(ids, id) + } + sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] }) + } + for _, id := range ids { if id == r.id { continue } @@ -883,12 +931,6 @@ func (r *raft) Step(m pb.Message) error { } case pb.MsgVote, pb.MsgPreVote: - if r.isLearner { - // TODO: learner may need to vote, in case of node down when confchange. - r.logger.Infof("%x [logterm: %d, index: %d, vote: %x] ignored %s from %x [logterm: %d, index: %d] at term %d: learner can not vote", - r.id, r.raftLog.lastTerm(), r.raftLog.lastIndex(), r.Vote, m.Type, m.From, m.LogTerm, m.Index, r.Term) - return nil - } // We can vote if this is a repeat of a vote we've already cast... canVote := r.Vote == m.From || // ...we haven't voted and we don't think there's a leader yet in this term... @@ -897,6 +939,24 @@ func (r *raft) Step(m pb.Message) error { (m.Type == pb.MsgPreVote && m.Term > r.Term) // ...and we believe the candidate is up to date. if canVote && r.raftLog.isUpToDate(m.Index, m.LogTerm) { + // Note: it turns out that that learners must be allowed to cast votes. + // This seems counter- intuitive but is necessary in the situation in which + // a learner has been promoted (i.e. is now a voter) but has not learned + // about this yet. + // For example, consider a group in which id=1 is a learner and id=2 and + // id=3 are voters. A configuration change promoting 1 can be committed on + // the quorum `{2,3}` without the config change being appended to the + // learner's log. If the leader (say 2) fails, there are de facto two + // voters remaining. Only 3 can win an election (due to its log containing + // all committed entries), but to do so it will need 1 to vote. But 1 + // considers itself a learner and will continue to do so until 3 has + // stepped up as leader, replicates the conf change to 1, and 1 applies it. + // Ultimately, by receiving a request to vote, the learner realizes that + // the candidate believes it to be a voter, and that it should act + // accordingly. The candidate's config may be stale, too; but in that case + // it won't win the election, at least in the absence of the bug discussed + // in: + // https://github.com/etcd-io/etcd/issues/7625#issuecomment-488798263. r.logger.Infof("%x [logterm: %d, index: %d, vote: %x] cast %s for %x [logterm: %d, index: %d] at term %d", r.id, r.raftLog.lastTerm(), r.raftLog.lastIndex(), r.Vote, m.Type, m.From, m.LogTerm, m.Index, r.Term) // When responding to Msg{Pre,}Vote messages we include the term @@ -976,10 +1036,36 @@ func stepLeader(r *raft, m pb.Message) error { for i := range m.Entries { e := &m.Entries[i] + var cc pb.ConfChangeI if e.Type == pb.EntryConfChange { - if r.pendingConfIndex > r.raftLog.applied { - r.logger.Infof("propose conf %s ignored since pending unapplied configuration [index %d, applied %d]", - e, r.pendingConfIndex, r.raftLog.applied) + var ccc pb.ConfChange + if err := ccc.Unmarshal(e.Data); err != nil { + panic(err) + } + cc = ccc + } else if e.Type == pb.EntryConfChangeV2 { + var ccc pb.ConfChangeV2 + if err := ccc.Unmarshal(e.Data); err != nil { + panic(err) + } + cc = ccc + } + if cc != nil { + alreadyPending := r.pendingConfIndex > r.raftLog.applied + alreadyJoint := len(r.prs.Config.Voters[1]) > 0 + wantsLeaveJoint := len(cc.AsV2().Changes) == 0 + + var refused string + if alreadyPending { + refused = fmt.Sprintf("possible unapplied conf change at index %d (applied to %d)", r.pendingConfIndex, r.raftLog.applied) + } else if alreadyJoint && !wantsLeaveJoint { + refused = "must transition out of joint config first" + } else if !alreadyJoint && wantsLeaveJoint { + refused = "not in joint state; refusing empty conf change" + } + + if refused != "" { + r.logger.Infof("%x ignoring conf change %v at config %s: %s", r.id, cc, r.prs.Config, refused) m.Entries[i] = pb.Entry{Type: pb.EntryNormal} } else { r.pendingConfIndex = r.raftLog.lastIndex() + uint64(i) + 1 @@ -1013,7 +1099,7 @@ func stepLeader(r *raft, m pb.Message) error { case ReadOnlyLeaseBased: ri := r.raftLog.committed if m.From == None || m.From == r.id { // from local member - r.readStates = append(r.readStates, ReadState{Index: r.raftLog.committed, RequestCtx: m.Entries[0].Data}) + r.readStates = append(r.readStates, ReadState{Index: ri, RequestCtx: m.Entries[0].Data}) } else { r.send(pb.Message{To: m.From, Type: pb.MsgReadIndexResp, Index: ri, Entries: m.Entries}) } @@ -1040,7 +1126,7 @@ func stepLeader(r *raft, m pb.Message) error { pr.RecentActive = true if m.Reject { - r.logger.Debugf("%x received msgApp rejection(lastindex: %d) from %x for index %d", + r.logger.Debugf("%x received MsgAppResp(MsgApp was rejected, lastindex: %d) from %x for index %d", r.id, m.RejectHint, m.From, m.Index) if pr.MaybeDecrTo(m.Index, m.RejectHint) { r.logger.Debugf("%x decreased progress of %x to [%s]", r.id, m.From, pr) @@ -1056,6 +1142,9 @@ func stepLeader(r *raft, m pb.Message) error { case pr.State == tracker.StateProbe: pr.BecomeReplicate() case pr.State == tracker.StateSnapshot && pr.Match >= pr.PendingSnapshot: + // TODO(tbg): we should also enter this branch if a snapshot is + // received that is below pr.PendingSnapshot but which makes it + // possible to use the log again. r.logger.Debugf("%x recovered from needing snapshot, resumed sending replication messages to %x [%s]", r.id, m.From, pr) // Transition back to replicating state via probing state // (which takes the snapshot into account). If we didn't @@ -1137,8 +1226,8 @@ func stepLeader(r *raft, m pb.Message) error { pr.BecomeProbe() r.logger.Debugf("%x snapshot failed, resumed sending replication messages to %x [%s]", r.id, m.From, pr) } - // If snapshot finish, wait for the msgAppResp from the remote node before sending - // out the next msgApp. + // If snapshot finish, wait for the MsgAppResp from the remote node before sending + // out the next MsgApp. // If snapshot failure, wait for a heartbeat interval before next try pr.ProbeSent = true case pb.MsgUnreachable: @@ -1297,7 +1386,7 @@ func (r *raft) handleAppendEntries(m pb.Message) { if mlastIndex, ok := r.raftLog.maybeAppend(m.Index, m.LogTerm, m.Commit, m.Entries...); ok { r.send(pb.Message{To: m.From, Type: pb.MsgAppResp, Index: mlastIndex}) } else { - r.logger.Debugf("%x [logterm: %d, index: %d] rejected msgApp [logterm: %d, index: %d] from %x", + r.logger.Debugf("%x [logterm: %d, index: %d] rejected MsgApp [logterm: %d, index: %d] from %x", r.id, r.raftLog.zeroTermOnErrCompacted(r.raftLog.term(m.Index)), m.Index, m.LogTerm, m.Index, m.From) r.send(pb.Message{To: m.From, Type: pb.MsgAppResp, Index: m.Index, Reject: true, RejectHint: r.raftLog.lastIndex()}) } @@ -1347,7 +1436,7 @@ func (r *raft) restore(s pb.Snapshot) bool { found := false cs := s.Metadata.ConfState for _, set := range [][]uint64{ - cs.Nodes, + cs.Voters, cs.Learners, } { for _, id := range set { @@ -1374,27 +1463,23 @@ func (r *raft) restore(s pb.Snapshot) bool { return false } - // The normal peer can't become learner. - if !r.isLearner { - for _, id := range s.Metadata.ConfState.Learners { - if id == r.id { - r.logger.Errorf("%x can't become learner when restores snapshot [index: %d, term: %d]", r.id, s.Metadata.Index, s.Metadata.Term) - return false - } - } - } - r.raftLog.restore(s) // Reset the configuration and add the (potentially updated) peers in anew. r.prs = tracker.MakeProgressTracker(r.prs.MaxInflight) - for _, id := range s.Metadata.ConfState.Nodes { - r.applyConfChange(pb.ConfChange{NodeID: id, Type: pb.ConfChangeAddNode}) - } - for _, id := range s.Metadata.ConfState.Learners { - r.applyConfChange(pb.ConfChange{NodeID: id, Type: pb.ConfChangeAddLearnerNode}) + cfg, prs, err := confchange.Restore(confchange.Changer{ + Tracker: r.prs, + LastIndex: r.raftLog.lastIndex(), + }, cs) + + if err != nil { + // This should never happen. Either there's a bug in our config change + // handling or the client corrupted the conf change. + panic(fmt.Sprintf("unable to restore config %+v: %s", cs, err)) } + assertConfStatesEquivalent(r.logger, cs, r.switchToConfig(cfg, prs)) + pr := r.prs.Progress[r.id] pr.MaybeUpdate(pr.Next - 1) // TODO(tbg): this is untested and likely unneeded @@ -1410,61 +1495,40 @@ func (r *raft) promotable() bool { return pr != nil && !pr.IsLearner } -func (r *raft) applyConfChange(cc pb.ConfChange) pb.ConfState { - addNodeOrLearnerNode := func(id uint64, isLearner bool) { - // NB: this method is intentionally hidden from view. All mutations of - // the conf state must call applyConfChange directly. - pr := r.prs.Progress[id] - if pr == nil { - r.prs.InitProgress(id, 0, r.raftLog.lastIndex()+1, isLearner) - } else { - if isLearner && !pr.IsLearner { - // Can only change Learner to Voter. - // - // TODO(tbg): why? - r.logger.Infof("%x ignored addLearner: do not support changing %x from raft peer to learner.", r.id, id) - return - } - - if isLearner == pr.IsLearner { - // Ignore any redundant addNode calls (which can happen because the - // initial bootstrapping entries are applied twice). - return - } - - // Change Learner to Voter, use origin Learner progress. - r.prs.RemoveAny(id) - r.prs.InitProgress(id, 0 /* match */, 1 /* next */, false /* isLearner */) - pr.IsLearner = false - *r.prs.Progress[id] = *pr +func (r *raft) applyConfChange(cc pb.ConfChangeV2) pb.ConfState { + cfg, prs, err := func() (tracker.Config, tracker.ProgressMap, error) { + changer := confchange.Changer{ + Tracker: r.prs, + LastIndex: r.raftLog.lastIndex(), } - - // When a node is first added, we should mark it as recently active. - // Otherwise, CheckQuorum may cause us to step down if it is invoked - // before the added node has had a chance to communicate with us. - r.prs.Progress[id].RecentActive = true - } - - var removed int - if cc.NodeID != None { - switch cc.Type { - case pb.ConfChangeAddNode: - addNodeOrLearnerNode(cc.NodeID, false /* isLearner */) - case pb.ConfChangeAddLearnerNode: - addNodeOrLearnerNode(cc.NodeID, true /* isLearner */) - case pb.ConfChangeRemoveNode: - removed++ - r.prs.RemoveAny(cc.NodeID) - case pb.ConfChangeUpdateNode: - default: - panic("unexpected conf type") + if cc.LeaveJoint() { + return changer.LeaveJoint() + } else if autoLeave, ok := cc.EnterJoint(); ok { + return changer.EnterJoint(autoLeave, cc.Changes...) } + return changer.Simple(cc.Changes...) + }() + + if err != nil { + // TODO(tbg): return the error to the caller. + panic(err) } - r.logger.Infof("%x switched to configuration %s", r.id, r.prs.Config) - // Now that the configuration is updated, handle any side effects. + return r.switchToConfig(cfg, prs) +} + +// switchToConfig reconfigures this node to use the provided configuration. It +// updates the in-memory state and, when necessary, carries out additional +// actions such as reacting to the removal of nodes or changed quorum +// requirements. +// +// The inputs usually result from restoring a ConfState or applying a ConfChange. +func (r *raft) switchToConfig(cfg tracker.Config, prs tracker.ProgressMap) pb.ConfState { + r.prs.Config = cfg + r.prs.Progress = prs - cs := pb.ConfState{Nodes: r.prs.VoterNodes(), Learners: r.prs.LearnerNodes()} + r.logger.Infof("%x switched to configuration %s", r.id, r.prs.Config) + cs := r.prs.ConfState() pr, ok := r.prs.Progress[r.id] // Update whether the node itself is a learner, resetting to false when the @@ -1486,15 +1550,21 @@ func (r *raft) applyConfChange(cc pb.ConfChange) pb.ConfState { // The remaining steps only make sense if this node is the leader and there // are other nodes. - if r.state != StateLeader || len(cs.Nodes) == 0 { + if r.state != StateLeader || len(cs.Voters) == 0 { return cs } - if removed > 0 { - // The quorum size may have been reduced (but not to zero), so see if - // any pending entries can be committed. - if r.maybeCommit() { - r.bcastAppend() - } + + if r.maybeCommit() { + // If the configuration change means that more entries are committed now, + // broadcast/append to everyone in the updated config. + r.bcastAppend() + } else { + // Otherwise, still probe the newly added replicas; there's no reason to + // let them wait out a heartbeat interval (or the next incoming + // proposal). + r.prs.Visit(func(id uint64, pr *tracker.Progress) { + r.maybeSendAppend(id, false /* sendIfEmpty */) + }) } // If the the leadTransferee was removed, abort the leadership transfer. if _, tOK := r.prs.Progress[r.leadTransferee]; !tOK && r.leadTransferee != 0 { diff --git a/vendor/go.etcd.io/etcd/raft/raftpb/confchange.go b/vendor/go.etcd.io/etcd/raft/raftpb/confchange.go new file mode 100644 index 0000000000..46a7a70212 --- /dev/null +++ b/vendor/go.etcd.io/etcd/raft/raftpb/confchange.go @@ -0,0 +1,170 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raftpb + +import ( + "fmt" + "strconv" + "strings" + + "github.com/gogo/protobuf/proto" +) + +// ConfChangeI abstracts over ConfChangeV2 and (legacy) ConfChange to allow +// treating them in a unified manner. +type ConfChangeI interface { + AsV2() ConfChangeV2 + AsV1() (ConfChange, bool) +} + +// MarshalConfChange calls Marshal on the underlying ConfChange or ConfChangeV2 +// and returns the result along with the corresponding EntryType. +func MarshalConfChange(c ConfChangeI) (EntryType, []byte, error) { + var typ EntryType + var ccdata []byte + var err error + if ccv1, ok := c.AsV1(); ok { + typ = EntryConfChange + ccdata, err = ccv1.Marshal() + } else { + ccv2 := c.AsV2() + typ = EntryConfChangeV2 + ccdata, err = ccv2.Marshal() + } + return typ, ccdata, err +} + +// AsV2 returns a V2 configuration change carrying out the same operation. +func (c ConfChange) AsV2() ConfChangeV2 { + return ConfChangeV2{ + Changes: []ConfChangeSingle{{ + Type: c.Type, + NodeID: c.NodeID, + }}, + Context: c.Context, + } +} + +// AsV1 returns the ConfChange and true. +func (c ConfChange) AsV1() (ConfChange, bool) { + return c, true +} + +// AsV2 is the identity. +func (c ConfChangeV2) AsV2() ConfChangeV2 { return c } + +// AsV1 returns ConfChange{} and false. +func (c ConfChangeV2) AsV1() (ConfChange, bool) { return ConfChange{}, false } + +// EnterJoint returns two bools. The second bool is true if and only if this +// config change will use Joint Consensus, which is the case if it contains more +// than one change or if the use of Joint Consensus was requested explicitly. +// The first bool can only be true if second one is, and indicates whether the +// Joint State will be left automatically. +func (c *ConfChangeV2) EnterJoint() (autoLeave bool, ok bool) { + // NB: in theory, more config changes could qualify for the "simple" + // protocol but it depends on the config on top of which the changes apply. + // For example, adding two learners is not OK if both nodes are part of the + // base config (i.e. two voters are turned into learners in the process of + // applying the conf change). In practice, these distinctions should not + // matter, so we keep it simple and use Joint Consensus liberally. + if c.Transition != ConfChangeTransitionAuto || len(c.Changes) > 1 { + // Use Joint Consensus. + var autoLeave bool + switch c.Transition { + case ConfChangeTransitionAuto: + autoLeave = true + case ConfChangeTransitionJointImplicit: + autoLeave = true + case ConfChangeTransitionJointExplicit: + default: + panic(fmt.Sprintf("unknown transition: %+v", c)) + } + return autoLeave, true + } + return false, false +} + +// LeaveJoint is true if the configuration change leaves a joint configuration. +// This is the case if the ConfChangeV2 is zero, with the possible exception of +// the Context field. +func (c *ConfChangeV2) LeaveJoint() bool { + cpy := *c + cpy.Context = nil + return proto.Equal(&cpy, &ConfChangeV2{}) +} + +// ConfChangesFromString parses a Space-delimited sequence of operations into a +// slice of ConfChangeSingle. The supported operations are: +// - vn: make n a voter, +// - ln: make n a learner, +// - rn: remove n, and +// - un: update n. +func ConfChangesFromString(s string) ([]ConfChangeSingle, error) { + var ccs []ConfChangeSingle + toks := strings.Split(strings.TrimSpace(s), " ") + if toks[0] == "" { + toks = nil + } + for _, tok := range toks { + if len(tok) < 2 { + return nil, fmt.Errorf("unknown token %s", tok) + } + var cc ConfChangeSingle + switch tok[0] { + case 'v': + cc.Type = ConfChangeAddNode + case 'l': + cc.Type = ConfChangeAddLearnerNode + case 'r': + cc.Type = ConfChangeRemoveNode + case 'u': + cc.Type = ConfChangeUpdateNode + default: + return nil, fmt.Errorf("unknown input: %s", tok) + } + id, err := strconv.ParseUint(tok[1:], 10, 64) + if err != nil { + return nil, err + } + cc.NodeID = id + ccs = append(ccs, cc) + } + return ccs, nil +} + +// ConfChangesToString is the inverse to ConfChangesFromString. +func ConfChangesToString(ccs []ConfChangeSingle) string { + var buf strings.Builder + for i, cc := range ccs { + if i > 0 { + buf.WriteByte(' ') + } + switch cc.Type { + case ConfChangeAddNode: + buf.WriteByte('v') + case ConfChangeAddLearnerNode: + buf.WriteByte('l') + case ConfChangeRemoveNode: + buf.WriteByte('r') + case ConfChangeUpdateNode: + buf.WriteByte('u') + default: + buf.WriteString("unknown") + } + fmt.Fprintf(&buf, "%d", cc.NodeID) + } + return buf.String() +} diff --git a/vendor/go.etcd.io/etcd/raft/raftpb/confstate.go b/vendor/go.etcd.io/etcd/raft/raftpb/confstate.go new file mode 100644 index 0000000000..4bda93214b --- /dev/null +++ b/vendor/go.etcd.io/etcd/raft/raftpb/confstate.go @@ -0,0 +1,45 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package raftpb + +import ( + "fmt" + "reflect" + "sort" +) + +// Equivalent returns a nil error if the inputs describe the same configuration. +// On mismatch, returns a descriptive error showing the differences. +func (cs ConfState) Equivalent(cs2 ConfState) error { + cs1 := cs + orig1, orig2 := cs1, cs2 + s := func(sl *[]uint64) { + *sl = append([]uint64(nil), *sl...) + sort.Slice(*sl, func(i, j int) bool { return (*sl)[i] < (*sl)[j] }) + } + + for _, cs := range []*ConfState{&cs1, &cs2} { + s(&cs.Voters) + s(&cs.Learners) + s(&cs.VotersOutgoing) + s(&cs.LearnersNext) + cs.XXX_unrecognized = nil + } + + if !reflect.DeepEqual(cs1, cs2) { + return fmt.Errorf("ConfStates not equivalent after sorting:\n%+#v\n%+#v\nInputs were:\n%+#v\n%+#v", cs1, cs2, orig1, orig2) + } + return nil +} diff --git a/vendor/go.etcd.io/etcd/raft/raftpb/raft.pb.go b/vendor/go.etcd.io/etcd/raft/raftpb/raft.pb.go index fd9ee3729e..fcf259c89b 100644 --- a/vendor/go.etcd.io/etcd/raft/raftpb/raft.pb.go +++ b/vendor/go.etcd.io/etcd/raft/raftpb/raft.pb.go @@ -15,6 +15,8 @@ HardState ConfState ConfChange + ConfChangeSingle + ConfChangeV2 */ package raftpb @@ -44,17 +46,20 @@ const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type EntryType int32 const ( - EntryNormal EntryType = 0 - EntryConfChange EntryType = 1 + EntryNormal EntryType = 0 + EntryConfChange EntryType = 1 + EntryConfChangeV2 EntryType = 2 ) var EntryType_name = map[int32]string{ 0: "EntryNormal", 1: "EntryConfChange", + 2: "EntryConfChangeV2", } var EntryType_value = map[string]int32{ - "EntryNormal": 0, - "EntryConfChange": 1, + "EntryNormal": 0, + "EntryConfChange": 1, + "EntryConfChangeV2": 2, } func (x EntryType) Enum() *EntryType { @@ -160,6 +165,57 @@ func (x *MessageType) UnmarshalJSON(data []byte) error { } func (MessageType) EnumDescriptor() ([]byte, []int) { return fileDescriptorRaft, []int{1} } +// ConfChangeTransition specifies the behavior of a configuration change with +// respect to joint consensus. +type ConfChangeTransition int32 + +const ( + // Automatically use the simple protocol if possible, otherwise fall back + // to ConfChangeJointImplicit. Most applications will want to use this. + ConfChangeTransitionAuto ConfChangeTransition = 0 + // Use joint consensus unconditionally, and transition out of them + // automatically (by proposing a zero configuration change). + // + // This option is suitable for applications that want to minimize the time + // spent in the joint configuration and do not store the joint configuration + // in the state machine (outside of InitialState). + ConfChangeTransitionJointImplicit ConfChangeTransition = 1 + // Use joint consensus and remain in the joint configuration until the + // application proposes a no-op configuration change. This is suitable for + // applications that want to explicitly control the transitions, for example + // to use a custom payload (via the Context field). + ConfChangeTransitionJointExplicit ConfChangeTransition = 2 +) + +var ConfChangeTransition_name = map[int32]string{ + 0: "ConfChangeTransitionAuto", + 1: "ConfChangeTransitionJointImplicit", + 2: "ConfChangeTransitionJointExplicit", +} +var ConfChangeTransition_value = map[string]int32{ + "ConfChangeTransitionAuto": 0, + "ConfChangeTransitionJointImplicit": 1, + "ConfChangeTransitionJointExplicit": 2, +} + +func (x ConfChangeTransition) Enum() *ConfChangeTransition { + p := new(ConfChangeTransition) + *p = x + return p +} +func (x ConfChangeTransition) String() string { + return proto.EnumName(ConfChangeTransition_name, int32(x)) +} +func (x *ConfChangeTransition) UnmarshalJSON(data []byte) error { + value, err := proto.UnmarshalJSONEnum(ConfChangeTransition_value, data, "ConfChangeTransition") + if err != nil { + return err + } + *x = ConfChangeTransition(value) + return nil +} +func (ConfChangeTransition) EnumDescriptor() ([]byte, []int) { return fileDescriptorRaft, []int{2} } + type ConfChangeType int32 const ( @@ -198,7 +254,7 @@ func (x *ConfChangeType) UnmarshalJSON(data []byte) error { *x = ConfChangeType(value) return nil } -func (ConfChangeType) EnumDescriptor() ([]byte, []int) { return fileDescriptorRaft, []int{2} } +func (ConfChangeType) EnumDescriptor() ([]byte, []int) { return fileDescriptorRaft, []int{3} } type Entry struct { Term uint64 `protobuf:"varint,2,opt,name=Term" json:"Term"` @@ -270,9 +326,21 @@ func (*HardState) ProtoMessage() {} func (*HardState) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{4} } type ConfState struct { - Nodes []uint64 `protobuf:"varint,1,rep,name=nodes" json:"nodes,omitempty"` - Learners []uint64 `protobuf:"varint,2,rep,name=learners" json:"learners,omitempty"` - XXX_unrecognized []byte `json:"-"` + // The voters in the incoming config. (If the configuration is not joint, + // then the outgoing config is empty). + Voters []uint64 `protobuf:"varint,1,rep,name=voters" json:"voters,omitempty"` + // The learners in the incoming config. + Learners []uint64 `protobuf:"varint,2,rep,name=learners" json:"learners,omitempty"` + // The voters in the outgoing config. + VotersOutgoing []uint64 `protobuf:"varint,3,rep,name=voters_outgoing,json=votersOutgoing" json:"voters_outgoing,omitempty"` + // The nodes that will become learners when the outgoing config is removed. + // These nodes are necessarily currently in nodes_joint (or they would have + // been added to the incoming config right away). + LearnersNext []uint64 `protobuf:"varint,4,rep,name=learners_next,json=learnersNext" json:"learners_next,omitempty"` + // If set, the config is joint and Raft will automatically transition into + // the final config (i.e. remove the outgoing config) when this is safe. + AutoLeave bool `protobuf:"varint,5,opt,name=auto_leave,json=autoLeave" json:"auto_leave"` + XXX_unrecognized []byte `json:"-"` } func (m *ConfState) Reset() { *m = ConfState{} } @@ -281,11 +349,14 @@ func (*ConfState) ProtoMessage() {} func (*ConfState) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{5} } type ConfChange struct { - ID uint64 `protobuf:"varint,1,opt,name=ID" json:"ID"` - Type ConfChangeType `protobuf:"varint,2,opt,name=Type,enum=raftpb.ConfChangeType" json:"Type"` - NodeID uint64 `protobuf:"varint,3,opt,name=NodeID" json:"NodeID"` - Context []byte `protobuf:"bytes,4,opt,name=Context" json:"Context,omitempty"` - XXX_unrecognized []byte `json:"-"` + Type ConfChangeType `protobuf:"varint,2,opt,name=type,enum=raftpb.ConfChangeType" json:"type"` + NodeID uint64 `protobuf:"varint,3,opt,name=node_id,json=nodeId" json:"node_id"` + Context []byte `protobuf:"bytes,4,opt,name=context" json:"context,omitempty"` + // NB: this is used only by etcd to thread through a unique identifier. + // Ideally it should really use the Context instead. No counterpart to + // this field exists in ConfChangeV2. + ID uint64 `protobuf:"varint,1,opt,name=id" json:"id"` + XXX_unrecognized []byte `json:"-"` } func (m *ConfChange) Reset() { *m = ConfChange{} } @@ -293,6 +364,63 @@ func (m *ConfChange) String() string { return proto.CompactTextString func (*ConfChange) ProtoMessage() {} func (*ConfChange) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{6} } +// ConfChangeSingle is an individual configuration change operation. Multiple +// such operations can be carried out atomically via a ConfChangeV2. +type ConfChangeSingle struct { + Type ConfChangeType `protobuf:"varint,1,opt,name=type,enum=raftpb.ConfChangeType" json:"type"` + NodeID uint64 `protobuf:"varint,2,opt,name=node_id,json=nodeId" json:"node_id"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *ConfChangeSingle) Reset() { *m = ConfChangeSingle{} } +func (m *ConfChangeSingle) String() string { return proto.CompactTextString(m) } +func (*ConfChangeSingle) ProtoMessage() {} +func (*ConfChangeSingle) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{7} } + +// ConfChangeV2 messages initiate configuration changes. They support both the +// simple "one at a time" membership change protocol and full Joint Consensus +// allowing for arbitrary changes in membership. +// +// The supplied context is treated as an opaque payload and can be used to +// attach an action on the state machine to the application of the config change +// proposal. Note that contrary to Joint Consensus as outlined in the Raft +// paper[1], configuration changes become active when they are *applied* to the +// state machine (not when they are appended to the log). +// +// The simple protocol can be used whenever only a single change is made. +// +// Non-simple changes require the use of Joint Consensus, for which two +// configuration changes are run. The first configuration change specifies the +// desired changes and transitions the Raft group into the joint configuration, +// in which quorum requires a majority of both the pre-changes and post-changes +// configuration. Joint Consensus avoids entering fragile intermediate +// configurations that could compromise survivability. For example, without the +// use of Joint Consensus and running across three availability zones with a +// replication factor of three, it is not possible to replace a voter without +// entering an intermediate configuration that does not survive the outage of +// one availability zone. +// +// The provided ConfChangeTransition specifies how (and whether) Joint Consensus +// is used, and assigns the task of leaving the joint configuration either to +// Raft or the application. Leaving the joint configuration is accomplished by +// proposing a ConfChangeV2 with only and optionally the Context field +// populated. +// +// For details on Raft membership changes, see: +// +// [1]: https://github.com/ongardie/dissertation/blob/master/online-trim.pdf +type ConfChangeV2 struct { + Transition ConfChangeTransition `protobuf:"varint,1,opt,name=transition,enum=raftpb.ConfChangeTransition" json:"transition"` + Changes []ConfChangeSingle `protobuf:"bytes,2,rep,name=changes" json:"changes"` + Context []byte `protobuf:"bytes,3,opt,name=context" json:"context,omitempty"` + XXX_unrecognized []byte `json:"-"` +} + +func (m *ConfChangeV2) Reset() { *m = ConfChangeV2{} } +func (m *ConfChangeV2) String() string { return proto.CompactTextString(m) } +func (*ConfChangeV2) ProtoMessage() {} +func (*ConfChangeV2) Descriptor() ([]byte, []int) { return fileDescriptorRaft, []int{8} } + func init() { proto.RegisterType((*Entry)(nil), "raftpb.Entry") proto.RegisterType((*SnapshotMetadata)(nil), "raftpb.SnapshotMetadata") @@ -301,8 +429,11 @@ func init() { proto.RegisterType((*HardState)(nil), "raftpb.HardState") proto.RegisterType((*ConfState)(nil), "raftpb.ConfState") proto.RegisterType((*ConfChange)(nil), "raftpb.ConfChange") + proto.RegisterType((*ConfChangeSingle)(nil), "raftpb.ConfChangeSingle") + proto.RegisterType((*ConfChangeV2)(nil), "raftpb.ConfChangeV2") proto.RegisterEnum("raftpb.EntryType", EntryType_name, EntryType_value) proto.RegisterEnum("raftpb.MessageType", MessageType_name, MessageType_value) + proto.RegisterEnum("raftpb.ConfChangeTransition", ConfChangeTransition_name, ConfChangeTransition_value) proto.RegisterEnum("raftpb.ConfChangeType", ConfChangeType_name, ConfChangeType_value) } func (m *Entry) Marshal() (dAtA []byte, err error) { @@ -535,8 +666,8 @@ func (m *ConfState) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.Nodes) > 0 { - for _, num := range m.Nodes { + if len(m.Voters) > 0 { + for _, num := range m.Voters { dAtA[i] = 0x8 i++ i = encodeVarintRaft(dAtA, i, uint64(num)) @@ -549,6 +680,28 @@ func (m *ConfState) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintRaft(dAtA, i, uint64(num)) } } + if len(m.VotersOutgoing) > 0 { + for _, num := range m.VotersOutgoing { + dAtA[i] = 0x18 + i++ + i = encodeVarintRaft(dAtA, i, uint64(num)) + } + } + if len(m.LearnersNext) > 0 { + for _, num := range m.LearnersNext { + dAtA[i] = 0x20 + i++ + i = encodeVarintRaft(dAtA, i, uint64(num)) + } + } + dAtA[i] = 0x28 + i++ + if m.AutoLeave { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ if m.XXX_unrecognized != nil { i += copy(dAtA[i:], m.XXX_unrecognized) } @@ -591,6 +744,75 @@ func (m *ConfChange) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *ConfChangeSingle) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ConfChangeSingle) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + dAtA[i] = 0x8 + i++ + i = encodeVarintRaft(dAtA, i, uint64(m.Type)) + dAtA[i] = 0x10 + i++ + i = encodeVarintRaft(dAtA, i, uint64(m.NodeID)) + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *ConfChangeV2) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ConfChangeV2) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + dAtA[i] = 0x8 + i++ + i = encodeVarintRaft(dAtA, i, uint64(m.Transition)) + if len(m.Changes) > 0 { + for _, msg := range m.Changes { + dAtA[i] = 0x12 + i++ + i = encodeVarintRaft(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if m.Context != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintRaft(dAtA, i, uint64(len(m.Context))) + i += copy(dAtA[i:], m.Context) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + func encodeVarintRaft(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -689,8 +911,8 @@ func (m *HardState) Size() (n int) { func (m *ConfState) Size() (n int) { var l int _ = l - if len(m.Nodes) > 0 { - for _, e := range m.Nodes { + if len(m.Voters) > 0 { + for _, e := range m.Voters { n += 1 + sovRaft(uint64(e)) } } @@ -699,6 +921,17 @@ func (m *ConfState) Size() (n int) { n += 1 + sovRaft(uint64(e)) } } + if len(m.VotersOutgoing) > 0 { + for _, e := range m.VotersOutgoing { + n += 1 + sovRaft(uint64(e)) + } + } + if len(m.LearnersNext) > 0 { + for _, e := range m.LearnersNext { + n += 1 + sovRaft(uint64(e)) + } + } + n += 2 if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -721,6 +954,37 @@ func (m *ConfChange) Size() (n int) { return n } +func (m *ConfChangeSingle) Size() (n int) { + var l int + _ = l + n += 1 + sovRaft(uint64(m.Type)) + n += 1 + sovRaft(uint64(m.NodeID)) + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ConfChangeV2) Size() (n int) { + var l int + _ = l + n += 1 + sovRaft(uint64(m.Transition)) + if len(m.Changes) > 0 { + for _, e := range m.Changes { + l = e.Size() + n += 1 + l + sovRaft(uint64(l)) + } + } + if m.Context != nil { + l = len(m.Context) + n += 1 + l + sovRaft(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func sovRaft(x uint64) (n int) { for { n++ @@ -1573,7 +1837,7 @@ func (m *ConfState) Unmarshal(dAtA []byte) error { break } } - m.Nodes = append(m.Nodes, v) + m.Voters = append(m.Voters, v) } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { @@ -1613,10 +1877,10 @@ func (m *ConfState) Unmarshal(dAtA []byte) error { break } } - m.Nodes = append(m.Nodes, v) + m.Voters = append(m.Voters, v) } } else { - return fmt.Errorf("proto: wrong wireType = %d for field Nodes", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Voters", wireType) } case 2: if wireType == 0 { @@ -1680,6 +1944,150 @@ func (m *ConfState) Unmarshal(dAtA []byte) error { } else { return fmt.Errorf("proto: wrong wireType = %d for field Learners", wireType) } + case 3: + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.VotersOutgoing = append(m.VotersOutgoing, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthRaft + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.VotersOutgoing = append(m.VotersOutgoing, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field VotersOutgoing", wireType) + } + case 4: + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.LearnersNext = append(m.LearnersNext, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthRaft + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.LearnersNext = append(m.LearnersNext, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field LearnersNext", wireType) + } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AutoLeave", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.AutoLeave = bool(v != 0) default: iNdEx = preIndex skippy, err := skipRaft(dAtA[iNdEx:]) @@ -1841,6 +2249,227 @@ func (m *ConfChange) Unmarshal(dAtA []byte) error { } return nil } +func (m *ConfChangeSingle) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ConfChangeSingle: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ConfChangeSingle: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) + } + m.Type = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Type |= (ConfChangeType(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field NodeID", wireType) + } + m.NodeID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.NodeID |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipRaft(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRaft + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ConfChangeV2) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ConfChangeV2: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ConfChangeV2: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Transition", wireType) + } + m.Transition = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Transition |= (ConfChangeTransition(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Changes", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthRaft + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Changes = append(m.Changes, ConfChangeSingle{}) + if err := m.Changes[len(m.Changes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Context", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRaft + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthRaft + } + postIndex := iNdEx + byteLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Context = append(m.Context[:0], dAtA[iNdEx:postIndex]...) + if m.Context == nil { + m.Context = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipRaft(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthRaft + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipRaft(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 @@ -1949,56 +2578,69 @@ var ( func init() { proto.RegisterFile("raft.proto", fileDescriptorRaft) } var fileDescriptorRaft = []byte{ - // 815 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x64, 0x54, 0xcd, 0x6e, 0x23, 0x45, - 0x10, 0xf6, 0x8c, 0xc7, 0x7f, 0x35, 0x8e, 0xd3, 0xa9, 0x35, 0xa8, 0x15, 0x45, 0xc6, 0xb2, 0x38, - 0x58, 0x41, 0x1b, 0x20, 0x07, 0x0e, 0x48, 0x1c, 0x36, 0x09, 0x52, 0x22, 0xad, 0xa3, 0xc5, 0x9b, - 0xe5, 0x80, 0x84, 0x50, 0xc7, 0x53, 0x9e, 0x18, 0x32, 0xd3, 0xa3, 0x9e, 0xf6, 0xb2, 0xb9, 0x20, - 0x1e, 0x80, 0x07, 0xe0, 0xc2, 0xfb, 0xe4, 0xb8, 0x12, 0x77, 0xc4, 0x86, 0x17, 0x41, 0xdd, 0xd3, - 0x63, 0xcf, 0x24, 0xb7, 0xae, 0xef, 0xab, 0xae, 0xfa, 0xea, 0xeb, 0x9a, 0x01, 0x50, 0x62, 0xa9, - 0x8f, 0x32, 0x25, 0xb5, 0xc4, 0xb6, 0x39, 0x67, 0xd7, 0xfb, 0xc3, 0x58, 0xc6, 0xd2, 0x42, 0x9f, - 0x9b, 0x53, 0xc1, 0x4e, 0x7e, 0x83, 0xd6, 0xb7, 0xa9, 0x56, 0x77, 0xf8, 0x19, 0x04, 0x57, 0x77, - 0x19, 0x71, 0x6f, 0xec, 0x4d, 0x07, 0xc7, 0x7b, 0x47, 0xc5, 0xad, 0x23, 0x4b, 0x1a, 0xe2, 0x24, - 0xb8, 0xff, 0xe7, 0x93, 0xc6, 0xdc, 0x26, 0x21, 0x87, 0xe0, 0x8a, 0x54, 0xc2, 0xfd, 0xb1, 0x37, - 0x0d, 0x36, 0x0c, 0xa9, 0x04, 0xf7, 0xa1, 0x75, 0x91, 0x46, 0xf4, 0x8e, 0x37, 0x2b, 0x54, 0x01, - 0x21, 0x42, 0x70, 0x26, 0xb4, 0xe0, 0xc1, 0xd8, 0x9b, 0xf6, 0xe7, 0xf6, 0x3c, 0xf9, 0xdd, 0x03, - 0xf6, 0x3a, 0x15, 0x59, 0x7e, 0x23, 0xf5, 0x8c, 0xb4, 0x88, 0x84, 0x16, 0xf8, 0x15, 0xc0, 0x42, - 0xa6, 0xcb, 0x9f, 0x72, 0x2d, 0x74, 0xa1, 0x28, 0xdc, 0x2a, 0x3a, 0x95, 0xe9, 0xf2, 0xb5, 0x21, - 0x5c, 0xf1, 0xde, 0xa2, 0x04, 0x4c, 0xf3, 0x95, 0x6d, 0x5e, 0xd5, 0x55, 0x40, 0x46, 0xb2, 0x36, - 0x92, 0xab, 0xba, 0x2c, 0x32, 0xf9, 0x01, 0xba, 0xa5, 0x02, 0x23, 0xd1, 0x28, 0xb0, 0x3d, 0xfb, - 0x73, 0x7b, 0xc6, 0xaf, 0xa1, 0x9b, 0x38, 0x65, 0xb6, 0x70, 0x78, 0xcc, 0x4b, 0x2d, 0x8f, 0x95, - 0xbb, 0xba, 0x9b, 0xfc, 0xc9, 0x5f, 0x4d, 0xe8, 0xcc, 0x28, 0xcf, 0x45, 0x4c, 0xf8, 0x1c, 0x02, - 0xbd, 0x75, 0xf8, 0x59, 0x59, 0xc3, 0xd1, 0x55, 0x8f, 0x4d, 0x1a, 0x0e, 0xc1, 0xd7, 0xb2, 0x36, - 0x89, 0xaf, 0xa5, 0x19, 0x63, 0xa9, 0xe4, 0xa3, 0x31, 0x0c, 0xb2, 0x19, 0x30, 0x78, 0x3c, 0x20, - 0x8e, 0xa0, 0x73, 0x2b, 0x63, 0xfb, 0x60, 0xad, 0x0a, 0x59, 0x82, 0x5b, 0xdb, 0xda, 0x4f, 0x6d, - 0x7b, 0x0e, 0x1d, 0x4a, 0xb5, 0x5a, 0x51, 0xce, 0x3b, 0xe3, 0xe6, 0x34, 0x3c, 0xde, 0xa9, 0x6d, - 0x46, 0x59, 0xca, 0xe5, 0xe0, 0x01, 0xb4, 0x17, 0x32, 0x49, 0x56, 0x9a, 0x77, 0x2b, 0xb5, 0x1c, - 0x86, 0xc7, 0xd0, 0xcd, 0x9d, 0x63, 0xbc, 0x67, 0x9d, 0x64, 0x8f, 0x9d, 0x2c, 0x1d, 0x2c, 0xf3, - 0x4c, 0x45, 0x45, 0x3f, 0xd3, 0x42, 0x73, 0x18, 0x7b, 0xd3, 0x6e, 0x59, 0xb1, 0xc0, 0xf0, 0x53, - 0x80, 0xe2, 0x74, 0xbe, 0x4a, 0x35, 0x0f, 0x2b, 0x3d, 0x2b, 0x38, 0x72, 0xe8, 0x2c, 0x64, 0xaa, - 0xe9, 0x9d, 0xe6, 0x7d, 0xfb, 0xb0, 0x65, 0x38, 0xf9, 0x11, 0x7a, 0xe7, 0x42, 0x45, 0xc5, 0xfa, - 0x94, 0x0e, 0x7a, 0x4f, 0x1c, 0xe4, 0x10, 0xbc, 0x95, 0x9a, 0xea, 0xfb, 0x6e, 0x90, 0xca, 0xc0, - 0xcd, 0xa7, 0x03, 0x4f, 0xbe, 0x81, 0xde, 0x66, 0x5d, 0x71, 0x08, 0xad, 0x54, 0x46, 0x94, 0x73, - 0x6f, 0xdc, 0x9c, 0x06, 0xf3, 0x22, 0xc0, 0x7d, 0xe8, 0xde, 0x92, 0x50, 0x29, 0xa9, 0x9c, 0xfb, - 0x96, 0xd8, 0xc4, 0x93, 0x3f, 0x3c, 0x00, 0x73, 0xff, 0xf4, 0x46, 0xa4, 0xb1, 0xdd, 0x88, 0x8b, - 0xb3, 0x9a, 0x3a, 0xff, 0xe2, 0x0c, 0xbf, 0x70, 0x1f, 0xae, 0x6f, 0xd7, 0xea, 0xe3, 0xea, 0x67, - 0x52, 0xdc, 0x7b, 0xf2, 0xf5, 0x1e, 0x40, 0xfb, 0x52, 0x46, 0x74, 0x71, 0x56, 0xd7, 0x5c, 0x60, - 0xc6, 0xac, 0x53, 0x67, 0x56, 0xf1, 0xa1, 0x96, 0xe1, 0xe1, 0x97, 0xd0, 0xdb, 0xfc, 0x0e, 0x70, - 0x17, 0x42, 0x1b, 0x5c, 0x4a, 0x95, 0x88, 0x5b, 0xd6, 0xc0, 0x67, 0xb0, 0x6b, 0x81, 0x6d, 0x63, - 0xe6, 0x1d, 0xfe, 0xed, 0x43, 0x58, 0x59, 0x70, 0x04, 0x68, 0xcf, 0xf2, 0xf8, 0x7c, 0x9d, 0xb1, - 0x06, 0x86, 0xd0, 0x99, 0xe5, 0xf1, 0x09, 0x09, 0xcd, 0x3c, 0x17, 0xbc, 0x52, 0x32, 0x63, 0xbe, - 0xcb, 0x7a, 0x91, 0x65, 0xac, 0x89, 0x03, 0x80, 0xe2, 0x3c, 0xa7, 0x3c, 0x63, 0x81, 0x4b, 0xfc, - 0x5e, 0x6a, 0x62, 0x2d, 0x23, 0xc2, 0x05, 0x96, 0x6d, 0x3b, 0xd6, 0x2c, 0x13, 0xeb, 0x20, 0x83, - 0xbe, 0x69, 0x46, 0x42, 0xe9, 0x6b, 0xd3, 0xa5, 0x8b, 0x43, 0x60, 0x55, 0xc4, 0x5e, 0xea, 0x21, - 0xc2, 0x60, 0x96, 0xc7, 0x6f, 0x52, 0x45, 0x62, 0x71, 0x23, 0xae, 0x6f, 0x89, 0x01, 0xee, 0xc1, - 0x8e, 0x2b, 0x64, 0x1e, 0x6f, 0x9d, 0xb3, 0xd0, 0xa5, 0x9d, 0xde, 0xd0, 0xe2, 0x97, 0xef, 0xd6, - 0x52, 0xad, 0x13, 0xd6, 0xc7, 0x8f, 0x60, 0x6f, 0x96, 0xc7, 0x57, 0x4a, 0xa4, 0xf9, 0x92, 0xd4, - 0x4b, 0x12, 0x11, 0x29, 0xb6, 0xe3, 0x6e, 0x5f, 0xad, 0x12, 0x92, 0x6b, 0x7d, 0x29, 0x7f, 0x65, - 0x03, 0x27, 0x66, 0x4e, 0x22, 0xb2, 0x3f, 0x43, 0xb6, 0xeb, 0xc4, 0x6c, 0x10, 0x2b, 0x86, 0xb9, - 0x79, 0x5f, 0x29, 0xb2, 0x23, 0xee, 0xb9, 0xae, 0x2e, 0xb6, 0x39, 0x78, 0x78, 0x07, 0x83, 0xfa, - 0xf3, 0x1a, 0x1d, 0x5b, 0xe4, 0x45, 0x14, 0x99, 0xb7, 0x64, 0x0d, 0xe4, 0x30, 0xdc, 0xc2, 0x73, - 0x4a, 0xe4, 0x5b, 0xb2, 0x8c, 0x57, 0x67, 0xde, 0x64, 0x91, 0xd0, 0x05, 0xe3, 0xe3, 0x01, 0xf0, - 0x5a, 0xa9, 0x97, 0xc5, 0x36, 0x5a, 0xb6, 0x79, 0xc2, 0xef, 0x3f, 0x8c, 0x1a, 0xef, 0x3f, 0x8c, - 0x1a, 0xf7, 0x0f, 0x23, 0xef, 0xfd, 0xc3, 0xc8, 0xfb, 0xf7, 0x61, 0xe4, 0xfd, 0xf9, 0xdf, 0xa8, - 0xf1, 0x7f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x86, 0x52, 0x5b, 0xe0, 0x74, 0x06, 0x00, 0x00, + // 1009 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x55, 0xcd, 0x6e, 0xe3, 0x36, + 0x17, 0xb5, 0x64, 0xc5, 0x3f, 0xd7, 0x8e, 0xc3, 0xdc, 0xc9, 0x37, 0x20, 0x82, 0xc0, 0xe3, 0xcf, + 0xd3, 0x62, 0x8c, 0x14, 0x93, 0x16, 0x5e, 0x14, 0x45, 0x77, 0xf9, 0x19, 0x20, 0x29, 0xe2, 0x74, + 0xea, 0x64, 0xb2, 0x28, 0x50, 0x04, 0x8c, 0x45, 0x2b, 0x6a, 0x2d, 0x51, 0xa0, 0xe8, 0x34, 0xd9, + 0x14, 0x45, 0x9f, 0xa2, 0x9b, 0xd9, 0xf6, 0x01, 0xfa, 0x14, 0x59, 0x0e, 0xd0, 0xfd, 0xa0, 0x93, + 0xbe, 0x48, 0x41, 0x8a, 0xb2, 0x65, 0x27, 0x98, 0x45, 0x77, 0xe4, 0x39, 0x87, 0xf7, 0x9e, 0x7b, + 0x79, 0x45, 0x01, 0x48, 0x36, 0x56, 0x3b, 0x89, 0x14, 0x4a, 0x60, 0x45, 0xaf, 0x93, 0xcb, 0xcd, + 0x8d, 0x40, 0x04, 0xc2, 0x40, 0x9f, 0xeb, 0x55, 0xc6, 0x76, 0x7f, 0x81, 0x95, 0x57, 0xb1, 0x92, + 0xb7, 0xf8, 0x19, 0x78, 0x67, 0xb7, 0x09, 0xa7, 0x4e, 0xc7, 0xe9, 0xb5, 0xfa, 0xeb, 0x3b, 0xd9, + 0xa9, 0x1d, 0x43, 0x6a, 0x62, 0xcf, 0xbb, 0x7b, 0xff, 0xac, 0x34, 0x34, 0x22, 0xa4, 0xe0, 0x9d, + 0x71, 0x19, 0x51, 0xb7, 0xe3, 0xf4, 0xbc, 0x19, 0xc3, 0x65, 0x84, 0x9b, 0xb0, 0x72, 0x14, 0xfb, + 0xfc, 0x86, 0x96, 0x0b, 0x54, 0x06, 0x21, 0x82, 0x77, 0xc0, 0x14, 0xa3, 0x5e, 0xc7, 0xe9, 0x35, + 0x87, 0x66, 0xdd, 0xfd, 0xd5, 0x01, 0x72, 0x1a, 0xb3, 0x24, 0xbd, 0x12, 0x6a, 0xc0, 0x15, 0xf3, + 0x99, 0x62, 0xf8, 0x25, 0xc0, 0x48, 0xc4, 0xe3, 0x8b, 0x54, 0x31, 0x95, 0x39, 0x6a, 0xcc, 0x1d, + 0xed, 0x8b, 0x78, 0x7c, 0xaa, 0x09, 0x1b, 0xbc, 0x3e, 0xca, 0x01, 0x9d, 0x3c, 0x34, 0xc9, 0x8b, + 0xbe, 0x32, 0x48, 0x5b, 0x56, 0xda, 0x72, 0xd1, 0x97, 0x41, 0xba, 0xdf, 0x43, 0x2d, 0x77, 0xa0, + 0x2d, 0x6a, 0x07, 0x26, 0x67, 0x73, 0x68, 0xd6, 0xf8, 0x35, 0xd4, 0x22, 0xeb, 0xcc, 0x04, 0x6e, + 0xf4, 0x69, 0xee, 0x65, 0xd9, 0xb9, 0x8d, 0x3b, 0xd3, 0x77, 0xdf, 0x96, 0xa1, 0x3a, 0xe0, 0x69, + 0xca, 0x02, 0x8e, 0x2f, 0xc1, 0x53, 0xf3, 0x0e, 0x3f, 0xc9, 0x63, 0x58, 0xba, 0xd8, 0x63, 0x2d, + 0xc3, 0x0d, 0x70, 0x95, 0x58, 0xa8, 0xc4, 0x55, 0x42, 0x97, 0x31, 0x96, 0x62, 0xa9, 0x0c, 0x8d, + 0xcc, 0x0a, 0xf4, 0x96, 0x0b, 0xc4, 0x36, 0x54, 0x27, 0x22, 0x30, 0x17, 0xb6, 0x52, 0x20, 0x73, + 0x70, 0xde, 0xb6, 0xca, 0xc3, 0xb6, 0xbd, 0x84, 0x2a, 0x8f, 0x95, 0x0c, 0x79, 0x4a, 0xab, 0x9d, + 0x72, 0xaf, 0xd1, 0x5f, 0x5d, 0x98, 0x8c, 0x3c, 0x94, 0xd5, 0xe0, 0x16, 0x54, 0x46, 0x22, 0x8a, + 0x42, 0x45, 0x6b, 0x85, 0x58, 0x16, 0xc3, 0x3e, 0xd4, 0x52, 0xdb, 0x31, 0x5a, 0x37, 0x9d, 0x24, + 0xcb, 0x9d, 0xcc, 0x3b, 0x98, 0xeb, 0x74, 0x44, 0xc9, 0x7f, 0xe4, 0x23, 0x45, 0xa1, 0xe3, 0xf4, + 0x6a, 0x79, 0xc4, 0x0c, 0xc3, 0x4f, 0x00, 0xb2, 0xd5, 0x61, 0x18, 0x2b, 0xda, 0x28, 0xe4, 0x2c, + 0xe0, 0x48, 0xa1, 0x3a, 0x12, 0xb1, 0xe2, 0x37, 0x8a, 0x36, 0xcd, 0xc5, 0xe6, 0xdb, 0xee, 0x0f, + 0x50, 0x3f, 0x64, 0xd2, 0xcf, 0xc6, 0x27, 0xef, 0xa0, 0xf3, 0xa0, 0x83, 0x14, 0xbc, 0x6b, 0xa1, + 0xf8, 0xe2, 0xbc, 0x6b, 0xa4, 0x50, 0x70, 0xf9, 0x61, 0xc1, 0xdd, 0x3f, 0x1d, 0xa8, 0xcf, 0xe6, + 0x15, 0x9f, 0x42, 0x45, 0x9f, 0x91, 0x29, 0x75, 0x3a, 0xe5, 0x9e, 0x37, 0xb4, 0x3b, 0xdc, 0x84, + 0xda, 0x84, 0x33, 0x19, 0x6b, 0xc6, 0x35, 0xcc, 0x6c, 0x8f, 0x2f, 0x60, 0x2d, 0x53, 0x5d, 0x88, + 0xa9, 0x0a, 0x44, 0x18, 0x07, 0xb4, 0x6c, 0x24, 0xad, 0x0c, 0xfe, 0xd6, 0xa2, 0xf8, 0x1c, 0x56, + 0xf3, 0x43, 0x17, 0xb1, 0xae, 0xd4, 0x33, 0xb2, 0x66, 0x0e, 0x9e, 0xf0, 0x1b, 0x85, 0xcf, 0x01, + 0xd8, 0x54, 0x89, 0x8b, 0x09, 0x67, 0xd7, 0xdc, 0x0c, 0x43, 0xde, 0xd0, 0xba, 0xc6, 0x8f, 0x35, + 0xdc, 0x7d, 0xeb, 0x00, 0x68, 0xd3, 0xfb, 0x57, 0x2c, 0x0e, 0xf4, 0x47, 0xe5, 0x86, 0xbe, 0xed, + 0x09, 0x68, 0xed, 0xfd, 0xfb, 0x67, 0xee, 0xd1, 0xc1, 0xd0, 0x0d, 0x7d, 0xfc, 0xc2, 0x8e, 0xb4, + 0x6b, 0x46, 0xfa, 0x69, 0xf1, 0x13, 0xcd, 0x4e, 0x3f, 0x98, 0xea, 0x17, 0x50, 0x8d, 0x85, 0xcf, + 0x2f, 0x42, 0xdf, 0x36, 0xac, 0x65, 0x43, 0x56, 0x4e, 0x84, 0xcf, 0x8f, 0x0e, 0x86, 0x15, 0x4d, + 0x1f, 0xf9, 0xc5, 0x3b, 0xf3, 0x16, 0xef, 0x2c, 0x02, 0x32, 0x4f, 0x70, 0x1a, 0xc6, 0xc1, 0x84, + 0xcf, 0x8c, 0x38, 0xff, 0xc5, 0x88, 0xfb, 0x31, 0x23, 0xdd, 0x3f, 0x1c, 0x68, 0xce, 0xe3, 0x9c, + 0xf7, 0x71, 0x0f, 0x40, 0x49, 0x16, 0xa7, 0xa1, 0x0a, 0x45, 0x6c, 0x33, 0x6e, 0x3d, 0x92, 0x71, + 0xa6, 0xc9, 0x27, 0x72, 0x7e, 0x0a, 0xbf, 0x82, 0xea, 0xc8, 0xa8, 0xb2, 0x1b, 0x2f, 0x3c, 0x29, + 0xcb, 0xa5, 0xe5, 0x5f, 0x98, 0x95, 0x17, 0xfb, 0x52, 0x5e, 0xe8, 0xcb, 0xf6, 0x21, 0xd4, 0x67, + 0xaf, 0x35, 0xae, 0x41, 0xc3, 0x6c, 0x4e, 0x84, 0x8c, 0xd8, 0x84, 0x94, 0xf0, 0x09, 0xac, 0x19, + 0x60, 0x1e, 0x9f, 0x38, 0xf8, 0x3f, 0x58, 0x5f, 0x02, 0xcf, 0xfb, 0xc4, 0xdd, 0xfe, 0xcb, 0x85, + 0x46, 0xe1, 0x59, 0x42, 0x80, 0xca, 0x20, 0x0d, 0x0e, 0xa7, 0x09, 0x29, 0x61, 0x03, 0xaa, 0x83, + 0x34, 0xd8, 0xe3, 0x4c, 0x11, 0xc7, 0x6e, 0x5e, 0x4b, 0x91, 0x10, 0xd7, 0xaa, 0x76, 0x93, 0x84, + 0x94, 0xb1, 0x05, 0x90, 0xad, 0x87, 0x3c, 0x4d, 0x88, 0x67, 0x85, 0xe7, 0x42, 0x71, 0xb2, 0xa2, + 0xbd, 0xd9, 0x8d, 0x61, 0x2b, 0x96, 0xd5, 0x4f, 0x00, 0xa9, 0x22, 0x81, 0xa6, 0x4e, 0xc6, 0x99, + 0x54, 0x97, 0x3a, 0x4b, 0x0d, 0x37, 0x80, 0x14, 0x11, 0x73, 0xa8, 0x8e, 0x08, 0xad, 0x41, 0x1a, + 0xbc, 0x89, 0x25, 0x67, 0xa3, 0x2b, 0x76, 0x39, 0xe1, 0x04, 0x70, 0x1d, 0x56, 0x6d, 0x20, 0xfd, + 0xc5, 0x4d, 0x53, 0xd2, 0xb0, 0xb2, 0xfd, 0x2b, 0x3e, 0xfa, 0xe9, 0xbb, 0xa9, 0x90, 0xd3, 0x88, + 0x34, 0x75, 0xd9, 0x83, 0x34, 0x30, 0x17, 0x34, 0xe6, 0xf2, 0x98, 0x33, 0x9f, 0x4b, 0xb2, 0x6a, + 0x4f, 0x9f, 0x85, 0x11, 0x17, 0x53, 0x75, 0x22, 0x7e, 0x26, 0x2d, 0x6b, 0x66, 0xc8, 0x99, 0x6f, + 0x7e, 0x61, 0x64, 0xcd, 0x9a, 0x99, 0x21, 0xc6, 0x0c, 0xb1, 0xf5, 0xbe, 0x96, 0xdc, 0x94, 0xb8, + 0x6e, 0xb3, 0xda, 0xbd, 0xd1, 0xe0, 0xf6, 0x6f, 0x0e, 0x6c, 0x3c, 0x36, 0x1e, 0xb8, 0x05, 0xf4, + 0x31, 0x7c, 0x77, 0xaa, 0x04, 0x29, 0xe1, 0xa7, 0xf0, 0xff, 0xc7, 0xd8, 0x6f, 0x44, 0x18, 0xab, + 0xa3, 0x28, 0x99, 0x84, 0xa3, 0x50, 0x5f, 0xc5, 0xc7, 0x64, 0xaf, 0x6e, 0xac, 0xcc, 0xdd, 0xbe, + 0x85, 0xd6, 0xe2, 0x47, 0xa1, 0x9b, 0x31, 0x47, 0x76, 0x7d, 0x5f, 0x8f, 0x3f, 0x29, 0x21, 0x2d, + 0x9a, 0x1d, 0xf2, 0x48, 0x5c, 0x73, 0xc3, 0x38, 0x8b, 0xcc, 0x9b, 0xc4, 0x67, 0x2a, 0x63, 0xdc, + 0xc5, 0x42, 0x76, 0x7d, 0xff, 0x38, 0x7b, 0x7b, 0x0c, 0x5b, 0xde, 0xa3, 0x77, 0x1f, 0xda, 0xa5, + 0x77, 0x1f, 0xda, 0xa5, 0xbb, 0xfb, 0xb6, 0xf3, 0xee, 0xbe, 0xed, 0xfc, 0x7d, 0xdf, 0x76, 0x7e, + 0xff, 0xa7, 0x5d, 0xfa, 0x37, 0x00, 0x00, 0xff, 0xff, 0x87, 0x11, 0x6d, 0xd6, 0xaf, 0x08, 0x00, + 0x00, } diff --git a/vendor/go.etcd.io/etcd/raft/raftpb/raft.proto b/vendor/go.etcd.io/etcd/raft/raftpb/raft.proto index 644ce7b8f2..23d62ec2fb 100644 --- a/vendor/go.etcd.io/etcd/raft/raftpb/raft.proto +++ b/vendor/go.etcd.io/etcd/raft/raftpb/raft.proto @@ -10,8 +10,9 @@ option (gogoproto.goproto_getters_all) = false; option (gogoproto.goproto_enum_prefix_all) = false; enum EntryType { - EntryNormal = 0; - EntryConfChange = 1; + EntryNormal = 0; + EntryConfChange = 1; // corresponds to pb.ConfChange + EntryConfChangeV2 = 2; // corresponds to pb.ConfChangeV2 } message Entry { @@ -75,9 +76,41 @@ message HardState { optional uint64 commit = 3 [(gogoproto.nullable) = false]; } +// ConfChangeTransition specifies the behavior of a configuration change with +// respect to joint consensus. +enum ConfChangeTransition { + // Automatically use the simple protocol if possible, otherwise fall back + // to ConfChangeJointImplicit. Most applications will want to use this. + ConfChangeTransitionAuto = 0; + // Use joint consensus unconditionally, and transition out of them + // automatically (by proposing a zero configuration change). + // + // This option is suitable for applications that want to minimize the time + // spent in the joint configuration and do not store the joint configuration + // in the state machine (outside of InitialState). + ConfChangeTransitionJointImplicit = 1; + // Use joint consensus and remain in the joint configuration until the + // application proposes a no-op configuration change. This is suitable for + // applications that want to explicitly control the transitions, for example + // to use a custom payload (via the Context field). + ConfChangeTransitionJointExplicit = 2; +} + message ConfState { - repeated uint64 nodes = 1; - repeated uint64 learners = 2; + // The voters in the incoming config. (If the configuration is not joint, + // then the outgoing config is empty). + repeated uint64 voters = 1; + // The learners in the incoming config. + repeated uint64 learners = 2; + // The voters in the outgoing config. + repeated uint64 voters_outgoing = 3; + // The nodes that will become learners when the outgoing config is removed. + // These nodes are necessarily currently in nodes_joint (or they would have + // been added to the incoming config right away). + repeated uint64 learners_next = 4; + // If set, the config is joint and Raft will automatically transition into + // the final config (i.e. remove the outgoing config) when this is safe. + optional bool auto_leave = 5 [(gogoproto.nullable) = false]; } enum ConfChangeType { @@ -88,8 +121,57 @@ enum ConfChangeType { } message ConfChange { - optional uint64 ID = 1 [(gogoproto.nullable) = false]; - optional ConfChangeType Type = 2 [(gogoproto.nullable) = false]; - optional uint64 NodeID = 3 [(gogoproto.nullable) = false]; - optional bytes Context = 4; + optional ConfChangeType type = 2 [(gogoproto.nullable) = false]; + optional uint64 node_id = 3 [(gogoproto.nullable) = false, (gogoproto.customname) = "NodeID" ]; + optional bytes context = 4; + + // NB: this is used only by etcd to thread through a unique identifier. + // Ideally it should really use the Context instead. No counterpart to + // this field exists in ConfChangeV2. + optional uint64 id = 1 [(gogoproto.nullable) = false, (gogoproto.customname) = "ID" ]; +} + +// ConfChangeSingle is an individual configuration change operation. Multiple +// such operations can be carried out atomically via a ConfChangeV2. +message ConfChangeSingle { + optional ConfChangeType type = 1 [(gogoproto.nullable) = false]; + optional uint64 node_id = 2 [(gogoproto.nullable) = false, (gogoproto.customname) = "NodeID"]; +} + +// ConfChangeV2 messages initiate configuration changes. They support both the +// simple "one at a time" membership change protocol and full Joint Consensus +// allowing for arbitrary changes in membership. +// +// The supplied context is treated as an opaque payload and can be used to +// attach an action on the state machine to the application of the config change +// proposal. Note that contrary to Joint Consensus as outlined in the Raft +// paper[1], configuration changes become active when they are *applied* to the +// state machine (not when they are appended to the log). +// +// The simple protocol can be used whenever only a single change is made. +// +// Non-simple changes require the use of Joint Consensus, for which two +// configuration changes are run. The first configuration change specifies the +// desired changes and transitions the Raft group into the joint configuration, +// in which quorum requires a majority of both the pre-changes and post-changes +// configuration. Joint Consensus avoids entering fragile intermediate +// configurations that could compromise survivability. For example, without the +// use of Joint Consensus and running across three availability zones with a +// replication factor of three, it is not possible to replace a voter without +// entering an intermediate configuration that does not survive the outage of +// one availability zone. +// +// The provided ConfChangeTransition specifies how (and whether) Joint Consensus +// is used, and assigns the task of leaving the joint configuration either to +// Raft or the application. Leaving the joint configuration is accomplished by +// proposing a ConfChangeV2 with only and optionally the Context field +// populated. +// +// For details on Raft membership changes, see: +// +// [1]: https://github.com/ongardie/dissertation/blob/master/online-trim.pdf +message ConfChangeV2 { + optional ConfChangeTransition transition = 1 [(gogoproto.nullable) = false]; + repeated ConfChangeSingle changes = 2 [(gogoproto.nullable) = false]; + optional bytes context = 3; } diff --git a/vendor/go.etcd.io/etcd/raft/rawnode.go b/vendor/go.etcd.io/etcd/raft/rawnode.go index 77183b793c..90eb69493c 100644 --- a/vendor/go.etcd.io/etcd/raft/rawnode.go +++ b/vendor/go.etcd.io/etcd/raft/rawnode.go @@ -37,82 +37,20 @@ type RawNode struct { prevHardSt pb.HardState } -func (rn *RawNode) newReady() Ready { - return newReady(rn.raft, rn.prevSoftSt, rn.prevHardSt) -} - -func (rn *RawNode) commitReady(rd Ready) { - if rd.SoftState != nil { - rn.prevSoftSt = rd.SoftState - } - if !IsEmptyHardState(rd.HardState) { - rn.prevHardSt = rd.HardState - } - - // If entries were applied (or a snapshot), update our cursor for - // the next Ready. Note that if the current HardState contains a - // new Commit index, this does not mean that we're also applying - // all of the new entries due to commit pagination by size. - if index := rd.appliedCursor(); index > 0 { - rn.raft.raftLog.appliedTo(index) - } - - if len(rd.Entries) > 0 { - e := rd.Entries[len(rd.Entries)-1] - rn.raft.raftLog.stableTo(e.Index, e.Term) - } - if !IsEmptySnap(rd.Snapshot) { - rn.raft.raftLog.stableSnapTo(rd.Snapshot.Metadata.Index) - } - if len(rd.ReadStates) != 0 { - rn.raft.readStates = nil - } -} - -// NewRawNode returns a new RawNode given configuration and a list of raft peers. -func NewRawNode(config *Config, peers []Peer) (*RawNode, error) { - if config.ID == 0 { - panic("config.ID must not be zero") - } +// NewRawNode instantiates a RawNode from the given configuration. +// +// See Bootstrap() for bootstrapping an initial state; this replaces the former +// 'peers' argument to this method (with identical behavior). However, It is +// recommended that instead of calling Bootstrap, applications bootstrap their +// state manually by setting up a Storage that has a first index > 1 and which +// stores the desired ConfState as its InitialState. +func NewRawNode(config *Config) (*RawNode, error) { r := newRaft(config) rn := &RawNode{ raft: r, } - lastIndex, err := config.Storage.LastIndex() - if err != nil { - panic(err) // TODO(bdarnell) - } - // If the log is empty, this is a new RawNode (like StartNode); otherwise it's - // restoring an existing RawNode (like RestartNode). - // TODO(bdarnell): rethink RawNode initialization and whether the application needs - // to be able to tell us when it expects the RawNode to exist. - if lastIndex == 0 { - r.becomeFollower(1, None) - ents := make([]pb.Entry, len(peers)) - for i, peer := range peers { - cc := pb.ConfChange{Type: pb.ConfChangeAddNode, NodeID: peer.ID, Context: peer.Context} - data, err := cc.Marshal() - if err != nil { - panic("unexpected marshal error") - } - - ents[i] = pb.Entry{Type: pb.EntryConfChange, Term: 1, Index: uint64(i + 1), Data: data} - } - r.raftLog.append(ents...) - r.raftLog.committed = uint64(len(ents)) - for _, peer := range peers { - r.applyConfChange(pb.ConfChange{NodeID: peer.ID, Type: pb.ConfChangeAddNode}) - } - } - - // Set the initial hard and soft states after performing all initialization. rn.prevSoftSt = r.softState() - if lastIndex == 0 { - rn.prevHardSt = emptyState - } else { - rn.prevHardSt = r.hardState() - } - + rn.prevHardSt = r.hardState() return rn, nil } @@ -150,23 +88,19 @@ func (rn *RawNode) Propose(data []byte) error { }}) } -// ProposeConfChange proposes a config change. -func (rn *RawNode) ProposeConfChange(cc pb.ConfChange) error { - data, err := cc.Marshal() +// ProposeConfChange proposes a config change. See (Node).ProposeConfChange for +// details. +func (rn *RawNode) ProposeConfChange(cc pb.ConfChangeI) error { + m, err := confChangeToMsg(cc) if err != nil { return err } - return rn.raft.Step(pb.Message{ - Type: pb.MsgProp, - Entries: []pb.Entry{ - {Type: pb.EntryConfChange, Data: data}, - }, - }) + return rn.raft.Step(m) } // ApplyConfChange applies a config change to the local node. -func (rn *RawNode) ApplyConfChange(cc pb.ConfChange) *pb.ConfState { - cs := rn.raft.applyConfChange(cc) +func (rn *RawNode) ApplyConfChange(cc pb.ConfChangeI) *pb.ConfState { + cs := rn.raft.applyConfChange(cc.AsV2()) return &cs } @@ -182,14 +116,35 @@ func (rn *RawNode) Step(m pb.Message) error { return ErrStepPeerNotFound } -// Ready returns the current point-in-time state of this RawNode. +// Ready returns the outstanding work that the application needs to handle. This +// includes appending and applying entries or a snapshot, updating the HardState, +// and sending messages. The returned Ready() *must* be handled and subsequently +// passed back via Advance(). func (rn *RawNode) Ready() Ready { - rd := rn.newReady() - rn.raft.msgs = nil - rn.raft.reduceUncommittedSize(rd.CommittedEntries) + rd := rn.readyWithoutAccept() + rn.acceptReady(rd) return rd } +// readyWithoutAccept returns a Ready. This is a read-only operation, i.e. there +// is no obligation that the Ready must be handled. +func (rn *RawNode) readyWithoutAccept() Ready { + return newReady(rn.raft, rn.prevSoftSt, rn.prevHardSt) +} + +// acceptReady is called when the consumer of the RawNode has decided to go +// ahead and handle a Ready. Nothing must alter the state of the RawNode between +// this call and the prior call to Ready(). +func (rn *RawNode) acceptReady(rd Ready) { + if rd.SoftState != nil { + rn.prevSoftSt = rd.SoftState + } + if len(rd.ReadStates) != 0 { + rn.raft.readStates = nil + } + rn.raft.msgs = nil +} + // HasReady called when RawNode user need to check if any Ready pending. // Checking logic in this method should be consistent with Ready.containsUpdates(). func (rn *RawNode) HasReady() bool { @@ -215,21 +170,23 @@ func (rn *RawNode) HasReady() bool { // Advance notifies the RawNode that the application has applied and saved progress in the // last Ready results. func (rn *RawNode) Advance(rd Ready) { - rn.commitReady(rd) + if !IsEmptyHardState(rd.HardState) { + rn.prevHardSt = rd.HardState + } + rn.raft.advance(rd) } -// Status returns the current status of the given group. -func (rn *RawNode) Status() *Status { +// Status returns the current status of the given group. This allocates, see +// BasicStatus and WithProgress for allocation-friendlier choices. +func (rn *RawNode) Status() Status { status := getStatus(rn.raft) - return &status + return status } -// StatusWithoutProgress returns a Status without populating the Progress field -// (and returns the Status as a value to avoid forcing it onto the heap). This -// is more performant if the Progress is not required. See WithProgress for an -// allocation-free way to introspect the Progress. -func (rn *RawNode) StatusWithoutProgress() Status { - return getStatusWithoutProgress(rn.raft) +// BasicStatus returns a BasicStatus. Notably this does not contain the +// Progress map; see WithProgress for an allocation-free way to inspect it. +func (rn *RawNode) BasicStatus() BasicStatus { + return getBasicStatus(rn.raft) } // ProgressType indicates the type of replica a Progress corresponds to. diff --git a/vendor/go.etcd.io/etcd/raft/status.go b/vendor/go.etcd.io/etcd/raft/status.go index bf4898c9e3..adc60486d9 100644 --- a/vendor/go.etcd.io/etcd/raft/status.go +++ b/vendor/go.etcd.io/etcd/raft/status.go @@ -21,14 +21,22 @@ import ( "go.etcd.io/etcd/raft/tracker" ) +// Status contains information about this Raft peer and its view of the system. +// The Progress is only populated on the leader. type Status struct { + BasicStatus + Config tracker.Config + Progress map[uint64]tracker.Progress +} + +// BasicStatus contains basic information about the Raft peer. It does not allocate. +type BasicStatus struct { ID uint64 pb.HardState SoftState - Applied uint64 - Progress map[uint64]tracker.Progress + Applied uint64 LeadTransferee uint64 } @@ -37,19 +45,17 @@ func getProgressCopy(r *raft) map[uint64]tracker.Progress { m := make(map[uint64]tracker.Progress) r.prs.Visit(func(id uint64, pr *tracker.Progress) { var p tracker.Progress - p, pr = *pr, nil /* avoid accidental reuse below */ - - // The inflight buffer is tricky to copy and besides, it isn't exposed - // to the client, so pretend it's nil. - p.Inflights = nil + p = *pr + p.Inflights = pr.Inflights.Clone() + pr = nil m[id] = p }) return m } -func getStatusWithoutProgress(r *raft) Status { - s := Status{ +func getBasicStatus(r *raft) BasicStatus { + s := BasicStatus{ ID: r.id, LeadTransferee: r.leadTransferee, } @@ -61,10 +67,12 @@ func getStatusWithoutProgress(r *raft) Status { // getStatus gets a copy of the current raft status. func getStatus(r *raft) Status { - s := getStatusWithoutProgress(r) + var s Status + s.BasicStatus = getBasicStatus(r) if s.RaftState == StateLeader { s.Progress = getProgressCopy(r) } + s.Config = r.prs.Config.Clone() return s } diff --git a/vendor/go.etcd.io/etcd/raft/storage.go b/vendor/go.etcd.io/etcd/raft/storage.go index 14ad686083..6be574590e 100644 --- a/vendor/go.etcd.io/etcd/raft/storage.go +++ b/vendor/go.etcd.io/etcd/raft/storage.go @@ -44,6 +44,8 @@ var ErrSnapshotTemporarilyUnavailable = errors.New("snapshot is temporarily unav // become inoperable and refuse to participate in elections; the // application is responsible for cleanup and recovery in this case. type Storage interface { + // TODO(tbg): split this into two interfaces, LogStorage and StateStorage. + // InitialState returns the saved HardState and ConfState information. InitialState() (pb.HardState, pb.ConfState, error) // Entries returns a slice of log entries in the range [lo,hi). diff --git a/vendor/go.etcd.io/etcd/raft/tracker/inflights.go b/vendor/go.etcd.io/etcd/raft/tracker/inflights.go index 9e209e21ea..1a056341ab 100644 --- a/vendor/go.etcd.io/etcd/raft/tracker/inflights.go +++ b/vendor/go.etcd.io/etcd/raft/tracker/inflights.go @@ -40,6 +40,14 @@ func NewInflights(size int) *Inflights { } } +// Clone returns an *Inflights that is identical to but shares no memory with +// the receiver. +func (in *Inflights) Clone() *Inflights { + ins := *in + ins.buffer = append([]uint64(nil), in.buffer...) + return &ins +} + // Add notifies the Inflights that a new message with the given index is being // dispatched. Full() must be called prior to Add() to verify that there is room // for one more message, and consecutive calls to add Add() must provide a diff --git a/vendor/go.etcd.io/etcd/raft/tracker/progress.go b/vendor/go.etcd.io/etcd/raft/tracker/progress.go index a7f1ab7d38..62c81f45af 100644 --- a/vendor/go.etcd.io/etcd/raft/tracker/progress.go +++ b/vendor/go.etcd.io/etcd/raft/tracker/progress.go @@ -16,6 +16,7 @@ package tracker import ( "fmt" + "sort" "strings" ) @@ -51,6 +52,8 @@ type Progress struct { // RecentActive is true if the progress is recently active. Receiving any messages // from the corresponding follower indicates the progress is active. // RecentActive can be reset to false after an election timeout. + // + // TODO(tbg): the leader should always have this set to true. RecentActive bool // ProbeSent is used while this follower is in StateProbe. When ProbeSent is @@ -235,3 +238,22 @@ func (pr *Progress) String() string { } return buf.String() } + +// ProgressMap is a map of *Progress. +type ProgressMap map[uint64]*Progress + +// String prints the ProgressMap in sorted key order, one Progress per line. +func (m ProgressMap) String() string { + ids := make([]uint64, 0, len(m)) + for k := range m { + ids = append(ids, k) + } + sort.Slice(ids, func(i, j int) bool { + return ids[i] < ids[j] + }) + var buf strings.Builder + for _, id := range ids { + fmt.Fprintf(&buf, "%d: %s\n", id, m[id]) + } + return buf.String() +} diff --git a/vendor/go.etcd.io/etcd/raft/tracker/tracker.go b/vendor/go.etcd.io/etcd/raft/tracker/tracker.go index 4b3396fbe1..a4581143d1 100644 --- a/vendor/go.etcd.io/etcd/raft/tracker/tracker.go +++ b/vendor/go.etcd.io/etcd/raft/tracker/tracker.go @@ -17,13 +17,20 @@ package tracker import ( "fmt" "sort" + "strings" "go.etcd.io/etcd/raft/quorum" + pb "go.etcd.io/etcd/raft/raftpb" ) // Config reflects the configuration tracked in a ProgressTracker. type Config struct { Voters quorum.JointConfig + // AutoLeave is true if the configuration is joint and a transition to the + // incoming configuration should be carried out automatically by Raft when + // this is possible. If false, the configuration will be joint until the + // application initiates the transition manually. + AutoLeave bool // Learners is a set of IDs corresponding to the learners active in the // current configuration. // @@ -33,12 +40,11 @@ type Config struct { // simplifies the implementation since it allows peers to have clarity about // its current role without taking into account joint consensus. Learners map[uint64]struct{} - // TODO(tbg): when we actually carry out joint consensus changes and turn a - // voter into a learner, we cannot add the learner when entering the joint - // state. This is because this would violate the invariant that the inter- - // section of voters and learners is empty. For example, assume a Voter is - // removed and immediately re-added as a learner (or in other words, it is - // demoted). + // When we turn a voter into a learner during a joint consensus transition, + // we cannot add the learner directly when entering the joint state. This is + // because this would violate the invariant that the intersection of + // voters and learners is empty. For example, assume a Voter is removed and + // immediately re-added as a learner (or in other words, it is demoted): // // Initially, the configuration will be // @@ -51,7 +57,7 @@ type Config struct { // learners: {3} // // but this violates the invariant (3 is both voter and learner). Instead, - // we have + // we get // // voters: {1 2} & {1 2 3} // learners: {} @@ -66,20 +72,43 @@ type Config struct { // // Note that next_learners is not used while adding a learner that is not // also a voter in the joint config. In this case, the learner is added - // to Learners right away when entering the joint configuration, so that it - // is caught up as soon as possible. - // - // NextLearners map[uint64]struct{} + // right away when entering the joint configuration, so that it is caught up + // as soon as possible. + LearnersNext map[uint64]struct{} } -func (c *Config) String() string { - if len(c.Learners) == 0 { - return fmt.Sprintf("voters=%s", c.Voters) +func (c Config) String() string { + var buf strings.Builder + fmt.Fprintf(&buf, "voters=%s", c.Voters) + if c.Learners != nil { + fmt.Fprintf(&buf, " learners=%s", quorum.MajorityConfig(c.Learners).String()) + } + if c.LearnersNext != nil { + fmt.Fprintf(&buf, " learners_next=%s", quorum.MajorityConfig(c.LearnersNext).String()) + } + if c.AutoLeave { + fmt.Fprintf(&buf, " autoleave") + } + return buf.String() +} + +// Clone returns a copy of the Config that shares no memory with the original. +func (c *Config) Clone() Config { + clone := func(m map[uint64]struct{}) map[uint64]struct{} { + if m == nil { + return nil + } + mm := make(map[uint64]struct{}, len(m)) + for k := range m { + mm[k] = struct{}{} + } + return mm + } + return Config{ + Voters: quorum.JointConfig{clone(c.Voters[0]), clone(c.Voters[1])}, + Learners: clone(c.Learners), + LearnersNext: clone(c.LearnersNext), } - return fmt.Sprintf( - "voters=%s learners=%s", - c.Voters, quorum.MajorityConfig(c.Learners).String(), - ) } // ProgressTracker tracks the currently active configuration and the information @@ -88,7 +117,7 @@ func (c *Config) String() string { type ProgressTracker struct { Config - Progress map[uint64]*Progress + Progress ProgressMap Votes map[uint64]bool @@ -102,11 +131,10 @@ func MakeProgressTracker(maxInflight int) ProgressTracker { Config: Config{ Voters: quorum.JointConfig{ quorum.MajorityConfig{}, - // TODO(tbg): this will be mostly empty, so make it a nil pointer - // in the common case. - quorum.MajorityConfig{}, + nil, // only populated when used }, - Learners: map[uint64]struct{}{}, + Learners: nil, // only populated when used + LearnersNext: nil, // only populated when used }, Votes: map[uint64]bool{}, Progress: map[uint64]*Progress{}, @@ -114,6 +142,17 @@ func MakeProgressTracker(maxInflight int) ProgressTracker { return p } +// ConfState returns a ConfState representing the active configuration. +func (p *ProgressTracker) ConfState() pb.ConfState { + return pb.ConfState{ + Voters: p.Voters[0].Slice(), + VotersOutgoing: p.Voters[1].Slice(), + Learners: quorum.MajorityConfig(p.Learners).Slice(), + LearnersNext: quorum.MajorityConfig(p.LearnersNext).Slice(), + AutoLeave: p.AutoLeave, + } +} + // IsSingleton returns true if (and only if) there is only one voting member // (i.e. the leader) in the current configuration. func (p *ProgressTracker) IsSingleton() bool { @@ -139,48 +178,35 @@ func (p *ProgressTracker) Committed() uint64 { return uint64(p.Voters.CommittedIndex(matchAckIndexer(p.Progress))) } -// RemoveAny removes this peer, which *must* be tracked as a voter or learner, -// from the tracker. -func (p *ProgressTracker) RemoveAny(id uint64) { - _, okPR := p.Progress[id] - _, okV1 := p.Voters[0][id] - _, okV2 := p.Voters[1][id] - _, okL := p.Learners[id] - - okV := okV1 || okV2 - - if !okPR { - panic("attempting to remove unknown peer %x") - } else if !okV && !okL { - panic("attempting to remove unknown peer %x") - } else if okV && okL { - panic(fmt.Sprintf("peer %x is both voter and learner", id)) +func insertionSort(sl []uint64) { + a, b := 0, len(sl) + for i := a + 1; i < b; i++ { + for j := i; j > a && sl[j] < sl[j-1]; j-- { + sl[j], sl[j-1] = sl[j-1], sl[j] + } } - - delete(p.Voters[0], id) - delete(p.Voters[1], id) - delete(p.Learners, id) - delete(p.Progress, id) } -// InitProgress initializes a new progress for the given node or learner. The -// node may not exist yet in either form or a panic will ensue. -func (p *ProgressTracker) InitProgress(id, match, next uint64, isLearner bool) { - if pr := p.Progress[id]; pr != nil { - panic(fmt.Sprintf("peer %x already tracked as node %v", id, pr)) - } - if !isLearner { - p.Voters[0][id] = struct{}{} +// Visit invokes the supplied closure for all tracked progresses in stable order. +func (p *ProgressTracker) Visit(f func(id uint64, pr *Progress)) { + n := len(p.Progress) + // We need to sort the IDs and don't want to allocate since this is hot code. + // The optimization here mirrors that in `(MajorityConfig).CommittedIndex`, + // see there for details. + var sl [7]uint64 + ids := sl[:] + if len(sl) >= n { + ids = sl[:n] } else { - p.Learners[id] = struct{}{} + ids = make([]uint64, n) } - p.Progress[id] = &Progress{Next: next, Match: match, Inflights: NewInflights(p.MaxInflight), IsLearner: isLearner} -} - -// Visit invokes the supplied closure for all tracked progresses. -func (p *ProgressTracker) Visit(f func(id uint64, pr *Progress)) { - for id, pr := range p.Progress { - f(id, pr) + for id := range p.Progress { + n-- + ids[n] = id + } + insertionSort(ids) + for _, id := range ids { + f(id, p.Progress[id]) } } @@ -211,6 +237,9 @@ func (p *ProgressTracker) VoterNodes() []uint64 { // LearnerNodes returns a sorted slice of learners. func (p *ProgressTracker) LearnerNodes() []uint64 { + if len(p.Learners) == 0 { + return nil + } nodes := make([]uint64, 0, len(p.Learners)) for id := range p.Learners { nodes = append(nodes, id) diff --git a/vendor/go.etcd.io/etcd/raft/util.go b/vendor/go.etcd.io/etcd/raft/util.go index c145d26dd7..785cf735d5 100644 --- a/vendor/go.etcd.io/etcd/raft/util.go +++ b/vendor/go.etcd.io/etcd/raft/util.go @@ -17,6 +17,7 @@ package raft import ( "bytes" "fmt" + "strings" pb "go.etcd.io/etcd/raft/raftpb" ) @@ -25,13 +26,6 @@ func (st StateType) MarshalJSON() ([]byte, error) { return []byte(fmt.Sprintf("%q", st.String())), nil } -// uint64Slice implements sort interface -type uint64Slice []uint64 - -func (p uint64Slice) Len() int { return len(p) } -func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] } -func (p uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } - func min(a, b uint64) uint64 { if a > b { return b @@ -67,6 +61,69 @@ func voteRespMsgType(msgt pb.MessageType) pb.MessageType { } } +func DescribeHardState(hs pb.HardState) string { + var buf strings.Builder + fmt.Fprintf(&buf, "Term:%d", hs.Term) + if hs.Vote != 0 { + fmt.Fprintf(&buf, " Vote:%d", hs.Vote) + } + fmt.Fprintf(&buf, " Commit:%d", hs.Commit) + return buf.String() +} + +func DescribeSoftState(ss SoftState) string { + return fmt.Sprintf("Lead:%d State:%s", ss.Lead, ss.RaftState) +} + +func DescribeConfState(state pb.ConfState) string { + return fmt.Sprintf( + "Voters:%v VotersOutgoing:%v Learners:%v LearnersNext:%v AutoLeave:%v", + state.Voters, state.VotersOutgoing, state.Learners, state.LearnersNext, state.AutoLeave, + ) +} + +func DescribeSnapshot(snap pb.Snapshot) string { + m := snap.Metadata + return fmt.Sprintf("Index:%d Term:%d ConfState:%s", m.Index, m.Term, DescribeConfState(m.ConfState)) +} + +func DescribeReady(rd Ready, f EntryFormatter) string { + var buf strings.Builder + if rd.SoftState != nil { + fmt.Fprint(&buf, DescribeSoftState(*rd.SoftState)) + buf.WriteByte('\n') + } + if !IsEmptyHardState(rd.HardState) { + fmt.Fprintf(&buf, "HardState %s", DescribeHardState(rd.HardState)) + buf.WriteByte('\n') + } + if len(rd.ReadStates) > 0 { + fmt.Fprintf(&buf, "ReadStates %v\n", rd.ReadStates) + } + if len(rd.Entries) > 0 { + buf.WriteString("Entries:\n") + fmt.Fprint(&buf, DescribeEntries(rd.Entries, f)) + } + if !IsEmptySnap(rd.Snapshot) { + fmt.Fprintf(&buf, "Snapshot %s\n", DescribeSnapshot(rd.Snapshot)) + } + if len(rd.CommittedEntries) > 0 { + buf.WriteString("CommittedEntries:\n") + fmt.Fprint(&buf, DescribeEntries(rd.CommittedEntries, f)) + } + if len(rd.Messages) > 0 { + buf.WriteString("Messages:\n") + for _, msg := range rd.Messages { + fmt.Fprint(&buf, DescribeMessage(msg, f)) + buf.WriteByte('\n') + } + } + if buf.Len() > 0 { + return fmt.Sprintf("Ready MustSync=%t:\n%s", rd.MustSync, buf.String()) + } + return "" +} + // EntryFormatter can be implemented by the application to provide human-readable formatting // of entry data. Nil is a valid EntryFormatter and will use a default format. type EntryFormatter func([]byte) string @@ -93,7 +150,7 @@ func DescribeMessage(m pb.Message, f EntryFormatter) string { fmt.Fprintf(&buf, "]") } if !IsEmptySnap(m.Snapshot) { - fmt.Fprintf(&buf, " Snapshot:%v", m.Snapshot) + fmt.Fprintf(&buf, " Snapshot: %s", DescribeSnapshot(m.Snapshot)) } return buf.String() } @@ -107,13 +164,39 @@ func PayloadSize(e pb.Entry) int { // DescribeEntry returns a concise human-readable description of an // Entry for debugging. func DescribeEntry(e pb.Entry, f EntryFormatter) string { + if f == nil { + f = func(data []byte) string { return fmt.Sprintf("%q", data) } + } + + formatConfChange := func(cc pb.ConfChangeI) string { + // TODO(tbg): give the EntryFormatter a type argument so that it gets + // a chance to expose the Context. + return pb.ConfChangesToString(cc.AsV2().Changes) + } + var formatted string - if e.Type == pb.EntryNormal && f != nil { + switch e.Type { + case pb.EntryNormal: formatted = f(e.Data) - } else { - formatted = fmt.Sprintf("%q", e.Data) + case pb.EntryConfChange: + var cc pb.ConfChange + if err := cc.Unmarshal(e.Data); err != nil { + formatted = err.Error() + } else { + formatted = formatConfChange(cc) + } + case pb.EntryConfChangeV2: + var cc pb.ConfChangeV2 + if err := cc.Unmarshal(e.Data); err != nil { + formatted = err.Error() + } else { + formatted = formatConfChange(cc) + } } - return fmt.Sprintf("%d/%d %s %s", e.Term, e.Index, e.Type, formatted) + if formatted != "" { + formatted = " " + formatted + } + return fmt.Sprintf("%d/%d %s%s", e.Term, e.Index, e.Type, formatted) } // DescribeEntries calls DescribeEntry for each Entry, adding a newline to @@ -140,3 +223,11 @@ func limitSize(ents []pb.Entry, maxSize uint64) []pb.Entry { } return ents[:limit] } + +func assertConfStatesEquivalent(l Logger, cs1, cs2 pb.ConfState) { + err := cs1.Equivalent(cs2) + if err == nil { + return + } + l.Panic(err) +} diff --git a/vendor/go.etcd.io/etcd/version/version.go b/vendor/go.etcd.io/etcd/version/version.go index c55a83579a..7d3a572028 100644 --- a/vendor/go.etcd.io/etcd/version/version.go +++ b/vendor/go.etcd.io/etcd/version/version.go @@ -26,7 +26,7 @@ import ( var ( // MinClusterVersion is the min cluster version this etcd binary is compatible with. MinClusterVersion = "3.0.0" - Version = "3.3.0+git" + Version = "3.4.3" APIVersion = "unknown" // Git SHA Value will be set during build diff --git a/vendor/go.uber.org/atomic/CHANGELOG.md b/vendor/go.uber.org/atomic/CHANGELOG.md index a88b023e48..6db846f102 100644 --- a/vendor/go.uber.org/atomic/CHANGELOG.md +++ b/vendor/go.uber.org/atomic/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.5.1] - 2019-11-19 +- Fix bug where `Bool.CAS` and `Bool.Toggle` do work correctly together + causing `CAS` to fail even though the old value matches. + ## [1.5.0] - 2019-10-29 ### Changed - With Go modules, only the `go.uber.org/atomic` import path is supported now. @@ -44,7 +48,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Initial release. -[1.4.0]: https://github.com/uber-go/atomic/compare/v1.4.0...v1.5.0 +[1.5.1]: https://github.com/uber-go/atomic/compare/v1.5.0...v1.5.1 +[1.5.0]: https://github.com/uber-go/atomic/compare/v1.4.0...v1.5.0 [1.4.0]: https://github.com/uber-go/atomic/compare/v1.3.2...v1.4.0 [1.3.2]: https://github.com/uber-go/atomic/compare/v1.3.1...v1.3.2 [1.3.1]: https://github.com/uber-go/atomic/compare/v1.3.0...v1.3.1 diff --git a/vendor/go.uber.org/atomic/README.md b/vendor/go.uber.org/atomic/README.md index 3cc368ba30..ade0c20f16 100644 --- a/vendor/go.uber.org/atomic/README.md +++ b/vendor/go.uber.org/atomic/README.md @@ -8,13 +8,25 @@ Simple wrappers for primitive types to enforce atomic access. $ go get -u go.uber.org/atomic@v1 ``` -Note: If you are using Go modules, this package will fail to compile with the -import path `github.com/uber-go/atomic`. To continue using that import path, -you will have to add a `replace` directive to your `go.mod`, replacing -`github.com/uber-go/atomic` with `go.uber.org/atomic`. +### Legacy Import Path + +As of v1.5.0, the import path `go.uber.org/atomic` is the only supported way +of using this package. If you are using Go modules, this package will fail to +compile with the legacy import path path `github.com/uber-go/atomic`. + +We recommend migrating your code to the new import path but if you're unable +to do so, or if your dependencies are still using the old import path, you +will have to add a `replace` directive to your `go.mod` file downgrading the +legacy import path to an older version. + +``` +replace github.com/uber-go/atomic => github.com/uber-go/atomic v1.4.0 +``` + +You can do so automatically by running the following command. ```shell -$ go mod edit -replace github.com/uber-go/atomic=go.uber.org/atomic@v1 +$ go mod edit -replace github.com/uber-go/atomic=github.com/uber-go/atomic@v1.4.0 ``` ## Usage diff --git a/vendor/go.uber.org/atomic/atomic.go b/vendor/go.uber.org/atomic/atomic.go index 1db6849fca..ad5fa0980a 100644 --- a/vendor/go.uber.org/atomic/atomic.go +++ b/vendor/go.uber.org/atomic/atomic.go @@ -250,11 +250,16 @@ func (b *Bool) Swap(new bool) bool { // Toggle atomically negates the Boolean and returns the previous value. func (b *Bool) Toggle() bool { - return truthy(atomic.AddUint32(&b.v, 1) - 1) + for { + old := b.Load() + if b.CAS(old, !old) { + return old + } + } } func truthy(n uint32) bool { - return n&1 == 1 + return n == 1 } func boolToInt(b bool) uint32 { diff --git a/vendor/go.uber.org/multierr/.gitignore b/vendor/go.uber.org/multierr/.gitignore index 61ead86667..b9a05e3da0 100644 --- a/vendor/go.uber.org/multierr/.gitignore +++ b/vendor/go.uber.org/multierr/.gitignore @@ -1 +1,4 @@ /vendor +cover.html +cover.out +/bin diff --git a/vendor/go.uber.org/multierr/.travis.yml b/vendor/go.uber.org/multierr/.travis.yml index 5ffa8fed48..786c917a39 100644 --- a/vendor/go.uber.org/multierr/.travis.yml +++ b/vendor/go.uber.org/multierr/.travis.yml @@ -5,11 +5,12 @@ go_import_path: go.uber.org/multierr env: global: - GO15VENDOREXPERIMENT=1 + - GO111MODULE=on go: - - 1.7 - - 1.8 - - tip + - 1.11.x + - 1.12.x + - 1.13.x cache: directories: @@ -18,16 +19,11 @@ cache: before_install: - go version -install: -- | - set -e - make install_ci - script: - | set -e make lint - make test_ci + make cover after_success: - bash <(curl -s https://codecov.io/bash) diff --git a/vendor/go.uber.org/multierr/CHANGELOG.md b/vendor/go.uber.org/multierr/CHANGELOG.md index 898445d063..abb355a064 100644 --- a/vendor/go.uber.org/multierr/CHANGELOG.md +++ b/vendor/go.uber.org/multierr/CHANGELOG.md @@ -1,6 +1,26 @@ Releases ======== +v1.4.0 (2019-11-04) +=================== + +- Add `AppendInto` function to more ergonomically build errors inside a + loop. + + +v1.3.0 (2019-10-29) +=================== + +- Switch to Go modules. + + +v1.2.0 (2019-09-26) +=================== + +- Support extracting and matching against wrapped errors with `errors.As` + and `errors.Is`. + + v1.1.0 (2017-06-30) =================== diff --git a/vendor/go.uber.org/multierr/Makefile b/vendor/go.uber.org/multierr/Makefile index a7437d061f..416018237e 100644 --- a/vendor/go.uber.org/multierr/Makefile +++ b/vendor/go.uber.org/multierr/Makefile @@ -1,23 +1,17 @@ -export GO15VENDOREXPERIMENT=1 - -PACKAGES := $(shell glide nv) +# Directory to put `go install`ed binaries in. +export GOBIN ?= $(shell pwd)/bin GO_FILES := $(shell \ find . '(' -path '*/.*' -o -path './vendor' ')' -prune \ -o -name '*.go' -print | cut -b3-) -.PHONY: install -install: - glide --version || go get github.com/Masterminds/glide - glide install - .PHONY: build build: - go build -i $(PACKAGES) + go build ./... .PHONY: test test: - go test -cover -race $(PACKAGES) + go test -race ./... .PHONY: gofmt gofmt: @@ -25,50 +19,24 @@ gofmt: @gofmt -e -s -l $(GO_FILES) > $(FMT_LOG) || true @[ ! -s "$(FMT_LOG)" ] || (echo "gofmt failed:" | cat - $(FMT_LOG) && false) -.PHONY: govet -govet: - $(eval VET_LOG := $(shell mktemp -t govet.XXXXX)) - @go vet $(PACKAGES) 2>&1 \ - | grep -v '^exit status' > $(VET_LOG) || true - @[ ! -s "$(VET_LOG)" ] || (echo "govet failed:" | cat - $(VET_LOG) && false) - .PHONY: golint golint: - @go get github.com/golang/lint/golint - $(eval LINT_LOG := $(shell mktemp -t golint.XXXXX)) - @cat /dev/null > $(LINT_LOG) - @$(foreach pkg, $(PACKAGES), golint $(pkg) >> $(LINT_LOG) || true;) - @[ ! -s "$(LINT_LOG)" ] || (echo "golint failed:" | cat - $(LINT_LOG) && false) + @go install golang.org/x/lint/golint + @$(GOBIN)/golint ./... .PHONY: staticcheck staticcheck: - @go get honnef.co/go/tools/cmd/staticcheck - $(eval STATICCHECK_LOG := $(shell mktemp -t staticcheck.XXXXX)) - @staticcheck $(PACKAGES) 2>&1 > $(STATICCHECK_LOG) || true - @[ ! -s "$(STATICCHECK_LOG)" ] || (echo "staticcheck failed:" | cat - $(STATICCHECK_LOG) && false) + @go install honnef.co/go/tools/cmd/staticcheck + @$(GOBIN)/staticcheck ./... .PHONY: lint -lint: gofmt govet golint staticcheck +lint: gofmt golint staticcheck .PHONY: cover cover: - ./scripts/cover.sh $(shell go list $(PACKAGES)) + go test -coverprofile=cover.out -coverpkg=./... -v ./... go tool cover -html=cover.out -o cover.html update-license: - @go get go.uber.org/tools/update-license - @update-license \ - $(shell go list -json $(PACKAGES) | \ - jq -r '.Dir + "/" + (.GoFiles | .[])') - -############################################################################## - -.PHONY: install_ci -install_ci: install - go get github.com/wadey/gocovmerge - go get github.com/mattn/goveralls - go get golang.org/x/tools/cmd/cover - -.PHONY: test_ci -test_ci: install_ci - ./scripts/cover.sh $(shell go list $(PACKAGES)) + @go install go.uber.org/tools/update-license + @$(GOBIN)/update-license $(GO_FILES) diff --git a/vendor/go.uber.org/multierr/README.md b/vendor/go.uber.org/multierr/README.md index 065088f641..751bd65e58 100644 --- a/vendor/go.uber.org/multierr/README.md +++ b/vendor/go.uber.org/multierr/README.md @@ -17,7 +17,7 @@ Released under the [MIT License]. [MIT License]: LICENSE.txt [doc-img]: https://godoc.org/go.uber.org/multierr?status.svg [doc]: https://godoc.org/go.uber.org/multierr -[ci-img]: https://travis-ci.org/uber-go/multierr.svg?branch=master +[ci-img]: https://travis-ci.com/uber-go/multierr.svg?branch=master [cov-img]: https://codecov.io/gh/uber-go/multierr/branch/master/graph/badge.svg -[ci]: https://travis-ci.org/uber-go/multierr +[ci]: https://travis-ci.com/uber-go/multierr [cov]: https://codecov.io/gh/uber-go/multierr diff --git a/vendor/go.uber.org/multierr/error.go b/vendor/go.uber.org/multierr/error.go index de6ce4736c..0ee6fe8bc7 100644 --- a/vendor/go.uber.org/multierr/error.go +++ b/vendor/go.uber.org/multierr/error.go @@ -1,4 +1,4 @@ -// Copyright (c) 2017 Uber Technologies, Inc. +// Copyright (c) 2019 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -33,7 +33,7 @@ // If only two errors are being combined, the Append function may be used // instead. // -// err = multierr.Combine(reader.Close(), writer.Close()) +// err = multierr.Append(reader.Close(), writer.Close()) // // This makes it possible to record resource cleanup failures from deferred // blocks with the help of named return values. @@ -99,8 +99,6 @@ var ( // Separator for single-line error messages. _singlelineSeparator = []byte("; ") - _newline = []byte("\n") - // Prefix for multi-line messages _multilinePrefix = []byte("the following errors occurred:") @@ -399,3 +397,53 @@ func Append(left error, right error) error { errors := [2]error{left, right} return fromSlice(errors[0:]) } + +// AppendInto appends an error into the destination of an error pointer and +// returns whether the error being appended was non-nil. +// +// var err error +// multierr.AppendInto(&err, r.Close()) +// multierr.AppendInto(&err, w.Close()) +// +// The above is equivalent to, +// +// err := multierr.Append(r.Close(), w.Close()) +// +// As AppendInto reports whether the provided error was non-nil, it may be +// used to build a multierr error in a loop more ergonomically. For example: +// +// var err error +// for line := range lines { +// var item Item +// if multierr.AppendInto(&err, parse(line, &item)) { +// continue +// } +// items = append(items, item) +// } +// +// Compare this with a verison that relies solely on Append: +// +// var err error +// for line := range lines { +// var item Item +// if parseErr := parse(line, &item); parseErr != nil { +// err = multierr.Append(err, parseErr) +// continue +// } +// items = append(items, item) +// } +func AppendInto(into *error, err error) (errored bool) { + if into == nil { + // We panic if 'into' is nil. This is not documented above + // because suggesting that the pointer must be non-nil may + // confuse users into thinking that the error that it points + // to must be non-nil. + panic("misuse of multierr.AppendInto: into pointer must not be nil") + } + + if err == nil { + return false + } + *into = Append(*into, err) + return true +} diff --git a/vendor/go.uber.org/multierr/glide.lock b/vendor/go.uber.org/multierr/glide.lock deleted file mode 100644 index f9ea94c334..0000000000 --- a/vendor/go.uber.org/multierr/glide.lock +++ /dev/null @@ -1,19 +0,0 @@ -hash: b53b5e9a84b9cb3cc4b2d0499e23da2feca1eec318ce9bb717ecf35bf24bf221 -updated: 2017-04-10T13:34:45.671678062-07:00 -imports: -- name: go.uber.org/atomic - version: 3b8db5e93c4c02efbc313e17b2e796b0914a01fb -testImports: -- name: github.com/davecgh/go-spew - version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9 - subpackages: - - spew -- name: github.com/pmezard/go-difflib - version: d8ed2627bdf02c080bf22230dbb337003b7aba2d - subpackages: - - difflib -- name: github.com/stretchr/testify - version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0 - subpackages: - - assert - - require diff --git a/vendor/go.uber.org/multierr/go.mod b/vendor/go.uber.org/multierr/go.mod new file mode 100644 index 0000000000..5463fac728 --- /dev/null +++ b/vendor/go.uber.org/multierr/go.mod @@ -0,0 +1,12 @@ +module go.uber.org/multierr + +go 1.12 + +require ( + github.com/stretchr/testify v1.3.0 + go.uber.org/atomic v1.5.0 + go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee + golang.org/x/lint v0.0.0-20190930215403-16217165b5de + golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 // indirect + honnef.co/go/tools v0.0.1-2019.2.3 +) diff --git a/vendor/go.uber.org/multierr/go.sum b/vendor/go.uber.org/multierr/go.sum new file mode 100644 index 0000000000..b460913d26 --- /dev/null +++ b/vendor/go.uber.org/multierr/go.sum @@ -0,0 +1,45 @@ +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c h1:IGkKhmfzcztjm6gYkykvu/NiS8kaqbCWAEWWAyf8J5U= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/vendor/go.uber.org/multierr/go113.go b/vendor/go.uber.org/multierr/go113.go new file mode 100644 index 0000000000..264b0eac0d --- /dev/null +++ b/vendor/go.uber.org/multierr/go113.go @@ -0,0 +1,52 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// +build go1.13 + +package multierr + +import "errors" + +// As attempts to find the first error in the error list that matches the type +// of the value that target points to. +// +// This function allows errors.As to traverse the values stored on the +// multierr error. +func (merr *multiError) As(target interface{}) bool { + for _, err := range merr.Errors() { + if errors.As(err, target) { + return true + } + } + return false +} + +// Is attempts to match the provided error against errors in the error list. +// +// This function allows errors.Is to traverse the values stored on the +// multierr error. +func (merr *multiError) Is(target error) bool { + for _, err := range merr.Errors() { + if errors.Is(err, target) { + return true + } + } + return false +} diff --git a/vendor/go.uber.org/multierr/tools.go b/vendor/go.uber.org/multierr/tools.go new file mode 100644 index 0000000000..df93f0723e --- /dev/null +++ b/vendor/go.uber.org/multierr/tools.go @@ -0,0 +1,30 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// +build tools + +package multierr + +import ( + // Tools we use during development. + _ "go.uber.org/tools/update-license" + _ "golang.org/x/lint/golint" + _ "honnef.co/go/tools/cmd/staticcheck" +) diff --git a/vendor/go.uber.org/tools/LICENSE b/vendor/go.uber.org/tools/LICENSE new file mode 100644 index 0000000000..858e02475f --- /dev/null +++ b/vendor/go.uber.org/tools/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2017 Uber Technologies, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/go.uber.org/tools/update-license/.gitignore b/vendor/go.uber.org/tools/update-license/.gitignore new file mode 100644 index 0000000000..b167772c82 --- /dev/null +++ b/vendor/go.uber.org/tools/update-license/.gitignore @@ -0,0 +1 @@ +update-license diff --git a/vendor/go.uber.org/tools/update-license/README.md b/vendor/go.uber.org/tools/update-license/README.md new file mode 100644 index 0000000000..5887df1dd3 --- /dev/null +++ b/vendor/go.uber.org/tools/update-license/README.md @@ -0,0 +1,24 @@ +# update-license + +This is a small tool that updates the license header for Uber's open source Golang files. + +## Installation + +``` +go get go.uber.org/tools/update-license +``` + +## Usage + +``` +update-license go_files... +``` + +## Further Work + +* Support more licenses by name (MIT, Apache 2.0, etc), file path, url (http GET) +* Support custom owner (not just "Uber Technologies, Inc.") +* Support more languages than go (cover go, java, js, py to start, along with LICENSE, LICENSE.txt) +* Talk about removing custom logic for header comments (ie `@generated`, `Code generated by`), it probably makes more sense just to put the license at the top +* Better detection support for existing licenses so they can be removed +* Verbose, dry run support diff --git a/vendor/go.uber.org/tools/update-license/licenses.go b/vendor/go.uber.org/tools/update-license/licenses.go new file mode 100644 index 0000000000..76957c21e7 --- /dev/null +++ b/vendor/go.uber.org/tools/update-license/licenses.go @@ -0,0 +1,56 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package main + +var licenseTemplates = map[string]string{ + "Apache-2.0": `// Copyright {{.Year}} {{.Owner}} +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.`, + "MIT": `// Copyright (c) {{.Year}} {{.Owner}} +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE.`, +} diff --git a/vendor/go.uber.org/tools/update-license/main.go b/vendor/go.uber.org/tools/update-license/main.go new file mode 100644 index 0000000000..269fd9b47f --- /dev/null +++ b/vendor/go.uber.org/tools/update-license/main.go @@ -0,0 +1,228 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package main + +import ( + "bytes" + "flag" + "fmt" + "html/template" + "io/ioutil" + "log" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +const ( + // how many lines to check for an existing copyright + // this logic is not great and we should probably do something else + // but this was copied from the python script + copyrightLineLimit = 5 + headerPrefix = "// Copyright" +) + +var ( + flagDryRun = flag.Bool("dry", false, "Do not edit files and just print out what files would be edited") + flagOwner = flag.String("owner", "Uber Technologies, Inc.", "Copyright owner") + flagLicense = flag.String( + "license", + "MIT", + fmt.Sprintf( + "Type of license to use [%s]", + strings.Join(validLicenses(), ", "), + ), + ) + + lineSkipPrefixes = []string{ + "// Code generated by", + "// @generated", + } +) + +func main() { + log.SetFlags(0) + log.SetOutput(os.Stdout) + log.SetPrefix("") + if err := do(); err != nil { + log.Fatal(err) + } +} + +func do() error { + flag.Parse() + + if len(flag.Args()) < 1 { + return fmt.Errorf("usage: %s GO_FILES", os.Args[0]) + } + + return updateFiles( + flag.Args(), + time.Now().UTC().Year(), + *flagLicense, + *flagOwner, + *flagDryRun, + ) +} + +func fullLicense(ts string, year int, owner string) string { + var buf bytes.Buffer + t, err := template.New("").Parse(ts) + if err != nil { + log.Panic("failed to parse license template", err) + } + + data := struct { + Year int + Owner string + }{year, owner} + if err := t.Execute(&buf, data); err != nil { + log.Panic("failed to execture license template", err) + } + + return strings.TrimSpace(buf.String()) +} + +// validLicenses grabs all the license templates from the folder +func validLicenses() []string { + res := make([]string, 0, len(licenseTemplates)) + + for k := range licenseTemplates { + res = append(res, k) + } + + sort.Strings(res) + return res +} + +func updateFiles( + filePaths []string, + year int, + license string, + owner string, + dryRun bool, +) error { + if err := checkFilePaths(filePaths); err != nil { + return err + } + for _, filePath := range filePaths { + if err := updateFile(filePath, year, license, owner, dryRun); err != nil { + return err + } + } + return nil +} + +func checkFilePaths(filePaths []string) error { + for _, filePath := range filePaths { + if filepath.Ext(filePath) != ".go" { + return fmt.Errorf("%s is not a go file", filePath) + } + } + return nil +} + +func updateFile( + filePath string, + year int, + license string, + owner string, + dryRun bool, +) error { + data, err := ioutil.ReadFile(filePath) + if err != nil { + return err + } + newData := updateData(data, year, license, owner) + if !bytes.Equal(data, newData) { + if dryRun { + log.Print(filePath) + return nil + } + // we could do something more complicated so that we do not + // need to pass 0644 as the file mode, but in this case it should + // never actually be used to create a file since we know the file + // already exists, and it's easier to use the ReadFile/WriteFile + // logic as it is right now, and since this is just a generation + // program, this should be acceptable + return ioutil.WriteFile(filePath, newData, 0644) + } + return nil +} + +func updateData( + data []byte, + year int, + license string, + owner string, +) []byte { + licenseText := fullLicense(string(licenseTemplates[license]), year, owner) + + return []byte( + strings.Join( + updateLines(strings.Split(string(data), "\n"), licenseText), + "\n", + ), + ) +} + +// a value in the returned slice may contain newlines itself +func updateLines(lines []string, license string) []string { + for i, line := range lines { + if i >= copyrightLineLimit { + break + } + if strings.HasPrefix(line, headerPrefix) { + // assume that the new license text always starts with the copyright + // string. Pretty safe to assume, right? RIGHT? + lines[i] = strings.Split(license, "\n")[0] + return lines + } + } + return addToLines(lines, license) +} + +// a value in the returned slice may contain newlines itself +func addToLines(lines []string, license string) []string { + i := 0 + for len(lines) > i && lineContainsSkipPrefix(lines[i]) { + i++ + // skip comments under the generated line too + for strings.HasPrefix(lines[i], "//") { + i++ + } + } + if i == 0 { + return append([]string{license, ""}, lines...) + } + return append(lines[0:i], append([]string{"", license}, lines[i:]...)...) +} + +func lineContainsSkipPrefix(line string) bool { + for _, skipPrefix := range lineSkipPrefixes { + if strings.HasPrefix(line, skipPrefix) { + return true + } + } + return false +} diff --git a/vendor/go.uber.org/zap/.gitignore b/vendor/go.uber.org/zap/.gitignore index 08fbde6ce2..da9d9d00b4 100644 --- a/vendor/go.uber.org/zap/.gitignore +++ b/vendor/go.uber.org/zap/.gitignore @@ -26,3 +26,7 @@ _testmain.go *.pprof *.out *.log + +/bin +cover.out +cover.html diff --git a/vendor/go.uber.org/zap/.readme.tmpl b/vendor/go.uber.org/zap/.readme.tmpl index c6440db8eb..3154a1e64c 100644 --- a/vendor/go.uber.org/zap/.readme.tmpl +++ b/vendor/go.uber.org/zap/.readme.tmpl @@ -100,9 +100,10 @@ pinned in zap's [glide.lock][] file. [↩](#anchor-versions) [doc-img]: https://godoc.org/go.uber.org/zap?status.svg [doc]: https://godoc.org/go.uber.org/zap -[ci-img]: https://travis-ci.org/uber-go/zap.svg?branch=master -[ci]: https://travis-ci.org/uber-go/zap +[ci-img]: https://travis-ci.com/uber-go/zap.svg?branch=master +[ci]: https://travis-ci.com/uber-go/zap [cov-img]: https://codecov.io/gh/uber-go/zap/branch/master/graph/badge.svg [cov]: https://codecov.io/gh/uber-go/zap [benchmarking suite]: https://github.com/uber-go/zap/tree/master/benchmarks [glide.lock]: https://github.com/uber-go/zap/blob/master/glide.lock + diff --git a/vendor/go.uber.org/zap/.travis.yml b/vendor/go.uber.org/zap/.travis.yml index ada5ebdcc9..647b4ee431 100644 --- a/vendor/go.uber.org/zap/.travis.yml +++ b/vendor/go.uber.org/zap/.travis.yml @@ -1,21 +1,23 @@ language: go sudo: false -go: - - 1.11.x - - 1.12.x + go_import_path: go.uber.org/zap env: global: - TEST_TIMEOUT_SCALE=10 -cache: - directories: - - vendor -install: - - make dependencies + - GO111MODULE=on + +matrix: + include: + - go: 1.12.x + - go: 1.13.x + env: LINT=1 + script: - - make lint + - test -z "$LINT" || make lint - make test - make bench + after_success: - make cover - bash <(curl -s https://codecov.io/bash) diff --git a/vendor/go.uber.org/zap/CHANGELOG.md b/vendor/go.uber.org/zap/CHANGELOG.md index 28d10677eb..bebdb748d8 100644 --- a/vendor/go.uber.org/zap/CHANGELOG.md +++ b/vendor/go.uber.org/zap/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +## 1.13.0 (13 Nov 2019) + +Enhancements: +* [#758][]: Add `Intp`, `Stringp`, and other similar `*p` field constructors + to log pointers to primitives with support for `nil` values. + +Thanks to @jbizzle for their contributions to this release. + +## 1.12.0 (29 Oct 2019) + +Enhancements: +* [#751][]: Migrate to Go modules. + +## 1.11.0 (21 Oct 2019) + +Enhancements: +* [#725][]: Add `zapcore.OmitKey` to omit keys in an `EncoderConfig`. +* [#736][]: Add `RFC3339` and `RFC3339Nano` time encoders. + +Thanks to @juicemia, @uhthomas for their contributions to this release. + ## 1.10.0 (29 Apr 2019) Bugfixes: @@ -325,3 +346,7 @@ upgrade to the upcoming stable release. [#610]: https://github.com/uber-go/zap/pull/610 [#675]: https://github.com/uber-go/zap/pull/675 [#704]: https://github.com/uber-go/zap/pull/704 +[#725]: https://github.com/uber-go/zap/pull/725 +[#736]: https://github.com/uber-go/zap/pull/736 +[#751]: https://github.com/uber-go/zap/pull/751 +[#758]: https://github.com/uber-go/zap/pull/758 diff --git a/vendor/go.uber.org/zap/Makefile b/vendor/go.uber.org/zap/Makefile index 073e9aa910..21e436c45f 100644 --- a/vendor/go.uber.org/zap/Makefile +++ b/vendor/go.uber.org/zap/Makefile @@ -1,74 +1,55 @@ -export GO15VENDOREXPERIMENT=1 +export GOBIN ?= $(shell pwd)/bin +GOLINT = $(GOBIN)/golint BENCH_FLAGS ?= -cpuprofile=cpu.pprof -memprofile=mem.pprof -benchmem -PKGS ?= $(shell glide novendor) -# Many Go tools take file globs or directories as arguments instead of packages. -PKG_FILES ?= *.go zapcore benchmarks buffer zapgrpc zaptest zaptest/observer internal/bufferpool internal/exit internal/color internal/ztest -# The linting tools evolve with each Go version, so run them only on the latest -# stable release. -GO_VERSION := $(shell go version | cut -d " " -f 3) -GO_MINOR_VERSION := $(word 2,$(subst ., ,$(GO_VERSION))) -LINTABLE_MINOR_VERSIONS := 12 -ifneq ($(filter $(LINTABLE_MINOR_VERSIONS),$(GO_MINOR_VERSION)),) -SHOULD_LINT := true -endif +# Directories containing independent Go modules. +# +# We track coverage only for the main module. +MODULE_DIRS = . ./benchmarks +# Many Go tools take file globs or directories as arguments instead of packages. +GO_FILES := $(shell \ + find . '(' -path '*/.*' -o -path './vendor' ')' -prune \ + -o -name '*.go' -print | cut -b3-) .PHONY: all all: lint test -.PHONY: dependencies -dependencies: - @echo "Installing Glide and locked dependencies..." - glide --version || go get -u -f github.com/Masterminds/glide - glide install - @echo "Installing test dependencies..." - go install ./vendor/github.com/axw/gocov/gocov - go install ./vendor/github.com/mattn/goveralls -ifdef SHOULD_LINT - @echo "Installing golint..." - go install ./vendor/github.com/golang/lint/golint -else - @echo "Not installing golint, since we don't expect to lint on" $(GO_VERSION) -endif - -# Disable printf-like invocation checking due to testify.assert.Error() -VET_RULES := -printf=false - .PHONY: lint -lint: -ifdef SHOULD_LINT +lint: $(GOLINT) @rm -rf lint.log @echo "Checking formatting..." - @gofmt -d -s $(PKG_FILES) 2>&1 | tee lint.log - @echo "Installing test dependencies for vet..." - @go test -i $(PKGS) + @gofmt -d -s $(GO_FILES) 2>&1 | tee lint.log @echo "Checking vet..." - @go vet $(VET_RULES) $(PKGS) 2>&1 | tee -a lint.log + @$(foreach dir,$(MODULE_DIRS),(cd $(dir) && go vet ./... 2>&1) &&) true | tee -a lint.log @echo "Checking lint..." - @$(foreach dir,$(PKGS),golint $(dir) 2>&1 | tee -a lint.log;) + @$(foreach dir,$(MODULE_DIRS),(cd $(dir) && $(GOLINT) ./... 2>&1) &&) true | tee -a lint.log @echo "Checking for unresolved FIXMEs..." - @git grep -i fixme | grep -v -e vendor -e Makefile | tee -a lint.log + @git grep -i fixme | grep -v -e Makefile | tee -a lint.log @echo "Checking for license headers..." - @./check_license.sh | tee -a lint.log + @./checklicense.sh | tee -a lint.log @[ ! -s lint.log ] -else - @echo "Skipping linters on" $(GO_VERSION) -endif + +$(GOLINT): + go install golang.org/x/lint/golint .PHONY: test test: - go test -race $(PKGS) + @$(foreach dir,$(MODULE_DIRS),(cd $(dir) && go test -race ./...) &&) true .PHONY: cover cover: - ./scripts/cover.sh $(PKGS) + go test -race -coverprofile=cover.out -coverpkg=./... ./... + go tool cover -html=cover.out -o cover.html .PHONY: bench BENCH ?= . bench: - @$(foreach pkg,$(PKGS),go test -bench=$(BENCH) -run="^$$" $(BENCH_FLAGS) $(pkg);) + @$(foreach dir,$(MODULE_DIRS), ( \ + cd $(dir) && \ + go list ./... | xargs -n1 go test -bench=$(BENCH) -run="^$$" $(BENCH_FLAGS) \ + ) &&) true .PHONY: updatereadme updatereadme: diff --git a/vendor/go.uber.org/zap/README.md b/vendor/go.uber.org/zap/README.md index f4fd1cb444..bcea28a196 100644 --- a/vendor/go.uber.org/zap/README.md +++ b/vendor/go.uber.org/zap/README.md @@ -64,43 +64,40 @@ id="anchor-versions">[1](#footnote-versions) Log a message and 10 fields: -| Package | Time | Objects Allocated | -| :--- | :---: | :---: | -| :zap: zap | 3131 ns/op | 5 allocs/op | -| :zap: zap (sugared) | 4173 ns/op | 21 allocs/op | -| zerolog | 16154 ns/op | 90 allocs/op | -| lion | 16341 ns/op | 111 allocs/op | -| go-kit | 17049 ns/op | 126 allocs/op | -| logrus | 23662 ns/op | 142 allocs/op | -| log15 | 36351 ns/op | 149 allocs/op | -| apex/log | 42530 ns/op | 126 allocs/op | +| Package | Time | Time % to zap | Objects Allocated | +| :------ | :--: | :-----------: | :---------------: | +| :zap: zap | 862 ns/op | +0% | 5 allocs/op +| :zap: zap (sugared) | 1250 ns/op | +45% | 11 allocs/op +| zerolog | 4021 ns/op | +366% | 76 allocs/op +| go-kit | 4542 ns/op | +427% | 105 allocs/op +| apex/log | 26785 ns/op | +3007% | 115 allocs/op +| logrus | 29501 ns/op | +3322% | 125 allocs/op +| log15 | 29906 ns/op | +3369% | 122 allocs/op Log a message with a logger that already has 10 fields of context: -| Package | Time | Objects Allocated | -| :--- | :---: | :---: | -| :zap: zap | 380 ns/op | 0 allocs/op | -| :zap: zap (sugared) | 564 ns/op | 2 allocs/op | -| zerolog | 321 ns/op | 0 allocs/op | -| lion | 7092 ns/op | 39 allocs/op | -| go-kit | 20226 ns/op | 115 allocs/op | -| logrus | 22312 ns/op | 130 allocs/op | -| log15 | 28788 ns/op | 79 allocs/op | -| apex/log | 42063 ns/op | 115 allocs/op | +| Package | Time | Time % to zap | Objects Allocated | +| :------ | :--: | :-----------: | :---------------: | +| :zap: zap | 126 ns/op | +0% | 0 allocs/op +| :zap: zap (sugared) | 187 ns/op | +48% | 2 allocs/op +| zerolog | 88 ns/op | -30% | 0 allocs/op +| go-kit | 5087 ns/op | +3937% | 103 allocs/op +| log15 | 18548 ns/op | +14621% | 73 allocs/op +| apex/log | 26012 ns/op | +20544% | 104 allocs/op +| logrus | 27236 ns/op | +21516% | 113 allocs/op Log a static string, without any context or `printf`-style templating: -| Package | Time | Objects Allocated | -| :--- | :---: | :---: | -| :zap: zap | 361 ns/op | 0 allocs/op | -| :zap: zap (sugared) | 534 ns/op | 2 allocs/op | -| zerolog | 323 ns/op | 0 allocs/op | -| standard library | 575 ns/op | 2 allocs/op | -| go-kit | 922 ns/op | 13 allocs/op | -| lion | 1413 ns/op | 10 allocs/op | -| logrus | 2291 ns/op | 27 allocs/op | -| apex/log | 3690 ns/op | 11 allocs/op | -| log15 | 5954 ns/op | 26 allocs/op | +| Package | Time | Time % to zap | Objects Allocated | +| :------ | :--: | :-----------: | :---------------: | +| :zap: zap | 118 ns/op | +0% | 0 allocs/op +| :zap: zap (sugared) | 191 ns/op | +62% | 2 allocs/op +| zerolog | 93 ns/op | -21% | 0 allocs/op +| go-kit | 280 ns/op | +137% | 11 allocs/op +| standard library | 499 ns/op | +323% | 2 allocs/op +| apex/log | 1990 ns/op | +1586% | 10 allocs/op +| logrus | 3129 ns/op | +2552% | 24 allocs/op +| log15 | 3887 ns/op | +3194% | 23 allocs/op ## Development Status: Stable @@ -124,13 +121,14 @@ Released under the [MIT License](LICENSE.txt). 1 In particular, keep in mind that we may be benchmarking against slightly older versions of other packages. Versions are -pinned in zap's [glide.lock][] file. [↩](#anchor-versions) +pinned in the [benchmarks/go.mod][] file. [↩](#anchor-versions) [doc-img]: https://godoc.org/go.uber.org/zap?status.svg [doc]: https://godoc.org/go.uber.org/zap -[ci-img]: https://travis-ci.org/uber-go/zap.svg?branch=master -[ci]: https://travis-ci.org/uber-go/zap +[ci-img]: https://travis-ci.com/uber-go/zap.svg?branch=master +[ci]: https://travis-ci.com/uber-go/zap [cov-img]: https://codecov.io/gh/uber-go/zap/branch/master/graph/badge.svg [cov]: https://codecov.io/gh/uber-go/zap [benchmarking suite]: https://github.com/uber-go/zap/tree/master/benchmarks -[glide.lock]: https://github.com/uber-go/zap/blob/master/glide.lock +[benchmarks/go.mod]: https://github.com/uber-go/zap/blob/master/benchmarks/go.mod + diff --git a/vendor/go.uber.org/zap/check_license.sh b/vendor/go.uber.org/zap/checklicense.sh similarity index 100% rename from vendor/go.uber.org/zap/check_license.sh rename to vendor/go.uber.org/zap/checklicense.sh diff --git a/vendor/go.uber.org/zap/field.go b/vendor/go.uber.org/zap/field.go index 5130e13477..83c1ea245a 100644 --- a/vendor/go.uber.org/zap/field.go +++ b/vendor/go.uber.org/zap/field.go @@ -38,6 +38,12 @@ func Skip() Field { return Field{Type: zapcore.SkipType} } +// nilField returns a field which will marshal explicitly as nil. See motivation +// in https://github.com/uber-go/zap/issues/753 . If we ever make breaking +// changes and add zapcore.NilType and zapcore.ObjectEncoder.AddNil, the +// implementation here should be changed to reflect that. +func nilField(key string) Field { return Reflect(key, nil) } + // Binary constructs a field that carries an opaque binary blob. // // Binary data is serialized in an encoding-appropriate format. For example, @@ -56,6 +62,15 @@ func Bool(key string, val bool) Field { return Field{Key: key, Type: zapcore.BoolType, Integer: ival} } +// Boolp constructs a field that carries a *bool. The returned Field will safely +// and explicitly represent `nil` when appropriate. +func Boolp(key string, val *bool) Field { + if val == nil { + return nilField(key) + } + return Bool(key, *val) +} + // ByteString constructs a field that carries UTF-8 encoded text as a []byte. // To log opaque binary blobs (which aren't necessarily valid UTF-8), use // Binary. @@ -70,6 +85,15 @@ func Complex128(key string, val complex128) Field { return Field{Key: key, Type: zapcore.Complex128Type, Interface: val} } +// Complex128p constructs a field that carries a *complex128. The returned Field will safely +// and explicitly represent `nil` when appropriate. +func Complex128p(key string, val *complex128) Field { + if val == nil { + return nilField(key) + } + return Complex128(key, *val) +} + // Complex64 constructs a field that carries a complex number. Unlike most // numeric fields, this costs an allocation (to convert the complex64 to // interface{}). @@ -77,6 +101,15 @@ func Complex64(key string, val complex64) Field { return Field{Key: key, Type: zapcore.Complex64Type, Interface: val} } +// Complex64p constructs a field that carries a *complex64. The returned Field will safely +// and explicitly represent `nil` when appropriate. +func Complex64p(key string, val *complex64) Field { + if val == nil { + return nilField(key) + } + return Complex64(key, *val) +} + // Float64 constructs a field that carries a float64. The way the // floating-point value is represented is encoder-dependent, so marshaling is // necessarily lazy. @@ -84,6 +117,15 @@ func Float64(key string, val float64) Field { return Field{Key: key, Type: zapcore.Float64Type, Integer: int64(math.Float64bits(val))} } +// Float64p constructs a field that carries a *float64. The returned Field will safely +// and explicitly represent `nil` when appropriate. +func Float64p(key string, val *float64) Field { + if val == nil { + return nilField(key) + } + return Float64(key, *val) +} + // Float32 constructs a field that carries a float32. The way the // floating-point value is represented is encoder-dependent, so marshaling is // necessarily lazy. @@ -91,66 +133,183 @@ func Float32(key string, val float32) Field { return Field{Key: key, Type: zapcore.Float32Type, Integer: int64(math.Float32bits(val))} } +// Float32p constructs a field that carries a *float32. The returned Field will safely +// and explicitly represent `nil` when appropriate. +func Float32p(key string, val *float32) Field { + if val == nil { + return nilField(key) + } + return Float32(key, *val) +} + // Int constructs a field with the given key and value. func Int(key string, val int) Field { return Int64(key, int64(val)) } +// Intp constructs a field that carries a *int. The returned Field will safely +// and explicitly represent `nil` when appropriate. +func Intp(key string, val *int) Field { + if val == nil { + return nilField(key) + } + return Int(key, *val) +} + // Int64 constructs a field with the given key and value. func Int64(key string, val int64) Field { return Field{Key: key, Type: zapcore.Int64Type, Integer: val} } +// Int64p constructs a field that carries a *int64. The returned Field will safely +// and explicitly represent `nil` when appropriate. +func Int64p(key string, val *int64) Field { + if val == nil { + return nilField(key) + } + return Int64(key, *val) +} + // Int32 constructs a field with the given key and value. func Int32(key string, val int32) Field { return Field{Key: key, Type: zapcore.Int32Type, Integer: int64(val)} } +// Int32p constructs a field that carries a *int32. The returned Field will safely +// and explicitly represent `nil` when appropriate. +func Int32p(key string, val *int32) Field { + if val == nil { + return nilField(key) + } + return Int32(key, *val) +} + // Int16 constructs a field with the given key and value. func Int16(key string, val int16) Field { return Field{Key: key, Type: zapcore.Int16Type, Integer: int64(val)} } +// Int16p constructs a field that carries a *int16. The returned Field will safely +// and explicitly represent `nil` when appropriate. +func Int16p(key string, val *int16) Field { + if val == nil { + return nilField(key) + } + return Int16(key, *val) +} + // Int8 constructs a field with the given key and value. func Int8(key string, val int8) Field { return Field{Key: key, Type: zapcore.Int8Type, Integer: int64(val)} } +// Int8p constructs a field that carries a *int8. The returned Field will safely +// and explicitly represent `nil` when appropriate. +func Int8p(key string, val *int8) Field { + if val == nil { + return nilField(key) + } + return Int8(key, *val) +} + // String constructs a field with the given key and value. func String(key string, val string) Field { return Field{Key: key, Type: zapcore.StringType, String: val} } +// Stringp constructs a field that carries a *string. The returned Field will safely +// and explicitly represent `nil` when appropriate. +func Stringp(key string, val *string) Field { + if val == nil { + return nilField(key) + } + return String(key, *val) +} + // Uint constructs a field with the given key and value. func Uint(key string, val uint) Field { return Uint64(key, uint64(val)) } +// Uintp constructs a field that carries a *uint. The returned Field will safely +// and explicitly represent `nil` when appropriate. +func Uintp(key string, val *uint) Field { + if val == nil { + return nilField(key) + } + return Uint(key, *val) +} + // Uint64 constructs a field with the given key and value. func Uint64(key string, val uint64) Field { return Field{Key: key, Type: zapcore.Uint64Type, Integer: int64(val)} } +// Uint64p constructs a field that carries a *uint64. The returned Field will safely +// and explicitly represent `nil` when appropriate. +func Uint64p(key string, val *uint64) Field { + if val == nil { + return nilField(key) + } + return Uint64(key, *val) +} + // Uint32 constructs a field with the given key and value. func Uint32(key string, val uint32) Field { return Field{Key: key, Type: zapcore.Uint32Type, Integer: int64(val)} } +// Uint32p constructs a field that carries a *uint32. The returned Field will safely +// and explicitly represent `nil` when appropriate. +func Uint32p(key string, val *uint32) Field { + if val == nil { + return nilField(key) + } + return Uint32(key, *val) +} + // Uint16 constructs a field with the given key and value. func Uint16(key string, val uint16) Field { return Field{Key: key, Type: zapcore.Uint16Type, Integer: int64(val)} } +// Uint16p constructs a field that carries a *uint16. The returned Field will safely +// and explicitly represent `nil` when appropriate. +func Uint16p(key string, val *uint16) Field { + if val == nil { + return nilField(key) + } + return Uint16(key, *val) +} + // Uint8 constructs a field with the given key and value. func Uint8(key string, val uint8) Field { return Field{Key: key, Type: zapcore.Uint8Type, Integer: int64(val)} } +// Uint8p constructs a field that carries a *uint8. The returned Field will safely +// and explicitly represent `nil` when appropriate. +func Uint8p(key string, val *uint8) Field { + if val == nil { + return nilField(key) + } + return Uint8(key, *val) +} + // Uintptr constructs a field with the given key and value. func Uintptr(key string, val uintptr) Field { return Field{Key: key, Type: zapcore.UintptrType, Integer: int64(val)} } +// Uintptrp constructs a field that carries a *uintptr. The returned Field will safely +// and explicitly represent `nil` when appropriate. +func Uintptrp(key string, val *uintptr) Field { + if val == nil { + return nilField(key) + } + return Uintptr(key, *val) +} + // Reflect constructs a field with the given key and an arbitrary object. It uses // an encoding-appropriate, reflection-based function to lazily serialize nearly // any object into the logging context, but it's relatively slow and @@ -183,6 +342,15 @@ func Time(key string, val time.Time) Field { return Field{Key: key, Type: zapcore.TimeType, Integer: val.UnixNano(), Interface: val.Location()} } +// Timep constructs a field that carries a *time.Time. The returned Field will safely +// and explicitly represent `nil` when appropriate. +func Timep(key string, val *time.Time) Field { + if val == nil { + return nilField(key) + } + return Time(key, *val) +} + // Stack constructs a field that stores a stacktrace of the current goroutine // under provided key. Keep in mind that taking a stacktrace is eager and // expensive (relatively speaking); this function both makes an allocation and @@ -201,6 +369,15 @@ func Duration(key string, val time.Duration) Field { return Field{Key: key, Type: zapcore.DurationType, Integer: int64(val)} } +// Durationp constructs a field that carries a *time.Duration. The returned Field will safely +// and explicitly represent `nil` when appropriate. +func Durationp(key string, val *time.Duration) Field { + if val == nil { + return nilField(key) + } + return Duration(key, *val) +} + // Object constructs a field with the given key and ObjectMarshaler. It // provides a flexible, but still type-safe and efficient, way to add map- or // struct-like user-defined types to the logging context. The struct's @@ -224,78 +401,116 @@ func Any(key string, value interface{}) Field { return Array(key, val) case bool: return Bool(key, val) + case *bool: + return Boolp(key, val) case []bool: return Bools(key, val) case complex128: return Complex128(key, val) + case *complex128: + return Complex128p(key, val) case []complex128: return Complex128s(key, val) case complex64: return Complex64(key, val) + case *complex64: + return Complex64p(key, val) case []complex64: return Complex64s(key, val) case float64: return Float64(key, val) + case *float64: + return Float64p(key, val) case []float64: return Float64s(key, val) case float32: return Float32(key, val) + case *float32: + return Float32p(key, val) case []float32: return Float32s(key, val) case int: return Int(key, val) + case *int: + return Intp(key, val) case []int: return Ints(key, val) case int64: return Int64(key, val) + case *int64: + return Int64p(key, val) case []int64: return Int64s(key, val) case int32: return Int32(key, val) + case *int32: + return Int32p(key, val) case []int32: return Int32s(key, val) case int16: return Int16(key, val) + case *int16: + return Int16p(key, val) case []int16: return Int16s(key, val) case int8: return Int8(key, val) + case *int8: + return Int8p(key, val) case []int8: return Int8s(key, val) case string: return String(key, val) + case *string: + return Stringp(key, val) case []string: return Strings(key, val) case uint: return Uint(key, val) + case *uint: + return Uintp(key, val) case []uint: return Uints(key, val) case uint64: return Uint64(key, val) + case *uint64: + return Uint64p(key, val) case []uint64: return Uint64s(key, val) case uint32: return Uint32(key, val) + case *uint32: + return Uint32p(key, val) case []uint32: return Uint32s(key, val) case uint16: return Uint16(key, val) + case *uint16: + return Uint16p(key, val) case []uint16: return Uint16s(key, val) case uint8: return Uint8(key, val) + case *uint8: + return Uint8p(key, val) case []byte: return Binary(key, val) case uintptr: return Uintptr(key, val) + case *uintptr: + return Uintptrp(key, val) case []uintptr: return Uintptrs(key, val) case time.Time: return Time(key, val) + case *time.Time: + return Timep(key, val) case []time.Time: return Times(key, val) case time.Duration: return Duration(key, val) + case *time.Duration: + return Durationp(key, val) case []time.Duration: return Durations(key, val) case error: diff --git a/vendor/go.uber.org/zap/glide.lock b/vendor/go.uber.org/zap/glide.lock deleted file mode 100644 index 881b462c0e..0000000000 --- a/vendor/go.uber.org/zap/glide.lock +++ /dev/null @@ -1,76 +0,0 @@ -hash: f073ba522c06c88ea3075bde32a8aaf0969a840a66cab6318a0897d141ffee92 -updated: 2017-07-22T18:06:49.598185334-07:00 -imports: -- name: go.uber.org/atomic - version: 4e336646b2ef9fc6e47be8e21594178f98e5ebcf -- name: go.uber.org/multierr - version: 3c4937480c32f4c13a875a1829af76c98ca3d40a -testImports: -- name: github.com/apex/log - version: d9b960447bfa720077b2da653cc79e533455b499 - subpackages: - - handlers/json -- name: github.com/axw/gocov - version: 3a69a0d2a4ef1f263e2d92b041a69593d6964fe8 - subpackages: - - gocov -- name: github.com/davecgh/go-spew - version: 04cdfd42973bb9c8589fd6a731800cf222fde1a9 - subpackages: - - spew -- name: github.com/fatih/color - version: 62e9147c64a1ed519147b62a56a14e83e2be02c1 -- name: github.com/go-kit/kit - version: e10f5bf035be9af21fd5b2fb4469d5716c6ab07d - subpackages: - - log -- name: github.com/go-logfmt/logfmt - version: 390ab7935ee28ec6b286364bba9b4dd6410cb3d5 -- name: github.com/go-stack/stack - version: 54be5f394ed2c3e19dac9134a40a95ba5a017f7b -- name: github.com/golang/lint - version: c5fb716d6688a859aae56d26d3e6070808df29f7 - subpackages: - - golint -- name: github.com/kr/logfmt - version: b84e30acd515aadc4b783ad4ff83aff3299bdfe0 -- name: github.com/mattn/go-colorable - version: 3fa8c76f9daed4067e4a806fb7e4dc86455c6d6a -- name: github.com/mattn/go-isatty - version: fc9e8d8ef48496124e79ae0df75490096eccf6fe -- name: github.com/mattn/goveralls - version: 6efce81852ad1b7567c17ad71b03aeccc9dd9ae0 -- name: github.com/pborman/uuid - version: e790cca94e6cc75c7064b1332e63811d4aae1a53 -- name: github.com/pkg/errors - version: 645ef00459ed84a119197bfb8d8205042c6df63d -- name: github.com/pmezard/go-difflib - version: d8ed2627bdf02c080bf22230dbb337003b7aba2d - subpackages: - - difflib -- name: github.com/rs/zerolog - version: eed4c2b94d945e0b2456ad6aa518a443986b5f22 -- name: github.com/satori/go.uuid - version: 5bf94b69c6b68ee1b541973bb8e1144db23a194b -- name: github.com/sirupsen/logrus - version: 7dd06bf38e1e13df288d471a57d5adbac106be9e -- name: github.com/stretchr/testify - version: f6abca593680b2315d2075e0f5e2a9751e3f431a - subpackages: - - assert - - require -- name: go.pedge.io/lion - version: 87958e8713f1fa138d993087133b97e976642159 -- name: golang.org/x/sys - version: c4489faa6e5ab84c0ef40d6ee878f7a030281f0f - subpackages: - - unix -- name: golang.org/x/tools - version: 496819729719f9d07692195e0a94d6edd2251389 - subpackages: - - cover -- name: gopkg.in/inconshreveable/log15.v2 - version: b105bd37f74e5d9dc7b6ad7806715c7a2b83fd3f - subpackages: - - stack - - term diff --git a/vendor/go.uber.org/zap/glide.yaml b/vendor/go.uber.org/zap/glide.yaml index 94412594ca..8e1d05e9ab 100644 --- a/vendor/go.uber.org/zap/glide.yaml +++ b/vendor/go.uber.org/zap/glide.yaml @@ -22,12 +22,11 @@ testImport: - package: github.com/mattn/goveralls - package: github.com/pborman/uuid - package: github.com/pkg/errors -- package: go.pedge.io/lion - package: github.com/rs/zerolog - package: golang.org/x/tools subpackages: - cover -- package: github.com/golang/lint +- package: golang.org/x/lint subpackages: - golint - package: github.com/axw/gocov diff --git a/vendor/go.uber.org/zap/go.mod b/vendor/go.uber.org/zap/go.mod new file mode 100644 index 0000000000..1fb6bba0b3 --- /dev/null +++ b/vendor/go.uber.org/zap/go.mod @@ -0,0 +1,11 @@ +module go.uber.org/zap + +go 1.13 + +require ( + github.com/pkg/errors v0.8.1 + github.com/stretchr/testify v1.4.0 + go.uber.org/atomic v1.5.0 + go.uber.org/multierr v1.3.0 + golang.org/x/lint v0.0.0-20190930215403-16217165b5de +) diff --git a/vendor/go.uber.org/zap/go.sum b/vendor/go.uber.org/zap/go.sum new file mode 100644 index 0000000000..9ff6735d8c --- /dev/null +++ b/vendor/go.uber.org/zap/go.sum @@ -0,0 +1,56 @@ +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.3.0 h1:sFPn2GLc3poCkfrpIXGhBD2X0CMIo4Q/zSULXrj/+uc= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c h1:IGkKhmfzcztjm6gYkykvu/NiS8kaqbCWAEWWAyf8J5U= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/vendor/go.uber.org/zap/tools.go b/vendor/go.uber.org/zap/tools.go new file mode 100644 index 0000000000..2b6366beac --- /dev/null +++ b/vendor/go.uber.org/zap/tools.go @@ -0,0 +1,28 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// +build tools + +package zap + +import ( + // Tools we use during development. + _ "golang.org/x/lint/golint" +) diff --git a/vendor/go.uber.org/zap/zapcore/encoder.go b/vendor/go.uber.org/zap/zapcore/encoder.go index f0509522b5..5e0a69be50 100644 --- a/vendor/go.uber.org/zap/zapcore/encoder.go +++ b/vendor/go.uber.org/zap/zapcore/encoder.go @@ -31,6 +31,9 @@ import ( // behavior. const DefaultLineEnding = "\n" +// OmitKey defines the key to use when callers want to remove a key from log output. +const OmitKey = "" + // A LevelEncoder serializes a Level to a primitive type. type LevelEncoder func(Level, PrimitiveArrayEncoder) @@ -115,11 +118,30 @@ func ISO8601TimeEncoder(t time.Time, enc PrimitiveArrayEncoder) { enc.AppendString(t.Format("2006-01-02T15:04:05.000Z0700")) } -// UnmarshalText unmarshals text to a TimeEncoder. "iso8601" and "ISO8601" are -// unmarshaled to ISO8601TimeEncoder, "millis" is unmarshaled to -// EpochMillisTimeEncoder, and anything else is unmarshaled to EpochTimeEncoder. +// RFC3339TimeEncoder serializes a time.Time to an RFC3339-formatted string. +func RFC3339TimeEncoder(t time.Time, enc PrimitiveArrayEncoder) { + enc.AppendString(t.Format(time.RFC3339)) +} + +// RFC3339NanoTimeEncoder serializes a time.Time to an RFC3339-formatted string +// with nanosecond precision. +func RFC3339NanoTimeEncoder(t time.Time, enc PrimitiveArrayEncoder) { + enc.AppendString(t.Format(time.RFC3339Nano)) +} + +// UnmarshalText unmarshals text to a TimeEncoder. +// "rfc3339nano" and "RFC3339Nano" are unmarshaled to RFC3339NanoTimeEncoder. +// "rfc3339" and "RFC3339" are unmarshaled to RFC3339TimeEncoder. +// "iso8601" and "ISO8601" are unmarshaled to ISO8601TimeEncoder. +// "millis" is unmarshaled to EpochMillisTimeEncoder. +// "nanos" is unmarshaled to EpochNanosEncoder. +// Anything else is unmarshaled to EpochTimeEncoder. func (e *TimeEncoder) UnmarshalText(text []byte) error { switch string(text) { + case "rfc3339nano", "RFC3339Nano": + *e = RFC3339NanoTimeEncoder + case "rfc3339", "RFC3339": + *e = RFC3339TimeEncoder case "iso8601", "ISO8601": *e = ISO8601TimeEncoder case "millis": @@ -272,8 +294,8 @@ type ObjectEncoder interface { AddUint8(key string, value uint8) AddUintptr(key string, value uintptr) - // AddReflected uses reflection to serialize arbitrary objects, so it's slow - // and allocation-heavy. + // AddReflected uses reflection to serialize arbitrary objects, so it can be + // slow and allocation-heavy. AddReflected(key string, value interface{}) error // OpenNamespace opens an isolated namespace where all subsequent fields will // be added. Applications can use namespaces to prevent key collisions when @@ -343,6 +365,7 @@ type Encoder interface { Clone() Encoder // EncodeEntry encodes an entry and fields, along with any accumulated - // context, into a byte buffer and returns it. + // context, into a byte buffer and returns it. Any fields that are empty, + // including fields on the `Entry` type, should be omitted. EncodeEntry(Entry, []Field) (*buffer.Buffer, error) } diff --git a/vendor/go.uber.org/zap/zapcore/entry.go b/vendor/go.uber.org/zap/zapcore/entry.go index 7d9893f331..8273abdf07 100644 --- a/vendor/go.uber.org/zap/zapcore/entry.go +++ b/vendor/go.uber.org/zap/zapcore/entry.go @@ -136,7 +136,8 @@ func (ec EntryCaller) TrimmedPath() string { // An Entry represents a complete log message. The entry's structured context // is already serialized, but the log level, time, message, and call site -// information are available for inspection and modification. +// information are available for inspection and modification. Any fields left +// empty will be omitted when encoding. // // Entries are pooled, so any functions that accept them MUST be careful not to // retain references to them. diff --git a/vendor/go.uber.org/zap/zapcore/json_encoder.go b/vendor/go.uber.org/zap/zapcore/json_encoder.go index 9aec4eada3..56256be8d5 100644 --- a/vendor/go.uber.org/zap/zapcore/json_encoder.go +++ b/vendor/go.uber.org/zap/zapcore/json_encoder.go @@ -145,15 +145,29 @@ func (enc *jsonEncoder) resetReflectBuf() { } } -func (enc *jsonEncoder) AddReflected(key string, obj interface{}) error { +var nullLiteralBytes = []byte("null") + +// Only invoke the standard JSON encoder if there is actually something to +// encode; otherwise write JSON null literal directly. +func (enc *jsonEncoder) encodeReflected(obj interface{}) ([]byte, error) { + if obj == nil { + return nullLiteralBytes, nil + } enc.resetReflectBuf() - err := enc.reflectEnc.Encode(obj) + if err := enc.reflectEnc.Encode(obj); err != nil { + return nil, err + } + enc.reflectBuf.TrimNewline() + return enc.reflectBuf.Bytes(), nil +} + +func (enc *jsonEncoder) AddReflected(key string, obj interface{}) error { + valueBytes, err := enc.encodeReflected(obj) if err != nil { return err } - enc.reflectBuf.TrimNewline() enc.addKey(key) - _, err = enc.buf.Write(enc.reflectBuf.Bytes()) + _, err = enc.buf.Write(valueBytes) return err } @@ -236,14 +250,12 @@ func (enc *jsonEncoder) AppendInt64(val int64) { } func (enc *jsonEncoder) AppendReflected(val interface{}) error { - enc.resetReflectBuf() - err := enc.reflectEnc.Encode(val) + valueBytes, err := enc.encodeReflected(val) if err != nil { return err } - enc.reflectBuf.TrimNewline() enc.addElementSeparator() - _, err = enc.buf.Write(enc.reflectBuf.Bytes()) + _, err = enc.buf.Write(valueBytes) return err } diff --git a/vendor/golang.org/x/lint/go.mod b/vendor/golang.org/x/lint/go.mod index d5ba4dbfd6..b32309c45f 100644 --- a/vendor/golang.org/x/lint/go.mod +++ b/vendor/golang.org/x/lint/go.mod @@ -1,3 +1,5 @@ module golang.org/x/lint -require golang.org/x/tools v0.0.0-20190311212946-11955173bddd +go 1.11 + +require golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7 diff --git a/vendor/golang.org/x/lint/go.sum b/vendor/golang.org/x/lint/go.sum index 7d0e2e6188..2ad45cae24 100644 --- a/vendor/golang.org/x/lint/go.sum +++ b/vendor/golang.org/x/lint/go.sum @@ -1,6 +1,12 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7 h1:EBZoQjiKKPaLbPrbpssUfuHtwM6KV/vb4U85g/cigFY= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/vendor/golang.org/x/sys/unix/mkerrors.sh b/vendor/golang.org/x/sys/unix/mkerrors.sh index 6ffac9250f..96bf2a9197 100644 --- a/vendor/golang.org/x/sys/unix/mkerrors.sh +++ b/vendor/golang.org/x/sys/unix/mkerrors.sh @@ -486,7 +486,7 @@ ccflags="$@" $2 ~ /^TCSET/ || $2 ~ /^TC(FLSH|SBRKP?|XONC)$/ || $2 !~ "RTF_BITS" && - $2 ~ /^(IFF|IFT|NET_RT|RTM|RTF|RTV|RTA|RTAX)_/ || + $2 ~ /^(IFF|IFT|NET_RT|RTM(GRP)?|RTF|RTV|RTA|RTAX)_/ || $2 ~ /^BIOC/ || $2 ~ /^RUSAGE_(SELF|CHILDREN|THREAD)/ || $2 ~ /^RLIMIT_(AS|CORE|CPU|DATA|FSIZE|LOCKS|MEMLOCK|MSGQUEUE|NICE|NOFILE|NPROC|RSS|RTPRIO|RTTIME|SIGPENDING|STACK)|RLIM_INFINITY/ || diff --git a/vendor/golang.org/x/sys/unix/syscall_linux_mips64x.go b/vendor/golang.org/x/sys/unix/syscall_linux_mips64x.go index e9d2f1d715..af77e6e25e 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux_mips64x.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux_mips64x.go @@ -216,6 +216,10 @@ func (cmsg *Cmsghdr) SetLen(length int) { cmsg.Len = uint64(length) } +func InotifyInit() (fd int, err error) { + return InotifyInit1(0) +} + //sys poll(fds *PollFd, nfds int, timeout int) (n int, err error) func Poll(fds []PollFd, timeout int) (n int, err error) { diff --git a/vendor/golang.org/x/sys/unix/syscall_openbsd.go b/vendor/golang.org/x/sys/unix/syscall_openbsd.go index 2629a32677..a266e92a9b 100644 --- a/vendor/golang.org/x/sys/unix/syscall_openbsd.go +++ b/vendor/golang.org/x/sys/unix/syscall_openbsd.go @@ -72,16 +72,20 @@ func SysctlUvmexp(name string) (*Uvmexp, error) { return &u, nil } -//sysnb pipe(p *[2]_C_int) (err error) func Pipe(p []int) (err error) { + return Pipe2(p, 0) +} + +//sysnb pipe2(p *[2]_C_int, flags int) (err error) +func Pipe2(p []int, flags int) error { if len(p) != 2 { return EINVAL } var pp [2]_C_int - err = pipe(&pp) + err := pipe2(&pp, flags) p[0] = int(pp[0]) p[1] = int(pp[1]) - return + return err } //sys Getdents(fd int, buf []byte) (n int, err error) diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_386.go b/vendor/golang.org/x/sys/unix/zerrors_linux_386.go index 62f110fa72..72066f808a 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_386.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_386.go @@ -243,6 +243,7 @@ const ( BPF_F_LOCK = 0x4 BPF_F_MARK_ENFORCE = 0x40 BPF_F_MARK_MANGLED_0 = 0x20 + BPF_F_MMAPABLE = 0x400 BPF_F_NO_COMMON_LRU = 0x2 BPF_F_NO_PREALLOC = 0x1 BPF_F_NUMA_NODE = 0x4 @@ -417,8 +418,10 @@ const ( CLOCK_TXFROMRX = 0x4 CLOCK_TXINT = 0x3 CLONE_ARGS_SIZE_VER0 = 0x40 + CLONE_ARGS_SIZE_VER1 = 0x50 CLONE_CHILD_CLEARTID = 0x200000 CLONE_CHILD_SETTID = 0x1000000 + CLONE_CLEAR_SIGHAND = 0x100000000 CLONE_DETACHED = 0x400000 CLONE_FILES = 0x400 CLONE_FS = 0x200 @@ -716,8 +719,9 @@ const ( FSCRYPT_POLICY_FLAGS_PAD_4 = 0x0 FSCRYPT_POLICY_FLAGS_PAD_8 = 0x1 FSCRYPT_POLICY_FLAGS_PAD_MASK = 0x3 - FSCRYPT_POLICY_FLAGS_VALID = 0x7 + FSCRYPT_POLICY_FLAGS_VALID = 0xf FSCRYPT_POLICY_FLAG_DIRECT_KEY = 0x4 + FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64 = 0x8 FSCRYPT_POLICY_V1 = 0x0 FSCRYPT_POLICY_V2 = 0x2 FS_ENCRYPTION_MODE_ADIANTUM = 0x9 @@ -747,7 +751,7 @@ const ( FS_POLICY_FLAGS_PAD_4 = 0x0 FS_POLICY_FLAGS_PAD_8 = 0x1 FS_POLICY_FLAGS_PAD_MASK = 0x3 - FS_POLICY_FLAGS_VALID = 0x7 + FS_POLICY_FLAGS_VALID = 0xf FUTEXFS_SUPER_MAGIC = 0xbad1dea F_ADD_SEALS = 0x409 F_DUPFD = 0x0 @@ -1229,6 +1233,7 @@ const ( LOOP_SET_STATUS64 = 0x4c04 LO_KEY_SIZE = 0x20 LO_NAME_SIZE = 0x40 + MADV_COLD = 0x14 MADV_DODUMP = 0x11 MADV_DOFORK = 0xb MADV_DONTDUMP = 0x10 @@ -1241,6 +1246,7 @@ const ( MADV_MERGEABLE = 0xc MADV_NOHUGEPAGE = 0xf MADV_NORMAL = 0x0 + MADV_PAGEOUT = 0x15 MADV_RANDOM = 0x1 MADV_REMOVE = 0x9 MADV_SEQUENTIAL = 0x2 @@ -1583,6 +1589,7 @@ const ( PERF_EVENT_IOC_SET_FILTER = 0x40042406 PERF_EVENT_IOC_SET_OUTPUT = 0x2405 PIPEFS_MAGIC = 0x50495045 + PPC_CMM_MAGIC = 0xc7571590 PPPIOCATTACH = 0x4004743d PPPIOCATTCHAN = 0x40047438 PPPIOCCONNECT = 0x4004743a @@ -1593,6 +1600,8 @@ const ( PPPIOCGDEBUG = 0x80047441 PPPIOCGFLAGS = 0x8004745a PPPIOCGIDLE = 0x8008743f + PPPIOCGIDLE32 = 0x8008743f + PPPIOCGIDLE64 = 0x8010743f PPPIOCGL2TPSTATS = 0x80487436 PPPIOCGMRU = 0x80047453 PPPIOCGNPMODE = 0xc008744c @@ -1929,12 +1938,28 @@ const ( RTF_UP = 0x1 RTF_WINDOW = 0x80 RTF_XRESOLVE = 0x800 + RTMGRP_DECnet_IFADDR = 0x1000 + RTMGRP_DECnet_ROUTE = 0x4000 + RTMGRP_IPV4_IFADDR = 0x10 + RTMGRP_IPV4_MROUTE = 0x20 + RTMGRP_IPV4_ROUTE = 0x40 + RTMGRP_IPV4_RULE = 0x80 + RTMGRP_IPV6_IFADDR = 0x100 + RTMGRP_IPV6_IFINFO = 0x800 + RTMGRP_IPV6_MROUTE = 0x200 + RTMGRP_IPV6_PREFIX = 0x20000 + RTMGRP_IPV6_ROUTE = 0x400 + RTMGRP_LINK = 0x1 + RTMGRP_NEIGH = 0x4 + RTMGRP_NOTIFY = 0x2 + RTMGRP_TC = 0x8 RTM_BASE = 0x10 RTM_DELACTION = 0x31 RTM_DELADDR = 0x15 RTM_DELADDRLABEL = 0x49 RTM_DELCHAIN = 0x65 RTM_DELLINK = 0x11 + RTM_DELLINKPROP = 0x6d RTM_DELMDB = 0x55 RTM_DELNEIGH = 0x1d RTM_DELNETCONF = 0x51 @@ -1958,6 +1983,7 @@ const ( RTM_GETCHAIN = 0x66 RTM_GETDCB = 0x4e RTM_GETLINK = 0x12 + RTM_GETLINKPROP = 0x6e RTM_GETMDB = 0x56 RTM_GETMULTICAST = 0x3a RTM_GETNEIGH = 0x1e @@ -1971,13 +1997,14 @@ const ( RTM_GETSTATS = 0x5e RTM_GETTCLASS = 0x2a RTM_GETTFILTER = 0x2e - RTM_MAX = 0x6b + RTM_MAX = 0x6f RTM_NEWACTION = 0x30 RTM_NEWADDR = 0x14 RTM_NEWADDRLABEL = 0x48 RTM_NEWCACHEREPORT = 0x60 RTM_NEWCHAIN = 0x64 RTM_NEWLINK = 0x10 + RTM_NEWLINKPROP = 0x6c RTM_NEWMDB = 0x54 RTM_NEWNDUSEROPT = 0x44 RTM_NEWNEIGH = 0x1c @@ -1992,8 +2019,8 @@ const ( RTM_NEWSTATS = 0x5c RTM_NEWTCLASS = 0x28 RTM_NEWTFILTER = 0x2c - RTM_NR_FAMILIES = 0x17 - RTM_NR_MSGTYPES = 0x5c + RTM_NR_FAMILIES = 0x18 + RTM_NR_MSGTYPES = 0x60 RTM_SETDCB = 0x4f RTM_SETLINK = 0x13 RTM_SETNEIGHTBL = 0x43 @@ -2208,7 +2235,7 @@ const ( SOL_TLS = 0x11a SOL_X25 = 0x106 SOL_XDP = 0x11b - SOMAXCONN = 0x80 + SOMAXCONN = 0x1000 SO_ACCEPTCONN = 0x1e SO_ATTACH_BPF = 0x32 SO_ATTACH_FILTER = 0x1a @@ -2314,6 +2341,7 @@ const ( STATX_ATTR_ENCRYPTED = 0x800 STATX_ATTR_IMMUTABLE = 0x10 STATX_ATTR_NODUMP = 0x40 + STATX_ATTR_VERITY = 0x100000 STATX_BASIC_STATS = 0x7ff STATX_BLOCKS = 0x400 STATX_BTIME = 0x800 @@ -2434,6 +2462,7 @@ const ( TCP_THIN_DUPACK = 0x11 TCP_THIN_LINEAR_TIMEOUTS = 0x10 TCP_TIMESTAMP = 0x18 + TCP_TX_DELAY = 0x25 TCP_ULP = 0x1f TCP_USER_TIMEOUT = 0x12 TCP_WINDOW_CLAMP = 0xa @@ -2530,6 +2559,10 @@ const ( TIPC_ADDR_MCAST = 0x1 TIPC_ADDR_NAME = 0x2 TIPC_ADDR_NAMESEQ = 0x1 + TIPC_AEAD_ALG_NAME = 0x20 + TIPC_AEAD_KEYLEN_MAX = 0x24 + TIPC_AEAD_KEYLEN_MIN = 0x14 + TIPC_AEAD_KEY_SIZE_MAX = 0x48 TIPC_CFG_SRV = 0x0 TIPC_CLUSTER_BITS = 0xc TIPC_CLUSTER_MASK = 0xfff000 @@ -2562,6 +2595,7 @@ const ( TIPC_MCAST_REPLICAST = 0x86 TIPC_MEDIUM_IMPORTANCE = 0x1 TIPC_NODEID_LEN = 0x10 + TIPC_NODELAY = 0x8a TIPC_NODE_BITS = 0xc TIPC_NODE_MASK = 0xfff TIPC_NODE_OFFSET = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go index 865154d943..ea6bb88dea 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go @@ -243,6 +243,7 @@ const ( BPF_F_LOCK = 0x4 BPF_F_MARK_ENFORCE = 0x40 BPF_F_MARK_MANGLED_0 = 0x20 + BPF_F_MMAPABLE = 0x400 BPF_F_NO_COMMON_LRU = 0x2 BPF_F_NO_PREALLOC = 0x1 BPF_F_NUMA_NODE = 0x4 @@ -417,8 +418,10 @@ const ( CLOCK_TXFROMRX = 0x4 CLOCK_TXINT = 0x3 CLONE_ARGS_SIZE_VER0 = 0x40 + CLONE_ARGS_SIZE_VER1 = 0x50 CLONE_CHILD_CLEARTID = 0x200000 CLONE_CHILD_SETTID = 0x1000000 + CLONE_CLEAR_SIGHAND = 0x100000000 CLONE_DETACHED = 0x400000 CLONE_FILES = 0x400 CLONE_FS = 0x200 @@ -716,8 +719,9 @@ const ( FSCRYPT_POLICY_FLAGS_PAD_4 = 0x0 FSCRYPT_POLICY_FLAGS_PAD_8 = 0x1 FSCRYPT_POLICY_FLAGS_PAD_MASK = 0x3 - FSCRYPT_POLICY_FLAGS_VALID = 0x7 + FSCRYPT_POLICY_FLAGS_VALID = 0xf FSCRYPT_POLICY_FLAG_DIRECT_KEY = 0x4 + FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64 = 0x8 FSCRYPT_POLICY_V1 = 0x0 FSCRYPT_POLICY_V2 = 0x2 FS_ENCRYPTION_MODE_ADIANTUM = 0x9 @@ -747,7 +751,7 @@ const ( FS_POLICY_FLAGS_PAD_4 = 0x0 FS_POLICY_FLAGS_PAD_8 = 0x1 FS_POLICY_FLAGS_PAD_MASK = 0x3 - FS_POLICY_FLAGS_VALID = 0x7 + FS_POLICY_FLAGS_VALID = 0xf FUTEXFS_SUPER_MAGIC = 0xbad1dea F_ADD_SEALS = 0x409 F_DUPFD = 0x0 @@ -1229,6 +1233,7 @@ const ( LOOP_SET_STATUS64 = 0x4c04 LO_KEY_SIZE = 0x20 LO_NAME_SIZE = 0x40 + MADV_COLD = 0x14 MADV_DODUMP = 0x11 MADV_DOFORK = 0xb MADV_DONTDUMP = 0x10 @@ -1241,6 +1246,7 @@ const ( MADV_MERGEABLE = 0xc MADV_NOHUGEPAGE = 0xf MADV_NORMAL = 0x0 + MADV_PAGEOUT = 0x15 MADV_RANDOM = 0x1 MADV_REMOVE = 0x9 MADV_SEQUENTIAL = 0x2 @@ -1583,6 +1589,7 @@ const ( PERF_EVENT_IOC_SET_FILTER = 0x40082406 PERF_EVENT_IOC_SET_OUTPUT = 0x2405 PIPEFS_MAGIC = 0x50495045 + PPC_CMM_MAGIC = 0xc7571590 PPPIOCATTACH = 0x4004743d PPPIOCATTCHAN = 0x40047438 PPPIOCCONNECT = 0x4004743a @@ -1593,6 +1600,8 @@ const ( PPPIOCGDEBUG = 0x80047441 PPPIOCGFLAGS = 0x8004745a PPPIOCGIDLE = 0x8010743f + PPPIOCGIDLE32 = 0x8008743f + PPPIOCGIDLE64 = 0x8010743f PPPIOCGL2TPSTATS = 0x80487436 PPPIOCGMRU = 0x80047453 PPPIOCGNPMODE = 0xc008744c @@ -1930,12 +1939,28 @@ const ( RTF_UP = 0x1 RTF_WINDOW = 0x80 RTF_XRESOLVE = 0x800 + RTMGRP_DECnet_IFADDR = 0x1000 + RTMGRP_DECnet_ROUTE = 0x4000 + RTMGRP_IPV4_IFADDR = 0x10 + RTMGRP_IPV4_MROUTE = 0x20 + RTMGRP_IPV4_ROUTE = 0x40 + RTMGRP_IPV4_RULE = 0x80 + RTMGRP_IPV6_IFADDR = 0x100 + RTMGRP_IPV6_IFINFO = 0x800 + RTMGRP_IPV6_MROUTE = 0x200 + RTMGRP_IPV6_PREFIX = 0x20000 + RTMGRP_IPV6_ROUTE = 0x400 + RTMGRP_LINK = 0x1 + RTMGRP_NEIGH = 0x4 + RTMGRP_NOTIFY = 0x2 + RTMGRP_TC = 0x8 RTM_BASE = 0x10 RTM_DELACTION = 0x31 RTM_DELADDR = 0x15 RTM_DELADDRLABEL = 0x49 RTM_DELCHAIN = 0x65 RTM_DELLINK = 0x11 + RTM_DELLINKPROP = 0x6d RTM_DELMDB = 0x55 RTM_DELNEIGH = 0x1d RTM_DELNETCONF = 0x51 @@ -1959,6 +1984,7 @@ const ( RTM_GETCHAIN = 0x66 RTM_GETDCB = 0x4e RTM_GETLINK = 0x12 + RTM_GETLINKPROP = 0x6e RTM_GETMDB = 0x56 RTM_GETMULTICAST = 0x3a RTM_GETNEIGH = 0x1e @@ -1972,13 +1998,14 @@ const ( RTM_GETSTATS = 0x5e RTM_GETTCLASS = 0x2a RTM_GETTFILTER = 0x2e - RTM_MAX = 0x6b + RTM_MAX = 0x6f RTM_NEWACTION = 0x30 RTM_NEWADDR = 0x14 RTM_NEWADDRLABEL = 0x48 RTM_NEWCACHEREPORT = 0x60 RTM_NEWCHAIN = 0x64 RTM_NEWLINK = 0x10 + RTM_NEWLINKPROP = 0x6c RTM_NEWMDB = 0x54 RTM_NEWNDUSEROPT = 0x44 RTM_NEWNEIGH = 0x1c @@ -1993,8 +2020,8 @@ const ( RTM_NEWSTATS = 0x5c RTM_NEWTCLASS = 0x28 RTM_NEWTFILTER = 0x2c - RTM_NR_FAMILIES = 0x17 - RTM_NR_MSGTYPES = 0x5c + RTM_NR_FAMILIES = 0x18 + RTM_NR_MSGTYPES = 0x60 RTM_SETDCB = 0x4f RTM_SETLINK = 0x13 RTM_SETNEIGHTBL = 0x43 @@ -2209,7 +2236,7 @@ const ( SOL_TLS = 0x11a SOL_X25 = 0x106 SOL_XDP = 0x11b - SOMAXCONN = 0x80 + SOMAXCONN = 0x1000 SO_ACCEPTCONN = 0x1e SO_ATTACH_BPF = 0x32 SO_ATTACH_FILTER = 0x1a @@ -2315,6 +2342,7 @@ const ( STATX_ATTR_ENCRYPTED = 0x800 STATX_ATTR_IMMUTABLE = 0x10 STATX_ATTR_NODUMP = 0x40 + STATX_ATTR_VERITY = 0x100000 STATX_BASIC_STATS = 0x7ff STATX_BLOCKS = 0x400 STATX_BTIME = 0x800 @@ -2435,6 +2463,7 @@ const ( TCP_THIN_DUPACK = 0x11 TCP_THIN_LINEAR_TIMEOUTS = 0x10 TCP_TIMESTAMP = 0x18 + TCP_TX_DELAY = 0x25 TCP_ULP = 0x1f TCP_USER_TIMEOUT = 0x12 TCP_WINDOW_CLAMP = 0xa @@ -2531,6 +2560,10 @@ const ( TIPC_ADDR_MCAST = 0x1 TIPC_ADDR_NAME = 0x2 TIPC_ADDR_NAMESEQ = 0x1 + TIPC_AEAD_ALG_NAME = 0x20 + TIPC_AEAD_KEYLEN_MAX = 0x24 + TIPC_AEAD_KEYLEN_MIN = 0x14 + TIPC_AEAD_KEY_SIZE_MAX = 0x48 TIPC_CFG_SRV = 0x0 TIPC_CLUSTER_BITS = 0xc TIPC_CLUSTER_MASK = 0xfff000 @@ -2563,6 +2596,7 @@ const ( TIPC_MCAST_REPLICAST = 0x86 TIPC_MEDIUM_IMPORTANCE = 0x1 TIPC_NODEID_LEN = 0x10 + TIPC_NODELAY = 0x8a TIPC_NODE_BITS = 0xc TIPC_NODE_MASK = 0xfff TIPC_NODE_OFFSET = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go b/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go index 386beb4501..76b3d3b0e2 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go @@ -243,6 +243,7 @@ const ( BPF_F_LOCK = 0x4 BPF_F_MARK_ENFORCE = 0x40 BPF_F_MARK_MANGLED_0 = 0x20 + BPF_F_MMAPABLE = 0x400 BPF_F_NO_COMMON_LRU = 0x2 BPF_F_NO_PREALLOC = 0x1 BPF_F_NUMA_NODE = 0x4 @@ -417,8 +418,10 @@ const ( CLOCK_TXFROMRX = 0x4 CLOCK_TXINT = 0x3 CLONE_ARGS_SIZE_VER0 = 0x40 + CLONE_ARGS_SIZE_VER1 = 0x50 CLONE_CHILD_CLEARTID = 0x200000 CLONE_CHILD_SETTID = 0x1000000 + CLONE_CLEAR_SIGHAND = 0x100000000 CLONE_DETACHED = 0x400000 CLONE_FILES = 0x400 CLONE_FS = 0x200 @@ -715,8 +718,9 @@ const ( FSCRYPT_POLICY_FLAGS_PAD_4 = 0x0 FSCRYPT_POLICY_FLAGS_PAD_8 = 0x1 FSCRYPT_POLICY_FLAGS_PAD_MASK = 0x3 - FSCRYPT_POLICY_FLAGS_VALID = 0x7 + FSCRYPT_POLICY_FLAGS_VALID = 0xf FSCRYPT_POLICY_FLAG_DIRECT_KEY = 0x4 + FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64 = 0x8 FSCRYPT_POLICY_V1 = 0x0 FSCRYPT_POLICY_V2 = 0x2 FS_ENCRYPTION_MODE_ADIANTUM = 0x9 @@ -746,7 +750,7 @@ const ( FS_POLICY_FLAGS_PAD_4 = 0x0 FS_POLICY_FLAGS_PAD_8 = 0x1 FS_POLICY_FLAGS_PAD_MASK = 0x3 - FS_POLICY_FLAGS_VALID = 0x7 + FS_POLICY_FLAGS_VALID = 0xf FUTEXFS_SUPER_MAGIC = 0xbad1dea F_ADD_SEALS = 0x409 F_DUPFD = 0x0 @@ -1228,6 +1232,7 @@ const ( LOOP_SET_STATUS64 = 0x4c04 LO_KEY_SIZE = 0x20 LO_NAME_SIZE = 0x40 + MADV_COLD = 0x14 MADV_DODUMP = 0x11 MADV_DOFORK = 0xb MADV_DONTDUMP = 0x10 @@ -1240,6 +1245,7 @@ const ( MADV_MERGEABLE = 0xc MADV_NOHUGEPAGE = 0xf MADV_NORMAL = 0x0 + MADV_PAGEOUT = 0x15 MADV_RANDOM = 0x1 MADV_REMOVE = 0x9 MADV_SEQUENTIAL = 0x2 @@ -1581,6 +1587,7 @@ const ( PERF_EVENT_IOC_SET_FILTER = 0x40042406 PERF_EVENT_IOC_SET_OUTPUT = 0x2405 PIPEFS_MAGIC = 0x50495045 + PPC_CMM_MAGIC = 0xc7571590 PPPIOCATTACH = 0x4004743d PPPIOCATTCHAN = 0x40047438 PPPIOCCONNECT = 0x4004743a @@ -1591,6 +1598,8 @@ const ( PPPIOCGDEBUG = 0x80047441 PPPIOCGFLAGS = 0x8004745a PPPIOCGIDLE = 0x8008743f + PPPIOCGIDLE32 = 0x8008743f + PPPIOCGIDLE64 = 0x8010743f PPPIOCGL2TPSTATS = 0x80487436 PPPIOCGMRU = 0x80047453 PPPIOCGNPMODE = 0xc008744c @@ -1936,12 +1945,28 @@ const ( RTF_UP = 0x1 RTF_WINDOW = 0x80 RTF_XRESOLVE = 0x800 + RTMGRP_DECnet_IFADDR = 0x1000 + RTMGRP_DECnet_ROUTE = 0x4000 + RTMGRP_IPV4_IFADDR = 0x10 + RTMGRP_IPV4_MROUTE = 0x20 + RTMGRP_IPV4_ROUTE = 0x40 + RTMGRP_IPV4_RULE = 0x80 + RTMGRP_IPV6_IFADDR = 0x100 + RTMGRP_IPV6_IFINFO = 0x800 + RTMGRP_IPV6_MROUTE = 0x200 + RTMGRP_IPV6_PREFIX = 0x20000 + RTMGRP_IPV6_ROUTE = 0x400 + RTMGRP_LINK = 0x1 + RTMGRP_NEIGH = 0x4 + RTMGRP_NOTIFY = 0x2 + RTMGRP_TC = 0x8 RTM_BASE = 0x10 RTM_DELACTION = 0x31 RTM_DELADDR = 0x15 RTM_DELADDRLABEL = 0x49 RTM_DELCHAIN = 0x65 RTM_DELLINK = 0x11 + RTM_DELLINKPROP = 0x6d RTM_DELMDB = 0x55 RTM_DELNEIGH = 0x1d RTM_DELNETCONF = 0x51 @@ -1965,6 +1990,7 @@ const ( RTM_GETCHAIN = 0x66 RTM_GETDCB = 0x4e RTM_GETLINK = 0x12 + RTM_GETLINKPROP = 0x6e RTM_GETMDB = 0x56 RTM_GETMULTICAST = 0x3a RTM_GETNEIGH = 0x1e @@ -1978,13 +2004,14 @@ const ( RTM_GETSTATS = 0x5e RTM_GETTCLASS = 0x2a RTM_GETTFILTER = 0x2e - RTM_MAX = 0x6b + RTM_MAX = 0x6f RTM_NEWACTION = 0x30 RTM_NEWADDR = 0x14 RTM_NEWADDRLABEL = 0x48 RTM_NEWCACHEREPORT = 0x60 RTM_NEWCHAIN = 0x64 RTM_NEWLINK = 0x10 + RTM_NEWLINKPROP = 0x6c RTM_NEWMDB = 0x54 RTM_NEWNDUSEROPT = 0x44 RTM_NEWNEIGH = 0x1c @@ -1999,8 +2026,8 @@ const ( RTM_NEWSTATS = 0x5c RTM_NEWTCLASS = 0x28 RTM_NEWTFILTER = 0x2c - RTM_NR_FAMILIES = 0x17 - RTM_NR_MSGTYPES = 0x5c + RTM_NR_FAMILIES = 0x18 + RTM_NR_MSGTYPES = 0x60 RTM_SETDCB = 0x4f RTM_SETLINK = 0x13 RTM_SETNEIGHTBL = 0x43 @@ -2215,7 +2242,7 @@ const ( SOL_TLS = 0x11a SOL_X25 = 0x106 SOL_XDP = 0x11b - SOMAXCONN = 0x80 + SOMAXCONN = 0x1000 SO_ACCEPTCONN = 0x1e SO_ATTACH_BPF = 0x32 SO_ATTACH_FILTER = 0x1a @@ -2321,6 +2348,7 @@ const ( STATX_ATTR_ENCRYPTED = 0x800 STATX_ATTR_IMMUTABLE = 0x10 STATX_ATTR_NODUMP = 0x40 + STATX_ATTR_VERITY = 0x100000 STATX_BASIC_STATS = 0x7ff STATX_BLOCKS = 0x400 STATX_BTIME = 0x800 @@ -2441,6 +2469,7 @@ const ( TCP_THIN_DUPACK = 0x11 TCP_THIN_LINEAR_TIMEOUTS = 0x10 TCP_TIMESTAMP = 0x18 + TCP_TX_DELAY = 0x25 TCP_ULP = 0x1f TCP_USER_TIMEOUT = 0x12 TCP_WINDOW_CLAMP = 0xa @@ -2537,6 +2566,10 @@ const ( TIPC_ADDR_MCAST = 0x1 TIPC_ADDR_NAME = 0x2 TIPC_ADDR_NAMESEQ = 0x1 + TIPC_AEAD_ALG_NAME = 0x20 + TIPC_AEAD_KEYLEN_MAX = 0x24 + TIPC_AEAD_KEYLEN_MIN = 0x14 + TIPC_AEAD_KEY_SIZE_MAX = 0x48 TIPC_CFG_SRV = 0x0 TIPC_CLUSTER_BITS = 0xc TIPC_CLUSTER_MASK = 0xfff000 @@ -2569,6 +2602,7 @@ const ( TIPC_MCAST_REPLICAST = 0x86 TIPC_MEDIUM_IMPORTANCE = 0x1 TIPC_NODEID_LEN = 0x10 + TIPC_NODELAY = 0x8a TIPC_NODE_BITS = 0xc TIPC_NODE_MASK = 0xfff TIPC_NODE_OFFSET = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go index 09e11cc93d..1405ea5709 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go @@ -243,6 +243,7 @@ const ( BPF_F_LOCK = 0x4 BPF_F_MARK_ENFORCE = 0x40 BPF_F_MARK_MANGLED_0 = 0x20 + BPF_F_MMAPABLE = 0x400 BPF_F_NO_COMMON_LRU = 0x2 BPF_F_NO_PREALLOC = 0x1 BPF_F_NUMA_NODE = 0x4 @@ -417,8 +418,10 @@ const ( CLOCK_TXFROMRX = 0x4 CLOCK_TXINT = 0x3 CLONE_ARGS_SIZE_VER0 = 0x40 + CLONE_ARGS_SIZE_VER1 = 0x50 CLONE_CHILD_CLEARTID = 0x200000 CLONE_CHILD_SETTID = 0x1000000 + CLONE_CLEAR_SIGHAND = 0x100000000 CLONE_DETACHED = 0x400000 CLONE_FILES = 0x400 CLONE_FS = 0x200 @@ -718,8 +721,9 @@ const ( FSCRYPT_POLICY_FLAGS_PAD_4 = 0x0 FSCRYPT_POLICY_FLAGS_PAD_8 = 0x1 FSCRYPT_POLICY_FLAGS_PAD_MASK = 0x3 - FSCRYPT_POLICY_FLAGS_VALID = 0x7 + FSCRYPT_POLICY_FLAGS_VALID = 0xf FSCRYPT_POLICY_FLAG_DIRECT_KEY = 0x4 + FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64 = 0x8 FSCRYPT_POLICY_V1 = 0x0 FSCRYPT_POLICY_V2 = 0x2 FS_ENCRYPTION_MODE_ADIANTUM = 0x9 @@ -749,7 +753,7 @@ const ( FS_POLICY_FLAGS_PAD_4 = 0x0 FS_POLICY_FLAGS_PAD_8 = 0x1 FS_POLICY_FLAGS_PAD_MASK = 0x3 - FS_POLICY_FLAGS_VALID = 0x7 + FS_POLICY_FLAGS_VALID = 0xf FUTEXFS_SUPER_MAGIC = 0xbad1dea F_ADD_SEALS = 0x409 F_DUPFD = 0x0 @@ -1231,6 +1235,7 @@ const ( LOOP_SET_STATUS64 = 0x4c04 LO_KEY_SIZE = 0x20 LO_NAME_SIZE = 0x40 + MADV_COLD = 0x14 MADV_DODUMP = 0x11 MADV_DOFORK = 0xb MADV_DONTDUMP = 0x10 @@ -1243,6 +1248,7 @@ const ( MADV_MERGEABLE = 0xc MADV_NOHUGEPAGE = 0xf MADV_NORMAL = 0x0 + MADV_PAGEOUT = 0x15 MADV_RANDOM = 0x1 MADV_REMOVE = 0x9 MADV_SEQUENTIAL = 0x2 @@ -1584,6 +1590,7 @@ const ( PERF_EVENT_IOC_SET_FILTER = 0x40082406 PERF_EVENT_IOC_SET_OUTPUT = 0x2405 PIPEFS_MAGIC = 0x50495045 + PPC_CMM_MAGIC = 0xc7571590 PPPIOCATTACH = 0x4004743d PPPIOCATTCHAN = 0x40047438 PPPIOCCONNECT = 0x4004743a @@ -1594,6 +1601,8 @@ const ( PPPIOCGDEBUG = 0x80047441 PPPIOCGFLAGS = 0x8004745a PPPIOCGIDLE = 0x8010743f + PPPIOCGIDLE32 = 0x8008743f + PPPIOCGIDLE64 = 0x8010743f PPPIOCGL2TPSTATS = 0x80487436 PPPIOCGMRU = 0x80047453 PPPIOCGNPMODE = 0xc008744c @@ -1922,12 +1931,28 @@ const ( RTF_UP = 0x1 RTF_WINDOW = 0x80 RTF_XRESOLVE = 0x800 + RTMGRP_DECnet_IFADDR = 0x1000 + RTMGRP_DECnet_ROUTE = 0x4000 + RTMGRP_IPV4_IFADDR = 0x10 + RTMGRP_IPV4_MROUTE = 0x20 + RTMGRP_IPV4_ROUTE = 0x40 + RTMGRP_IPV4_RULE = 0x80 + RTMGRP_IPV6_IFADDR = 0x100 + RTMGRP_IPV6_IFINFO = 0x800 + RTMGRP_IPV6_MROUTE = 0x200 + RTMGRP_IPV6_PREFIX = 0x20000 + RTMGRP_IPV6_ROUTE = 0x400 + RTMGRP_LINK = 0x1 + RTMGRP_NEIGH = 0x4 + RTMGRP_NOTIFY = 0x2 + RTMGRP_TC = 0x8 RTM_BASE = 0x10 RTM_DELACTION = 0x31 RTM_DELADDR = 0x15 RTM_DELADDRLABEL = 0x49 RTM_DELCHAIN = 0x65 RTM_DELLINK = 0x11 + RTM_DELLINKPROP = 0x6d RTM_DELMDB = 0x55 RTM_DELNEIGH = 0x1d RTM_DELNETCONF = 0x51 @@ -1951,6 +1976,7 @@ const ( RTM_GETCHAIN = 0x66 RTM_GETDCB = 0x4e RTM_GETLINK = 0x12 + RTM_GETLINKPROP = 0x6e RTM_GETMDB = 0x56 RTM_GETMULTICAST = 0x3a RTM_GETNEIGH = 0x1e @@ -1964,13 +1990,14 @@ const ( RTM_GETSTATS = 0x5e RTM_GETTCLASS = 0x2a RTM_GETTFILTER = 0x2e - RTM_MAX = 0x6b + RTM_MAX = 0x6f RTM_NEWACTION = 0x30 RTM_NEWADDR = 0x14 RTM_NEWADDRLABEL = 0x48 RTM_NEWCACHEREPORT = 0x60 RTM_NEWCHAIN = 0x64 RTM_NEWLINK = 0x10 + RTM_NEWLINKPROP = 0x6c RTM_NEWMDB = 0x54 RTM_NEWNDUSEROPT = 0x44 RTM_NEWNEIGH = 0x1c @@ -1985,8 +2012,8 @@ const ( RTM_NEWSTATS = 0x5c RTM_NEWTCLASS = 0x28 RTM_NEWTFILTER = 0x2c - RTM_NR_FAMILIES = 0x17 - RTM_NR_MSGTYPES = 0x5c + RTM_NR_FAMILIES = 0x18 + RTM_NR_MSGTYPES = 0x60 RTM_SETDCB = 0x4f RTM_SETLINK = 0x13 RTM_SETNEIGHTBL = 0x43 @@ -2201,7 +2228,7 @@ const ( SOL_TLS = 0x11a SOL_X25 = 0x106 SOL_XDP = 0x11b - SOMAXCONN = 0x80 + SOMAXCONN = 0x1000 SO_ACCEPTCONN = 0x1e SO_ATTACH_BPF = 0x32 SO_ATTACH_FILTER = 0x1a @@ -2307,6 +2334,7 @@ const ( STATX_ATTR_ENCRYPTED = 0x800 STATX_ATTR_IMMUTABLE = 0x10 STATX_ATTR_NODUMP = 0x40 + STATX_ATTR_VERITY = 0x100000 STATX_BASIC_STATS = 0x7ff STATX_BLOCKS = 0x400 STATX_BTIME = 0x800 @@ -2428,6 +2456,7 @@ const ( TCP_THIN_DUPACK = 0x11 TCP_THIN_LINEAR_TIMEOUTS = 0x10 TCP_TIMESTAMP = 0x18 + TCP_TX_DELAY = 0x25 TCP_ULP = 0x1f TCP_USER_TIMEOUT = 0x12 TCP_WINDOW_CLAMP = 0xa @@ -2524,6 +2553,10 @@ const ( TIPC_ADDR_MCAST = 0x1 TIPC_ADDR_NAME = 0x2 TIPC_ADDR_NAMESEQ = 0x1 + TIPC_AEAD_ALG_NAME = 0x20 + TIPC_AEAD_KEYLEN_MAX = 0x24 + TIPC_AEAD_KEYLEN_MIN = 0x14 + TIPC_AEAD_KEY_SIZE_MAX = 0x48 TIPC_CFG_SRV = 0x0 TIPC_CLUSTER_BITS = 0xc TIPC_CLUSTER_MASK = 0xfff000 @@ -2556,6 +2589,7 @@ const ( TIPC_MCAST_REPLICAST = 0x86 TIPC_MEDIUM_IMPORTANCE = 0x1 TIPC_NODEID_LEN = 0x10 + TIPC_NODELAY = 0x8a TIPC_NODE_BITS = 0xc TIPC_NODE_MASK = 0xfff TIPC_NODE_OFFSET = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_mips.go b/vendor/golang.org/x/sys/unix/zerrors_linux_mips.go index e3910aa112..a6cd090e93 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_mips.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_mips.go @@ -243,6 +243,7 @@ const ( BPF_F_LOCK = 0x4 BPF_F_MARK_ENFORCE = 0x40 BPF_F_MARK_MANGLED_0 = 0x20 + BPF_F_MMAPABLE = 0x400 BPF_F_NO_COMMON_LRU = 0x2 BPF_F_NO_PREALLOC = 0x1 BPF_F_NUMA_NODE = 0x4 @@ -417,8 +418,10 @@ const ( CLOCK_TXFROMRX = 0x4 CLOCK_TXINT = 0x3 CLONE_ARGS_SIZE_VER0 = 0x40 + CLONE_ARGS_SIZE_VER1 = 0x50 CLONE_CHILD_CLEARTID = 0x200000 CLONE_CHILD_SETTID = 0x1000000 + CLONE_CLEAR_SIGHAND = 0x100000000 CLONE_DETACHED = 0x400000 CLONE_FILES = 0x400 CLONE_FS = 0x200 @@ -715,8 +718,9 @@ const ( FSCRYPT_POLICY_FLAGS_PAD_4 = 0x0 FSCRYPT_POLICY_FLAGS_PAD_8 = 0x1 FSCRYPT_POLICY_FLAGS_PAD_MASK = 0x3 - FSCRYPT_POLICY_FLAGS_VALID = 0x7 + FSCRYPT_POLICY_FLAGS_VALID = 0xf FSCRYPT_POLICY_FLAG_DIRECT_KEY = 0x4 + FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64 = 0x8 FSCRYPT_POLICY_V1 = 0x0 FSCRYPT_POLICY_V2 = 0x2 FS_ENCRYPTION_MODE_ADIANTUM = 0x9 @@ -746,7 +750,7 @@ const ( FS_POLICY_FLAGS_PAD_4 = 0x0 FS_POLICY_FLAGS_PAD_8 = 0x1 FS_POLICY_FLAGS_PAD_MASK = 0x3 - FS_POLICY_FLAGS_VALID = 0x7 + FS_POLICY_FLAGS_VALID = 0xf FUTEXFS_SUPER_MAGIC = 0xbad1dea F_ADD_SEALS = 0x409 F_DUPFD = 0x0 @@ -1228,6 +1232,7 @@ const ( LOOP_SET_STATUS64 = 0x4c04 LO_KEY_SIZE = 0x20 LO_NAME_SIZE = 0x40 + MADV_COLD = 0x14 MADV_DODUMP = 0x11 MADV_DOFORK = 0xb MADV_DONTDUMP = 0x10 @@ -1240,6 +1245,7 @@ const ( MADV_MERGEABLE = 0xc MADV_NOHUGEPAGE = 0xf MADV_NORMAL = 0x0 + MADV_PAGEOUT = 0x15 MADV_RANDOM = 0x1 MADV_REMOVE = 0x9 MADV_SEQUENTIAL = 0x2 @@ -1581,6 +1587,7 @@ const ( PERF_EVENT_IOC_SET_FILTER = 0x80042406 PERF_EVENT_IOC_SET_OUTPUT = 0x20002405 PIPEFS_MAGIC = 0x50495045 + PPC_CMM_MAGIC = 0xc7571590 PPPIOCATTACH = 0x8004743d PPPIOCATTCHAN = 0x80047438 PPPIOCCONNECT = 0x8004743a @@ -1591,6 +1598,8 @@ const ( PPPIOCGDEBUG = 0x40047441 PPPIOCGFLAGS = 0x4004745a PPPIOCGIDLE = 0x4008743f + PPPIOCGIDLE32 = 0x4008743f + PPPIOCGIDLE64 = 0x4010743f PPPIOCGL2TPSTATS = 0x40487436 PPPIOCGMRU = 0x40047453 PPPIOCGNPMODE = 0xc008744c @@ -1929,12 +1938,28 @@ const ( RTF_UP = 0x1 RTF_WINDOW = 0x80 RTF_XRESOLVE = 0x800 + RTMGRP_DECnet_IFADDR = 0x1000 + RTMGRP_DECnet_ROUTE = 0x4000 + RTMGRP_IPV4_IFADDR = 0x10 + RTMGRP_IPV4_MROUTE = 0x20 + RTMGRP_IPV4_ROUTE = 0x40 + RTMGRP_IPV4_RULE = 0x80 + RTMGRP_IPV6_IFADDR = 0x100 + RTMGRP_IPV6_IFINFO = 0x800 + RTMGRP_IPV6_MROUTE = 0x200 + RTMGRP_IPV6_PREFIX = 0x20000 + RTMGRP_IPV6_ROUTE = 0x400 + RTMGRP_LINK = 0x1 + RTMGRP_NEIGH = 0x4 + RTMGRP_NOTIFY = 0x2 + RTMGRP_TC = 0x8 RTM_BASE = 0x10 RTM_DELACTION = 0x31 RTM_DELADDR = 0x15 RTM_DELADDRLABEL = 0x49 RTM_DELCHAIN = 0x65 RTM_DELLINK = 0x11 + RTM_DELLINKPROP = 0x6d RTM_DELMDB = 0x55 RTM_DELNEIGH = 0x1d RTM_DELNETCONF = 0x51 @@ -1958,6 +1983,7 @@ const ( RTM_GETCHAIN = 0x66 RTM_GETDCB = 0x4e RTM_GETLINK = 0x12 + RTM_GETLINKPROP = 0x6e RTM_GETMDB = 0x56 RTM_GETMULTICAST = 0x3a RTM_GETNEIGH = 0x1e @@ -1971,13 +1997,14 @@ const ( RTM_GETSTATS = 0x5e RTM_GETTCLASS = 0x2a RTM_GETTFILTER = 0x2e - RTM_MAX = 0x6b + RTM_MAX = 0x6f RTM_NEWACTION = 0x30 RTM_NEWADDR = 0x14 RTM_NEWADDRLABEL = 0x48 RTM_NEWCACHEREPORT = 0x60 RTM_NEWCHAIN = 0x64 RTM_NEWLINK = 0x10 + RTM_NEWLINKPROP = 0x6c RTM_NEWMDB = 0x54 RTM_NEWNDUSEROPT = 0x44 RTM_NEWNEIGH = 0x1c @@ -1992,8 +2019,8 @@ const ( RTM_NEWSTATS = 0x5c RTM_NEWTCLASS = 0x28 RTM_NEWTFILTER = 0x2c - RTM_NR_FAMILIES = 0x17 - RTM_NR_MSGTYPES = 0x5c + RTM_NR_FAMILIES = 0x18 + RTM_NR_MSGTYPES = 0x60 RTM_SETDCB = 0x4f RTM_SETLINK = 0x13 RTM_SETNEIGHTBL = 0x43 @@ -2208,7 +2235,7 @@ const ( SOL_TLS = 0x11a SOL_X25 = 0x106 SOL_XDP = 0x11b - SOMAXCONN = 0x80 + SOMAXCONN = 0x1000 SO_ACCEPTCONN = 0x1009 SO_ATTACH_BPF = 0x32 SO_ATTACH_FILTER = 0x1a @@ -2315,6 +2342,7 @@ const ( STATX_ATTR_ENCRYPTED = 0x800 STATX_ATTR_IMMUTABLE = 0x10 STATX_ATTR_NODUMP = 0x40 + STATX_ATTR_VERITY = 0x100000 STATX_BASIC_STATS = 0x7ff STATX_BLOCKS = 0x400 STATX_BTIME = 0x800 @@ -2434,6 +2462,7 @@ const ( TCP_THIN_DUPACK = 0x11 TCP_THIN_LINEAR_TIMEOUTS = 0x10 TCP_TIMESTAMP = 0x18 + TCP_TX_DELAY = 0x25 TCP_ULP = 0x1f TCP_USER_TIMEOUT = 0x12 TCP_WINDOW_CLAMP = 0xa @@ -2532,6 +2561,10 @@ const ( TIPC_ADDR_MCAST = 0x1 TIPC_ADDR_NAME = 0x2 TIPC_ADDR_NAMESEQ = 0x1 + TIPC_AEAD_ALG_NAME = 0x20 + TIPC_AEAD_KEYLEN_MAX = 0x24 + TIPC_AEAD_KEYLEN_MIN = 0x14 + TIPC_AEAD_KEY_SIZE_MAX = 0x48 TIPC_CFG_SRV = 0x0 TIPC_CLUSTER_BITS = 0xc TIPC_CLUSTER_MASK = 0xfff000 @@ -2564,6 +2597,7 @@ const ( TIPC_MCAST_REPLICAST = 0x86 TIPC_MEDIUM_IMPORTANCE = 0x1 TIPC_NODEID_LEN = 0x10 + TIPC_NODELAY = 0x8a TIPC_NODE_BITS = 0xc TIPC_NODE_MASK = 0xfff TIPC_NODE_OFFSET = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go index 9877b7c754..9152b5f12f 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go @@ -243,6 +243,7 @@ const ( BPF_F_LOCK = 0x4 BPF_F_MARK_ENFORCE = 0x40 BPF_F_MARK_MANGLED_0 = 0x20 + BPF_F_MMAPABLE = 0x400 BPF_F_NO_COMMON_LRU = 0x2 BPF_F_NO_PREALLOC = 0x1 BPF_F_NUMA_NODE = 0x4 @@ -417,8 +418,10 @@ const ( CLOCK_TXFROMRX = 0x4 CLOCK_TXINT = 0x3 CLONE_ARGS_SIZE_VER0 = 0x40 + CLONE_ARGS_SIZE_VER1 = 0x50 CLONE_CHILD_CLEARTID = 0x200000 CLONE_CHILD_SETTID = 0x1000000 + CLONE_CLEAR_SIGHAND = 0x100000000 CLONE_DETACHED = 0x400000 CLONE_FILES = 0x400 CLONE_FS = 0x200 @@ -715,8 +718,9 @@ const ( FSCRYPT_POLICY_FLAGS_PAD_4 = 0x0 FSCRYPT_POLICY_FLAGS_PAD_8 = 0x1 FSCRYPT_POLICY_FLAGS_PAD_MASK = 0x3 - FSCRYPT_POLICY_FLAGS_VALID = 0x7 + FSCRYPT_POLICY_FLAGS_VALID = 0xf FSCRYPT_POLICY_FLAG_DIRECT_KEY = 0x4 + FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64 = 0x8 FSCRYPT_POLICY_V1 = 0x0 FSCRYPT_POLICY_V2 = 0x2 FS_ENCRYPTION_MODE_ADIANTUM = 0x9 @@ -746,7 +750,7 @@ const ( FS_POLICY_FLAGS_PAD_4 = 0x0 FS_POLICY_FLAGS_PAD_8 = 0x1 FS_POLICY_FLAGS_PAD_MASK = 0x3 - FS_POLICY_FLAGS_VALID = 0x7 + FS_POLICY_FLAGS_VALID = 0xf FUTEXFS_SUPER_MAGIC = 0xbad1dea F_ADD_SEALS = 0x409 F_DUPFD = 0x0 @@ -1228,6 +1232,7 @@ const ( LOOP_SET_STATUS64 = 0x4c04 LO_KEY_SIZE = 0x20 LO_NAME_SIZE = 0x40 + MADV_COLD = 0x14 MADV_DODUMP = 0x11 MADV_DOFORK = 0xb MADV_DONTDUMP = 0x10 @@ -1240,6 +1245,7 @@ const ( MADV_MERGEABLE = 0xc MADV_NOHUGEPAGE = 0xf MADV_NORMAL = 0x0 + MADV_PAGEOUT = 0x15 MADV_RANDOM = 0x1 MADV_REMOVE = 0x9 MADV_SEQUENTIAL = 0x2 @@ -1581,6 +1587,7 @@ const ( PERF_EVENT_IOC_SET_FILTER = 0x80082406 PERF_EVENT_IOC_SET_OUTPUT = 0x20002405 PIPEFS_MAGIC = 0x50495045 + PPC_CMM_MAGIC = 0xc7571590 PPPIOCATTACH = 0x8004743d PPPIOCATTCHAN = 0x80047438 PPPIOCCONNECT = 0x8004743a @@ -1591,6 +1598,8 @@ const ( PPPIOCGDEBUG = 0x40047441 PPPIOCGFLAGS = 0x4004745a PPPIOCGIDLE = 0x4010743f + PPPIOCGIDLE32 = 0x4008743f + PPPIOCGIDLE64 = 0x4010743f PPPIOCGL2TPSTATS = 0x40487436 PPPIOCGMRU = 0x40047453 PPPIOCGNPMODE = 0xc008744c @@ -1929,12 +1938,28 @@ const ( RTF_UP = 0x1 RTF_WINDOW = 0x80 RTF_XRESOLVE = 0x800 + RTMGRP_DECnet_IFADDR = 0x1000 + RTMGRP_DECnet_ROUTE = 0x4000 + RTMGRP_IPV4_IFADDR = 0x10 + RTMGRP_IPV4_MROUTE = 0x20 + RTMGRP_IPV4_ROUTE = 0x40 + RTMGRP_IPV4_RULE = 0x80 + RTMGRP_IPV6_IFADDR = 0x100 + RTMGRP_IPV6_IFINFO = 0x800 + RTMGRP_IPV6_MROUTE = 0x200 + RTMGRP_IPV6_PREFIX = 0x20000 + RTMGRP_IPV6_ROUTE = 0x400 + RTMGRP_LINK = 0x1 + RTMGRP_NEIGH = 0x4 + RTMGRP_NOTIFY = 0x2 + RTMGRP_TC = 0x8 RTM_BASE = 0x10 RTM_DELACTION = 0x31 RTM_DELADDR = 0x15 RTM_DELADDRLABEL = 0x49 RTM_DELCHAIN = 0x65 RTM_DELLINK = 0x11 + RTM_DELLINKPROP = 0x6d RTM_DELMDB = 0x55 RTM_DELNEIGH = 0x1d RTM_DELNETCONF = 0x51 @@ -1958,6 +1983,7 @@ const ( RTM_GETCHAIN = 0x66 RTM_GETDCB = 0x4e RTM_GETLINK = 0x12 + RTM_GETLINKPROP = 0x6e RTM_GETMDB = 0x56 RTM_GETMULTICAST = 0x3a RTM_GETNEIGH = 0x1e @@ -1971,13 +1997,14 @@ const ( RTM_GETSTATS = 0x5e RTM_GETTCLASS = 0x2a RTM_GETTFILTER = 0x2e - RTM_MAX = 0x6b + RTM_MAX = 0x6f RTM_NEWACTION = 0x30 RTM_NEWADDR = 0x14 RTM_NEWADDRLABEL = 0x48 RTM_NEWCACHEREPORT = 0x60 RTM_NEWCHAIN = 0x64 RTM_NEWLINK = 0x10 + RTM_NEWLINKPROP = 0x6c RTM_NEWMDB = 0x54 RTM_NEWNDUSEROPT = 0x44 RTM_NEWNEIGH = 0x1c @@ -1992,8 +2019,8 @@ const ( RTM_NEWSTATS = 0x5c RTM_NEWTCLASS = 0x28 RTM_NEWTFILTER = 0x2c - RTM_NR_FAMILIES = 0x17 - RTM_NR_MSGTYPES = 0x5c + RTM_NR_FAMILIES = 0x18 + RTM_NR_MSGTYPES = 0x60 RTM_SETDCB = 0x4f RTM_SETLINK = 0x13 RTM_SETNEIGHTBL = 0x43 @@ -2208,7 +2235,7 @@ const ( SOL_TLS = 0x11a SOL_X25 = 0x106 SOL_XDP = 0x11b - SOMAXCONN = 0x80 + SOMAXCONN = 0x1000 SO_ACCEPTCONN = 0x1009 SO_ATTACH_BPF = 0x32 SO_ATTACH_FILTER = 0x1a @@ -2315,6 +2342,7 @@ const ( STATX_ATTR_ENCRYPTED = 0x800 STATX_ATTR_IMMUTABLE = 0x10 STATX_ATTR_NODUMP = 0x40 + STATX_ATTR_VERITY = 0x100000 STATX_BASIC_STATS = 0x7ff STATX_BLOCKS = 0x400 STATX_BTIME = 0x800 @@ -2434,6 +2462,7 @@ const ( TCP_THIN_DUPACK = 0x11 TCP_THIN_LINEAR_TIMEOUTS = 0x10 TCP_TIMESTAMP = 0x18 + TCP_TX_DELAY = 0x25 TCP_ULP = 0x1f TCP_USER_TIMEOUT = 0x12 TCP_WINDOW_CLAMP = 0xa @@ -2532,6 +2561,10 @@ const ( TIPC_ADDR_MCAST = 0x1 TIPC_ADDR_NAME = 0x2 TIPC_ADDR_NAMESEQ = 0x1 + TIPC_AEAD_ALG_NAME = 0x20 + TIPC_AEAD_KEYLEN_MAX = 0x24 + TIPC_AEAD_KEYLEN_MIN = 0x14 + TIPC_AEAD_KEY_SIZE_MAX = 0x48 TIPC_CFG_SRV = 0x0 TIPC_CLUSTER_BITS = 0xc TIPC_CLUSTER_MASK = 0xfff000 @@ -2564,6 +2597,7 @@ const ( TIPC_MCAST_REPLICAST = 0x86 TIPC_MEDIUM_IMPORTANCE = 0x1 TIPC_NODEID_LEN = 0x10 + TIPC_NODELAY = 0x8a TIPC_NODE_BITS = 0xc TIPC_NODE_MASK = 0xfff TIPC_NODE_OFFSET = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go b/vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go index b77dd84317..e1aa146bcf 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go @@ -243,6 +243,7 @@ const ( BPF_F_LOCK = 0x4 BPF_F_MARK_ENFORCE = 0x40 BPF_F_MARK_MANGLED_0 = 0x20 + BPF_F_MMAPABLE = 0x400 BPF_F_NO_COMMON_LRU = 0x2 BPF_F_NO_PREALLOC = 0x1 BPF_F_NUMA_NODE = 0x4 @@ -417,8 +418,10 @@ const ( CLOCK_TXFROMRX = 0x4 CLOCK_TXINT = 0x3 CLONE_ARGS_SIZE_VER0 = 0x40 + CLONE_ARGS_SIZE_VER1 = 0x50 CLONE_CHILD_CLEARTID = 0x200000 CLONE_CHILD_SETTID = 0x1000000 + CLONE_CLEAR_SIGHAND = 0x100000000 CLONE_DETACHED = 0x400000 CLONE_FILES = 0x400 CLONE_FS = 0x200 @@ -715,8 +718,9 @@ const ( FSCRYPT_POLICY_FLAGS_PAD_4 = 0x0 FSCRYPT_POLICY_FLAGS_PAD_8 = 0x1 FSCRYPT_POLICY_FLAGS_PAD_MASK = 0x3 - FSCRYPT_POLICY_FLAGS_VALID = 0x7 + FSCRYPT_POLICY_FLAGS_VALID = 0xf FSCRYPT_POLICY_FLAG_DIRECT_KEY = 0x4 + FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64 = 0x8 FSCRYPT_POLICY_V1 = 0x0 FSCRYPT_POLICY_V2 = 0x2 FS_ENCRYPTION_MODE_ADIANTUM = 0x9 @@ -746,7 +750,7 @@ const ( FS_POLICY_FLAGS_PAD_4 = 0x0 FS_POLICY_FLAGS_PAD_8 = 0x1 FS_POLICY_FLAGS_PAD_MASK = 0x3 - FS_POLICY_FLAGS_VALID = 0x7 + FS_POLICY_FLAGS_VALID = 0xf FUTEXFS_SUPER_MAGIC = 0xbad1dea F_ADD_SEALS = 0x409 F_DUPFD = 0x0 @@ -1228,6 +1232,7 @@ const ( LOOP_SET_STATUS64 = 0x4c04 LO_KEY_SIZE = 0x20 LO_NAME_SIZE = 0x40 + MADV_COLD = 0x14 MADV_DODUMP = 0x11 MADV_DOFORK = 0xb MADV_DONTDUMP = 0x10 @@ -1240,6 +1245,7 @@ const ( MADV_MERGEABLE = 0xc MADV_NOHUGEPAGE = 0xf MADV_NORMAL = 0x0 + MADV_PAGEOUT = 0x15 MADV_RANDOM = 0x1 MADV_REMOVE = 0x9 MADV_SEQUENTIAL = 0x2 @@ -1581,6 +1587,7 @@ const ( PERF_EVENT_IOC_SET_FILTER = 0x80082406 PERF_EVENT_IOC_SET_OUTPUT = 0x20002405 PIPEFS_MAGIC = 0x50495045 + PPC_CMM_MAGIC = 0xc7571590 PPPIOCATTACH = 0x8004743d PPPIOCATTCHAN = 0x80047438 PPPIOCCONNECT = 0x8004743a @@ -1591,6 +1598,8 @@ const ( PPPIOCGDEBUG = 0x40047441 PPPIOCGFLAGS = 0x4004745a PPPIOCGIDLE = 0x4010743f + PPPIOCGIDLE32 = 0x4008743f + PPPIOCGIDLE64 = 0x4010743f PPPIOCGL2TPSTATS = 0x40487436 PPPIOCGMRU = 0x40047453 PPPIOCGNPMODE = 0xc008744c @@ -1929,12 +1938,28 @@ const ( RTF_UP = 0x1 RTF_WINDOW = 0x80 RTF_XRESOLVE = 0x800 + RTMGRP_DECnet_IFADDR = 0x1000 + RTMGRP_DECnet_ROUTE = 0x4000 + RTMGRP_IPV4_IFADDR = 0x10 + RTMGRP_IPV4_MROUTE = 0x20 + RTMGRP_IPV4_ROUTE = 0x40 + RTMGRP_IPV4_RULE = 0x80 + RTMGRP_IPV6_IFADDR = 0x100 + RTMGRP_IPV6_IFINFO = 0x800 + RTMGRP_IPV6_MROUTE = 0x200 + RTMGRP_IPV6_PREFIX = 0x20000 + RTMGRP_IPV6_ROUTE = 0x400 + RTMGRP_LINK = 0x1 + RTMGRP_NEIGH = 0x4 + RTMGRP_NOTIFY = 0x2 + RTMGRP_TC = 0x8 RTM_BASE = 0x10 RTM_DELACTION = 0x31 RTM_DELADDR = 0x15 RTM_DELADDRLABEL = 0x49 RTM_DELCHAIN = 0x65 RTM_DELLINK = 0x11 + RTM_DELLINKPROP = 0x6d RTM_DELMDB = 0x55 RTM_DELNEIGH = 0x1d RTM_DELNETCONF = 0x51 @@ -1958,6 +1983,7 @@ const ( RTM_GETCHAIN = 0x66 RTM_GETDCB = 0x4e RTM_GETLINK = 0x12 + RTM_GETLINKPROP = 0x6e RTM_GETMDB = 0x56 RTM_GETMULTICAST = 0x3a RTM_GETNEIGH = 0x1e @@ -1971,13 +1997,14 @@ const ( RTM_GETSTATS = 0x5e RTM_GETTCLASS = 0x2a RTM_GETTFILTER = 0x2e - RTM_MAX = 0x6b + RTM_MAX = 0x6f RTM_NEWACTION = 0x30 RTM_NEWADDR = 0x14 RTM_NEWADDRLABEL = 0x48 RTM_NEWCACHEREPORT = 0x60 RTM_NEWCHAIN = 0x64 RTM_NEWLINK = 0x10 + RTM_NEWLINKPROP = 0x6c RTM_NEWMDB = 0x54 RTM_NEWNDUSEROPT = 0x44 RTM_NEWNEIGH = 0x1c @@ -1992,8 +2019,8 @@ const ( RTM_NEWSTATS = 0x5c RTM_NEWTCLASS = 0x28 RTM_NEWTFILTER = 0x2c - RTM_NR_FAMILIES = 0x17 - RTM_NR_MSGTYPES = 0x5c + RTM_NR_FAMILIES = 0x18 + RTM_NR_MSGTYPES = 0x60 RTM_SETDCB = 0x4f RTM_SETLINK = 0x13 RTM_SETNEIGHTBL = 0x43 @@ -2208,7 +2235,7 @@ const ( SOL_TLS = 0x11a SOL_X25 = 0x106 SOL_XDP = 0x11b - SOMAXCONN = 0x80 + SOMAXCONN = 0x1000 SO_ACCEPTCONN = 0x1009 SO_ATTACH_BPF = 0x32 SO_ATTACH_FILTER = 0x1a @@ -2315,6 +2342,7 @@ const ( STATX_ATTR_ENCRYPTED = 0x800 STATX_ATTR_IMMUTABLE = 0x10 STATX_ATTR_NODUMP = 0x40 + STATX_ATTR_VERITY = 0x100000 STATX_BASIC_STATS = 0x7ff STATX_BLOCKS = 0x400 STATX_BTIME = 0x800 @@ -2434,6 +2462,7 @@ const ( TCP_THIN_DUPACK = 0x11 TCP_THIN_LINEAR_TIMEOUTS = 0x10 TCP_TIMESTAMP = 0x18 + TCP_TX_DELAY = 0x25 TCP_ULP = 0x1f TCP_USER_TIMEOUT = 0x12 TCP_WINDOW_CLAMP = 0xa @@ -2532,6 +2561,10 @@ const ( TIPC_ADDR_MCAST = 0x1 TIPC_ADDR_NAME = 0x2 TIPC_ADDR_NAMESEQ = 0x1 + TIPC_AEAD_ALG_NAME = 0x20 + TIPC_AEAD_KEYLEN_MAX = 0x24 + TIPC_AEAD_KEYLEN_MIN = 0x14 + TIPC_AEAD_KEY_SIZE_MAX = 0x48 TIPC_CFG_SRV = 0x0 TIPC_CLUSTER_BITS = 0xc TIPC_CLUSTER_MASK = 0xfff000 @@ -2564,6 +2597,7 @@ const ( TIPC_MCAST_REPLICAST = 0x86 TIPC_MEDIUM_IMPORTANCE = 0x1 TIPC_NODEID_LEN = 0x10 + TIPC_NODELAY = 0x8a TIPC_NODE_BITS = 0xc TIPC_NODE_MASK = 0xfff TIPC_NODE_OFFSET = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go b/vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go index 966d025f71..d23b3a9441 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go @@ -243,6 +243,7 @@ const ( BPF_F_LOCK = 0x4 BPF_F_MARK_ENFORCE = 0x40 BPF_F_MARK_MANGLED_0 = 0x20 + BPF_F_MMAPABLE = 0x400 BPF_F_NO_COMMON_LRU = 0x2 BPF_F_NO_PREALLOC = 0x1 BPF_F_NUMA_NODE = 0x4 @@ -417,8 +418,10 @@ const ( CLOCK_TXFROMRX = 0x4 CLOCK_TXINT = 0x3 CLONE_ARGS_SIZE_VER0 = 0x40 + CLONE_ARGS_SIZE_VER1 = 0x50 CLONE_CHILD_CLEARTID = 0x200000 CLONE_CHILD_SETTID = 0x1000000 + CLONE_CLEAR_SIGHAND = 0x100000000 CLONE_DETACHED = 0x400000 CLONE_FILES = 0x400 CLONE_FS = 0x200 @@ -715,8 +718,9 @@ const ( FSCRYPT_POLICY_FLAGS_PAD_4 = 0x0 FSCRYPT_POLICY_FLAGS_PAD_8 = 0x1 FSCRYPT_POLICY_FLAGS_PAD_MASK = 0x3 - FSCRYPT_POLICY_FLAGS_VALID = 0x7 + FSCRYPT_POLICY_FLAGS_VALID = 0xf FSCRYPT_POLICY_FLAG_DIRECT_KEY = 0x4 + FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64 = 0x8 FSCRYPT_POLICY_V1 = 0x0 FSCRYPT_POLICY_V2 = 0x2 FS_ENCRYPTION_MODE_ADIANTUM = 0x9 @@ -746,7 +750,7 @@ const ( FS_POLICY_FLAGS_PAD_4 = 0x0 FS_POLICY_FLAGS_PAD_8 = 0x1 FS_POLICY_FLAGS_PAD_MASK = 0x3 - FS_POLICY_FLAGS_VALID = 0x7 + FS_POLICY_FLAGS_VALID = 0xf FUTEXFS_SUPER_MAGIC = 0xbad1dea F_ADD_SEALS = 0x409 F_DUPFD = 0x0 @@ -1228,6 +1232,7 @@ const ( LOOP_SET_STATUS64 = 0x4c04 LO_KEY_SIZE = 0x20 LO_NAME_SIZE = 0x40 + MADV_COLD = 0x14 MADV_DODUMP = 0x11 MADV_DOFORK = 0xb MADV_DONTDUMP = 0x10 @@ -1240,6 +1245,7 @@ const ( MADV_MERGEABLE = 0xc MADV_NOHUGEPAGE = 0xf MADV_NORMAL = 0x0 + MADV_PAGEOUT = 0x15 MADV_RANDOM = 0x1 MADV_REMOVE = 0x9 MADV_SEQUENTIAL = 0x2 @@ -1581,6 +1587,7 @@ const ( PERF_EVENT_IOC_SET_FILTER = 0x80042406 PERF_EVENT_IOC_SET_OUTPUT = 0x20002405 PIPEFS_MAGIC = 0x50495045 + PPC_CMM_MAGIC = 0xc7571590 PPPIOCATTACH = 0x8004743d PPPIOCATTCHAN = 0x80047438 PPPIOCCONNECT = 0x8004743a @@ -1591,6 +1598,8 @@ const ( PPPIOCGDEBUG = 0x40047441 PPPIOCGFLAGS = 0x4004745a PPPIOCGIDLE = 0x4008743f + PPPIOCGIDLE32 = 0x4008743f + PPPIOCGIDLE64 = 0x4010743f PPPIOCGL2TPSTATS = 0x40487436 PPPIOCGMRU = 0x40047453 PPPIOCGNPMODE = 0xc008744c @@ -1929,12 +1938,28 @@ const ( RTF_UP = 0x1 RTF_WINDOW = 0x80 RTF_XRESOLVE = 0x800 + RTMGRP_DECnet_IFADDR = 0x1000 + RTMGRP_DECnet_ROUTE = 0x4000 + RTMGRP_IPV4_IFADDR = 0x10 + RTMGRP_IPV4_MROUTE = 0x20 + RTMGRP_IPV4_ROUTE = 0x40 + RTMGRP_IPV4_RULE = 0x80 + RTMGRP_IPV6_IFADDR = 0x100 + RTMGRP_IPV6_IFINFO = 0x800 + RTMGRP_IPV6_MROUTE = 0x200 + RTMGRP_IPV6_PREFIX = 0x20000 + RTMGRP_IPV6_ROUTE = 0x400 + RTMGRP_LINK = 0x1 + RTMGRP_NEIGH = 0x4 + RTMGRP_NOTIFY = 0x2 + RTMGRP_TC = 0x8 RTM_BASE = 0x10 RTM_DELACTION = 0x31 RTM_DELADDR = 0x15 RTM_DELADDRLABEL = 0x49 RTM_DELCHAIN = 0x65 RTM_DELLINK = 0x11 + RTM_DELLINKPROP = 0x6d RTM_DELMDB = 0x55 RTM_DELNEIGH = 0x1d RTM_DELNETCONF = 0x51 @@ -1958,6 +1983,7 @@ const ( RTM_GETCHAIN = 0x66 RTM_GETDCB = 0x4e RTM_GETLINK = 0x12 + RTM_GETLINKPROP = 0x6e RTM_GETMDB = 0x56 RTM_GETMULTICAST = 0x3a RTM_GETNEIGH = 0x1e @@ -1971,13 +1997,14 @@ const ( RTM_GETSTATS = 0x5e RTM_GETTCLASS = 0x2a RTM_GETTFILTER = 0x2e - RTM_MAX = 0x6b + RTM_MAX = 0x6f RTM_NEWACTION = 0x30 RTM_NEWADDR = 0x14 RTM_NEWADDRLABEL = 0x48 RTM_NEWCACHEREPORT = 0x60 RTM_NEWCHAIN = 0x64 RTM_NEWLINK = 0x10 + RTM_NEWLINKPROP = 0x6c RTM_NEWMDB = 0x54 RTM_NEWNDUSEROPT = 0x44 RTM_NEWNEIGH = 0x1c @@ -1992,8 +2019,8 @@ const ( RTM_NEWSTATS = 0x5c RTM_NEWTCLASS = 0x28 RTM_NEWTFILTER = 0x2c - RTM_NR_FAMILIES = 0x17 - RTM_NR_MSGTYPES = 0x5c + RTM_NR_FAMILIES = 0x18 + RTM_NR_MSGTYPES = 0x60 RTM_SETDCB = 0x4f RTM_SETLINK = 0x13 RTM_SETNEIGHTBL = 0x43 @@ -2208,7 +2235,7 @@ const ( SOL_TLS = 0x11a SOL_X25 = 0x106 SOL_XDP = 0x11b - SOMAXCONN = 0x80 + SOMAXCONN = 0x1000 SO_ACCEPTCONN = 0x1009 SO_ATTACH_BPF = 0x32 SO_ATTACH_FILTER = 0x1a @@ -2315,6 +2342,7 @@ const ( STATX_ATTR_ENCRYPTED = 0x800 STATX_ATTR_IMMUTABLE = 0x10 STATX_ATTR_NODUMP = 0x40 + STATX_ATTR_VERITY = 0x100000 STATX_BASIC_STATS = 0x7ff STATX_BLOCKS = 0x400 STATX_BTIME = 0x800 @@ -2434,6 +2462,7 @@ const ( TCP_THIN_DUPACK = 0x11 TCP_THIN_LINEAR_TIMEOUTS = 0x10 TCP_TIMESTAMP = 0x18 + TCP_TX_DELAY = 0x25 TCP_ULP = 0x1f TCP_USER_TIMEOUT = 0x12 TCP_WINDOW_CLAMP = 0xa @@ -2532,6 +2561,10 @@ const ( TIPC_ADDR_MCAST = 0x1 TIPC_ADDR_NAME = 0x2 TIPC_ADDR_NAMESEQ = 0x1 + TIPC_AEAD_ALG_NAME = 0x20 + TIPC_AEAD_KEYLEN_MAX = 0x24 + TIPC_AEAD_KEYLEN_MIN = 0x14 + TIPC_AEAD_KEY_SIZE_MAX = 0x48 TIPC_CFG_SRV = 0x0 TIPC_CLUSTER_BITS = 0xc TIPC_CLUSTER_MASK = 0xfff000 @@ -2564,6 +2597,7 @@ const ( TIPC_MCAST_REPLICAST = 0x86 TIPC_MEDIUM_IMPORTANCE = 0x1 TIPC_NODEID_LEN = 0x10 + TIPC_NODELAY = 0x8a TIPC_NODE_BITS = 0xc TIPC_NODE_MASK = 0xfff TIPC_NODE_OFFSET = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go index 3baadbe2fc..ab6134ebb9 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go @@ -243,6 +243,7 @@ const ( BPF_F_LOCK = 0x4 BPF_F_MARK_ENFORCE = 0x40 BPF_F_MARK_MANGLED_0 = 0x20 + BPF_F_MMAPABLE = 0x400 BPF_F_NO_COMMON_LRU = 0x2 BPF_F_NO_PREALLOC = 0x1 BPF_F_NUMA_NODE = 0x4 @@ -417,8 +418,10 @@ const ( CLOCK_TXFROMRX = 0x4 CLOCK_TXINT = 0x3 CLONE_ARGS_SIZE_VER0 = 0x40 + CLONE_ARGS_SIZE_VER1 = 0x50 CLONE_CHILD_CLEARTID = 0x200000 CLONE_CHILD_SETTID = 0x1000000 + CLONE_CLEAR_SIGHAND = 0x100000000 CLONE_DETACHED = 0x400000 CLONE_FILES = 0x400 CLONE_FS = 0x200 @@ -715,8 +718,9 @@ const ( FSCRYPT_POLICY_FLAGS_PAD_4 = 0x0 FSCRYPT_POLICY_FLAGS_PAD_8 = 0x1 FSCRYPT_POLICY_FLAGS_PAD_MASK = 0x3 - FSCRYPT_POLICY_FLAGS_VALID = 0x7 + FSCRYPT_POLICY_FLAGS_VALID = 0xf FSCRYPT_POLICY_FLAG_DIRECT_KEY = 0x4 + FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64 = 0x8 FSCRYPT_POLICY_V1 = 0x0 FSCRYPT_POLICY_V2 = 0x2 FS_ENCRYPTION_MODE_ADIANTUM = 0x9 @@ -746,7 +750,7 @@ const ( FS_POLICY_FLAGS_PAD_4 = 0x0 FS_POLICY_FLAGS_PAD_8 = 0x1 FS_POLICY_FLAGS_PAD_MASK = 0x3 - FS_POLICY_FLAGS_VALID = 0x7 + FS_POLICY_FLAGS_VALID = 0xf FUTEXFS_SUPER_MAGIC = 0xbad1dea F_ADD_SEALS = 0x409 F_DUPFD = 0x0 @@ -1228,6 +1232,7 @@ const ( LOOP_SET_STATUS64 = 0x4c04 LO_KEY_SIZE = 0x20 LO_NAME_SIZE = 0x40 + MADV_COLD = 0x14 MADV_DODUMP = 0x11 MADV_DOFORK = 0xb MADV_DONTDUMP = 0x10 @@ -1240,6 +1245,7 @@ const ( MADV_MERGEABLE = 0xc MADV_NOHUGEPAGE = 0xf MADV_NORMAL = 0x0 + MADV_PAGEOUT = 0x15 MADV_RANDOM = 0x1 MADV_REMOVE = 0x9 MADV_SEQUENTIAL = 0x2 @@ -1265,6 +1271,7 @@ const ( MAP_SHARED = 0x1 MAP_SHARED_VALIDATE = 0x3 MAP_STACK = 0x20000 + MAP_SYNC = 0x80000 MAP_TYPE = 0xf MCAST_BLOCK_SOURCE = 0x2b MCAST_EXCLUDE = 0x0 @@ -1582,6 +1589,7 @@ const ( PERF_EVENT_IOC_SET_FILTER = 0x80082406 PERF_EVENT_IOC_SET_OUTPUT = 0x20002405 PIPEFS_MAGIC = 0x50495045 + PPC_CMM_MAGIC = 0xc7571590 PPPIOCATTACH = 0x8004743d PPPIOCATTCHAN = 0x80047438 PPPIOCCONNECT = 0x8004743a @@ -1592,6 +1600,8 @@ const ( PPPIOCGDEBUG = 0x40047441 PPPIOCGFLAGS = 0x4004745a PPPIOCGIDLE = 0x4010743f + PPPIOCGIDLE32 = 0x4008743f + PPPIOCGIDLE64 = 0x4010743f PPPIOCGL2TPSTATS = 0x40487436 PPPIOCGMRU = 0x40047453 PPPIOCGNPMODE = 0xc008744c @@ -1987,12 +1997,28 @@ const ( RTF_UP = 0x1 RTF_WINDOW = 0x80 RTF_XRESOLVE = 0x800 + RTMGRP_DECnet_IFADDR = 0x1000 + RTMGRP_DECnet_ROUTE = 0x4000 + RTMGRP_IPV4_IFADDR = 0x10 + RTMGRP_IPV4_MROUTE = 0x20 + RTMGRP_IPV4_ROUTE = 0x40 + RTMGRP_IPV4_RULE = 0x80 + RTMGRP_IPV6_IFADDR = 0x100 + RTMGRP_IPV6_IFINFO = 0x800 + RTMGRP_IPV6_MROUTE = 0x200 + RTMGRP_IPV6_PREFIX = 0x20000 + RTMGRP_IPV6_ROUTE = 0x400 + RTMGRP_LINK = 0x1 + RTMGRP_NEIGH = 0x4 + RTMGRP_NOTIFY = 0x2 + RTMGRP_TC = 0x8 RTM_BASE = 0x10 RTM_DELACTION = 0x31 RTM_DELADDR = 0x15 RTM_DELADDRLABEL = 0x49 RTM_DELCHAIN = 0x65 RTM_DELLINK = 0x11 + RTM_DELLINKPROP = 0x6d RTM_DELMDB = 0x55 RTM_DELNEIGH = 0x1d RTM_DELNETCONF = 0x51 @@ -2016,6 +2042,7 @@ const ( RTM_GETCHAIN = 0x66 RTM_GETDCB = 0x4e RTM_GETLINK = 0x12 + RTM_GETLINKPROP = 0x6e RTM_GETMDB = 0x56 RTM_GETMULTICAST = 0x3a RTM_GETNEIGH = 0x1e @@ -2029,13 +2056,14 @@ const ( RTM_GETSTATS = 0x5e RTM_GETTCLASS = 0x2a RTM_GETTFILTER = 0x2e - RTM_MAX = 0x6b + RTM_MAX = 0x6f RTM_NEWACTION = 0x30 RTM_NEWADDR = 0x14 RTM_NEWADDRLABEL = 0x48 RTM_NEWCACHEREPORT = 0x60 RTM_NEWCHAIN = 0x64 RTM_NEWLINK = 0x10 + RTM_NEWLINKPROP = 0x6c RTM_NEWMDB = 0x54 RTM_NEWNDUSEROPT = 0x44 RTM_NEWNEIGH = 0x1c @@ -2050,8 +2078,8 @@ const ( RTM_NEWSTATS = 0x5c RTM_NEWTCLASS = 0x28 RTM_NEWTFILTER = 0x2c - RTM_NR_FAMILIES = 0x17 - RTM_NR_MSGTYPES = 0x5c + RTM_NR_FAMILIES = 0x18 + RTM_NR_MSGTYPES = 0x60 RTM_SETDCB = 0x4f RTM_SETLINK = 0x13 RTM_SETNEIGHTBL = 0x43 @@ -2266,7 +2294,7 @@ const ( SOL_TLS = 0x11a SOL_X25 = 0x106 SOL_XDP = 0x11b - SOMAXCONN = 0x80 + SOMAXCONN = 0x1000 SO_ACCEPTCONN = 0x1e SO_ATTACH_BPF = 0x32 SO_ATTACH_FILTER = 0x1a @@ -2372,6 +2400,7 @@ const ( STATX_ATTR_ENCRYPTED = 0x800 STATX_ATTR_IMMUTABLE = 0x10 STATX_ATTR_NODUMP = 0x40 + STATX_ATTR_VERITY = 0x100000 STATX_BASIC_STATS = 0x7ff STATX_BLOCKS = 0x400 STATX_BTIME = 0x800 @@ -2490,6 +2519,7 @@ const ( TCP_THIN_DUPACK = 0x11 TCP_THIN_LINEAR_TIMEOUTS = 0x10 TCP_TIMESTAMP = 0x18 + TCP_TX_DELAY = 0x25 TCP_ULP = 0x1f TCP_USER_TIMEOUT = 0x12 TCP_WINDOW_CLAMP = 0xa @@ -2592,6 +2622,10 @@ const ( TIPC_ADDR_MCAST = 0x1 TIPC_ADDR_NAME = 0x2 TIPC_ADDR_NAMESEQ = 0x1 + TIPC_AEAD_ALG_NAME = 0x20 + TIPC_AEAD_KEYLEN_MAX = 0x24 + TIPC_AEAD_KEYLEN_MIN = 0x14 + TIPC_AEAD_KEY_SIZE_MAX = 0x48 TIPC_CFG_SRV = 0x0 TIPC_CLUSTER_BITS = 0xc TIPC_CLUSTER_MASK = 0xfff000 @@ -2624,6 +2658,7 @@ const ( TIPC_MCAST_REPLICAST = 0x86 TIPC_MEDIUM_IMPORTANCE = 0x1 TIPC_NODEID_LEN = 0x10 + TIPC_NODELAY = 0x8a TIPC_NODE_BITS = 0xc TIPC_NODE_MASK = 0xfff TIPC_NODE_OFFSET = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go index b0bc487687..dc8cafffe1 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go @@ -243,6 +243,7 @@ const ( BPF_F_LOCK = 0x4 BPF_F_MARK_ENFORCE = 0x40 BPF_F_MARK_MANGLED_0 = 0x20 + BPF_F_MMAPABLE = 0x400 BPF_F_NO_COMMON_LRU = 0x2 BPF_F_NO_PREALLOC = 0x1 BPF_F_NUMA_NODE = 0x4 @@ -417,8 +418,10 @@ const ( CLOCK_TXFROMRX = 0x4 CLOCK_TXINT = 0x3 CLONE_ARGS_SIZE_VER0 = 0x40 + CLONE_ARGS_SIZE_VER1 = 0x50 CLONE_CHILD_CLEARTID = 0x200000 CLONE_CHILD_SETTID = 0x1000000 + CLONE_CLEAR_SIGHAND = 0x100000000 CLONE_DETACHED = 0x400000 CLONE_FILES = 0x400 CLONE_FS = 0x200 @@ -715,8 +718,9 @@ const ( FSCRYPT_POLICY_FLAGS_PAD_4 = 0x0 FSCRYPT_POLICY_FLAGS_PAD_8 = 0x1 FSCRYPT_POLICY_FLAGS_PAD_MASK = 0x3 - FSCRYPT_POLICY_FLAGS_VALID = 0x7 + FSCRYPT_POLICY_FLAGS_VALID = 0xf FSCRYPT_POLICY_FLAG_DIRECT_KEY = 0x4 + FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64 = 0x8 FSCRYPT_POLICY_V1 = 0x0 FSCRYPT_POLICY_V2 = 0x2 FS_ENCRYPTION_MODE_ADIANTUM = 0x9 @@ -746,7 +750,7 @@ const ( FS_POLICY_FLAGS_PAD_4 = 0x0 FS_POLICY_FLAGS_PAD_8 = 0x1 FS_POLICY_FLAGS_PAD_MASK = 0x3 - FS_POLICY_FLAGS_VALID = 0x7 + FS_POLICY_FLAGS_VALID = 0xf FUTEXFS_SUPER_MAGIC = 0xbad1dea F_ADD_SEALS = 0x409 F_DUPFD = 0x0 @@ -1228,6 +1232,7 @@ const ( LOOP_SET_STATUS64 = 0x4c04 LO_KEY_SIZE = 0x20 LO_NAME_SIZE = 0x40 + MADV_COLD = 0x14 MADV_DODUMP = 0x11 MADV_DOFORK = 0xb MADV_DONTDUMP = 0x10 @@ -1240,6 +1245,7 @@ const ( MADV_MERGEABLE = 0xc MADV_NOHUGEPAGE = 0xf MADV_NORMAL = 0x0 + MADV_PAGEOUT = 0x15 MADV_RANDOM = 0x1 MADV_REMOVE = 0x9 MADV_SEQUENTIAL = 0x2 @@ -1265,6 +1271,7 @@ const ( MAP_SHARED = 0x1 MAP_SHARED_VALIDATE = 0x3 MAP_STACK = 0x20000 + MAP_SYNC = 0x80000 MAP_TYPE = 0xf MCAST_BLOCK_SOURCE = 0x2b MCAST_EXCLUDE = 0x0 @@ -1582,6 +1589,7 @@ const ( PERF_EVENT_IOC_SET_FILTER = 0x80082406 PERF_EVENT_IOC_SET_OUTPUT = 0x20002405 PIPEFS_MAGIC = 0x50495045 + PPC_CMM_MAGIC = 0xc7571590 PPPIOCATTACH = 0x8004743d PPPIOCATTCHAN = 0x80047438 PPPIOCCONNECT = 0x8004743a @@ -1592,6 +1600,8 @@ const ( PPPIOCGDEBUG = 0x40047441 PPPIOCGFLAGS = 0x4004745a PPPIOCGIDLE = 0x4010743f + PPPIOCGIDLE32 = 0x4008743f + PPPIOCGIDLE64 = 0x4010743f PPPIOCGL2TPSTATS = 0x40487436 PPPIOCGMRU = 0x40047453 PPPIOCGNPMODE = 0xc008744c @@ -1987,12 +1997,28 @@ const ( RTF_UP = 0x1 RTF_WINDOW = 0x80 RTF_XRESOLVE = 0x800 + RTMGRP_DECnet_IFADDR = 0x1000 + RTMGRP_DECnet_ROUTE = 0x4000 + RTMGRP_IPV4_IFADDR = 0x10 + RTMGRP_IPV4_MROUTE = 0x20 + RTMGRP_IPV4_ROUTE = 0x40 + RTMGRP_IPV4_RULE = 0x80 + RTMGRP_IPV6_IFADDR = 0x100 + RTMGRP_IPV6_IFINFO = 0x800 + RTMGRP_IPV6_MROUTE = 0x200 + RTMGRP_IPV6_PREFIX = 0x20000 + RTMGRP_IPV6_ROUTE = 0x400 + RTMGRP_LINK = 0x1 + RTMGRP_NEIGH = 0x4 + RTMGRP_NOTIFY = 0x2 + RTMGRP_TC = 0x8 RTM_BASE = 0x10 RTM_DELACTION = 0x31 RTM_DELADDR = 0x15 RTM_DELADDRLABEL = 0x49 RTM_DELCHAIN = 0x65 RTM_DELLINK = 0x11 + RTM_DELLINKPROP = 0x6d RTM_DELMDB = 0x55 RTM_DELNEIGH = 0x1d RTM_DELNETCONF = 0x51 @@ -2016,6 +2042,7 @@ const ( RTM_GETCHAIN = 0x66 RTM_GETDCB = 0x4e RTM_GETLINK = 0x12 + RTM_GETLINKPROP = 0x6e RTM_GETMDB = 0x56 RTM_GETMULTICAST = 0x3a RTM_GETNEIGH = 0x1e @@ -2029,13 +2056,14 @@ const ( RTM_GETSTATS = 0x5e RTM_GETTCLASS = 0x2a RTM_GETTFILTER = 0x2e - RTM_MAX = 0x6b + RTM_MAX = 0x6f RTM_NEWACTION = 0x30 RTM_NEWADDR = 0x14 RTM_NEWADDRLABEL = 0x48 RTM_NEWCACHEREPORT = 0x60 RTM_NEWCHAIN = 0x64 RTM_NEWLINK = 0x10 + RTM_NEWLINKPROP = 0x6c RTM_NEWMDB = 0x54 RTM_NEWNDUSEROPT = 0x44 RTM_NEWNEIGH = 0x1c @@ -2050,8 +2078,8 @@ const ( RTM_NEWSTATS = 0x5c RTM_NEWTCLASS = 0x28 RTM_NEWTFILTER = 0x2c - RTM_NR_FAMILIES = 0x17 - RTM_NR_MSGTYPES = 0x5c + RTM_NR_FAMILIES = 0x18 + RTM_NR_MSGTYPES = 0x60 RTM_SETDCB = 0x4f RTM_SETLINK = 0x13 RTM_SETNEIGHTBL = 0x43 @@ -2266,7 +2294,7 @@ const ( SOL_TLS = 0x11a SOL_X25 = 0x106 SOL_XDP = 0x11b - SOMAXCONN = 0x80 + SOMAXCONN = 0x1000 SO_ACCEPTCONN = 0x1e SO_ATTACH_BPF = 0x32 SO_ATTACH_FILTER = 0x1a @@ -2372,6 +2400,7 @@ const ( STATX_ATTR_ENCRYPTED = 0x800 STATX_ATTR_IMMUTABLE = 0x10 STATX_ATTR_NODUMP = 0x40 + STATX_ATTR_VERITY = 0x100000 STATX_BASIC_STATS = 0x7ff STATX_BLOCKS = 0x400 STATX_BTIME = 0x800 @@ -2490,6 +2519,7 @@ const ( TCP_THIN_DUPACK = 0x11 TCP_THIN_LINEAR_TIMEOUTS = 0x10 TCP_TIMESTAMP = 0x18 + TCP_TX_DELAY = 0x25 TCP_ULP = 0x1f TCP_USER_TIMEOUT = 0x12 TCP_WINDOW_CLAMP = 0xa @@ -2592,6 +2622,10 @@ const ( TIPC_ADDR_MCAST = 0x1 TIPC_ADDR_NAME = 0x2 TIPC_ADDR_NAMESEQ = 0x1 + TIPC_AEAD_ALG_NAME = 0x20 + TIPC_AEAD_KEYLEN_MAX = 0x24 + TIPC_AEAD_KEYLEN_MIN = 0x14 + TIPC_AEAD_KEY_SIZE_MAX = 0x48 TIPC_CFG_SRV = 0x0 TIPC_CLUSTER_BITS = 0xc TIPC_CLUSTER_MASK = 0xfff000 @@ -2624,6 +2658,7 @@ const ( TIPC_MCAST_REPLICAST = 0x86 TIPC_MEDIUM_IMPORTANCE = 0x1 TIPC_NODEID_LEN = 0x10 + TIPC_NODELAY = 0x8a TIPC_NODE_BITS = 0xc TIPC_NODE_MASK = 0xfff TIPC_NODE_OFFSET = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_riscv64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_riscv64.go index 86976e4198..5d964445d8 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_riscv64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_riscv64.go @@ -243,6 +243,7 @@ const ( BPF_F_LOCK = 0x4 BPF_F_MARK_ENFORCE = 0x40 BPF_F_MARK_MANGLED_0 = 0x20 + BPF_F_MMAPABLE = 0x400 BPF_F_NO_COMMON_LRU = 0x2 BPF_F_NO_PREALLOC = 0x1 BPF_F_NUMA_NODE = 0x4 @@ -417,8 +418,10 @@ const ( CLOCK_TXFROMRX = 0x4 CLOCK_TXINT = 0x3 CLONE_ARGS_SIZE_VER0 = 0x40 + CLONE_ARGS_SIZE_VER1 = 0x50 CLONE_CHILD_CLEARTID = 0x200000 CLONE_CHILD_SETTID = 0x1000000 + CLONE_CLEAR_SIGHAND = 0x100000000 CLONE_DETACHED = 0x400000 CLONE_FILES = 0x400 CLONE_FS = 0x200 @@ -715,8 +718,9 @@ const ( FSCRYPT_POLICY_FLAGS_PAD_4 = 0x0 FSCRYPT_POLICY_FLAGS_PAD_8 = 0x1 FSCRYPT_POLICY_FLAGS_PAD_MASK = 0x3 - FSCRYPT_POLICY_FLAGS_VALID = 0x7 + FSCRYPT_POLICY_FLAGS_VALID = 0xf FSCRYPT_POLICY_FLAG_DIRECT_KEY = 0x4 + FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64 = 0x8 FSCRYPT_POLICY_V1 = 0x0 FSCRYPT_POLICY_V2 = 0x2 FS_ENCRYPTION_MODE_ADIANTUM = 0x9 @@ -746,7 +750,7 @@ const ( FS_POLICY_FLAGS_PAD_4 = 0x0 FS_POLICY_FLAGS_PAD_8 = 0x1 FS_POLICY_FLAGS_PAD_MASK = 0x3 - FS_POLICY_FLAGS_VALID = 0x7 + FS_POLICY_FLAGS_VALID = 0xf FUTEXFS_SUPER_MAGIC = 0xbad1dea F_ADD_SEALS = 0x409 F_DUPFD = 0x0 @@ -1228,6 +1232,7 @@ const ( LOOP_SET_STATUS64 = 0x4c04 LO_KEY_SIZE = 0x20 LO_NAME_SIZE = 0x40 + MADV_COLD = 0x14 MADV_DODUMP = 0x11 MADV_DOFORK = 0xb MADV_DONTDUMP = 0x10 @@ -1240,6 +1245,7 @@ const ( MADV_MERGEABLE = 0xc MADV_NOHUGEPAGE = 0xf MADV_NORMAL = 0x0 + MADV_PAGEOUT = 0x15 MADV_RANDOM = 0x1 MADV_REMOVE = 0x9 MADV_SEQUENTIAL = 0x2 @@ -1581,6 +1587,7 @@ const ( PERF_EVENT_IOC_SET_FILTER = 0x40082406 PERF_EVENT_IOC_SET_OUTPUT = 0x2405 PIPEFS_MAGIC = 0x50495045 + PPC_CMM_MAGIC = 0xc7571590 PPPIOCATTACH = 0x4004743d PPPIOCATTCHAN = 0x40047438 PPPIOCCONNECT = 0x4004743a @@ -1591,6 +1598,8 @@ const ( PPPIOCGDEBUG = 0x80047441 PPPIOCGFLAGS = 0x8004745a PPPIOCGIDLE = 0x8010743f + PPPIOCGIDLE32 = 0x8008743f + PPPIOCGIDLE64 = 0x8010743f PPPIOCGL2TPSTATS = 0x80487436 PPPIOCGMRU = 0x80047453 PPPIOCGNPMODE = 0xc008744c @@ -1917,12 +1926,28 @@ const ( RTF_UP = 0x1 RTF_WINDOW = 0x80 RTF_XRESOLVE = 0x800 + RTMGRP_DECnet_IFADDR = 0x1000 + RTMGRP_DECnet_ROUTE = 0x4000 + RTMGRP_IPV4_IFADDR = 0x10 + RTMGRP_IPV4_MROUTE = 0x20 + RTMGRP_IPV4_ROUTE = 0x40 + RTMGRP_IPV4_RULE = 0x80 + RTMGRP_IPV6_IFADDR = 0x100 + RTMGRP_IPV6_IFINFO = 0x800 + RTMGRP_IPV6_MROUTE = 0x200 + RTMGRP_IPV6_PREFIX = 0x20000 + RTMGRP_IPV6_ROUTE = 0x400 + RTMGRP_LINK = 0x1 + RTMGRP_NEIGH = 0x4 + RTMGRP_NOTIFY = 0x2 + RTMGRP_TC = 0x8 RTM_BASE = 0x10 RTM_DELACTION = 0x31 RTM_DELADDR = 0x15 RTM_DELADDRLABEL = 0x49 RTM_DELCHAIN = 0x65 RTM_DELLINK = 0x11 + RTM_DELLINKPROP = 0x6d RTM_DELMDB = 0x55 RTM_DELNEIGH = 0x1d RTM_DELNETCONF = 0x51 @@ -1946,6 +1971,7 @@ const ( RTM_GETCHAIN = 0x66 RTM_GETDCB = 0x4e RTM_GETLINK = 0x12 + RTM_GETLINKPROP = 0x6e RTM_GETMDB = 0x56 RTM_GETMULTICAST = 0x3a RTM_GETNEIGH = 0x1e @@ -1959,13 +1985,14 @@ const ( RTM_GETSTATS = 0x5e RTM_GETTCLASS = 0x2a RTM_GETTFILTER = 0x2e - RTM_MAX = 0x6b + RTM_MAX = 0x6f RTM_NEWACTION = 0x30 RTM_NEWADDR = 0x14 RTM_NEWADDRLABEL = 0x48 RTM_NEWCACHEREPORT = 0x60 RTM_NEWCHAIN = 0x64 RTM_NEWLINK = 0x10 + RTM_NEWLINKPROP = 0x6c RTM_NEWMDB = 0x54 RTM_NEWNDUSEROPT = 0x44 RTM_NEWNEIGH = 0x1c @@ -1980,8 +2007,8 @@ const ( RTM_NEWSTATS = 0x5c RTM_NEWTCLASS = 0x28 RTM_NEWTFILTER = 0x2c - RTM_NR_FAMILIES = 0x17 - RTM_NR_MSGTYPES = 0x5c + RTM_NR_FAMILIES = 0x18 + RTM_NR_MSGTYPES = 0x60 RTM_SETDCB = 0x4f RTM_SETLINK = 0x13 RTM_SETNEIGHTBL = 0x43 @@ -2196,7 +2223,7 @@ const ( SOL_TLS = 0x11a SOL_X25 = 0x106 SOL_XDP = 0x11b - SOMAXCONN = 0x80 + SOMAXCONN = 0x1000 SO_ACCEPTCONN = 0x1e SO_ATTACH_BPF = 0x32 SO_ATTACH_FILTER = 0x1a @@ -2302,6 +2329,7 @@ const ( STATX_ATTR_ENCRYPTED = 0x800 STATX_ATTR_IMMUTABLE = 0x10 STATX_ATTR_NODUMP = 0x40 + STATX_ATTR_VERITY = 0x100000 STATX_BASIC_STATS = 0x7ff STATX_BLOCKS = 0x400 STATX_BTIME = 0x800 @@ -2422,6 +2450,7 @@ const ( TCP_THIN_DUPACK = 0x11 TCP_THIN_LINEAR_TIMEOUTS = 0x10 TCP_TIMESTAMP = 0x18 + TCP_TX_DELAY = 0x25 TCP_ULP = 0x1f TCP_USER_TIMEOUT = 0x12 TCP_WINDOW_CLAMP = 0xa @@ -2518,6 +2547,10 @@ const ( TIPC_ADDR_MCAST = 0x1 TIPC_ADDR_NAME = 0x2 TIPC_ADDR_NAMESEQ = 0x1 + TIPC_AEAD_ALG_NAME = 0x20 + TIPC_AEAD_KEYLEN_MAX = 0x24 + TIPC_AEAD_KEYLEN_MIN = 0x14 + TIPC_AEAD_KEY_SIZE_MAX = 0x48 TIPC_CFG_SRV = 0x0 TIPC_CLUSTER_BITS = 0xc TIPC_CLUSTER_MASK = 0xfff000 @@ -2550,6 +2583,7 @@ const ( TIPC_MCAST_REPLICAST = 0x86 TIPC_MEDIUM_IMPORTANCE = 0x1 TIPC_NODEID_LEN = 0x10 + TIPC_NODELAY = 0x8a TIPC_NODE_BITS = 0xc TIPC_NODE_MASK = 0xfff TIPC_NODE_OFFSET = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_s390x.go b/vendor/golang.org/x/sys/unix/zerrors_linux_s390x.go index cd6a046411..7d64d6e798 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_s390x.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_s390x.go @@ -243,6 +243,7 @@ const ( BPF_F_LOCK = 0x4 BPF_F_MARK_ENFORCE = 0x40 BPF_F_MARK_MANGLED_0 = 0x20 + BPF_F_MMAPABLE = 0x400 BPF_F_NO_COMMON_LRU = 0x2 BPF_F_NO_PREALLOC = 0x1 BPF_F_NUMA_NODE = 0x4 @@ -417,8 +418,10 @@ const ( CLOCK_TXFROMRX = 0x4 CLOCK_TXINT = 0x3 CLONE_ARGS_SIZE_VER0 = 0x40 + CLONE_ARGS_SIZE_VER1 = 0x50 CLONE_CHILD_CLEARTID = 0x200000 CLONE_CHILD_SETTID = 0x1000000 + CLONE_CLEAR_SIGHAND = 0x100000000 CLONE_DETACHED = 0x400000 CLONE_FILES = 0x400 CLONE_FS = 0x200 @@ -715,8 +718,9 @@ const ( FSCRYPT_POLICY_FLAGS_PAD_4 = 0x0 FSCRYPT_POLICY_FLAGS_PAD_8 = 0x1 FSCRYPT_POLICY_FLAGS_PAD_MASK = 0x3 - FSCRYPT_POLICY_FLAGS_VALID = 0x7 + FSCRYPT_POLICY_FLAGS_VALID = 0xf FSCRYPT_POLICY_FLAG_DIRECT_KEY = 0x4 + FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64 = 0x8 FSCRYPT_POLICY_V1 = 0x0 FSCRYPT_POLICY_V2 = 0x2 FS_ENCRYPTION_MODE_ADIANTUM = 0x9 @@ -746,7 +750,7 @@ const ( FS_POLICY_FLAGS_PAD_4 = 0x0 FS_POLICY_FLAGS_PAD_8 = 0x1 FS_POLICY_FLAGS_PAD_MASK = 0x3 - FS_POLICY_FLAGS_VALID = 0x7 + FS_POLICY_FLAGS_VALID = 0xf FUTEXFS_SUPER_MAGIC = 0xbad1dea F_ADD_SEALS = 0x409 F_DUPFD = 0x0 @@ -1228,6 +1232,7 @@ const ( LOOP_SET_STATUS64 = 0x4c04 LO_KEY_SIZE = 0x20 LO_NAME_SIZE = 0x40 + MADV_COLD = 0x14 MADV_DODUMP = 0x11 MADV_DOFORK = 0xb MADV_DONTDUMP = 0x10 @@ -1240,6 +1245,7 @@ const ( MADV_MERGEABLE = 0xc MADV_NOHUGEPAGE = 0xf MADV_NORMAL = 0x0 + MADV_PAGEOUT = 0x15 MADV_RANDOM = 0x1 MADV_REMOVE = 0x9 MADV_SEQUENTIAL = 0x2 @@ -1581,6 +1587,7 @@ const ( PERF_EVENT_IOC_SET_FILTER = 0x40082406 PERF_EVENT_IOC_SET_OUTPUT = 0x2405 PIPEFS_MAGIC = 0x50495045 + PPC_CMM_MAGIC = 0xc7571590 PPPIOCATTACH = 0x4004743d PPPIOCATTCHAN = 0x40047438 PPPIOCCONNECT = 0x4004743a @@ -1591,6 +1598,8 @@ const ( PPPIOCGDEBUG = 0x80047441 PPPIOCGFLAGS = 0x8004745a PPPIOCGIDLE = 0x8010743f + PPPIOCGIDLE32 = 0x8008743f + PPPIOCGIDLE64 = 0x8010743f PPPIOCGL2TPSTATS = 0x80487436 PPPIOCGMRU = 0x80047453 PPPIOCGNPMODE = 0xc008744c @@ -1990,12 +1999,28 @@ const ( RTF_UP = 0x1 RTF_WINDOW = 0x80 RTF_XRESOLVE = 0x800 + RTMGRP_DECnet_IFADDR = 0x1000 + RTMGRP_DECnet_ROUTE = 0x4000 + RTMGRP_IPV4_IFADDR = 0x10 + RTMGRP_IPV4_MROUTE = 0x20 + RTMGRP_IPV4_ROUTE = 0x40 + RTMGRP_IPV4_RULE = 0x80 + RTMGRP_IPV6_IFADDR = 0x100 + RTMGRP_IPV6_IFINFO = 0x800 + RTMGRP_IPV6_MROUTE = 0x200 + RTMGRP_IPV6_PREFIX = 0x20000 + RTMGRP_IPV6_ROUTE = 0x400 + RTMGRP_LINK = 0x1 + RTMGRP_NEIGH = 0x4 + RTMGRP_NOTIFY = 0x2 + RTMGRP_TC = 0x8 RTM_BASE = 0x10 RTM_DELACTION = 0x31 RTM_DELADDR = 0x15 RTM_DELADDRLABEL = 0x49 RTM_DELCHAIN = 0x65 RTM_DELLINK = 0x11 + RTM_DELLINKPROP = 0x6d RTM_DELMDB = 0x55 RTM_DELNEIGH = 0x1d RTM_DELNETCONF = 0x51 @@ -2019,6 +2044,7 @@ const ( RTM_GETCHAIN = 0x66 RTM_GETDCB = 0x4e RTM_GETLINK = 0x12 + RTM_GETLINKPROP = 0x6e RTM_GETMDB = 0x56 RTM_GETMULTICAST = 0x3a RTM_GETNEIGH = 0x1e @@ -2032,13 +2058,14 @@ const ( RTM_GETSTATS = 0x5e RTM_GETTCLASS = 0x2a RTM_GETTFILTER = 0x2e - RTM_MAX = 0x6b + RTM_MAX = 0x6f RTM_NEWACTION = 0x30 RTM_NEWADDR = 0x14 RTM_NEWADDRLABEL = 0x48 RTM_NEWCACHEREPORT = 0x60 RTM_NEWCHAIN = 0x64 RTM_NEWLINK = 0x10 + RTM_NEWLINKPROP = 0x6c RTM_NEWMDB = 0x54 RTM_NEWNDUSEROPT = 0x44 RTM_NEWNEIGH = 0x1c @@ -2053,8 +2080,8 @@ const ( RTM_NEWSTATS = 0x5c RTM_NEWTCLASS = 0x28 RTM_NEWTFILTER = 0x2c - RTM_NR_FAMILIES = 0x17 - RTM_NR_MSGTYPES = 0x5c + RTM_NR_FAMILIES = 0x18 + RTM_NR_MSGTYPES = 0x60 RTM_SETDCB = 0x4f RTM_SETLINK = 0x13 RTM_SETNEIGHTBL = 0x43 @@ -2269,7 +2296,7 @@ const ( SOL_TLS = 0x11a SOL_X25 = 0x106 SOL_XDP = 0x11b - SOMAXCONN = 0x80 + SOMAXCONN = 0x1000 SO_ACCEPTCONN = 0x1e SO_ATTACH_BPF = 0x32 SO_ATTACH_FILTER = 0x1a @@ -2375,6 +2402,7 @@ const ( STATX_ATTR_ENCRYPTED = 0x800 STATX_ATTR_IMMUTABLE = 0x10 STATX_ATTR_NODUMP = 0x40 + STATX_ATTR_VERITY = 0x100000 STATX_BASIC_STATS = 0x7ff STATX_BLOCKS = 0x400 STATX_BTIME = 0x800 @@ -2495,6 +2523,7 @@ const ( TCP_THIN_DUPACK = 0x11 TCP_THIN_LINEAR_TIMEOUTS = 0x10 TCP_TIMESTAMP = 0x18 + TCP_TX_DELAY = 0x25 TCP_ULP = 0x1f TCP_USER_TIMEOUT = 0x12 TCP_WINDOW_CLAMP = 0xa @@ -2591,6 +2620,10 @@ const ( TIPC_ADDR_MCAST = 0x1 TIPC_ADDR_NAME = 0x2 TIPC_ADDR_NAMESEQ = 0x1 + TIPC_AEAD_ALG_NAME = 0x20 + TIPC_AEAD_KEYLEN_MAX = 0x24 + TIPC_AEAD_KEYLEN_MIN = 0x14 + TIPC_AEAD_KEY_SIZE_MAX = 0x48 TIPC_CFG_SRV = 0x0 TIPC_CLUSTER_BITS = 0xc TIPC_CLUSTER_MASK = 0xfff000 @@ -2623,6 +2656,7 @@ const ( TIPC_MCAST_REPLICAST = 0x86 TIPC_MEDIUM_IMPORTANCE = 0x1 TIPC_NODEID_LEN = 0x10 + TIPC_NODELAY = 0x8a TIPC_NODE_BITS = 0xc TIPC_NODE_MASK = 0xfff TIPC_NODE_OFFSET = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go index 89da22b793..084d55be5f 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go @@ -246,6 +246,7 @@ const ( BPF_F_LOCK = 0x4 BPF_F_MARK_ENFORCE = 0x40 BPF_F_MARK_MANGLED_0 = 0x20 + BPF_F_MMAPABLE = 0x400 BPF_F_NO_COMMON_LRU = 0x2 BPF_F_NO_PREALLOC = 0x1 BPF_F_NUMA_NODE = 0x4 @@ -420,8 +421,10 @@ const ( CLOCK_TXFROMRX = 0x4 CLOCK_TXINT = 0x3 CLONE_ARGS_SIZE_VER0 = 0x40 + CLONE_ARGS_SIZE_VER1 = 0x50 CLONE_CHILD_CLEARTID = 0x200000 CLONE_CHILD_SETTID = 0x1000000 + CLONE_CLEAR_SIGHAND = 0x100000000 CLONE_DETACHED = 0x400000 CLONE_FILES = 0x400 CLONE_FS = 0x200 @@ -719,8 +722,9 @@ const ( FSCRYPT_POLICY_FLAGS_PAD_4 = 0x0 FSCRYPT_POLICY_FLAGS_PAD_8 = 0x1 FSCRYPT_POLICY_FLAGS_PAD_MASK = 0x3 - FSCRYPT_POLICY_FLAGS_VALID = 0x7 + FSCRYPT_POLICY_FLAGS_VALID = 0xf FSCRYPT_POLICY_FLAG_DIRECT_KEY = 0x4 + FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64 = 0x8 FSCRYPT_POLICY_V1 = 0x0 FSCRYPT_POLICY_V2 = 0x2 FS_ENCRYPTION_MODE_ADIANTUM = 0x9 @@ -750,7 +754,7 @@ const ( FS_POLICY_FLAGS_PAD_4 = 0x0 FS_POLICY_FLAGS_PAD_8 = 0x1 FS_POLICY_FLAGS_PAD_MASK = 0x3 - FS_POLICY_FLAGS_VALID = 0x7 + FS_POLICY_FLAGS_VALID = 0xf FUTEXFS_SUPER_MAGIC = 0xbad1dea F_ADD_SEALS = 0x409 F_DUPFD = 0x0 @@ -1232,6 +1236,7 @@ const ( LOOP_SET_STATUS64 = 0x4c04 LO_KEY_SIZE = 0x20 LO_NAME_SIZE = 0x40 + MADV_COLD = 0x14 MADV_DODUMP = 0x11 MADV_DOFORK = 0xb MADV_DONTDUMP = 0x10 @@ -1244,6 +1249,7 @@ const ( MADV_MERGEABLE = 0xc MADV_NOHUGEPAGE = 0xf MADV_NORMAL = 0x0 + MADV_PAGEOUT = 0x15 MADV_RANDOM = 0x1 MADV_REMOVE = 0x9 MADV_SEQUENTIAL = 0x2 @@ -1270,6 +1276,7 @@ const ( MAP_SHARED = 0x1 MAP_SHARED_VALIDATE = 0x3 MAP_STACK = 0x20000 + MAP_SYNC = 0x80000 MAP_TYPE = 0xf MCAST_BLOCK_SOURCE = 0x2b MCAST_EXCLUDE = 0x0 @@ -1585,6 +1592,7 @@ const ( PERF_EVENT_IOC_SET_FILTER = 0x80082406 PERF_EVENT_IOC_SET_OUTPUT = 0x20002405 PIPEFS_MAGIC = 0x50495045 + PPC_CMM_MAGIC = 0xc7571590 PPPIOCATTACH = 0x8004743d PPPIOCATTCHAN = 0x80047438 PPPIOCCONNECT = 0x8004743a @@ -1595,6 +1603,8 @@ const ( PPPIOCGDEBUG = 0x40047441 PPPIOCGFLAGS = 0x4004745a PPPIOCGIDLE = 0x4010743f + PPPIOCGIDLE32 = 0x4008743f + PPPIOCGIDLE64 = 0x4010743f PPPIOCGL2TPSTATS = 0x40487436 PPPIOCGMRU = 0x40047453 PPPIOCGNPMODE = 0xc008744c @@ -1982,12 +1992,28 @@ const ( RTF_UP = 0x1 RTF_WINDOW = 0x80 RTF_XRESOLVE = 0x800 + RTMGRP_DECnet_IFADDR = 0x1000 + RTMGRP_DECnet_ROUTE = 0x4000 + RTMGRP_IPV4_IFADDR = 0x10 + RTMGRP_IPV4_MROUTE = 0x20 + RTMGRP_IPV4_ROUTE = 0x40 + RTMGRP_IPV4_RULE = 0x80 + RTMGRP_IPV6_IFADDR = 0x100 + RTMGRP_IPV6_IFINFO = 0x800 + RTMGRP_IPV6_MROUTE = 0x200 + RTMGRP_IPV6_PREFIX = 0x20000 + RTMGRP_IPV6_ROUTE = 0x400 + RTMGRP_LINK = 0x1 + RTMGRP_NEIGH = 0x4 + RTMGRP_NOTIFY = 0x2 + RTMGRP_TC = 0x8 RTM_BASE = 0x10 RTM_DELACTION = 0x31 RTM_DELADDR = 0x15 RTM_DELADDRLABEL = 0x49 RTM_DELCHAIN = 0x65 RTM_DELLINK = 0x11 + RTM_DELLINKPROP = 0x6d RTM_DELMDB = 0x55 RTM_DELNEIGH = 0x1d RTM_DELNETCONF = 0x51 @@ -2011,6 +2037,7 @@ const ( RTM_GETCHAIN = 0x66 RTM_GETDCB = 0x4e RTM_GETLINK = 0x12 + RTM_GETLINKPROP = 0x6e RTM_GETMDB = 0x56 RTM_GETMULTICAST = 0x3a RTM_GETNEIGH = 0x1e @@ -2024,13 +2051,14 @@ const ( RTM_GETSTATS = 0x5e RTM_GETTCLASS = 0x2a RTM_GETTFILTER = 0x2e - RTM_MAX = 0x6b + RTM_MAX = 0x6f RTM_NEWACTION = 0x30 RTM_NEWADDR = 0x14 RTM_NEWADDRLABEL = 0x48 RTM_NEWCACHEREPORT = 0x60 RTM_NEWCHAIN = 0x64 RTM_NEWLINK = 0x10 + RTM_NEWLINKPROP = 0x6c RTM_NEWMDB = 0x54 RTM_NEWNDUSEROPT = 0x44 RTM_NEWNEIGH = 0x1c @@ -2045,8 +2073,8 @@ const ( RTM_NEWSTATS = 0x5c RTM_NEWTCLASS = 0x28 RTM_NEWTFILTER = 0x2c - RTM_NR_FAMILIES = 0x17 - RTM_NR_MSGTYPES = 0x5c + RTM_NR_FAMILIES = 0x18 + RTM_NR_MSGTYPES = 0x60 RTM_SETDCB = 0x4f RTM_SETLINK = 0x13 RTM_SETNEIGHTBL = 0x43 @@ -2261,7 +2289,7 @@ const ( SOL_TLS = 0x11a SOL_X25 = 0x106 SOL_XDP = 0x11b - SOMAXCONN = 0x80 + SOMAXCONN = 0x1000 SO_ACCEPTCONN = 0x8000 SO_ATTACH_BPF = 0x34 SO_ATTACH_FILTER = 0x1a @@ -2367,6 +2395,7 @@ const ( STATX_ATTR_ENCRYPTED = 0x800 STATX_ATTR_IMMUTABLE = 0x10 STATX_ATTR_NODUMP = 0x40 + STATX_ATTR_VERITY = 0x100000 STATX_BASIC_STATS = 0x7ff STATX_BLOCKS = 0x400 STATX_BTIME = 0x800 @@ -2486,6 +2515,7 @@ const ( TCP_THIN_DUPACK = 0x11 TCP_THIN_LINEAR_TIMEOUTS = 0x10 TCP_TIMESTAMP = 0x18 + TCP_TX_DELAY = 0x25 TCP_ULP = 0x1f TCP_USER_TIMEOUT = 0x12 TCP_WINDOW_CLAMP = 0xa @@ -2580,6 +2610,10 @@ const ( TIPC_ADDR_MCAST = 0x1 TIPC_ADDR_NAME = 0x2 TIPC_ADDR_NAMESEQ = 0x1 + TIPC_AEAD_ALG_NAME = 0x20 + TIPC_AEAD_KEYLEN_MAX = 0x24 + TIPC_AEAD_KEYLEN_MIN = 0x14 + TIPC_AEAD_KEY_SIZE_MAX = 0x48 TIPC_CFG_SRV = 0x0 TIPC_CLUSTER_BITS = 0xc TIPC_CLUSTER_MASK = 0xfff000 @@ -2612,6 +2646,7 @@ const ( TIPC_MCAST_REPLICAST = 0x86 TIPC_MEDIUM_IMPORTANCE = 0x1 TIPC_NODEID_LEN = 0x10 + TIPC_NODELAY = 0x8a TIPC_NODE_BITS = 0xc TIPC_NODE_MASK = 0xfff TIPC_NODE_OFFSET = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.go index aad7b9c80d..b44b31aeb1 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.go @@ -350,8 +350,8 @@ func Munlockall() (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func pipe(p *[2]_C_int) (err error) { - _, _, e1 := RawSyscall(SYS_PIPE, uintptr(unsafe.Pointer(p)), 0, 0) +func pipe2(p *[2]_C_int, flags int) (err error) { + _, _, e1 := RawSyscall(SYS_PIPE2, uintptr(unsafe.Pointer(p)), uintptr(flags), 0) if e1 != 0 { err = errnoErr(e1) } diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.go index 365b33d28f..67f93ee76d 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.go @@ -350,8 +350,8 @@ func Munlockall() (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func pipe(p *[2]_C_int) (err error) { - _, _, e1 := RawSyscall(SYS_PIPE, uintptr(unsafe.Pointer(p)), 0, 0) +func pipe2(p *[2]_C_int, flags int) (err error) { + _, _, e1 := RawSyscall(SYS_PIPE2, uintptr(unsafe.Pointer(p)), uintptr(flags), 0) if e1 != 0 { err = errnoErr(e1) } diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.go index 07cba9659c..d7c878b1d0 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.go @@ -350,8 +350,8 @@ func Munlockall() (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func pipe(p *[2]_C_int) (err error) { - _, _, e1 := RawSyscall(SYS_PIPE, uintptr(unsafe.Pointer(p)), 0, 0) +func pipe2(p *[2]_C_int, flags int) (err error) { + _, _, e1 := RawSyscall(SYS_PIPE2, uintptr(unsafe.Pointer(p)), uintptr(flags), 0) if e1 != 0 { err = errnoErr(e1) } diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.go index 78951abf2c..8facd695d5 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.go @@ -350,8 +350,8 @@ func Munlockall() (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func pipe(p *[2]_C_int) (err error) { - _, _, e1 := RawSyscall(SYS_PIPE, uintptr(unsafe.Pointer(p)), 0, 0) +func pipe2(p *[2]_C_int, flags int) (err error) { + _, _, e1 := RawSyscall(SYS_PIPE2, uintptr(unsafe.Pointer(p)), uintptr(flags), 0) if e1 != 0 { err = errnoErr(e1) } diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go index 753def987e..1f3b4d150f 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go @@ -297,4 +297,5 @@ const ( SYS_FSMOUNT = 432 SYS_FSPICK = 433 SYS_PIDFD_OPEN = 434 + SYS_CLONE3 = 435 ) diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_386.go b/vendor/golang.org/x/sys/unix/ztypes_linux_386.go index d2306df42a..d8089584bb 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_386.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_386.go @@ -592,7 +592,7 @@ const ( IFLA_NEW_IFINDEX = 0x31 IFLA_MIN_MTU = 0x32 IFLA_MAX_MTU = 0x33 - IFLA_MAX = 0x33 + IFLA_MAX = 0x35 IFLA_INFO_KIND = 0x1 IFLA_INFO_DATA = 0x2 IFLA_INFO_XSTATS = 0x3 @@ -2794,7 +2794,7 @@ const ( DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE = 0x3c DEVLINK_ATTR_PAD = 0x3d DEVLINK_ATTR_ESWITCH_ENCAP_MODE = 0x3e - DEVLINK_ATTR_MAX = 0x89 + DEVLINK_ATTR_MAX = 0x8c DEVLINK_DPIPE_FIELD_MAPPING_TYPE_NONE = 0x0 DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX = 0x1 DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT = 0x0 diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_amd64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_amd64.go index 888870584f..88c76390b7 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_amd64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_amd64.go @@ -593,7 +593,7 @@ const ( IFLA_NEW_IFINDEX = 0x31 IFLA_MIN_MTU = 0x32 IFLA_MAX_MTU = 0x33 - IFLA_MAX = 0x33 + IFLA_MAX = 0x35 IFLA_INFO_KIND = 0x1 IFLA_INFO_DATA = 0x2 IFLA_INFO_XSTATS = 0x3 @@ -2809,7 +2809,7 @@ const ( DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE = 0x3c DEVLINK_ATTR_PAD = 0x3d DEVLINK_ATTR_ESWITCH_ENCAP_MODE = 0x3e - DEVLINK_ATTR_MAX = 0x89 + DEVLINK_ATTR_MAX = 0x8c DEVLINK_DPIPE_FIELD_MAPPING_TYPE_NONE = 0x0 DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX = 0x1 DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT = 0x0 diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_arm.go b/vendor/golang.org/x/sys/unix/ztypes_linux_arm.go index 5f6f5945da..0c0f24c77f 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_arm.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_arm.go @@ -596,7 +596,7 @@ const ( IFLA_NEW_IFINDEX = 0x31 IFLA_MIN_MTU = 0x32 IFLA_MAX_MTU = 0x33 - IFLA_MAX = 0x33 + IFLA_MAX = 0x35 IFLA_INFO_KIND = 0x1 IFLA_INFO_DATA = 0x2 IFLA_INFO_XSTATS = 0x3 @@ -2786,7 +2786,7 @@ const ( DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE = 0x3c DEVLINK_ATTR_PAD = 0x3d DEVLINK_ATTR_ESWITCH_ENCAP_MODE = 0x3e - DEVLINK_ATTR_MAX = 0x89 + DEVLINK_ATTR_MAX = 0x8c DEVLINK_DPIPE_FIELD_MAPPING_TYPE_NONE = 0x0 DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX = 0x1 DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT = 0x0 diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_arm64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_arm64.go index e27030c5d2..6065d2d5bd 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_arm64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_arm64.go @@ -594,7 +594,7 @@ const ( IFLA_NEW_IFINDEX = 0x31 IFLA_MIN_MTU = 0x32 IFLA_MAX_MTU = 0x33 - IFLA_MAX = 0x33 + IFLA_MAX = 0x35 IFLA_INFO_KIND = 0x1 IFLA_INFO_DATA = 0x2 IFLA_INFO_XSTATS = 0x3 @@ -2788,7 +2788,7 @@ const ( DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE = 0x3c DEVLINK_ATTR_PAD = 0x3d DEVLINK_ATTR_ESWITCH_ENCAP_MODE = 0x3e - DEVLINK_ATTR_MAX = 0x89 + DEVLINK_ATTR_MAX = 0x8c DEVLINK_DPIPE_FIELD_MAPPING_TYPE_NONE = 0x0 DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX = 0x1 DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT = 0x0 diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_mips.go b/vendor/golang.org/x/sys/unix/ztypes_linux_mips.go index 9b1c03da3c..29d4408d37 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_mips.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_mips.go @@ -595,7 +595,7 @@ const ( IFLA_NEW_IFINDEX = 0x31 IFLA_MIN_MTU = 0x32 IFLA_MAX_MTU = 0x33 - IFLA_MAX = 0x33 + IFLA_MAX = 0x35 IFLA_INFO_KIND = 0x1 IFLA_INFO_DATA = 0x2 IFLA_INFO_XSTATS = 0x3 @@ -2792,7 +2792,7 @@ const ( DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE = 0x3c DEVLINK_ATTR_PAD = 0x3d DEVLINK_ATTR_ESWITCH_ENCAP_MODE = 0x3e - DEVLINK_ATTR_MAX = 0x89 + DEVLINK_ATTR_MAX = 0x8c DEVLINK_DPIPE_FIELD_MAPPING_TYPE_NONE = 0x0 DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX = 0x1 DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT = 0x0 diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_mips64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_mips64.go index 3b88e1ffa2..9cac9ff84a 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_mips64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_mips64.go @@ -594,7 +594,7 @@ const ( IFLA_NEW_IFINDEX = 0x31 IFLA_MIN_MTU = 0x32 IFLA_MAX_MTU = 0x33 - IFLA_MAX = 0x33 + IFLA_MAX = 0x35 IFLA_INFO_KIND = 0x1 IFLA_INFO_DATA = 0x2 IFLA_INFO_XSTATS = 0x3 @@ -2791,7 +2791,7 @@ const ( DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE = 0x3c DEVLINK_ATTR_PAD = 0x3d DEVLINK_ATTR_ESWITCH_ENCAP_MODE = 0x3e - DEVLINK_ATTR_MAX = 0x89 + DEVLINK_ATTR_MAX = 0x8c DEVLINK_DPIPE_FIELD_MAPPING_TYPE_NONE = 0x0 DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX = 0x1 DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT = 0x0 diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_mips64le.go b/vendor/golang.org/x/sys/unix/ztypes_linux_mips64le.go index 7277dc3b74..dbc21cf3d6 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_mips64le.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_mips64le.go @@ -594,7 +594,7 @@ const ( IFLA_NEW_IFINDEX = 0x31 IFLA_MIN_MTU = 0x32 IFLA_MAX_MTU = 0x33 - IFLA_MAX = 0x33 + IFLA_MAX = 0x35 IFLA_INFO_KIND = 0x1 IFLA_INFO_DATA = 0x2 IFLA_INFO_XSTATS = 0x3 @@ -2791,7 +2791,7 @@ const ( DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE = 0x3c DEVLINK_ATTR_PAD = 0x3d DEVLINK_ATTR_ESWITCH_ENCAP_MODE = 0x3e - DEVLINK_ATTR_MAX = 0x89 + DEVLINK_ATTR_MAX = 0x8c DEVLINK_DPIPE_FIELD_MAPPING_TYPE_NONE = 0x0 DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX = 0x1 DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT = 0x0 diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_mipsle.go b/vendor/golang.org/x/sys/unix/ztypes_linux_mipsle.go index 6f32c17533..a26623700e 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_mipsle.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_mipsle.go @@ -595,7 +595,7 @@ const ( IFLA_NEW_IFINDEX = 0x31 IFLA_MIN_MTU = 0x32 IFLA_MAX_MTU = 0x33 - IFLA_MAX = 0x33 + IFLA_MAX = 0x35 IFLA_INFO_KIND = 0x1 IFLA_INFO_DATA = 0x2 IFLA_INFO_XSTATS = 0x3 @@ -2792,7 +2792,7 @@ const ( DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE = 0x3c DEVLINK_ATTR_PAD = 0x3d DEVLINK_ATTR_ESWITCH_ENCAP_MODE = 0x3e - DEVLINK_ATTR_MAX = 0x89 + DEVLINK_ATTR_MAX = 0x8c DEVLINK_DPIPE_FIELD_MAPPING_TYPE_NONE = 0x0 DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX = 0x1 DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT = 0x0 diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64.go index 8a9484d687..e93b73cd74 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64.go @@ -595,7 +595,7 @@ const ( IFLA_NEW_IFINDEX = 0x31 IFLA_MIN_MTU = 0x32 IFLA_MAX_MTU = 0x33 - IFLA_MAX = 0x33 + IFLA_MAX = 0x35 IFLA_INFO_KIND = 0x1 IFLA_INFO_DATA = 0x2 IFLA_INFO_XSTATS = 0x3 @@ -2798,7 +2798,7 @@ const ( DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE = 0x3c DEVLINK_ATTR_PAD = 0x3d DEVLINK_ATTR_ESWITCH_ENCAP_MODE = 0x3e - DEVLINK_ATTR_MAX = 0x89 + DEVLINK_ATTR_MAX = 0x8c DEVLINK_DPIPE_FIELD_MAPPING_TYPE_NONE = 0x0 DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX = 0x1 DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT = 0x0 diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64le.go b/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64le.go index 7190a186aa..1f431b6ff6 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64le.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64le.go @@ -595,7 +595,7 @@ const ( IFLA_NEW_IFINDEX = 0x31 IFLA_MIN_MTU = 0x32 IFLA_MAX_MTU = 0x33 - IFLA_MAX = 0x33 + IFLA_MAX = 0x35 IFLA_INFO_KIND = 0x1 IFLA_INFO_DATA = 0x2 IFLA_INFO_XSTATS = 0x3 @@ -2798,7 +2798,7 @@ const ( DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE = 0x3c DEVLINK_ATTR_PAD = 0x3d DEVLINK_ATTR_ESWITCH_ENCAP_MODE = 0x3e - DEVLINK_ATTR_MAX = 0x89 + DEVLINK_ATTR_MAX = 0x8c DEVLINK_DPIPE_FIELD_MAPPING_TYPE_NONE = 0x0 DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX = 0x1 DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT = 0x0 diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_riscv64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_riscv64.go index 502d98aa2b..81a5dc1475 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_riscv64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_riscv64.go @@ -594,7 +594,7 @@ const ( IFLA_NEW_IFINDEX = 0x31 IFLA_MIN_MTU = 0x32 IFLA_MAX_MTU = 0x33 - IFLA_MAX = 0x33 + IFLA_MAX = 0x35 IFLA_INFO_KIND = 0x1 IFLA_INFO_DATA = 0x2 IFLA_INFO_XSTATS = 0x3 @@ -2816,7 +2816,7 @@ const ( DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE = 0x3c DEVLINK_ATTR_PAD = 0x3d DEVLINK_ATTR_ESWITCH_ENCAP_MODE = 0x3e - DEVLINK_ATTR_MAX = 0x89 + DEVLINK_ATTR_MAX = 0x8c DEVLINK_DPIPE_FIELD_MAPPING_TYPE_NONE = 0x0 DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX = 0x1 DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT = 0x0 diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_s390x.go b/vendor/golang.org/x/sys/unix/ztypes_linux_s390x.go index 65e12d7b46..ae765d47cf 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_s390x.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_s390x.go @@ -593,7 +593,7 @@ const ( IFLA_NEW_IFINDEX = 0x31 IFLA_MIN_MTU = 0x32 IFLA_MAX_MTU = 0x33 - IFLA_MAX = 0x33 + IFLA_MAX = 0x35 IFLA_INFO_KIND = 0x1 IFLA_INFO_DATA = 0x2 IFLA_INFO_XSTATS = 0x3 @@ -2812,7 +2812,7 @@ const ( DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE = 0x3c DEVLINK_ATTR_PAD = 0x3d DEVLINK_ATTR_ESWITCH_ENCAP_MODE = 0x3e - DEVLINK_ATTR_MAX = 0x89 + DEVLINK_ATTR_MAX = 0x8c DEVLINK_DPIPE_FIELD_MAPPING_TYPE_NONE = 0x0 DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX = 0x1 DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT = 0x0 diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_sparc64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_sparc64.go index 26e8a2bb74..63685ca878 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_sparc64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_sparc64.go @@ -597,7 +597,7 @@ const ( IFLA_NEW_IFINDEX = 0x31 IFLA_MIN_MTU = 0x32 IFLA_MAX_MTU = 0x33 - IFLA_MAX = 0x33 + IFLA_MAX = 0x35 IFLA_INFO_KIND = 0x1 IFLA_INFO_DATA = 0x2 IFLA_INFO_XSTATS = 0x3 @@ -2793,7 +2793,7 @@ const ( DEVLINK_ATTR_DPIPE_FIELD_MAPPING_TYPE = 0x3c DEVLINK_ATTR_PAD = 0x3d DEVLINK_ATTR_ESWITCH_ENCAP_MODE = 0x3e - DEVLINK_ATTR_MAX = 0x89 + DEVLINK_ATTR_MAX = 0x8c DEVLINK_DPIPE_FIELD_MAPPING_TYPE_NONE = 0x0 DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX = 0x1 DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT = 0x0 diff --git a/vendor/golang.org/x/sys/windows/svc/eventlog/install.go b/vendor/golang.org/x/sys/windows/svc/eventlog/install.go new file mode 100644 index 0000000000..c76a3760a4 --- /dev/null +++ b/vendor/golang.org/x/sys/windows/svc/eventlog/install.go @@ -0,0 +1,80 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build windows + +package eventlog + +import ( + "errors" + + "golang.org/x/sys/windows" + "golang.org/x/sys/windows/registry" +) + +const ( + // Log levels. + Info = windows.EVENTLOG_INFORMATION_TYPE + Warning = windows.EVENTLOG_WARNING_TYPE + Error = windows.EVENTLOG_ERROR_TYPE +) + +const addKeyName = `SYSTEM\CurrentControlSet\Services\EventLog\Application` + +// Install modifies PC registry to allow logging with an event source src. +// It adds all required keys and values to the event log registry key. +// Install uses msgFile as the event message file. If useExpandKey is true, +// the event message file is installed as REG_EXPAND_SZ value, +// otherwise as REG_SZ. Use bitwise of log.Error, log.Warning and +// log.Info to specify events supported by the new event source. +func Install(src, msgFile string, useExpandKey bool, eventsSupported uint32) error { + appkey, err := registry.OpenKey(registry.LOCAL_MACHINE, addKeyName, registry.CREATE_SUB_KEY) + if err != nil { + return err + } + defer appkey.Close() + + sk, alreadyExist, err := registry.CreateKey(appkey, src, registry.SET_VALUE) + if err != nil { + return err + } + defer sk.Close() + if alreadyExist { + return errors.New(addKeyName + `\` + src + " registry key already exists") + } + + err = sk.SetDWordValue("CustomSource", 1) + if err != nil { + return err + } + if useExpandKey { + err = sk.SetExpandStringValue("EventMessageFile", msgFile) + } else { + err = sk.SetStringValue("EventMessageFile", msgFile) + } + if err != nil { + return err + } + err = sk.SetDWordValue("TypesSupported", eventsSupported) + if err != nil { + return err + } + return nil +} + +// InstallAsEventCreate is the same as Install, but uses +// %SystemRoot%\System32\EventCreate.exe as the event message file. +func InstallAsEventCreate(src string, eventsSupported uint32) error { + return Install(src, "%SystemRoot%\\System32\\EventCreate.exe", true, eventsSupported) +} + +// Remove deletes all registry elements installed by the correspondent Install. +func Remove(src string) error { + appkey, err := registry.OpenKey(registry.LOCAL_MACHINE, addKeyName, registry.SET_VALUE) + if err != nil { + return err + } + defer appkey.Close() + return registry.DeleteKey(appkey, src) +} diff --git a/vendor/golang.org/x/sys/windows/svc/eventlog/log.go b/vendor/golang.org/x/sys/windows/svc/eventlog/log.go new file mode 100644 index 0000000000..46e5153d02 --- /dev/null +++ b/vendor/golang.org/x/sys/windows/svc/eventlog/log.go @@ -0,0 +1,70 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build windows + +// Package eventlog implements access to Windows event log. +// +package eventlog + +import ( + "errors" + "syscall" + + "golang.org/x/sys/windows" +) + +// Log provides access to the system log. +type Log struct { + Handle windows.Handle +} + +// Open retrieves a handle to the specified event log. +func Open(source string) (*Log, error) { + return OpenRemote("", source) +} + +// OpenRemote does the same as Open, but on different computer host. +func OpenRemote(host, source string) (*Log, error) { + if source == "" { + return nil, errors.New("Specify event log source") + } + var s *uint16 + if host != "" { + s = syscall.StringToUTF16Ptr(host) + } + h, err := windows.RegisterEventSource(s, syscall.StringToUTF16Ptr(source)) + if err != nil { + return nil, err + } + return &Log{Handle: h}, nil +} + +// Close closes event log l. +func (l *Log) Close() error { + return windows.DeregisterEventSource(l.Handle) +} + +func (l *Log) report(etype uint16, eid uint32, msg string) error { + ss := []*uint16{syscall.StringToUTF16Ptr(msg)} + return windows.ReportEvent(l.Handle, etype, 0, eid, 0, 1, 0, &ss[0], nil) +} + +// Info writes an information event msg with event id eid to the end of event log l. +// When EventCreate.exe is used, eid must be between 1 and 1000. +func (l *Log) Info(eid uint32, msg string) error { + return l.report(windows.EVENTLOG_INFORMATION_TYPE, eid, msg) +} + +// Warning writes an warning event msg with event id eid to the end of event log l. +// When EventCreate.exe is used, eid must be between 1 and 1000. +func (l *Log) Warning(eid uint32, msg string) error { + return l.report(windows.EVENTLOG_WARNING_TYPE, eid, msg) +} + +// Error writes an error event msg with event id eid to the end of event log l. +// When EventCreate.exe is used, eid must be between 1 and 1000. +func (l *Log) Error(eid uint32, msg string) error { + return l.report(windows.EVENTLOG_ERROR_TYPE, eid, msg) +} diff --git a/vendor/google.golang.org/grpc/.travis.yml b/vendor/google.golang.org/grpc/.travis.yml index 87f40b83d9..819c1e2a92 100644 --- a/vendor/google.golang.org/grpc/.travis.yml +++ b/vendor/google.golang.org/grpc/.travis.yml @@ -11,7 +11,7 @@ matrix: - go: 1.13.x env: GRPC_GO_RETRY=on - go: 1.13.x - env: TESTEXAMPLES=1 + env: TESTEXTRAS=1 - go: 1.12.x env: GO111MODULE=on - go: 1.11.x @@ -35,7 +35,7 @@ install: script: - set -e - - if [[ -n "${TESTEXAMPLES}" ]]; then examples/examples_test.sh; exit 0; fi + - if [[ -n "${TESTEXTRAS}" ]]; then examples/examples_test.sh; interop/interop_test.sh; exit 0; fi - if [[ -n "${VET}" ]]; then ./vet.sh; fi - if [[ -n "${GAE}" ]]; then make testappengine; exit 0; fi - if [[ -n "${RACE}" ]]; then make testrace; exit 0; fi diff --git a/vendor/google.golang.org/grpc/attributes/attributes.go b/vendor/google.golang.org/grpc/attributes/attributes.go new file mode 100644 index 0000000000..68ffc62013 --- /dev/null +++ b/vendor/google.golang.org/grpc/attributes/attributes.go @@ -0,0 +1,70 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package attributes defines a generic key/value store used in various gRPC +// components. +// +// All APIs in this package are EXPERIMENTAL. +package attributes + +import "fmt" + +// Attributes is an immutable struct for storing and retrieving generic +// key/value pairs. Keys must be hashable, and users should define their own +// types for keys. +type Attributes struct { + m map[interface{}]interface{} +} + +// New returns a new Attributes containing all key/value pairs in kvs. If the +// same key appears multiple times, the last value overwrites all previous +// values for that key. Panics if len(kvs) is not even. +func New(kvs ...interface{}) *Attributes { + if len(kvs)%2 != 0 { + panic(fmt.Sprintf("attributes.New called with unexpected input: len(kvs) = %v", len(kvs))) + } + a := &Attributes{m: make(map[interface{}]interface{}, len(kvs)/2)} + for i := 0; i < len(kvs)/2; i++ { + a.m[kvs[i*2]] = kvs[i*2+1] + } + return a +} + +// WithValues returns a new Attributes containing all key/value pairs in a and +// kvs. Panics if len(kvs) is not even. If the same key appears multiple +// times, the last value overwrites all previous values for that key. To +// remove an existing key, use a nil value. +func (a *Attributes) WithValues(kvs ...interface{}) *Attributes { + if len(kvs)%2 != 0 { + panic(fmt.Sprintf("attributes.New called with unexpected input: len(kvs) = %v", len(kvs))) + } + n := &Attributes{m: make(map[interface{}]interface{}, len(a.m)+len(kvs)/2)} + for k, v := range a.m { + n.m[k] = v + } + for i := 0; i < len(kvs)/2; i++ { + n.m[kvs[i*2]] = kvs[i*2+1] + } + return n +} + +// Value returns the value associated with these attributes for key, or nil if +// no value is associated with key. +func (a *Attributes) Value(key interface{}) interface{} { + return a.m[key] +} diff --git a/vendor/google.golang.org/grpc/balancer/balancer.go b/vendor/google.golang.org/grpc/balancer/balancer.go index 917c242f8a..531a174a83 100644 --- a/vendor/google.golang.org/grpc/balancer/balancer.go +++ b/vendor/google.golang.org/grpc/balancer/balancer.go @@ -117,6 +117,15 @@ type NewSubConnOptions struct { HealthCheckEnabled bool } +// State contains the balancer's state relevant to the gRPC ClientConn. +type State struct { + // State contains the connectivity state of the balancer, which is used to + // determine the state of the ClientConn. + ConnectivityState connectivity.State + // Picker is used to choose connections (SubConns) for RPCs. + Picker V2Picker +} + // ClientConn represents a gRPC ClientConn. // // This interface is to be implemented by gRPC. Users should not need a @@ -137,10 +146,19 @@ type ClientConn interface { // // gRPC will update the connectivity state of the ClientConn, and will call pick // on the new picker to pick new SubConn. + // + // Deprecated: use UpdateState instead UpdateBalancerState(s connectivity.State, p Picker) + // UpdateState notifies gRPC that the balancer's internal state has + // changed. + // + // gRPC will update the connectivity state of the ClientConn, and will call pick + // on the new picker to pick new SubConns. + UpdateState(State) + // ResolveNow is called by balancer to notify gRPC to do a name resolving. - ResolveNow(resolver.ResolveNowOption) + ResolveNow(resolver.ResolveNowOptions) // Target returns the dial target for this ClientConn. // @@ -185,11 +203,19 @@ type ConfigParser interface { ParseConfig(LoadBalancingConfigJSON json.RawMessage) (serviceconfig.LoadBalancingConfig, error) } -// PickOptions contains addition information for the Pick operation. -type PickOptions struct { +// PickOptions is a type alias of PickInfo for legacy reasons. +// +// Deprecated: use PickInfo instead. +type PickOptions = PickInfo + +// PickInfo contains additional information for the Pick operation. +type PickInfo struct { // FullMethodName is the method name that NewClientStream() is called // with. The canonical format is /service/Method. FullMethodName string + // Ctx is the RPC's context, and may contain relevant RPC-level information + // like the outgoing header metadata. + Ctx context.Context } // DoneInfo contains additional information for done. @@ -215,7 +241,7 @@ var ( ErrNoSubConnAvailable = errors.New("no SubConn is available") // ErrTransientFailure indicates all SubConns are in TransientFailure. // WaitForReady RPCs will block, non-WaitForReady RPCs will fail. - ErrTransientFailure = errors.New("all SubConns are in TransientFailure") + ErrTransientFailure = TransientFailureError(errors.New("all SubConns are in TransientFailure")) ) // Picker is used by gRPC to pick a SubConn to send an RPC. @@ -223,6 +249,8 @@ var ( // internal state has changed. // // The pickers used by gRPC can be updated by ClientConn.UpdateBalancerState(). +// +// Deprecated: use V2Picker instead type Picker interface { // Pick returns the SubConn to be used to send the RPC. // The returned SubConn must be one returned by NewSubConn(). @@ -243,18 +271,76 @@ type Picker interface { // // If the returned error is not nil: // - If the error is ErrNoSubConnAvailable, gRPC will block until UpdateBalancerState() - // - If the error is ErrTransientFailure: + // - If the error is ErrTransientFailure or implements IsTransientFailure() + // bool, returning true: // - If the RPC is wait-for-ready, gRPC will block until UpdateBalancerState() // is called to pick again; // - Otherwise, RPC will fail with unavailable error. // - Else (error is other non-nil error): - // - The RPC will fail with unavailable error. + // - The RPC will fail with the error's status code, or Unknown if it is + // not a status error. // // The returned done() function will be called once the rpc has finished, // with the final status of that RPC. If the SubConn returned is not a // valid SubConn type, done may not be called. done may be nil if balancer // doesn't care about the RPC status. - Pick(ctx context.Context, opts PickOptions) (conn SubConn, done func(DoneInfo), err error) + Pick(ctx context.Context, info PickInfo) (conn SubConn, done func(DoneInfo), err error) +} + +// PickResult contains information related to a connection chosen for an RPC. +type PickResult struct { + // SubConn is the connection to use for this pick, if its state is Ready. + // If the state is not Ready, gRPC will block the RPC until a new Picker is + // provided by the balancer (using ClientConn.UpdateState). The SubConn + // must be one returned by ClientConn.NewSubConn. + SubConn SubConn + + // Done is called when the RPC is completed. If the SubConn is not ready, + // this will be called with a nil parameter. If the SubConn is not a valid + // type, Done may not be called. May be nil if the balancer does not wish + // to be notified when the RPC completes. + Done func(DoneInfo) +} + +type transientFailureError struct { + error +} + +func (e *transientFailureError) IsTransientFailure() bool { return true } + +// TransientFailureError wraps err in an error implementing +// IsTransientFailure() bool, returning true. +func TransientFailureError(err error) error { + return &transientFailureError{error: err} +} + +// V2Picker is used by gRPC to pick a SubConn to send an RPC. +// Balancer is expected to generate a new picker from its snapshot every time its +// internal state has changed. +// +// The pickers used by gRPC can be updated by ClientConn.UpdateBalancerState(). +type V2Picker interface { + // Pick returns the connection to use for this RPC and related information. + // + // Pick should not block. If the balancer needs to do I/O or any blocking + // or time-consuming work to service this call, it should return + // ErrNoSubConnAvailable, and the Pick call will be repeated by gRPC when + // the Picker is updated (using ClientConn.UpdateState). + // + // If an error is returned: + // + // - If the error is ErrNoSubConnAvailable, gRPC will block until a new + // Picker is provided by the balancer (using ClientConn.UpdateState). + // + // - If the error implements IsTransientFailure() bool, returning true, + // wait for ready RPCs will wait, but non-wait for ready RPCs will be + // terminated with this error's Error() string and status code + // Unavailable. + // + // - Any other errors terminate all RPCs with the code and message + // provided. If the error is not a status error, it will be converted by + // gRPC to a status error with code Unknown. + Pick(info PickInfo) (PickResult, error) } // Balancer takes input from gRPC, manages SubConns, and collects and aggregates @@ -292,8 +378,11 @@ type Balancer interface { // SubConnState describes the state of a SubConn. type SubConnState struct { + // ConnectivityState is the connectivity state of the SubConn. ConnectivityState connectivity.State - // TODO: add last connection error + // ConnectionError is set if the ConnectivityState is TransientFailure, + // describing the reason the SubConn failed. Otherwise, it is nil. + ConnectionError error } // ClientConnState describes the state of a ClientConn relevant to the @@ -335,9 +424,8 @@ type V2Balancer interface { // // It's not thread safe. type ConnectivityStateEvaluator struct { - numReady uint64 // Number of addrConns in ready state. - numConnecting uint64 // Number of addrConns in connecting state. - numTransientFailure uint64 // Number of addrConns in transientFailure. + numReady uint64 // Number of addrConns in ready state. + numConnecting uint64 // Number of addrConns in connecting state. } // RecordTransition records state change happening in subConn and based on that @@ -357,8 +445,6 @@ func (cse *ConnectivityStateEvaluator) RecordTransition(oldState, newState conne cse.numReady += updateVal case connectivity.Connecting: cse.numConnecting += updateVal - case connectivity.TransientFailure: - cse.numTransientFailure += updateVal } } diff --git a/vendor/google.golang.org/grpc/balancer/base/balancer.go b/vendor/google.golang.org/grpc/balancer/base/balancer.go index 1a5c1aa7e9..d952f09f34 100644 --- a/vendor/google.golang.org/grpc/balancer/base/balancer.go +++ b/vendor/google.golang.org/grpc/balancer/base/balancer.go @@ -20,6 +20,7 @@ package base import ( "context" + "errors" "google.golang.org/grpc/balancer" "google.golang.org/grpc/connectivity" @@ -28,25 +29,32 @@ import ( ) type baseBuilder struct { - name string - pickerBuilder PickerBuilder - config Config + name string + pickerBuilder PickerBuilder + v2PickerBuilder V2PickerBuilder + config Config } func (bb *baseBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) balancer.Balancer { - return &baseBalancer{ - cc: cc, - pickerBuilder: bb.pickerBuilder, + bal := &baseBalancer{ + cc: cc, + pickerBuilder: bb.pickerBuilder, + v2PickerBuilder: bb.v2PickerBuilder, subConns: make(map[resolver.Address]balancer.SubConn), scStates: make(map[balancer.SubConn]connectivity.State), csEvltr: &balancer.ConnectivityStateEvaluator{}, - // Initialize picker to a picker that always return - // ErrNoSubConnAvailable, because when state of a SubConn changes, we - // may call UpdateBalancerState with this picker. - picker: NewErrPicker(balancer.ErrNoSubConnAvailable), - config: bb.config, + config: bb.config, } + // Initialize picker to a picker that always returns + // ErrNoSubConnAvailable, because when state of a SubConn changes, we + // may call UpdateState with this picker. + if bb.pickerBuilder != nil { + bal.picker = NewErrPicker(balancer.ErrNoSubConnAvailable) + } else { + bal.v2Picker = NewErrPickerV2(balancer.ErrNoSubConnAvailable) + } + return bal } func (bb *baseBuilder) Name() string { @@ -56,8 +64,9 @@ func (bb *baseBuilder) Name() string { var _ balancer.V2Balancer = (*baseBalancer)(nil) // Assert that we implement V2Balancer type baseBalancer struct { - cc balancer.ClientConn - pickerBuilder PickerBuilder + cc balancer.ClientConn + pickerBuilder PickerBuilder + v2PickerBuilder V2PickerBuilder csEvltr *balancer.ConnectivityStateEvaluator state connectivity.State @@ -65,6 +74,7 @@ type baseBalancer struct { subConns map[resolver.Address]balancer.SubConn scStates map[balancer.SubConn]connectivity.State picker balancer.Picker + v2Picker balancer.V2Picker config Config } @@ -72,8 +82,15 @@ func (b *baseBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) panic("not implemented") } -func (b *baseBalancer) ResolverError(error) { - // Ignore +func (b *baseBalancer) ResolverError(err error) { + switch b.state { + case connectivity.TransientFailure, connectivity.Idle, connectivity.Connecting: + if b.picker != nil { + b.picker = NewErrPicker(err) + } else { + b.v2Picker = NewErrPickerV2(err) + } + } } func (b *baseBalancer) UpdateClientConnState(s balancer.ClientConnState) error { @@ -114,20 +131,44 @@ func (b *baseBalancer) UpdateClientConnState(s balancer.ClientConnState) error { // from it. The picker is // - errPicker with ErrTransientFailure if the balancer is in TransientFailure, // - built by the pickerBuilder with all READY SubConns otherwise. -func (b *baseBalancer) regeneratePicker() { +func (b *baseBalancer) regeneratePicker(err error) { if b.state == connectivity.TransientFailure { - b.picker = NewErrPicker(balancer.ErrTransientFailure) + if b.pickerBuilder != nil { + b.picker = NewErrPicker(balancer.ErrTransientFailure) + } else { + if err != nil { + b.v2Picker = NewErrPickerV2(balancer.TransientFailureError(err)) + } else { + // This means the last subchannel transition was not to + // TransientFailure (otherwise err must be set), but the + // aggregate state of the balancer is TransientFailure, meaning + // there are no other addresses. + b.v2Picker = NewErrPickerV2(balancer.TransientFailureError(errors.New("resolver returned no addresses"))) + } + } return } - readySCs := make(map[resolver.Address]balancer.SubConn) + if b.pickerBuilder != nil { + readySCs := make(map[resolver.Address]balancer.SubConn) - // Filter out all ready SCs from full subConn map. - for addr, sc := range b.subConns { - if st, ok := b.scStates[sc]; ok && st == connectivity.Ready { - readySCs[addr] = sc + // Filter out all ready SCs from full subConn map. + for addr, sc := range b.subConns { + if st, ok := b.scStates[sc]; ok && st == connectivity.Ready { + readySCs[addr] = sc + } + } + b.picker = b.pickerBuilder.Build(readySCs) + } else { + readySCs := make(map[balancer.SubConn]SubConnInfo) + + // Filter out all ready SCs from full subConn map. + for addr, sc := range b.subConns { + if st, ok := b.scStates[sc]; ok && st == connectivity.Ready { + readySCs[sc] = SubConnInfo{Address: addr} + } } + b.v2Picker = b.v2PickerBuilder.Build(PickerBuildInfo{ReadySCs: readySCs}) } - b.picker = b.pickerBuilder.Build(readySCs) } func (b *baseBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) { @@ -166,10 +207,14 @@ func (b *baseBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.Su // - the aggregated state of balancer became non-TransientFailure from TransientFailure if (s == connectivity.Ready) != (oldS == connectivity.Ready) || (b.state == connectivity.TransientFailure) != (oldAggrState == connectivity.TransientFailure) { - b.regeneratePicker() + b.regeneratePicker(state.ConnectionError) } - b.cc.UpdateBalancerState(b.state, b.picker) + if b.picker != nil { + b.cc.UpdateBalancerState(b.state, b.picker) + } else { + b.cc.UpdateState(balancer.State{ConnectivityState: b.state, Picker: b.v2Picker}) + } } // Close is a nop because base balancer doesn't have internal state to clean up, @@ -186,6 +231,19 @@ type errPicker struct { err error // Pick() always returns this err. } -func (p *errPicker) Pick(ctx context.Context, opts balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) { +func (p *errPicker) Pick(context.Context, balancer.PickInfo) (balancer.SubConn, func(balancer.DoneInfo), error) { return nil, nil, p.err } + +// NewErrPickerV2 returns a V2Picker that always returns err on Pick(). +func NewErrPickerV2(err error) balancer.V2Picker { + return &errPickerV2{err: err} +} + +type errPickerV2 struct { + err error // Pick() always returns this err. +} + +func (p *errPickerV2) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + return balancer.PickResult{}, p.err +} diff --git a/vendor/google.golang.org/grpc/balancer/base/base.go b/vendor/google.golang.org/grpc/balancer/base/base.go index 34b1f2994a..4192918b9e 100644 --- a/vendor/google.golang.org/grpc/balancer/base/base.go +++ b/vendor/google.golang.org/grpc/balancer/base/base.go @@ -42,6 +42,26 @@ type PickerBuilder interface { Build(readySCs map[resolver.Address]balancer.SubConn) balancer.Picker } +// V2PickerBuilder creates balancer.V2Picker. +type V2PickerBuilder interface { + // Build returns a picker that will be used by gRPC to pick a SubConn. + Build(info PickerBuildInfo) balancer.V2Picker +} + +// PickerBuildInfo contains information needed by the picker builder to +// construct a picker. +type PickerBuildInfo struct { + // ReadySCs is a map from all ready SubConns to the Addresses used to + // create them. + ReadySCs map[balancer.SubConn]SubConnInfo +} + +// SubConnInfo contains information about a SubConn created by the base +// balancer. +type SubConnInfo struct { + Address resolver.Address // the address used to create this SubConn +} + // NewBalancerBuilder returns a balancer builder. The balancers // built by this builder will use the picker builder to build pickers. func NewBalancerBuilder(name string, pb PickerBuilder) balancer.Builder { @@ -62,3 +82,12 @@ func NewBalancerBuilderWithConfig(name string, pb PickerBuilder, config Config) config: config, } } + +// NewBalancerBuilderV2 returns a base balancer builder configured by the provided config. +func NewBalancerBuilderV2(name string, pb V2PickerBuilder, config Config) balancer.Builder { + return &baseBuilder{ + name: name, + v2PickerBuilder: pb, + config: config, + } +} diff --git a/vendor/google.golang.org/grpc/balancer/grpclb/grpclb.go b/vendor/google.golang.org/grpc/balancer/grpclb/grpclb.go index e79b1a7bfd..1e34786834 100644 --- a/vendor/google.golang.org/grpc/balancer/grpclb/grpclb.go +++ b/vendor/google.golang.org/grpc/balancer/grpclb/grpclb.go @@ -40,6 +40,7 @@ import ( "google.golang.org/grpc/grpclog" "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/internal/resolver/dns" "google.golang.org/grpc/resolver" ) @@ -97,6 +98,7 @@ func (x *balanceLoadClientStream) Recv() (*lbpb.LoadBalanceResponse, error) { func init() { balancer.Register(newLBBuilder()) + dns.EnableSRVLookups = true } // newLBBuilder creates a builder for grpclb. @@ -212,7 +214,7 @@ type lbBalancer struct { state connectivity.State subConns map[resolver.Address]balancer.SubConn // Used to new/remove SubConn. scStates map[balancer.SubConn]connectivity.State // Used to filter READY SubConns. - picker balancer.Picker + picker balancer.V2Picker // Support fallback to resolved backend addresses if there's no response // from remote balancer within fallbackTimeout. remoteBalancerConnected bool @@ -367,7 +369,7 @@ func (lb *lbBalancer) updateStateAndPicker(forceRegeneratePicker bool, resetDrop lb.regeneratePicker(resetDrop) } - lb.cc.UpdateBalancerState(lb.state, lb.picker) + lb.cc.UpdateState(balancer.State{ConnectivityState: lb.state, Picker: lb.picker}) } // fallbackToBackendsAfter blocks for fallbackTimeout and falls back to use diff --git a/vendor/google.golang.org/grpc/balancer/grpclb/grpclb_picker.go b/vendor/google.golang.org/grpc/balancer/grpclb/grpclb_picker.go index 6f023bc5ee..22213cc0be 100644 --- a/vendor/google.golang.org/grpc/balancer/grpclb/grpclb_picker.go +++ b/vendor/google.golang.org/grpc/balancer/grpclb/grpclb_picker.go @@ -19,7 +19,6 @@ package grpclb import ( - "context" "sync" "sync/atomic" @@ -50,6 +49,14 @@ func newRPCStats() *rpcStats { } } +func isZeroStats(stats *lbpb.ClientStats) bool { + return len(stats.CallsFinishedWithDrop) == 0 && + stats.NumCallsStarted == 0 && + stats.NumCallsFinished == 0 && + stats.NumCallsFinishedWithClientFailedToSend == 0 && + stats.NumCallsFinishedKnownReceived == 0 +} + // toClientStats converts rpcStats to lbpb.ClientStats, and clears rpcStats. func (s *rpcStats) toClientStats() *lbpb.ClientStats { stats := &lbpb.ClientStats{ @@ -96,8 +103,8 @@ type errPicker struct { err error } -func (p *errPicker) Pick(ctx context.Context, opts balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) { - return nil, nil, p.err +func (p *errPicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { + return balancer.PickResult{}, p.err } // rrPicker does roundrobin on subConns. It's typically used when there's no @@ -118,12 +125,12 @@ func newRRPicker(readySCs []balancer.SubConn) *rrPicker { } } -func (p *rrPicker) Pick(ctx context.Context, opts balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) { +func (p *rrPicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { p.mu.Lock() defer p.mu.Unlock() sc := p.subConns[p.subConnsNext] p.subConnsNext = (p.subConnsNext + 1) % len(p.subConns) - return sc, nil, nil + return balancer.PickResult{SubConn: sc}, nil } // lbPicker does two layers of picks: @@ -154,7 +161,7 @@ func newLBPicker(serverList []*lbpb.Server, readySCs []balancer.SubConn, stats * } } -func (p *lbPicker) Pick(ctx context.Context, opts balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) { +func (p *lbPicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { p.mu.Lock() defer p.mu.Unlock() @@ -165,12 +172,12 @@ func (p *lbPicker) Pick(ctx context.Context, opts balancer.PickOptions) (balance // If it's a drop, return an error and fail the RPC. if s.Drop { p.stats.drop(s.LoadBalanceToken) - return nil, nil, status.Errorf(codes.Unavailable, "request dropped by grpclb") + return balancer.PickResult{}, status.Errorf(codes.Unavailable, "request dropped by grpclb") } // If not a drop but there's no ready subConns. if len(p.subConns) <= 0 { - return nil, nil, balancer.ErrNoSubConnAvailable + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable } // Return the next ready subConn in the list, also collect rpc stats. @@ -183,7 +190,7 @@ func (p *lbPicker) Pick(ctx context.Context, opts balancer.PickOptions) (balance p.stats.knownReceived() } } - return sc, done, nil + return balancer.PickResult{SubConn: sc, Done: done}, nil } func (p *lbPicker) updateReadySCs(readySCs []balancer.SubConn) { diff --git a/vendor/google.golang.org/grpc/balancer/grpclb/grpclb_remote_balancer.go b/vendor/google.golang.org/grpc/balancer/grpclb/grpclb_remote_balancer.go index 897ca8ff47..ecaa1e7878 100644 --- a/vendor/google.golang.org/grpc/balancer/grpclb/grpclb_remote_balancer.go +++ b/vendor/google.golang.org/grpc/balancer/grpclb/grpclb_remote_balancer.go @@ -286,6 +286,7 @@ func (ccw *remoteBalancerCCWrapper) readServerList(s *balanceLoadClientStream) e func (ccw *remoteBalancerCCWrapper) sendLoadReport(s *balanceLoadClientStream, interval time.Duration) { ticker := time.NewTicker(interval) defer ticker.Stop() + lastZero := false for { select { case <-ticker.C: @@ -293,6 +294,12 @@ func (ccw *remoteBalancerCCWrapper) sendLoadReport(s *balanceLoadClientStream, i return } stats := ccw.lb.clientStats.toClientStats() + zero := isZeroStats(stats) + if zero && lastZero { + // Quash redundant empty load reports. + continue + } + lastZero = zero t := time.Now() stats.Timestamp = ×tamppb.Timestamp{ Seconds: t.Unix(), @@ -372,7 +379,7 @@ func (ccw *remoteBalancerCCWrapper) watchRemoteBalancer() { } } // Trigger a re-resolve when the stream errors. - ccw.lb.cc.cc.ResolveNow(resolver.ResolveNowOption{}) + ccw.lb.cc.cc.ResolveNow(resolver.ResolveNowOptions{}) ccw.lb.mu.Lock() ccw.lb.remoteBalancerConnected = false diff --git a/vendor/google.golang.org/grpc/balancer/grpclb/grpclb_util.go b/vendor/google.golang.org/grpc/balancer/grpclb/grpclb_util.go index 6d44ae5c09..636725e541 100644 --- a/vendor/google.golang.org/grpc/balancer/grpclb/grpclb_util.go +++ b/vendor/google.golang.org/grpc/balancer/grpclb/grpclb_util.go @@ -24,7 +24,6 @@ import ( "time" "google.golang.org/grpc/balancer" - "google.golang.org/grpc/connectivity" "google.golang.org/grpc/resolver" ) @@ -67,7 +66,7 @@ type lbManualResolver struct { ccb balancer.ClientConn } -func (r *lbManualResolver) Build(_ resolver.Target, cc resolver.ClientConn, _ resolver.BuildOption) (resolver.Resolver, error) { +func (r *lbManualResolver) Build(_ resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) (resolver.Resolver, error) { r.ccr = cc return r, nil } @@ -77,7 +76,7 @@ func (r *lbManualResolver) Scheme() string { } // ResolveNow calls resolveNow on the parent ClientConn. -func (r *lbManualResolver) ResolveNow(o resolver.ResolveNowOption) { +func (r *lbManualResolver) ResolveNow(o resolver.ResolveNowOptions) { r.ccb.ResolveNow(o) } @@ -195,8 +194,8 @@ func (ccc *lbCacheClientConn) RemoveSubConn(sc balancer.SubConn) { } } -func (ccc *lbCacheClientConn) UpdateBalancerState(s connectivity.State, p balancer.Picker) { - ccc.cc.UpdateBalancerState(s, p) +func (ccc *lbCacheClientConn) UpdateState(s balancer.State) { + ccc.cc.UpdateState(s) } func (ccc *lbCacheClientConn) close() { diff --git a/vendor/google.golang.org/grpc/balancer/roundrobin/roundrobin.go b/vendor/google.golang.org/grpc/balancer/roundrobin/roundrobin.go index 29f7a4ddd6..d4d645501c 100644 --- a/vendor/google.golang.org/grpc/balancer/roundrobin/roundrobin.go +++ b/vendor/google.golang.org/grpc/balancer/roundrobin/roundrobin.go @@ -22,14 +22,12 @@ package roundrobin import ( - "context" "sync" "google.golang.org/grpc/balancer" "google.golang.org/grpc/balancer/base" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/internal/grpcrand" - "google.golang.org/grpc/resolver" ) // Name is the name of round_robin balancer. @@ -37,7 +35,7 @@ const Name = "round_robin" // newBuilder creates a new roundrobin balancer builder. func newBuilder() balancer.Builder { - return base.NewBalancerBuilderWithConfig(Name, &rrPickerBuilder{}, base.Config{HealthCheck: true}) + return base.NewBalancerBuilderV2(Name, &rrPickerBuilder{}, base.Config{HealthCheck: true}) } func init() { @@ -46,13 +44,13 @@ func init() { type rrPickerBuilder struct{} -func (*rrPickerBuilder) Build(readySCs map[resolver.Address]balancer.SubConn) balancer.Picker { - grpclog.Infof("roundrobinPicker: newPicker called with readySCs: %v", readySCs) - if len(readySCs) == 0 { - return base.NewErrPicker(balancer.ErrNoSubConnAvailable) +func (*rrPickerBuilder) Build(info base.PickerBuildInfo) balancer.V2Picker { + grpclog.Infof("roundrobinPicker: newPicker called with info: %v", info) + if len(info.ReadySCs) == 0 { + return base.NewErrPickerV2(balancer.ErrNoSubConnAvailable) } var scs []balancer.SubConn - for _, sc := range readySCs { + for sc := range info.ReadySCs { scs = append(scs, sc) } return &rrPicker{ @@ -74,10 +72,10 @@ type rrPicker struct { next int } -func (p *rrPicker) Pick(ctx context.Context, opts balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) { +func (p *rrPicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { p.mu.Lock() sc := p.subConns[p.next] p.next = (p.next + 1) % len(p.subConns) p.mu.Unlock() - return sc, nil, nil + return balancer.PickResult{SubConn: sc}, nil } diff --git a/vendor/google.golang.org/grpc/balancer_conn_wrappers.go b/vendor/google.golang.org/grpc/balancer_conn_wrappers.go index 5356194c34..824f28e740 100644 --- a/vendor/google.golang.org/grpc/balancer_conn_wrappers.go +++ b/vendor/google.golang.org/grpc/balancer_conn_wrappers.go @@ -34,6 +34,7 @@ import ( type scStateUpdate struct { sc balancer.SubConn state connectivity.State + err error } // ccBalancerWrapper is a wrapper on top of cc for balancers. @@ -74,7 +75,7 @@ func (ccb *ccBalancerWrapper) watcher() { ccb.balancerMu.Lock() su := t.(*scStateUpdate) if ub, ok := ccb.balancer.(balancer.V2Balancer); ok { - ub.UpdateSubConnState(su.sc, balancer.SubConnState{ConnectivityState: su.state}) + ub.UpdateSubConnState(su.sc, balancer.SubConnState{ConnectivityState: su.state, ConnectionError: su.err}) } else { ccb.balancer.HandleSubConnStateChange(su.sc, su.state) } @@ -91,7 +92,7 @@ func (ccb *ccBalancerWrapper) watcher() { for acbw := range scs { ccb.cc.removeAddrConn(acbw.getAddrConn(), errConnDrain) } - ccb.UpdateBalancerState(connectivity.Connecting, nil) + ccb.UpdateState(balancer.State{ConnectivityState: connectivity.Connecting, Picker: nil}) return } } @@ -101,7 +102,7 @@ func (ccb *ccBalancerWrapper) close() { ccb.done.Fire() } -func (ccb *ccBalancerWrapper) handleSubConnStateChange(sc balancer.SubConn, s connectivity.State) { +func (ccb *ccBalancerWrapper) handleSubConnStateChange(sc balancer.SubConn, s connectivity.State, err error) { // When updating addresses for a SubConn, if the address in use is not in // the new addresses, the old ac will be tearDown() and a new ac will be // created. tearDown() generates a state change with Shutdown state, we @@ -115,6 +116,7 @@ func (ccb *ccBalancerWrapper) handleSubConnStateChange(sc balancer.SubConn, s co ccb.scBuffer.Put(&scStateUpdate{ sc: sc, state: s, + err: err, }) } @@ -186,7 +188,22 @@ func (ccb *ccBalancerWrapper) UpdateBalancerState(s connectivity.State, p balanc ccb.cc.csMgr.updateState(s) } -func (ccb *ccBalancerWrapper) ResolveNow(o resolver.ResolveNowOption) { +func (ccb *ccBalancerWrapper) UpdateState(s balancer.State) { + ccb.mu.Lock() + defer ccb.mu.Unlock() + if ccb.subConns == nil { + return + } + // Update picker before updating state. Even though the ordering here does + // not matter, it can lead to multiple calls of Pick in the common start-up + // case where we wait for ready and then perform an RPC. If the picker is + // updated later, we could call the "connecting" picker when the state is + // updated, and then call the "ready" picker after the picker gets updated. + ccb.cc.blockingpicker.updatePickerV2(s.Picker) + ccb.cc.csMgr.updateState(s.ConnectivityState) +} + +func (ccb *ccBalancerWrapper) ResolveNow(o resolver.ResolveNowOptions) { ccb.cc.resolveNow(o) } diff --git a/vendor/google.golang.org/grpc/balancer_v1_wrapper.go b/vendor/google.golang.org/grpc/balancer_v1_wrapper.go index 66e9a44ac4..db04b08b84 100644 --- a/vendor/google.golang.org/grpc/balancer_v1_wrapper.go +++ b/vendor/google.golang.org/grpc/balancer_v1_wrapper.go @@ -19,7 +19,6 @@ package grpc import ( - "context" "sync" "google.golang.org/grpc/balancer" @@ -49,7 +48,7 @@ func (bwb *balancerWrapperBuilder) Build(cc balancer.ClientConn, opts balancer.B csEvltr: &balancer.ConnectivityStateEvaluator{}, state: connectivity.Idle, } - cc.UpdateBalancerState(connectivity.Idle, bw) + cc.UpdateState(balancer.State{ConnectivityState: connectivity.Idle, Picker: bw}) go bw.lbWatcher() return bw } @@ -243,7 +242,7 @@ func (bw *balancerWrapper) HandleSubConnStateChange(sc balancer.SubConn, s conne if bw.state != sa { bw.state = sa } - bw.cc.UpdateBalancerState(bw.state, bw) + bw.cc.UpdateState(balancer.State{ConnectivityState: bw.state, Picker: bw}) if s == connectivity.Shutdown { // Remove state for this sc. delete(bw.connSt, sc) @@ -275,17 +274,17 @@ func (bw *balancerWrapper) Close() { // The picker is the balancerWrapper itself. // It either blocks or returns error, consistent with v1 balancer Get(). -func (bw *balancerWrapper) Pick(ctx context.Context, opts balancer.PickOptions) (sc balancer.SubConn, done func(balancer.DoneInfo), err error) { +func (bw *balancerWrapper) Pick(info balancer.PickInfo) (result balancer.PickResult, err error) { failfast := true // Default failfast is true. - if ss, ok := rpcInfoFromContext(ctx); ok { + if ss, ok := rpcInfoFromContext(info.Ctx); ok { failfast = ss.failfast } - a, p, err := bw.balancer.Get(ctx, BalancerGetOptions{BlockingWait: !failfast}) + a, p, err := bw.balancer.Get(info.Ctx, BalancerGetOptions{BlockingWait: !failfast}) if err != nil { - return nil, nil, err + return balancer.PickResult{}, toRPCErr(err) } if p != nil { - done = func(balancer.DoneInfo) { p() } + result.Done = func(balancer.DoneInfo) { p() } defer func() { if err != nil { p() @@ -297,38 +296,39 @@ func (bw *balancerWrapper) Pick(ctx context.Context, opts balancer.PickOptions) defer bw.mu.Unlock() if bw.pickfirst { // Get the first sc in conns. - for _, sc := range bw.conns { - return sc, done, nil + for _, result.SubConn = range bw.conns { + return result, nil } - return nil, nil, balancer.ErrNoSubConnAvailable + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable } - sc, ok1 := bw.conns[resolver.Address{ + var ok1 bool + result.SubConn, ok1 = bw.conns[resolver.Address{ Addr: a.Addr, Type: resolver.Backend, ServerName: "", Metadata: a.Metadata, }] - s, ok2 := bw.connSt[sc] + s, ok2 := bw.connSt[result.SubConn] if !ok1 || !ok2 { // This can only happen due to a race where Get() returned an address // that was subsequently removed by Notify. In this case we should // retry always. - return nil, nil, balancer.ErrNoSubConnAvailable + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable } switch s.s { case connectivity.Ready, connectivity.Idle: - return sc, done, nil + return result, nil case connectivity.Shutdown, connectivity.TransientFailure: // If the returned sc has been shut down or is in transient failure, // return error, and this RPC will fail or wait for another picker (if // non-failfast). - return nil, nil, balancer.ErrTransientFailure + return balancer.PickResult{}, balancer.ErrTransientFailure default: // For other states (connecting or unknown), the v1 balancer would // traditionally wait until ready and then issue the RPC. Returning // ErrNoSubConnAvailable will be a slight improvement in that it will // allow the balancer to choose another address in case others are // connected. - return nil, nil, balancer.ErrNoSubConnAvailable + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable } } diff --git a/vendor/google.golang.org/grpc/clientconn.go b/vendor/google.golang.org/grpc/clientconn.go index 4414ba87fa..14ce9c76aa 100644 --- a/vendor/google.golang.org/grpc/clientconn.go +++ b/vendor/google.golang.org/grpc/clientconn.go @@ -688,7 +688,7 @@ func (cc *ClientConn) switchBalancer(name string) { cc.balancerWrapper = newCCBalancerWrapper(cc, builder, cc.balancerBuildOpts) } -func (cc *ClientConn) handleSubConnStateChange(sc balancer.SubConn, s connectivity.State) { +func (cc *ClientConn) handleSubConnStateChange(sc balancer.SubConn, s connectivity.State, err error) { cc.mu.Lock() if cc.conns == nil { cc.mu.Unlock() @@ -696,7 +696,7 @@ func (cc *ClientConn) handleSubConnStateChange(sc balancer.SubConn, s connectivi } // TODO(bar switching) send updates to all balancer wrappers when balancer // gracefully switching is supported. - cc.balancerWrapper.handleSubConnStateChange(sc, s) + cc.balancerWrapper.handleSubConnStateChange(sc, s, err) cc.mu.Unlock() } @@ -793,7 +793,7 @@ func (ac *addrConn) connect() error { } // Update connectivity state within the lock to prevent subsequent or // concurrent calls from resetting the transport more than once. - ac.updateConnectivityState(connectivity.Connecting) + ac.updateConnectivityState(connectivity.Connecting, nil) ac.mu.Unlock() // Start a goroutine connecting to the server asynchronously. @@ -879,7 +879,8 @@ func (cc *ClientConn) healthCheckConfig() *healthCheckConfig { } func (cc *ClientConn) getTransport(ctx context.Context, failfast bool, method string) (transport.ClientTransport, func(balancer.DoneInfo), error) { - t, done, err := cc.blockingpicker.pick(ctx, failfast, balancer.PickOptions{ + t, done, err := cc.blockingpicker.pick(ctx, failfast, balancer.PickInfo{ + Ctx: ctx, FullMethodName: method, }) if err != nil { @@ -938,7 +939,7 @@ func (cc *ClientConn) applyServiceConfigAndBalancer(sc *ServiceConfig, addrs []r } } -func (cc *ClientConn) resolveNow(o resolver.ResolveNowOption) { +func (cc *ClientConn) resolveNow(o resolver.ResolveNowOptions) { cc.mu.RLock() r := cc.resolverWrapper cc.mu.RUnlock() @@ -1048,7 +1049,7 @@ type addrConn struct { } // Note: this requires a lock on ac.mu. -func (ac *addrConn) updateConnectivityState(s connectivity.State) { +func (ac *addrConn) updateConnectivityState(s connectivity.State, lastErr error) { if ac.state == s { return } @@ -1061,7 +1062,7 @@ func (ac *addrConn) updateConnectivityState(s connectivity.State) { Severity: channelz.CtINFO, }) } - ac.cc.handleSubConnStateChange(ac.acbw, s) + ac.cc.handleSubConnStateChange(ac.acbw, s, lastErr) } // adjustParams updates parameters used to create transports upon @@ -1081,7 +1082,7 @@ func (ac *addrConn) adjustParams(r transport.GoAwayReason) { func (ac *addrConn) resetTransport() { for i := 0; ; i++ { if i > 0 { - ac.cc.resolveNow(resolver.ResolveNowOption{}) + ac.cc.resolveNow(resolver.ResolveNowOptions{}) } ac.mu.Lock() @@ -1110,7 +1111,7 @@ func (ac *addrConn) resetTransport() { // https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md#proposed-backoff-algorithm connectDeadline := time.Now().Add(dialDuration) - ac.updateConnectivityState(connectivity.Connecting) + ac.updateConnectivityState(connectivity.Connecting, nil) ac.transport = nil ac.mu.Unlock() @@ -1123,7 +1124,7 @@ func (ac *addrConn) resetTransport() { ac.mu.Unlock() return } - ac.updateConnectivityState(connectivity.TransientFailure) + ac.updateConnectivityState(connectivity.TransientFailure, err) // Backoff. b := ac.resetBackoff @@ -1179,6 +1180,7 @@ func (ac *addrConn) resetTransport() { // first successful one. It returns the transport, the address and a Event in // the successful case. The Event fires when the returned transport disconnects. func (ac *addrConn) tryAllAddrs(addrs []resolver.Address, connectDeadline time.Time) (transport.ClientTransport, resolver.Address, *grpcsync.Event, error) { + var firstConnErr error for _, addr := range addrs { ac.mu.Lock() if ac.state == connectivity.Shutdown { @@ -1207,11 +1209,14 @@ func (ac *addrConn) tryAllAddrs(addrs []resolver.Address, connectDeadline time.T if err == nil { return newTr, addr, reconnect, nil } + if firstConnErr == nil { + firstConnErr = err + } ac.cc.blockingpicker.updateConnectionError(err) } // Couldn't connect to any address. - return nil, resolver.Address{}, nil, fmt.Errorf("couldn't connect to any address") + return nil, resolver.Address{}, nil, firstConnErr } // createTransport creates a connection to addr. It returns the transport and a @@ -1244,7 +1249,7 @@ func (ac *addrConn) createTransport(addr resolver.Address, copts transport.Conne // state to Connecting. // // TODO: this should be Idle when grpc-go properly supports it. - ac.updateConnectivityState(connectivity.Connecting) + ac.updateConnectivityState(connectivity.Connecting, nil) } }) ac.mu.Unlock() @@ -1259,7 +1264,7 @@ func (ac *addrConn) createTransport(addr resolver.Address, copts transport.Conne // state to Connecting. // // TODO: this should be Idle when grpc-go properly supports it. - ac.updateConnectivityState(connectivity.Connecting) + ac.updateConnectivityState(connectivity.Connecting, nil) } }) ac.mu.Unlock() @@ -1285,7 +1290,7 @@ func (ac *addrConn) createTransport(addr resolver.Address, copts transport.Conne } select { - case <-time.After(connectDeadline.Sub(time.Now())): + case <-time.After(time.Until(connectDeadline)): // We didn't get the preface in time. newTr.Close() grpclog.Warningf("grpc: addrConn.createTransport failed to connect to %v: didn't receive server preface in time. Reconnecting...", addr) @@ -1316,7 +1321,7 @@ func (ac *addrConn) startHealthCheck(ctx context.Context) { var healthcheckManagingState bool defer func() { if !healthcheckManagingState { - ac.updateConnectivityState(connectivity.Ready) + ac.updateConnectivityState(connectivity.Ready, nil) } }() @@ -1352,13 +1357,13 @@ func (ac *addrConn) startHealthCheck(ctx context.Context) { ac.mu.Unlock() return newNonRetryClientStream(ctx, &StreamDesc{ServerStreams: true}, method, currentTr, ac) } - setConnectivityState := func(s connectivity.State) { + setConnectivityState := func(s connectivity.State, lastErr error) { ac.mu.Lock() defer ac.mu.Unlock() if ac.transport != currentTr { return } - ac.updateConnectivityState(s) + ac.updateConnectivityState(s, lastErr) } // Start the health checking stream. go func() { @@ -1424,7 +1429,7 @@ func (ac *addrConn) tearDown(err error) { ac.transport = nil // We have to set the state to Shutdown before anything else to prevent races // between setting the state and logic that waits on context cancellation / etc. - ac.updateConnectivityState(connectivity.Shutdown) + ac.updateConnectivityState(connectivity.Shutdown, nil) ac.cancel() ac.curAddr = resolver.Address{} if err == errConnDrain && curTr != nil { diff --git a/vendor/google.golang.org/grpc/credentials/alts/internal/handshaker/handshaker.go b/vendor/google.golang.org/grpc/credentials/alts/internal/handshaker/handshaker.go index 49c22c1e89..8bc7ceee0a 100644 --- a/vendor/google.golang.org/grpc/credentials/alts/internal/handshaker/handshaker.go +++ b/vendor/google.golang.org/grpc/credentials/alts/internal/handshaker/handshaker.go @@ -64,6 +64,9 @@ var ( concurrentHandshakes = int64(0) // errDropped occurs when maxPendingHandshakes is reached. errDropped = errors.New("maximum number of concurrent ALTS handshakes is reached") + // errOutOfBound occurs when the handshake service returns a consumed + // bytes value larger than the buffer that was passed to it originally. + errOutOfBound = errors.New("handshaker service consumed bytes value is out-of-bound") ) func init() { @@ -74,8 +77,10 @@ func init() { } } -func acquire(n int64) bool { +func acquire() bool { mu.Lock() + // If we need n to be configurable, we can pass it as an argument. + n := int64(1) success := maxPendingHandshakes-concurrentHandshakes >= n if success { concurrentHandshakes += n @@ -84,8 +89,10 @@ func acquire(n int64) bool { return success } -func release(n int64) { +func release() { mu.Lock() + // If we need n to be configurable, we can pass it as an argument. + n := int64(1) concurrentHandshakes -= n if concurrentHandshakes < 0 { mu.Unlock() @@ -182,10 +189,10 @@ func NewServerHandshaker(ctx context.Context, conn *grpc.ClientConn, c net.Conn, // ClientHandshake starts and completes a client ALTS handshaking for GCP. Once // done, ClientHandshake returns a secure connection. func (h *altsHandshaker) ClientHandshake(ctx context.Context) (net.Conn, credentials.AuthInfo, error) { - if !acquire(1) { + if !acquire() { return nil, nil, errDropped } - defer release(1) + defer release() if h.side != core.ClientSide { return nil, nil, errors.New("only handshakers created using NewClientHandshaker can perform a client handshaker") @@ -225,10 +232,10 @@ func (h *altsHandshaker) ClientHandshake(ctx context.Context) (net.Conn, credent // ServerHandshake starts and completes a server ALTS handshaking for GCP. Once // done, ServerHandshake returns a secure connection. func (h *altsHandshaker) ServerHandshake(ctx context.Context) (net.Conn, credentials.AuthInfo, error) { - if !acquire(1) { + if !acquire() { return nil, nil, errDropped } - defer release(1) + defer release() if h.side != core.ServerSide { return nil, nil, errors.New("only handshakers created using NewServerHandshaker can perform a server handshaker") @@ -280,6 +287,9 @@ func (h *altsHandshaker) doHandshake(req *altspb.HandshakerReq) (net.Conn, *alts var extra []byte if req.GetServerStart() != nil { + if resp.GetBytesConsumed() > uint32(len(req.GetServerStart().GetInBytes())) { + return nil, nil, errOutOfBound + } extra = req.GetServerStart().GetInBytes()[resp.GetBytesConsumed():] } result, extra, err := h.processUntilDone(resp, extra) @@ -339,6 +349,7 @@ func (h *altsHandshaker) processUntilDone(resp *altspb.HandshakerResp, extra []b // Append extra bytes from the previous interaction with the // handshaker service with the current buffer read from conn. p := append(extra, buf[:n]...) + // From here on, p and extra point to the same slice. resp, err = h.accessHandshakerService(&altspb.HandshakerReq{ ReqOneof: &altspb.HandshakerReq_Next{ Next: &altspb.NextHandshakeMessageReq{ @@ -350,11 +361,10 @@ func (h *altsHandshaker) processUntilDone(resp *altspb.HandshakerResp, extra []b return nil, nil, err } // Set extra based on handshaker service response. - if n == 0 { - extra = nil - } else { - extra = buf[resp.GetBytesConsumed():n] + if resp.GetBytesConsumed() > uint32(len(p)) { + return nil, nil, errOutOfBound } + extra = p[resp.GetBytesConsumed():] } } diff --git a/vendor/google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go b/vendor/google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go index 021646ed76..4d1fc4c4d8 100644 --- a/vendor/google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go +++ b/vendor/google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go @@ -258,10 +258,12 @@ type StartClientHandshakeReq struct { // the target name. TargetName string `protobuf:"bytes,8,opt,name=target_name,json=targetName,proto3" json:"target_name,omitempty"` // (Optional) RPC protocol versions supported by the client. - RpcVersions *RpcProtocolVersions `protobuf:"bytes,9,opt,name=rpc_versions,json=rpcVersions,proto3" json:"rpc_versions,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + RpcVersions *RpcProtocolVersions `protobuf:"bytes,9,opt,name=rpc_versions,json=rpcVersions,proto3" json:"rpc_versions,omitempty"` + // (Optional) Maximum frame size supported by the client. + MaxFrameSize uint32 `protobuf:"varint,10,opt,name=max_frame_size,json=maxFrameSize,proto3" json:"max_frame_size,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *StartClientHandshakeReq) Reset() { *m = StartClientHandshakeReq{} } @@ -352,6 +354,13 @@ func (m *StartClientHandshakeReq) GetRpcVersions() *RpcProtocolVersions { return nil } +func (m *StartClientHandshakeReq) GetMaxFrameSize() uint32 { + if m != nil { + return m.MaxFrameSize + } + return 0 +} + type ServerHandshakeParameters struct { // The record protocols supported by the server, e.g., // "ALTSRP_GCM_AES128". @@ -423,10 +432,12 @@ type StartServerHandshakeReq struct { // port number, and network protocol. RemoteEndpoint *Endpoint `protobuf:"bytes,5,opt,name=remote_endpoint,json=remoteEndpoint,proto3" json:"remote_endpoint,omitempty"` // (Optional) RPC protocol versions supported by the server. - RpcVersions *RpcProtocolVersions `protobuf:"bytes,6,opt,name=rpc_versions,json=rpcVersions,proto3" json:"rpc_versions,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + RpcVersions *RpcProtocolVersions `protobuf:"bytes,6,opt,name=rpc_versions,json=rpcVersions,proto3" json:"rpc_versions,omitempty"` + // (Optional) Maximum frame size supported by the server. + MaxFrameSize uint32 `protobuf:"varint,7,opt,name=max_frame_size,json=maxFrameSize,proto3" json:"max_frame_size,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *StartServerHandshakeReq) Reset() { *m = StartServerHandshakeReq{} } @@ -496,6 +507,13 @@ func (m *StartServerHandshakeReq) GetRpcVersions() *RpcProtocolVersions { return nil } +func (m *StartServerHandshakeReq) GetMaxFrameSize() uint32 { + if m != nil { + return m.MaxFrameSize + } + return 0 +} + type NextHandshakeMessageReq struct { // Bytes in out_frames returned from the peer's HandshakerResp. It is possible // that the peer's out_frames are split into multiple NextHandshakerMessageReq @@ -651,10 +669,12 @@ type HandshakerResult struct { // post-handshake messages in the future. KeepChannelOpen bool `protobuf:"varint,6,opt,name=keep_channel_open,json=keepChannelOpen,proto3" json:"keep_channel_open,omitempty"` // The RPC protocol versions supported by the peer. - PeerRpcVersions *RpcProtocolVersions `protobuf:"bytes,7,opt,name=peer_rpc_versions,json=peerRpcVersions,proto3" json:"peer_rpc_versions,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + PeerRpcVersions *RpcProtocolVersions `protobuf:"bytes,7,opt,name=peer_rpc_versions,json=peerRpcVersions,proto3" json:"peer_rpc_versions,omitempty"` + // The maximum frame size of the peer. + MaxFrameSize uint32 `protobuf:"varint,8,opt,name=max_frame_size,json=maxFrameSize,proto3" json:"max_frame_size,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *HandshakerResult) Reset() { *m = HandshakerResult{} } @@ -731,6 +751,13 @@ func (m *HandshakerResult) GetPeerRpcVersions() *RpcProtocolVersions { return nil } +func (m *HandshakerResult) GetMaxFrameSize() uint32 { + if m != nil { + return m.MaxFrameSize + } + return 0 +} + type HandshakerStatus struct { // The status code. This could be the gRPC status code. Code uint32 `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"` @@ -874,80 +901,83 @@ func init() { func init() { proto.RegisterFile("grpc/gcp/handshaker.proto", fileDescriptor_54c074f40c7c7e99) } var fileDescriptor_54c074f40c7c7e99 = []byte{ - // 1168 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xdf, 0x6e, 0x1a, 0xc7, - 0x17, 0xf6, 0x02, 0xb6, 0xf1, 0xc1, 0xfc, 0xf1, 0xc4, 0x51, 0xd6, 0x4e, 0xf2, 0xfb, 0x51, 0xaa, - 0xaa, 0x24, 0x17, 0xd0, 0x92, 0x56, 0x69, 0x52, 0x45, 0x09, 0x60, 0x2c, 0xdc, 0xa4, 0x18, 0x2d, - 0x4e, 0x2b, 0x35, 0x17, 0xab, 0xc9, 0x32, 0xc1, 0x2b, 0x96, 0x99, 0xf5, 0xcc, 0xe0, 0x86, 0x07, - 0xe8, 0xe3, 0xf4, 0x15, 0xfa, 0x36, 0x95, 0xfa, 0x00, 0xbd, 0x6f, 0xb5, 0xb3, 0xb3, 0x7f, 0xc0, - 0x10, 0x25, 0xea, 0xdd, 0xee, 0x99, 0xef, 0x3b, 0x7b, 0xe6, 0x3b, 0xdf, 0x9c, 0x1d, 0x38, 0x9a, - 0x70, 0xdf, 0x69, 0x4e, 0x1c, 0xbf, 0x79, 0x89, 0xe9, 0x58, 0x5c, 0xe2, 0x29, 0xe1, 0x0d, 0x9f, - 0x33, 0xc9, 0x50, 0x3e, 0x58, 0x6a, 0x4c, 0x1c, 0xff, 0xb8, 0x1e, 0x83, 0x24, 0xc7, 0x54, 0xf8, - 0x8c, 0x4b, 0x5b, 0x10, 0x67, 0xce, 0x5d, 0xb9, 0xb0, 0x1d, 0x36, 0x9b, 0x31, 0x1a, 0x72, 0x6a, - 0x12, 0xf2, 0x3d, 0x3a, 0xf6, 0x99, 0x4b, 0x25, 0xba, 0x0f, 0xe0, 0xfa, 0x36, 0x1e, 0x8f, 0x39, - 0x11, 0xc2, 0x34, 0xaa, 0x46, 0x7d, 0xcf, 0xda, 0x73, 0xfd, 0x76, 0x18, 0x40, 0x08, 0x72, 0x41, - 0x22, 0x33, 0x53, 0x35, 0xea, 0xdb, 0x96, 0x7a, 0x46, 0xdf, 0x42, 0x5e, 0xe5, 0x71, 0x98, 0x67, - 0x66, 0xab, 0x46, 0xbd, 0xd4, 0x3a, 0x6a, 0x44, 0x55, 0x34, 0x06, 0x44, 0xfe, 0xca, 0xf8, 0x74, - 0xa8, 0x01, 0x56, 0x0c, 0xad, 0xfd, 0x65, 0x40, 0xfe, 0x6c, 0x4c, 0xa8, 0x74, 0xe5, 0x02, 0x3d, - 0x80, 0xb2, 0x20, 0xfc, 0xda, 0x75, 0x88, 0x8d, 0x1d, 0x87, 0xcd, 0xa9, 0x0c, 0xbf, 0xdd, 0xdf, - 0xb2, 0x4a, 0x7a, 0xa1, 0x1d, 0xc6, 0xd1, 0x3d, 0xc8, 0x5f, 0x32, 0x21, 0x29, 0x9e, 0x11, 0x55, - 0x46, 0x80, 0x89, 0x23, 0xa8, 0x03, 0x80, 0xa5, 0xe4, 0xee, 0xdb, 0xb9, 0x24, 0xc2, 0xcc, 0x56, - 0xb3, 0xf5, 0x42, 0xab, 0x96, 0x94, 0x13, 0x7d, 0xb0, 0xd1, 0x8e, 0x41, 0x3d, 0x2a, 0xf9, 0xc2, - 0x4a, 0xb1, 0x8e, 0x9f, 0x41, 0x79, 0x65, 0x19, 0x55, 0x20, 0x3b, 0x25, 0x0b, 0xad, 0x47, 0xf0, - 0x88, 0x0e, 0x61, 0xfb, 0x1a, 0x7b, 0x73, 0x5d, 0x83, 0x15, 0xbe, 0x3c, 0xcd, 0x7c, 0x67, 0x74, - 0x2a, 0x50, 0x72, 0xf5, 0x67, 0x6c, 0x46, 0x09, 0x7b, 0x57, 0xfb, 0x3d, 0x07, 0x77, 0x46, 0x12, - 0x73, 0xd9, 0xf5, 0x5c, 0x42, 0x65, 0x3f, 0x6a, 0x9a, 0x45, 0xae, 0xd0, 0x1b, 0xb8, 0x1b, 0x37, - 0x31, 0xe9, 0x4f, 0x2c, 0xa8, 0xa1, 0x04, 0xbd, 0x9b, 0xec, 0x20, 0x26, 0xc7, 0x92, 0x1e, 0xc5, - 0xfc, 0x91, 0xa6, 0x47, 0x4b, 0xe8, 0x11, 0xdc, 0xc6, 0xbe, 0xef, 0xb9, 0x0e, 0x96, 0x2e, 0xa3, - 0x71, 0x56, 0x61, 0x66, 0xaa, 0xd9, 0xfa, 0x9e, 0x75, 0x98, 0x5a, 0x8c, 0x38, 0x02, 0x3d, 0x80, - 0x0a, 0x27, 0x0e, 0xe3, 0xe3, 0x14, 0x3e, 0xab, 0xf0, 0xe5, 0x30, 0x9e, 0x40, 0x9f, 0xc3, 0x81, - 0xc4, 0x7c, 0x42, 0xa4, 0xad, 0x77, 0xec, 0x12, 0x61, 0xe6, 0x94, 0xe8, 0xe8, 0xa6, 0xe8, 0x56, - 0x25, 0x04, 0x9f, 0xc5, 0x58, 0xf4, 0x04, 0x4a, 0x1e, 0x73, 0xb0, 0x17, 0xf1, 0x17, 0xe6, 0x76, - 0xd5, 0xd8, 0xc0, 0x2e, 0x2a, 0x64, 0x6c, 0x99, 0x98, 0x4a, 0xb4, 0x77, 0xcd, 0x9d, 0x55, 0x6a, - 0xe4, 0x6a, 0x4d, 0x8d, 0x4d, 0xfe, 0x3d, 0x94, 0x39, 0x99, 0x31, 0x49, 0x12, 0xee, 0xee, 0x46, - 0x6e, 0x29, 0x84, 0xc6, 0xe4, 0xff, 0x43, 0x41, 0xef, 0x59, 0x59, 0x30, 0xaf, 0xda, 0x0f, 0x61, - 0x68, 0x10, 0x58, 0xf0, 0x05, 0xec, 0x73, 0xdf, 0xb1, 0xaf, 0x09, 0x17, 0x2e, 0xa3, 0xc2, 0xdc, - 0x53, 0xa9, 0xef, 0x27, 0xa9, 0x2d, 0xdf, 0x89, 0x24, 0xfc, 0x49, 0x83, 0xac, 0x02, 0xf7, 0x9d, - 0xe8, 0xa5, 0xf6, 0x9b, 0x01, 0x47, 0x23, 0xc2, 0xaf, 0x09, 0x4f, 0xba, 0x8d, 0x39, 0x9e, 0x11, - 0x49, 0xf8, 0xfa, 0xfe, 0x18, 0xeb, 0xfb, 0xf3, 0x0c, 0x2a, 0x4b, 0xf2, 0x06, 0xed, 0xc9, 0x6c, - 0x6c, 0x4f, 0x39, 0x2d, 0xb0, 0x4b, 0x44, 0xed, 0x9f, 0xac, 0xf6, 0xed, 0x4a, 0x31, 0x81, 0x6f, - 0x37, 0x5a, 0xcb, 0xf8, 0x80, 0xb5, 0x66, 0x70, 0x98, 0x98, 0xdd, 0x8f, 0xb7, 0xa4, 0x6b, 0x7a, - 0x9a, 0xd4, 0xb4, 0xe1, 0xab, 0x8d, 0x35, 0x7a, 0x84, 0xe7, 0xf7, 0xd6, 0xe5, 0x1a, 0xa5, 0x8e, - 0x20, 0xef, 0x52, 0xfb, 0xed, 0x22, 0x1c, 0x05, 0x46, 0x7d, 0xdf, 0xda, 0x75, 0x69, 0x27, 0x78, - 0x5d, 0xe3, 0x9e, 0xdc, 0x7f, 0x70, 0xcf, 0xf6, 0x47, 0xbb, 0x67, 0xd5, 0x1c, 0x3b, 0x9f, 0x6a, - 0x8e, 0xe3, 0x29, 0x98, 0x9b, 0x54, 0x48, 0x8f, 0xa9, 0xed, 0x70, 0x4c, 0x3d, 0x49, 0x8f, 0xa9, - 0x42, 0xeb, 0xf3, 0x94, 0xc4, 0x9b, 0x0c, 0x96, 0x9a, 0x65, 0xb5, 0x6f, 0xe0, 0xce, 0x80, 0xbc, - 0x4f, 0x26, 0xd6, 0x8f, 0x44, 0x08, 0x3c, 0x51, 0x06, 0x48, 0x8b, 0x6b, 0x2c, 0x89, 0x5b, 0xfb, - 0xd3, 0x80, 0x62, 0x4c, 0xe1, 0x01, 0xf8, 0x14, 0xf6, 0x1d, 0x35, 0xfb, 0x6c, 0x11, 0x74, 0x56, - 0x11, 0x0a, 0xad, 0xcf, 0x56, 0x1a, 0x7e, 0x73, 0x3c, 0xf6, 0xb7, 0xac, 0x42, 0x48, 0x54, 0x80, - 0x20, 0x8f, 0x50, 0x75, 0xeb, 0x3c, 0x99, 0xb5, 0x79, 0x6e, 0x1a, 0x27, 0xc8, 0x13, 0x12, 0xc3, - 0x3c, 0x8f, 0x21, 0x47, 0xc9, 0x7b, 0xa9, 0x5c, 0xb1, 0xc4, 0xdf, 0xb0, 0xdb, 0xfe, 0x96, 0xa5, - 0x08, 0x9d, 0x02, 0xec, 0x71, 0x72, 0xa5, 0xe7, 0xfa, 0xdf, 0x19, 0xa8, 0xa4, 0xf7, 0x29, 0xe6, - 0x9e, 0x44, 0x5f, 0xc3, 0xe1, 0xba, 0x83, 0xa1, 0xff, 0x1d, 0xb7, 0xd6, 0x9c, 0x0b, 0xf4, 0x25, - 0x94, 0x57, 0x4e, 0xb4, 0xfe, 0xab, 0x94, 0x96, 0x0f, 0x74, 0xa0, 0xf9, 0x94, 0x2c, 0xec, 0x31, - 0x96, 0x38, 0x32, 0xf4, 0x94, 0x2c, 0x4e, 0xb0, 0xc4, 0xe8, 0x31, 0x14, 0x7d, 0x42, 0x78, 0x32, - 0x48, 0x73, 0x1b, 0x07, 0xe9, 0x7e, 0x00, 0xbc, 0x39, 0x47, 0x3f, 0x7d, 0x04, 0x3f, 0x84, 0x83, - 0x29, 0x21, 0xbe, 0xed, 0x5c, 0x62, 0x4a, 0x89, 0x67, 0x33, 0x9f, 0x50, 0xe5, 0xe8, 0xbc, 0x55, - 0x0e, 0x16, 0xba, 0x61, 0xfc, 0xdc, 0x27, 0x14, 0x9d, 0xc1, 0x81, 0xaa, 0x6f, 0xc9, 0xfd, 0xbb, - 0x1f, 0xe3, 0xfe, 0x72, 0xc0, 0xb3, 0x52, 0xe3, 0xf1, 0x45, 0x5a, 0xf5, 0x91, 0xc4, 0x72, 0xae, - 0x2e, 0x26, 0x0e, 0x1b, 0x13, 0xa5, 0x72, 0xd1, 0x52, 0xcf, 0xc8, 0x84, 0xdd, 0x31, 0x91, 0xd8, - 0x55, 0xff, 0xbb, 0x40, 0xce, 0xe8, 0xb5, 0xf6, 0x87, 0x01, 0xa5, 0xa5, 0xc6, 0xf9, 0xc1, 0xc5, - 0x87, 0xcd, 0xa5, 0xfd, 0x2e, 0x38, 0x05, 0x91, 0xa1, 0xf7, 0xd8, 0x5c, 0x9e, 0xaa, 0x00, 0xfa, - 0x02, 0x4a, 0xca, 0xea, 0xb6, 0xc3, 0xa8, 0x98, 0xcf, 0xc8, 0x58, 0xa5, 0x2c, 0x5a, 0x45, 0x15, - 0xed, 0xea, 0x20, 0x6a, 0xc1, 0x0e, 0x57, 0x36, 0xd0, 0xce, 0x3a, 0x5e, 0xf3, 0xe3, 0xd6, 0x46, - 0xb1, 0x34, 0x32, 0xe0, 0x08, 0xb5, 0x09, 0xdd, 0xb2, 0xb5, 0x9c, 0x70, 0x9b, 0x96, 0x46, 0x3e, - 0xfc, 0x01, 0x0e, 0x6e, 0x5c, 0x04, 0x50, 0x0d, 0xfe, 0xd7, 0x6f, 0x0f, 0x4e, 0x46, 0xfd, 0xf6, - 0xcb, 0x9e, 0x3d, 0xb4, 0xce, 0x2f, 0xce, 0xbb, 0xe7, 0xaf, 0xec, 0xd7, 0x83, 0xd1, 0xb0, 0xd7, - 0x3d, 0x3b, 0x3d, 0xeb, 0x9d, 0x54, 0xb6, 0xd0, 0x2e, 0x64, 0x2f, 0x5e, 0x8d, 0x2a, 0x06, 0xca, - 0x43, 0xae, 0xfd, 0xea, 0x62, 0x54, 0xc9, 0x3c, 0xec, 0x41, 0x79, 0xe5, 0x96, 0x86, 0xaa, 0x70, - 0x6f, 0xd0, 0xbb, 0xf8, 0xf9, 0xdc, 0x7a, 0xf9, 0xa1, 0x3c, 0xdd, 0x61, 0xc5, 0x08, 0x1e, 0x5e, - 0x9f, 0x0c, 0x2b, 0x99, 0xd6, 0x9b, 0x54, 0x49, 0x7c, 0x14, 0xde, 0xd9, 0xd0, 0x29, 0x14, 0x4e, - 0x58, 0x1c, 0x46, 0x77, 0xd6, 0xcb, 0x71, 0x75, 0x6c, 0x6e, 0xd0, 0xc9, 0xaf, 0x6d, 0xd5, 0x8d, - 0xaf, 0x8c, 0xce, 0x14, 0x6e, 0xbb, 0x2c, 0xc4, 0x60, 0x4f, 0x8a, 0x86, 0x4b, 0x25, 0xe1, 0x14, - 0x7b, 0x9d, 0x72, 0x02, 0x57, 0xd5, 0x0f, 0x8d, 0x5f, 0x9e, 0x4f, 0x18, 0x9b, 0x78, 0xa4, 0x31, - 0x61, 0x1e, 0xa6, 0x93, 0x06, 0xe3, 0x93, 0xa6, 0xba, 0x0a, 0x3b, 0x9c, 0x28, 0xe3, 0x62, 0x4f, - 0x34, 0x83, 0x24, 0xcd, 0x28, 0x49, 0x53, 0x9d, 0x3a, 0x05, 0xb2, 0x27, 0x8e, 0xff, 0x76, 0x47, - 0xbd, 0x3f, 0xfa, 0x37, 0x00, 0x00, 0xff, 0xff, 0x6e, 0x37, 0x34, 0x9b, 0x67, 0x0b, 0x00, 0x00, + // 1203 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x6e, 0x1b, 0x45, + 0x14, 0xce, 0xda, 0x4e, 0xe2, 0x1c, 0xc7, 0x3f, 0x99, 0xa6, 0xea, 0x26, 0x6d, 0xc1, 0x18, 0x10, + 0x6e, 0x2f, 0x6c, 0x70, 0x41, 0xa5, 0x45, 0x55, 0x6b, 0x3b, 0x8e, 0x1c, 0x5a, 0x1c, 0x6b, 0x9d, + 0x82, 0x44, 0x2f, 0x56, 0xd3, 0xf5, 0xd4, 0x59, 0x79, 0x3d, 0xb3, 0x9d, 0x19, 0x87, 0xb8, 0xf7, + 0xbc, 0x04, 0xf7, 0xbc, 0x06, 0x2f, 0xc1, 0x33, 0x20, 0xf1, 0x18, 0x68, 0x67, 0x7f, 0x6d, 0xaf, + 0xab, 0x22, 0xb8, 0xdb, 0x39, 0xf3, 0x7d, 0x67, 0xce, 0x9c, 0xf3, 0x9d, 0xb3, 0x03, 0x47, 0x13, + 0xee, 0x5a, 0xcd, 0x89, 0xe5, 0x36, 0x2f, 0x31, 0x1d, 0x8b, 0x4b, 0x3c, 0x25, 0xbc, 0xe1, 0x72, + 0x26, 0x19, 0xca, 0x7b, 0x5b, 0x8d, 0x89, 0xe5, 0x1e, 0xd7, 0x23, 0x90, 0xe4, 0x98, 0x0a, 0x97, + 0x71, 0x69, 0x0a, 0x62, 0xcd, 0xb9, 0x2d, 0x17, 0xa6, 0xc5, 0x66, 0x33, 0x46, 0x7d, 0x4e, 0x4d, + 0x42, 0xbe, 0x47, 0xc7, 0x2e, 0xb3, 0xa9, 0x44, 0x77, 0x01, 0x6c, 0xd7, 0xc4, 0xe3, 0x31, 0x27, + 0x42, 0xe8, 0x5a, 0x55, 0xab, 0xef, 0x19, 0x7b, 0xb6, 0xdb, 0xf6, 0x0d, 0x08, 0x41, 0xce, 0x73, + 0xa4, 0x67, 0xaa, 0x5a, 0x7d, 0xdb, 0x50, 0xdf, 0xe8, 0x1b, 0xc8, 0x2b, 0x3f, 0x16, 0x73, 0xf4, + 0x6c, 0x55, 0xab, 0x97, 0x5a, 0x47, 0x8d, 0x30, 0x8a, 0xc6, 0x80, 0xc8, 0x5f, 0x18, 0x9f, 0x0e, + 0x03, 0x80, 0x11, 0x41, 0x6b, 0x7f, 0x6b, 0x90, 0x3f, 0x1b, 0x13, 0x2a, 0x6d, 0xb9, 0x40, 0xf7, + 0xa0, 0x2c, 0x08, 0xbf, 0xb2, 0x2d, 0x62, 0x62, 0xcb, 0x62, 0x73, 0x2a, 0xfd, 0xb3, 0xfb, 0x5b, + 0x46, 0x29, 0xd8, 0x68, 0xfb, 0x76, 0x74, 0x07, 0xf2, 0x97, 0x4c, 0x48, 0x8a, 0x67, 0x44, 0x85, + 0xe1, 0x61, 0x22, 0x0b, 0xea, 0x00, 0x60, 0x29, 0xb9, 0xfd, 0x7a, 0x2e, 0x89, 0xd0, 0xb3, 0xd5, + 0x6c, 0xbd, 0xd0, 0xaa, 0xc5, 0xe1, 0x84, 0x07, 0x36, 0xda, 0x11, 0xa8, 0x47, 0x25, 0x5f, 0x18, + 0x09, 0xd6, 0xf1, 0x13, 0x28, 0xaf, 0x6c, 0xa3, 0x0a, 0x64, 0xa7, 0x64, 0x11, 0xe4, 0xc3, 0xfb, + 0x44, 0x87, 0xb0, 0x7d, 0x85, 0x9d, 0x79, 0x10, 0x83, 0xe1, 0x2f, 0x1e, 0x67, 0xbe, 0xd5, 0x3a, + 0x15, 0x28, 0xd9, 0xc1, 0x31, 0x26, 0xa3, 0x84, 0xbd, 0xa9, 0xfd, 0x99, 0x83, 0x5b, 0x23, 0x89, + 0xb9, 0xec, 0x3a, 0x36, 0xa1, 0xb2, 0x1f, 0x16, 0xcd, 0x20, 0x6f, 0xd1, 0x2b, 0xb8, 0x1d, 0x15, + 0x31, 0xae, 0x4f, 0x94, 0x50, 0x4d, 0x25, 0xf4, 0x76, 0x7c, 0x83, 0x88, 0x1c, 0xa5, 0xf4, 0x28, + 0xe2, 0x8f, 0x02, 0x7a, 0xb8, 0x85, 0x1e, 0xc0, 0x4d, 0xec, 0xba, 0x8e, 0x6d, 0x61, 0x69, 0x33, + 0x1a, 0x79, 0x15, 0x7a, 0xa6, 0x9a, 0xad, 0xef, 0x19, 0x87, 0x89, 0xcd, 0x90, 0x23, 0xd0, 0x3d, + 0xa8, 0x70, 0x62, 0x31, 0x3e, 0x4e, 0xe0, 0xb3, 0x0a, 0x5f, 0xf6, 0xed, 0x31, 0xf4, 0x29, 0x1c, + 0x48, 0xcc, 0x27, 0x44, 0x9a, 0xc1, 0x8d, 0x6d, 0x22, 0xf4, 0x9c, 0x4a, 0x3a, 0x5a, 0x4f, 0xba, + 0x51, 0xf1, 0xc1, 0x67, 0x11, 0x16, 0x3d, 0x82, 0x92, 0xc3, 0x2c, 0xec, 0x84, 0xfc, 0x85, 0xbe, + 0x5d, 0xd5, 0x36, 0xb0, 0x8b, 0x0a, 0x19, 0x49, 0x26, 0xa2, 0x92, 0x40, 0xbb, 0xfa, 0xce, 0x2a, + 0x35, 0x54, 0x75, 0x40, 0x8d, 0x44, 0xfe, 0x1d, 0x94, 0x39, 0x99, 0x31, 0x49, 0x62, 0xee, 0xee, + 0x46, 0x6e, 0xc9, 0x87, 0x46, 0xe4, 0x8f, 0xa1, 0x10, 0xdc, 0x59, 0x49, 0x30, 0xaf, 0xca, 0x0f, + 0xbe, 0x69, 0xe0, 0x49, 0xf0, 0x19, 0xec, 0x73, 0xd7, 0x32, 0xaf, 0x08, 0x17, 0x36, 0xa3, 0x42, + 0xdf, 0x53, 0xae, 0xef, 0xc6, 0xae, 0x0d, 0xd7, 0x0a, 0x53, 0xf8, 0x63, 0x00, 0x32, 0x0a, 0xdc, + 0xb5, 0xc2, 0x05, 0xfa, 0x0c, 0x4a, 0x33, 0x7c, 0x6d, 0xbe, 0xe1, 0x78, 0x46, 0x4c, 0x61, 0xbf, + 0x23, 0x3a, 0x54, 0xb5, 0x7a, 0xd1, 0xd8, 0x9f, 0xe1, 0xeb, 0x53, 0xcf, 0x38, 0xb2, 0xdf, 0x91, + 0xda, 0xaf, 0x1a, 0x1c, 0x8d, 0x08, 0xbf, 0x22, 0x3c, 0xd6, 0x04, 0xf6, 0x76, 0x25, 0xe1, 0xe9, + 0x55, 0xd4, 0xd2, 0xab, 0xf8, 0x04, 0x2a, 0x4b, 0x45, 0xf0, 0x8a, 0x98, 0xd9, 0x58, 0xc4, 0x72, + 0xb2, 0x0c, 0x36, 0x11, 0xb5, 0xdf, 0x43, 0x75, 0xaf, 0x04, 0xe3, 0xa9, 0x7b, 0xa3, 0x00, 0xb5, + 0xf7, 0x08, 0x70, 0x06, 0x87, 0x71, 0x4b, 0xb8, 0xd1, 0x95, 0x82, 0x98, 0x1e, 0xc7, 0x31, 0x6d, + 0x38, 0xb5, 0x91, 0x92, 0x0f, 0xbf, 0xcb, 0x6f, 0x5c, 0xa6, 0x64, 0xea, 0x08, 0xf2, 0x36, 0x35, + 0x5f, 0x2f, 0xfc, 0x81, 0xa1, 0xd5, 0xf7, 0x8d, 0x5d, 0x9b, 0x76, 0xbc, 0x65, 0x8a, 0xc6, 0x72, + 0xff, 0x41, 0x63, 0xdb, 0x1f, 0xac, 0xb1, 0x55, 0x09, 0xed, 0xfc, 0x0f, 0x12, 0xda, 0x5d, 0x97, + 0xd0, 0xf1, 0x14, 0xf4, 0x4d, 0xb9, 0x4a, 0x8e, 0xbc, 0x6d, 0x7f, 0xe4, 0x3d, 0x4a, 0x8e, 0xbc, + 0x42, 0xeb, 0xd3, 0x44, 0x21, 0x36, 0xc9, 0x30, 0x31, 0x17, 0x6b, 0x5f, 0xc3, 0xad, 0x01, 0xb9, + 0x8e, 0xa7, 0xdf, 0x0f, 0x44, 0x08, 0x3c, 0x51, 0x32, 0x49, 0x96, 0x40, 0x5b, 0x2a, 0x41, 0xed, + 0x2f, 0x0d, 0x8a, 0x11, 0x85, 0x7b, 0xe0, 0x53, 0xd8, 0xb7, 0xd4, 0x1c, 0x35, 0x85, 0x57, 0x7f, + 0x45, 0x28, 0xb4, 0x3e, 0x59, 0x91, 0xc5, 0xfa, 0xa8, 0xed, 0x6f, 0x19, 0x05, 0x9f, 0xa8, 0x00, + 0x9e, 0x1f, 0xa1, 0xe2, 0x0e, 0xfc, 0x64, 0x52, 0xfd, 0xac, 0xcb, 0xcb, 0xf3, 0xe3, 0x13, 0x7d, + 0x3f, 0x0f, 0x21, 0x47, 0xc9, 0xb5, 0x54, 0xda, 0x59, 0xe2, 0x6f, 0xb8, 0x6d, 0x7f, 0xcb, 0x50, + 0x84, 0x4e, 0x01, 0xf6, 0x38, 0x79, 0x1b, 0xfc, 0x23, 0x7e, 0xcb, 0x42, 0x25, 0x79, 0x4f, 0x31, + 0x77, 0x24, 0xfa, 0x0a, 0x0e, 0xd3, 0xda, 0x27, 0xf8, 0x0f, 0xdd, 0x48, 0xe9, 0x1e, 0xf4, 0x05, + 0x94, 0x57, 0xfa, 0x3e, 0xf8, 0x43, 0x95, 0x96, 0xdb, 0xde, 0xcb, 0xf9, 0x94, 0x2c, 0xcc, 0x31, + 0x96, 0x38, 0x94, 0xfd, 0x94, 0x2c, 0x4e, 0xb0, 0xc4, 0xe8, 0x21, 0x14, 0x5d, 0x42, 0x78, 0x3c, + 0x94, 0x73, 0x1b, 0x87, 0xf2, 0xbe, 0x07, 0x5c, 0x9f, 0xc9, 0xff, 0x7e, 0x9c, 0xdf, 0x87, 0x83, + 0x29, 0x21, 0xae, 0x69, 0x5d, 0x62, 0x4a, 0x89, 0x63, 0x32, 0x97, 0x50, 0xa5, 0xfb, 0xbc, 0x51, + 0xf6, 0x36, 0xba, 0xbe, 0xfd, 0xdc, 0x25, 0x14, 0x9d, 0xc1, 0x81, 0x8a, 0x6f, 0xa9, 0x47, 0x76, + 0x3f, 0xa4, 0x47, 0xca, 0x1e, 0xcf, 0x78, 0x6f, 0x9f, 0xe4, 0x53, 0x46, 0xed, 0xb3, 0x64, 0x6d, + 0x46, 0x12, 0xcb, 0xb9, 0x7a, 0x0a, 0x59, 0x6c, 0x4c, 0x54, 0x2d, 0x8a, 0x86, 0xfa, 0x46, 0x3a, + 0xec, 0x8e, 0x89, 0xc4, 0xb6, 0xfa, 0xc3, 0x7a, 0x49, 0x0f, 0x97, 0xb5, 0x3f, 0x34, 0x28, 0x2d, + 0x95, 0xd7, 0xf5, 0x9e, 0x5a, 0x6c, 0x2e, 0xfd, 0xa3, 0x43, 0xd9, 0xef, 0xb1, 0xb9, 0x54, 0xc7, + 0x0a, 0xf4, 0x39, 0x94, 0x54, 0x43, 0x98, 0x16, 0xa3, 0x62, 0x3e, 0x23, 0x63, 0xe5, 0xb2, 0x68, + 0x14, 0x95, 0xb5, 0x1b, 0x18, 0x51, 0x0b, 0x76, 0xb8, 0x12, 0x4b, 0xa0, 0xbf, 0xe3, 0x94, 0xa7, + 0x42, 0x20, 0x27, 0x23, 0x40, 0x7a, 0x1c, 0xa1, 0x2e, 0x11, 0x14, 0x36, 0x95, 0xe3, 0x5f, 0xd3, + 0x08, 0x90, 0xf7, 0xbf, 0x87, 0x83, 0xb5, 0xa7, 0x07, 0xaa, 0xc1, 0x47, 0xfd, 0xf6, 0xe0, 0x64, + 0xd4, 0x6f, 0x3f, 0xef, 0x99, 0x43, 0xe3, 0xfc, 0xe2, 0xbc, 0x7b, 0xfe, 0xc2, 0x7c, 0x39, 0x18, + 0x0d, 0x7b, 0xdd, 0xb3, 0xd3, 0xb3, 0xde, 0x49, 0x65, 0x0b, 0xed, 0x42, 0xf6, 0xe2, 0xc5, 0xa8, + 0xa2, 0xa1, 0x3c, 0xe4, 0xda, 0x2f, 0x2e, 0x46, 0x95, 0xcc, 0xfd, 0x1e, 0x94, 0x57, 0xde, 0x85, + 0xa8, 0x0a, 0x77, 0x06, 0xbd, 0x8b, 0x9f, 0xce, 0x8d, 0xe7, 0xef, 0xf3, 0xd3, 0x1d, 0x56, 0x34, + 0xef, 0xe3, 0xe5, 0xc9, 0xb0, 0x92, 0x69, 0xbd, 0x4a, 0x84, 0xc4, 0x47, 0xfe, 0x2b, 0x11, 0x9d, + 0x42, 0xe1, 0x84, 0x45, 0x66, 0x74, 0x2b, 0x3d, 0x1d, 0x6f, 0x8f, 0xf5, 0x0d, 0x79, 0x72, 0x6b, + 0x5b, 0x75, 0xed, 0x4b, 0xad, 0x33, 0x85, 0x9b, 0x36, 0xf3, 0x31, 0xd8, 0x91, 0xa2, 0x61, 0x53, + 0x49, 0x38, 0xc5, 0x4e, 0xa7, 0x1c, 0xc3, 0x55, 0xf4, 0x43, 0xed, 0xe7, 0xa7, 0x13, 0xc6, 0x26, + 0x0e, 0x69, 0x4c, 0x98, 0x83, 0xe9, 0xa4, 0xc1, 0xf8, 0xa4, 0xa9, 0x1e, 0xdf, 0x16, 0x27, 0x4a, + 0xde, 0xd8, 0x11, 0x4d, 0xcf, 0x49, 0x33, 0x74, 0xd2, 0x54, 0xbd, 0xa9, 0x40, 0xe6, 0xc4, 0x72, + 0x5f, 0xef, 0xa8, 0xf5, 0x83, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0xc1, 0xf9, 0x9d, 0xf2, 0xd9, + 0x0b, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/vendor/google.golang.org/grpc/credentials/credentials.go b/vendor/google.golang.org/grpc/credentials/credentials.go index c690161242..667cf6b336 100644 --- a/vendor/google.golang.org/grpc/credentials/credentials.go +++ b/vendor/google.golang.org/grpc/credentials/credentials.go @@ -24,17 +24,11 @@ package credentials // import "google.golang.org/grpc/credentials" import ( "context" - "crypto/tls" - "crypto/x509" "errors" - "fmt" - "io/ioutil" "net" "github.com/golang/protobuf/proto" - - "google.golang.org/grpc/credentials/internal" - ginternal "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal" ) // PerRPCCredentials defines the common interface for the credentials which need to @@ -127,146 +121,35 @@ type Bundle interface { NewWithMode(mode string) (Bundle, error) } -// TLSInfo contains the auth information for a TLS authenticated connection. -// It implements the AuthInfo interface. -type TLSInfo struct { - State tls.ConnectionState -} - -// AuthType returns the type of TLSInfo as a string. -func (t TLSInfo) AuthType() string { - return "tls" -} - -// GetSecurityValue returns security info requested by channelz. -func (t TLSInfo) GetSecurityValue() ChannelzSecurityValue { - v := &TLSChannelzSecurityValue{ - StandardName: cipherSuiteLookup[t.State.CipherSuite], - } - // Currently there's no way to get LocalCertificate info from tls package. - if len(t.State.PeerCertificates) > 0 { - v.RemoteCertificate = t.State.PeerCertificates[0].Raw - } - return v -} - -// tlsCreds is the credentials required for authenticating a connection using TLS. -type tlsCreds struct { - // TLS configuration - config *tls.Config -} - -func (c tlsCreds) Info() ProtocolInfo { - return ProtocolInfo{ - SecurityProtocol: "tls", - SecurityVersion: "1.2", - ServerName: c.config.ServerName, - } -} - -func (c *tlsCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (_ net.Conn, _ AuthInfo, err error) { - // use local cfg to avoid clobbering ServerName if using multiple endpoints - cfg := cloneTLSConfig(c.config) - if cfg.ServerName == "" { - serverName, _, err := net.SplitHostPort(authority) - if err != nil { - // If the authority had no host port or if the authority cannot be parsed, use it as-is. - serverName = authority - } - cfg.ServerName = serverName - } - conn := tls.Client(rawConn, cfg) - errChannel := make(chan error, 1) - go func() { - errChannel <- conn.Handshake() - }() - select { - case err := <-errChannel: - if err != nil { - return nil, nil, err - } - case <-ctx.Done(): - return nil, nil, ctx.Err() - } - return internal.WrapSyscallConn(rawConn, conn), TLSInfo{conn.ConnectionState()}, nil -} - -func (c *tlsCreds) ServerHandshake(rawConn net.Conn) (net.Conn, AuthInfo, error) { - conn := tls.Server(rawConn, c.config) - if err := conn.Handshake(); err != nil { - return nil, nil, err - } - return internal.WrapSyscallConn(rawConn, conn), TLSInfo{conn.ConnectionState()}, nil -} - -func (c *tlsCreds) Clone() TransportCredentials { - return NewTLS(c.config) -} - -func (c *tlsCreds) OverrideServerName(serverNameOverride string) error { - c.config.ServerName = serverNameOverride - return nil -} - -const alpnProtoStrH2 = "h2" - -func appendH2ToNextProtos(ps []string) []string { - for _, p := range ps { - if p == alpnProtoStrH2 { - return ps - } - } - ret := make([]string, 0, len(ps)+1) - ret = append(ret, ps...) - return append(ret, alpnProtoStrH2) -} - -// NewTLS uses c to construct a TransportCredentials based on TLS. -func NewTLS(c *tls.Config) TransportCredentials { - tc := &tlsCreds{cloneTLSConfig(c)} - tc.config.NextProtos = appendH2ToNextProtos(tc.config.NextProtos) - return tc -} - -// NewClientTLSFromCert constructs TLS credentials from the input certificate for client. -// serverNameOverride is for testing only. If set to a non empty string, -// it will override the virtual host name of authority (e.g. :authority header field) in requests. -func NewClientTLSFromCert(cp *x509.CertPool, serverNameOverride string) TransportCredentials { - return NewTLS(&tls.Config{ServerName: serverNameOverride, RootCAs: cp}) +// RequestInfo contains request data attached to the context passed to GetRequestMetadata calls. +// +// This API is experimental. +type RequestInfo struct { + // The method passed to Invoke or NewStream for this RPC. (For proto methods, this has the format "/some.Service/Method") + Method string } -// NewClientTLSFromFile constructs TLS credentials from the input certificate file for client. -// serverNameOverride is for testing only. If set to a non empty string, -// it will override the virtual host name of authority (e.g. :authority header field) in requests. -func NewClientTLSFromFile(certFile, serverNameOverride string) (TransportCredentials, error) { - b, err := ioutil.ReadFile(certFile) - if err != nil { - return nil, err - } - cp := x509.NewCertPool() - if !cp.AppendCertsFromPEM(b) { - return nil, fmt.Errorf("credentials: failed to append certificates") - } - return NewTLS(&tls.Config{ServerName: serverNameOverride, RootCAs: cp}), nil -} +// requestInfoKey is a struct to be used as the key when attaching a RequestInfo to a context object. +type requestInfoKey struct{} -// NewServerTLSFromCert constructs TLS credentials from the input certificate for server. -func NewServerTLSFromCert(cert *tls.Certificate) TransportCredentials { - return NewTLS(&tls.Config{Certificates: []tls.Certificate{*cert}}) +// RequestInfoFromContext extracts the RequestInfo from the context if it exists. +// +// This API is experimental. +func RequestInfoFromContext(ctx context.Context) (ri RequestInfo, ok bool) { + ri, ok = ctx.Value(requestInfoKey{}).(RequestInfo) + return } -// NewServerTLSFromFile constructs TLS credentials from the input certificate file and key -// file for server. -func NewServerTLSFromFile(certFile, keyFile string) (TransportCredentials, error) { - cert, err := tls.LoadX509KeyPair(certFile, keyFile) - if err != nil { - return nil, err +func init() { + internal.NewRequestInfoContext = func(ctx context.Context, ri RequestInfo) context.Context { + return context.WithValue(ctx, requestInfoKey{}, ri) } - return NewTLS(&tls.Config{Certificates: []tls.Certificate{cert}}), nil } // ChannelzSecurityInfo defines the interface that security protocols should implement // in order to provide security info to channelz. +// +// This API is experimental. type ChannelzSecurityInfo interface { GetSecurityValue() ChannelzSecurityValue } @@ -274,91 +157,20 @@ type ChannelzSecurityInfo interface { // ChannelzSecurityValue defines the interface that GetSecurityValue() return value // should satisfy. This interface should only be satisfied by *TLSChannelzSecurityValue // and *OtherChannelzSecurityValue. +// +// This API is experimental. type ChannelzSecurityValue interface { isChannelzSecurityValue() } -// TLSChannelzSecurityValue defines the struct that TLS protocol should return -// from GetSecurityValue(), containing security info like cipher and certificate used. -type TLSChannelzSecurityValue struct { - ChannelzSecurityValue - StandardName string - LocalCertificate []byte - RemoteCertificate []byte -} - // OtherChannelzSecurityValue defines the struct that non-TLS protocol should return // from GetSecurityValue(), which contains protocol specific security info. Note // the Value field will be sent to users of channelz requesting channel info, and // thus sensitive info should better be avoided. +// +// This API is experimental. type OtherChannelzSecurityValue struct { ChannelzSecurityValue Name string Value proto.Message } - -var cipherSuiteLookup = map[uint16]string{ - tls.TLS_RSA_WITH_RC4_128_SHA: "TLS_RSA_WITH_RC4_128_SHA", - tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA: "TLS_RSA_WITH_3DES_EDE_CBC_SHA", - tls.TLS_RSA_WITH_AES_128_CBC_SHA: "TLS_RSA_WITH_AES_128_CBC_SHA", - tls.TLS_RSA_WITH_AES_256_CBC_SHA: "TLS_RSA_WITH_AES_256_CBC_SHA", - tls.TLS_RSA_WITH_AES_128_GCM_SHA256: "TLS_RSA_WITH_AES_128_GCM_SHA256", - tls.TLS_RSA_WITH_AES_256_GCM_SHA384: "TLS_RSA_WITH_AES_256_GCM_SHA384", - tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA: "TLS_ECDHE_ECDSA_WITH_RC4_128_SHA", - tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA: "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA", - tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA: "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA", - tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA: "TLS_ECDHE_RSA_WITH_RC4_128_SHA", - tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA: "TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA", - tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA: "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", - tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA: "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA", - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256: "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", - tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384: "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", - tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384: "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", - tls.TLS_FALLBACK_SCSV: "TLS_FALLBACK_SCSV", - tls.TLS_RSA_WITH_AES_128_CBC_SHA256: "TLS_RSA_WITH_AES_128_CBC_SHA256", - tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256: "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256", - tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256: "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256", - tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305: "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", - tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305: "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", -} - -// cloneTLSConfig returns a shallow clone of the exported -// fields of cfg, ignoring the unexported sync.Once, which -// contains a mutex and must not be copied. -// -// If cfg is nil, a new zero tls.Config is returned. -// -// TODO: inline this function if possible. -func cloneTLSConfig(cfg *tls.Config) *tls.Config { - if cfg == nil { - return &tls.Config{} - } - - return cfg.Clone() -} - -// RequestInfo contains request data attached to the context passed to GetRequestMetadata calls. -// -// This API is experimental. -type RequestInfo struct { - // The method passed to Invoke or NewStream for this RPC. (For proto methods, this has the format "/some.Service/Method") - Method string -} - -// requestInfoKey is a struct to be used as the key when attaching a RequestInfo to a context object. -type requestInfoKey struct{} - -// RequestInfoFromContext extracts the RequestInfo from the context if it exists. -// -// This API is experimental. -func RequestInfoFromContext(ctx context.Context) (ri RequestInfo, ok bool) { - ri, ok = ctx.Value(requestInfoKey{}).(RequestInfo) - return -} - -func init() { - ginternal.NewRequestInfoContext = func(ctx context.Context, ri RequestInfo) context.Context { - return context.WithValue(ctx, requestInfoKey{}, ri) - } -} diff --git a/vendor/google.golang.org/grpc/credentials/tls13.go b/vendor/google.golang.org/grpc/credentials/go12.go similarity index 100% rename from vendor/google.golang.org/grpc/credentials/tls13.go rename to vendor/google.golang.org/grpc/credentials/go12.go diff --git a/vendor/google.golang.org/grpc/credentials/tls.go b/vendor/google.golang.org/grpc/credentials/tls.go new file mode 100644 index 0000000000..7c33613685 --- /dev/null +++ b/vendor/google.golang.org/grpc/credentials/tls.go @@ -0,0 +1,220 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package credentials + +import ( + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" + "net" + + "google.golang.org/grpc/credentials/internal" +) + +// TLSInfo contains the auth information for a TLS authenticated connection. +// It implements the AuthInfo interface. +type TLSInfo struct { + State tls.ConnectionState +} + +// AuthType returns the type of TLSInfo as a string. +func (t TLSInfo) AuthType() string { + return "tls" +} + +// GetSecurityValue returns security info requested by channelz. +func (t TLSInfo) GetSecurityValue() ChannelzSecurityValue { + v := &TLSChannelzSecurityValue{ + StandardName: cipherSuiteLookup[t.State.CipherSuite], + } + // Currently there's no way to get LocalCertificate info from tls package. + if len(t.State.PeerCertificates) > 0 { + v.RemoteCertificate = t.State.PeerCertificates[0].Raw + } + return v +} + +// tlsCreds is the credentials required for authenticating a connection using TLS. +type tlsCreds struct { + // TLS configuration + config *tls.Config +} + +func (c tlsCreds) Info() ProtocolInfo { + return ProtocolInfo{ + SecurityProtocol: "tls", + SecurityVersion: "1.2", + ServerName: c.config.ServerName, + } +} + +func (c *tlsCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (_ net.Conn, _ AuthInfo, err error) { + // use local cfg to avoid clobbering ServerName if using multiple endpoints + cfg := cloneTLSConfig(c.config) + if cfg.ServerName == "" { + serverName, _, err := net.SplitHostPort(authority) + if err != nil { + // If the authority had no host port or if the authority cannot be parsed, use it as-is. + serverName = authority + } + cfg.ServerName = serverName + } + conn := tls.Client(rawConn, cfg) + errChannel := make(chan error, 1) + go func() { + errChannel <- conn.Handshake() + }() + select { + case err := <-errChannel: + if err != nil { + return nil, nil, err + } + case <-ctx.Done(): + return nil, nil, ctx.Err() + } + return internal.WrapSyscallConn(rawConn, conn), TLSInfo{conn.ConnectionState()}, nil +} + +func (c *tlsCreds) ServerHandshake(rawConn net.Conn) (net.Conn, AuthInfo, error) { + conn := tls.Server(rawConn, c.config) + if err := conn.Handshake(); err != nil { + return nil, nil, err + } + return internal.WrapSyscallConn(rawConn, conn), TLSInfo{conn.ConnectionState()}, nil +} + +func (c *tlsCreds) Clone() TransportCredentials { + return NewTLS(c.config) +} + +func (c *tlsCreds) OverrideServerName(serverNameOverride string) error { + c.config.ServerName = serverNameOverride + return nil +} + +const alpnProtoStrH2 = "h2" + +func appendH2ToNextProtos(ps []string) []string { + for _, p := range ps { + if p == alpnProtoStrH2 { + return ps + } + } + ret := make([]string, 0, len(ps)+1) + ret = append(ret, ps...) + return append(ret, alpnProtoStrH2) +} + +// NewTLS uses c to construct a TransportCredentials based on TLS. +func NewTLS(c *tls.Config) TransportCredentials { + tc := &tlsCreds{cloneTLSConfig(c)} + tc.config.NextProtos = appendH2ToNextProtos(tc.config.NextProtos) + return tc +} + +// NewClientTLSFromCert constructs TLS credentials from the input certificate for client. +// serverNameOverride is for testing only. If set to a non empty string, +// it will override the virtual host name of authority (e.g. :authority header field) in requests. +func NewClientTLSFromCert(cp *x509.CertPool, serverNameOverride string) TransportCredentials { + return NewTLS(&tls.Config{ServerName: serverNameOverride, RootCAs: cp}) +} + +// NewClientTLSFromFile constructs TLS credentials from the input certificate file for client. +// serverNameOverride is for testing only. If set to a non empty string, +// it will override the virtual host name of authority (e.g. :authority header field) in requests. +func NewClientTLSFromFile(certFile, serverNameOverride string) (TransportCredentials, error) { + b, err := ioutil.ReadFile(certFile) + if err != nil { + return nil, err + } + cp := x509.NewCertPool() + if !cp.AppendCertsFromPEM(b) { + return nil, fmt.Errorf("credentials: failed to append certificates") + } + return NewTLS(&tls.Config{ServerName: serverNameOverride, RootCAs: cp}), nil +} + +// NewServerTLSFromCert constructs TLS credentials from the input certificate for server. +func NewServerTLSFromCert(cert *tls.Certificate) TransportCredentials { + return NewTLS(&tls.Config{Certificates: []tls.Certificate{*cert}}) +} + +// NewServerTLSFromFile constructs TLS credentials from the input certificate file and key +// file for server. +func NewServerTLSFromFile(certFile, keyFile string) (TransportCredentials, error) { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, err + } + return NewTLS(&tls.Config{Certificates: []tls.Certificate{cert}}), nil +} + +// TLSChannelzSecurityValue defines the struct that TLS protocol should return +// from GetSecurityValue(), containing security info like cipher and certificate used. +// +// This API is EXPERIMENTAL. +type TLSChannelzSecurityValue struct { + ChannelzSecurityValue + StandardName string + LocalCertificate []byte + RemoteCertificate []byte +} + +var cipherSuiteLookup = map[uint16]string{ + tls.TLS_RSA_WITH_RC4_128_SHA: "TLS_RSA_WITH_RC4_128_SHA", + tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA: "TLS_RSA_WITH_3DES_EDE_CBC_SHA", + tls.TLS_RSA_WITH_AES_128_CBC_SHA: "TLS_RSA_WITH_AES_128_CBC_SHA", + tls.TLS_RSA_WITH_AES_256_CBC_SHA: "TLS_RSA_WITH_AES_256_CBC_SHA", + tls.TLS_RSA_WITH_AES_128_GCM_SHA256: "TLS_RSA_WITH_AES_128_GCM_SHA256", + tls.TLS_RSA_WITH_AES_256_GCM_SHA384: "TLS_RSA_WITH_AES_256_GCM_SHA384", + tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA: "TLS_ECDHE_ECDSA_WITH_RC4_128_SHA", + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA: "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA", + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA: "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA", + tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA: "TLS_ECDHE_RSA_WITH_RC4_128_SHA", + tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA: "TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA", + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA: "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA", + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA: "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA", + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256: "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384: "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384: "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + tls.TLS_FALLBACK_SCSV: "TLS_FALLBACK_SCSV", + tls.TLS_RSA_WITH_AES_128_CBC_SHA256: "TLS_RSA_WITH_AES_128_CBC_SHA256", + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256: "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256", + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256: "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256", + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305: "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305: "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", +} + +// cloneTLSConfig returns a shallow clone of the exported +// fields of cfg, ignoring the unexported sync.Once, which +// contains a mutex and must not be copied. +// +// If cfg is nil, a new zero tls.Config is returned. +// +// TODO: inline this function if possible. +func cloneTLSConfig(cfg *tls.Config) *tls.Config { + if cfg == nil { + return &tls.Config{} + } + + return cfg.Clone() +} diff --git a/vendor/google.golang.org/grpc/dialoptions.go b/vendor/google.golang.org/grpc/dialoptions.go index 9f872df8b7..9af3eef7ab 100644 --- a/vendor/google.golang.org/grpc/dialoptions.go +++ b/vendor/google.golang.org/grpc/dialoptions.go @@ -346,8 +346,8 @@ func WithCredentialsBundle(b credentials.Bundle) DialOption { // WithTimeout returns a DialOption that configures a timeout for dialing a // ClientConn initially. This is valid if and only if WithBlock() is present. // -// Deprecated: use DialContext and context.WithTimeout instead. Will be -// supported throughout 1.x. +// Deprecated: use DialContext instead of Dial and context.WithTimeout +// instead. Will be supported throughout 1.x. func WithTimeout(d time.Duration) DialOption { return newFuncDialOption(func(o *dialOptions) { o.timeout = d @@ -479,6 +479,8 @@ func WithAuthority(a string) DialOption { // WithChannelzParentID returns a DialOption that specifies the channelz ID of // current ClientConn's parent. This function is used in nested channel creation // (e.g. grpclb dial). +// +// This API is EXPERIMENTAL. func WithChannelzParentID(id int64) DialOption { return newFuncDialOption(func(o *dialOptions) { o.channelzParentID = id diff --git a/vendor/google.golang.org/grpc/go.mod b/vendor/google.golang.org/grpc/go.mod index dc56aa79d9..2378361302 100644 --- a/vendor/google.golang.org/grpc/go.mod +++ b/vendor/google.golang.org/grpc/go.mod @@ -3,7 +3,7 @@ module google.golang.org/grpc go 1.11 require ( - github.com/envoyproxy/go-control-plane v0.9.0 + github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473 github.com/envoyproxy/protoc-gen-validate v0.1.0 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b github.com/golang/mock v1.1.1 diff --git a/vendor/google.golang.org/grpc/go.sum b/vendor/google.golang.org/grpc/go.sum index f6a4784d0b..dd5d0cee7a 100644 --- a/vendor/google.golang.org/grpc/go.sum +++ b/vendor/google.golang.org/grpc/go.sum @@ -3,8 +3,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/envoyproxy/go-control-plane v0.9.0 h1:67WMNTvGrl7V1dWdKCeTwxDr7nio9clKoTlLhwIPnT4= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473 h1:4cmBvAEBNJaGARUEs3/suWRyfyBfhf7I60WBZq+bv2w= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= diff --git a/vendor/google.golang.org/grpc/health/client.go b/vendor/google.golang.org/grpc/health/client.go index 419f06040d..b5bee48380 100644 --- a/vendor/google.golang.org/grpc/health/client.go +++ b/vendor/google.golang.org/grpc/health/client.go @@ -56,7 +56,7 @@ const healthCheckMethod = "/grpc.health.v1.Health/Watch" // This function implements the protocol defined at: // https://github.com/grpc/grpc/blob/master/doc/health-checking.md -func clientHealthCheck(ctx context.Context, newStream func(string) (interface{}, error), setConnectivityState func(connectivity.State), service string) error { +func clientHealthCheck(ctx context.Context, newStream func(string) (interface{}, error), setConnectivityState func(connectivity.State, error), service string) error { tryCnt := 0 retryConnection: @@ -70,7 +70,7 @@ retryConnection: if ctx.Err() != nil { return nil } - setConnectivityState(connectivity.Connecting) + setConnectivityState(connectivity.Connecting, nil) rawS, err := newStream(healthCheckMethod) if err != nil { continue retryConnection @@ -79,7 +79,7 @@ retryConnection: s, ok := rawS.(grpc.ClientStream) // Ideally, this should never happen. But if it happens, the server is marked as healthy for LBing purposes. if !ok { - setConnectivityState(connectivity.Ready) + setConnectivityState(connectivity.Ready, nil) return fmt.Errorf("newStream returned %v (type %T); want grpc.ClientStream", rawS, rawS) } @@ -95,22 +95,22 @@ retryConnection: // Reports healthy for the LBing purposes if health check is not implemented in the server. if status.Code(err) == codes.Unimplemented { - setConnectivityState(connectivity.Ready) + setConnectivityState(connectivity.Ready, nil) return err } // Reports unhealthy if server's Watch method gives an error other than UNIMPLEMENTED. if err != nil { - setConnectivityState(connectivity.TransientFailure) + setConnectivityState(connectivity.TransientFailure, fmt.Errorf("connection active but received health check RPC error: %v", err)) continue retryConnection } // As a message has been received, removes the need for backoff for the next retry by resetting the try count. tryCnt = 0 if resp.Status == healthpb.HealthCheckResponse_SERVING { - setConnectivityState(connectivity.Ready) + setConnectivityState(connectivity.Ready, nil) } else { - setConnectivityState(connectivity.TransientFailure) + setConnectivityState(connectivity.TransientFailure, fmt.Errorf("connection active but health check failed. status=%s", resp.Status)) } } } diff --git a/vendor/google.golang.org/grpc/internal/binarylog/binarylog.go b/vendor/google.golang.org/grpc/internal/binarylog/binarylog.go index 4062c025c7..8b10516749 100644 --- a/vendor/google.golang.org/grpc/internal/binarylog/binarylog.go +++ b/vendor/google.golang.org/grpc/internal/binarylog/binarylog.go @@ -98,7 +98,7 @@ func (l *logger) setDefaultMethodLogger(ml *methodLoggerConfig) error { // New methodLogger with same service overrides the old one. func (l *logger) setServiceMethodLogger(service string, ml *methodLoggerConfig) error { if _, ok := l.services[service]; ok { - return fmt.Errorf("conflicting rules for service %v found", service) + return fmt.Errorf("conflicting service rules for service %v found", service) } if l.services == nil { l.services = make(map[string]*methodLoggerConfig) @@ -112,10 +112,10 @@ func (l *logger) setServiceMethodLogger(service string, ml *methodLoggerConfig) // New methodLogger with same method overrides the old one. func (l *logger) setMethodMethodLogger(method string, ml *methodLoggerConfig) error { if _, ok := l.blacklist[method]; ok { - return fmt.Errorf("conflicting rules for method %v found", method) + return fmt.Errorf("conflicting blacklist rules for method %v found", method) } if _, ok := l.methods[method]; ok { - return fmt.Errorf("conflicting rules for method %v found", method) + return fmt.Errorf("conflicting method rules for method %v found", method) } if l.methods == nil { l.methods = make(map[string]*methodLoggerConfig) @@ -127,10 +127,10 @@ func (l *logger) setMethodMethodLogger(method string, ml *methodLoggerConfig) er // Set blacklist method for "-service/method". func (l *logger) setBlacklist(method string) error { if _, ok := l.blacklist[method]; ok { - return fmt.Errorf("conflicting rules for method %v found", method) + return fmt.Errorf("conflicting blacklist rules for method %v found", method) } if _, ok := l.methods[method]; ok { - return fmt.Errorf("conflicting rules for method %v found", method) + return fmt.Errorf("conflicting method rules for method %v found", method) } if l.blacklist == nil { l.blacklist = make(map[string]struct{}) diff --git a/vendor/google.golang.org/grpc/internal/buffer/unbounded.go b/vendor/google.golang.org/grpc/internal/buffer/unbounded.go index 2cb3109d80..9f6a0c1200 100644 --- a/vendor/google.golang.org/grpc/internal/buffer/unbounded.go +++ b/vendor/google.golang.org/grpc/internal/buffer/unbounded.go @@ -26,6 +26,13 @@ import "sync" // // All methods on this type are thread-safe and don't block on anything except // the underlying mutex used for synchronization. +// +// Unbounded supports values of any type to be stored in it by using a channel +// of `interface{}`. This means that a call to Put() incurs an extra memory +// allocation, and also that users need a type assertion while reading. For +// performance critical code paths, using Unbounded is strongly discouraged and +// defining a new type specific implementation of this buffer is preferred. See +// internal/transport/transport.go for an example of this. type Unbounded struct { c chan interface{} mu sync.Mutex diff --git a/vendor/google.golang.org/grpc/internal/internal.go b/vendor/google.golang.org/grpc/internal/internal.go index b96b3597ca..eae18e18cc 100644 --- a/vendor/google.golang.org/grpc/internal/internal.go +++ b/vendor/google.golang.org/grpc/internal/internal.go @@ -60,7 +60,7 @@ var ( // // The health checking protocol is defined at: // https://github.com/grpc/grpc/blob/master/doc/health-checking.md -type HealthChecker func(ctx context.Context, newStream func(string) (interface{}, error), setConnectivityState func(connectivity.State), serviceName string) error +type HealthChecker func(ctx context.Context, newStream func(string) (interface{}, error), setConnectivityState func(connectivity.State, error), serviceName string) error const ( // CredsBundleModeFallback switches GoogleDefaultCreds to fallback mode. diff --git a/vendor/google.golang.org/grpc/internal/resolver/dns/dns_resolver.go b/vendor/google.golang.org/grpc/internal/resolver/dns/dns_resolver.go index abc0f92ca5..7705ca22eb 100644 --- a/vendor/google.golang.org/grpc/internal/resolver/dns/dns_resolver.go +++ b/vendor/google.golang.org/grpc/internal/resolver/dns/dns_resolver.go @@ -32,20 +32,22 @@ import ( "sync" "time" - "google.golang.org/grpc/backoff" "google.golang.org/grpc/grpclog" - internalbackoff "google.golang.org/grpc/internal/backoff" "google.golang.org/grpc/internal/grpcrand" "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" ) +// EnableSRVLookups controls whether the DNS resolver attempts to fetch gRPCLB +// addresses from SRV records. Must not be changed after init time. +var EnableSRVLookups = false + func init() { resolver.Register(NewBuilder()) } const ( defaultPort = "443" - defaultFreq = time.Minute * 30 defaultDNSSvrPort = "53" golang = "GO" // txtPrefix is the prefix string to be prepended to the host name for txt record lookup. @@ -95,49 +97,33 @@ var customAuthorityResolver = func(authority string) (netResolver, error) { // NewBuilder creates a dnsBuilder which is used to factory DNS resolvers. func NewBuilder() resolver.Builder { - return &dnsBuilder{minFreq: defaultFreq} + return &dnsBuilder{} } -type dnsBuilder struct { - // minimum frequency of polling the DNS server. - minFreq time.Duration -} +type dnsBuilder struct{} // Build creates and starts a DNS resolver that watches the name resolution of the target. -func (b *dnsBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOption) (resolver.Resolver, error) { +func (b *dnsBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) { host, port, err := parseTarget(target.Endpoint, defaultPort) if err != nil { return nil, err } // IP address. - if net.ParseIP(host) != nil { - host, _ = formatIP(host) - addr := []resolver.Address{{Addr: host + ":" + port}} - i := &ipResolver{ - cc: cc, - ip: addr, - rn: make(chan struct{}, 1), - q: make(chan struct{}), - } - cc.NewAddress(addr) - go i.watcher() - return i, nil + if ipAddr, ok := formatIP(host); ok { + addr := []resolver.Address{{Addr: ipAddr + ":" + port}} + cc.UpdateState(resolver.State{Addresses: addr}) + return deadResolver{}, nil } // DNS address (non-IP). ctx, cancel := context.WithCancel(context.Background()) - bc := backoff.DefaultConfig - bc.MaxDelay = b.minFreq d := &dnsResolver{ - freq: b.minFreq, - backoff: internalbackoff.Exponential{Config: bc}, host: host, port: port, ctx: ctx, cancel: cancel, cc: cc, - t: time.NewTimer(0), rn: make(chan struct{}, 1), disableServiceConfig: opts.DisableServiceConfig, } @@ -153,6 +139,7 @@ func (b *dnsBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts d.wg.Add(1) go d.watcher() + d.ResolveNow(resolver.ResolveNowOptions{}) return d, nil } @@ -167,53 +154,23 @@ type netResolver interface { LookupTXT(ctx context.Context, name string) (txts []string, err error) } -// ipResolver watches for the name resolution update for an IP address. -type ipResolver struct { - cc resolver.ClientConn - ip []resolver.Address - // rn channel is used by ResolveNow() to force an immediate resolution of the target. - rn chan struct{} - q chan struct{} -} +// deadResolver is a resolver that does nothing. +type deadResolver struct{} -// ResolveNow resend the address it stores, no resolution is needed. -func (i *ipResolver) ResolveNow(opt resolver.ResolveNowOption) { - select { - case i.rn <- struct{}{}: - default: - } -} +func (deadResolver) ResolveNow(resolver.ResolveNowOptions) {} -// Close closes the ipResolver. -func (i *ipResolver) Close() { - close(i.q) -} - -func (i *ipResolver) watcher() { - for { - select { - case <-i.rn: - i.cc.NewAddress(i.ip) - case <-i.q: - return - } - } -} +func (deadResolver) Close() {} // dnsResolver watches for the name resolution update for a non-IP target. type dnsResolver struct { - freq time.Duration - backoff internalbackoff.Exponential - retryCount int - host string - port string - resolver netResolver - ctx context.Context - cancel context.CancelFunc - cc resolver.ClientConn + host string + port string + resolver netResolver + ctx context.Context + cancel context.CancelFunc + cc resolver.ClientConn // rn channel is used by ResolveNow() to force an immediate resolution of the target. rn chan struct{} - t *time.Timer // wg is used to enforce Close() to return after the watcher() goroutine has finished. // Otherwise, data race will be possible. [Race Example] in dns_resolver_test we // replace the real lookup functions with mocked ones to facilitate testing. @@ -225,7 +182,7 @@ type dnsResolver struct { } // ResolveNow invoke an immediate resolution of the target that this dnsResolver watches. -func (d *dnsResolver) ResolveNow(opt resolver.ResolveNowOption) { +func (d *dnsResolver) ResolveNow(resolver.ResolveNowOptions) { select { case d.rn <- struct{}{}: default: @@ -236,7 +193,6 @@ func (d *dnsResolver) ResolveNow(opt resolver.ResolveNowOption) { func (d *dnsResolver) Close() { d.cancel() d.wg.Wait() - d.t.Stop() } func (d *dnsResolver) watcher() { @@ -245,27 +201,11 @@ func (d *dnsResolver) watcher() { select { case <-d.ctx.Done(): return - case <-d.t.C: case <-d.rn: - if !d.t.Stop() { - // Before resetting a timer, it should be stopped to prevent racing with - // reads on it's channel. - <-d.t.C - } } - result, sc := d.lookup() - // Next lookup should happen within an interval defined by d.freq. It may be - // more often due to exponential retry on empty address list. - if len(result) == 0 { - d.retryCount++ - d.t.Reset(d.backoff.Backoff(d.retryCount)) - } else { - d.retryCount = 0 - d.t.Reset(d.freq) - } - d.cc.NewServiceConfig(sc) - d.cc.NewAddress(result) + state := d.lookup() + d.cc.UpdateState(*state) // Sleep to prevent excessive re-resolutions. Incoming resolution requests // will be queued in d.rn. @@ -280,6 +220,9 @@ func (d *dnsResolver) watcher() { } func (d *dnsResolver) lookupSRV() []resolver.Address { + if !EnableSRVLookups { + return nil + } var newAddrs []resolver.Address _, srvs, err := d.resolver.LookupSRV(d.ctx, "grpclb", "tcp", d.host) if err != nil { @@ -305,11 +248,26 @@ func (d *dnsResolver) lookupSRV() []resolver.Address { return newAddrs } -func (d *dnsResolver) lookupTXT() string { +var filterError = func(err error) error { + if dnsErr, ok := err.(*net.DNSError); ok && !dnsErr.IsTimeout && !dnsErr.IsTemporary { + // Timeouts and temporary errors should be communicated to gRPC to + // attempt another DNS query (with backoff). Other errors should be + // suppressed (they may represent the absence of a TXT record). + return nil + } + return err +} + +func (d *dnsResolver) lookupTXT() *serviceconfig.ParseResult { ss, err := d.resolver.LookupTXT(d.ctx, txtPrefix+d.host) if err != nil { - grpclog.Infof("grpc: failed dns TXT record lookup due to %v.\n", err) - return "" + err = filterError(err) + if err != nil { + err = fmt.Errorf("error from DNS TXT record lookup: %v", err) + grpclog.Infoln("grpc:", err) + return &serviceconfig.ParseResult{Err: err} + } + return nil } var res string for _, s := range ss { @@ -318,10 +276,12 @@ func (d *dnsResolver) lookupTXT() string { // TXT record must have "grpc_config=" attribute in order to be used as service config. if !strings.HasPrefix(res, txtAttribute) { - grpclog.Warningf("grpc: TXT record %v missing %v attribute", res, txtAttribute) - return "" + grpclog.Warningf("grpc: DNS TXT record %v missing %v attribute", res, txtAttribute) + // This is not an error; it is the equivalent of not having a service config. + return nil } - return strings.TrimPrefix(res, txtAttribute) + sc := canaryingSC(strings.TrimPrefix(res, txtAttribute)) + return d.cc.ParseServiceConfig(sc) } func (d *dnsResolver) lookupHost() []resolver.Address { @@ -343,15 +303,15 @@ func (d *dnsResolver) lookupHost() []resolver.Address { return newAddrs } -func (d *dnsResolver) lookup() ([]resolver.Address, string) { - newAddrs := d.lookupSRV() - // Support fallback to non-balancer address. - newAddrs = append(newAddrs, d.lookupHost()...) - if d.disableServiceConfig { - return newAddrs, "" +func (d *dnsResolver) lookup() *resolver.State { + srv := d.lookupSRV() + state := &resolver.State{ + Addresses: append(d.lookupHost(), srv...), + } + if !d.disableServiceConfig { + state.ServiceConfig = d.lookupTXT() } - sc := d.lookupTXT() - return newAddrs, canaryingSC(sc) + return state } // formatIP returns ok = false if addr is not a valid textual representation of an IP address. diff --git a/vendor/google.golang.org/grpc/internal/resolver/dns/go113.go b/vendor/google.golang.org/grpc/internal/resolver/dns/go113.go new file mode 100644 index 0000000000..8783a8cf82 --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/resolver/dns/go113.go @@ -0,0 +1,33 @@ +// +build go1.13 + +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package dns + +import "net" + +func init() { + filterError = func(err error) error { + if dnsErr, ok := err.(*net.DNSError); ok && dnsErr.IsNotFound { + // The name does not exist; not an error. + return nil + } + return err + } +} diff --git a/vendor/google.golang.org/grpc/internal/resolver/passthrough/passthrough.go b/vendor/google.golang.org/grpc/internal/resolver/passthrough/passthrough.go index 893d5d12cb..520d9229e1 100644 --- a/vendor/google.golang.org/grpc/internal/resolver/passthrough/passthrough.go +++ b/vendor/google.golang.org/grpc/internal/resolver/passthrough/passthrough.go @@ -26,7 +26,7 @@ const scheme = "passthrough" type passthroughBuilder struct{} -func (*passthroughBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOption) (resolver.Resolver, error) { +func (*passthroughBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) { r := &passthroughResolver{ target: target, cc: cc, @@ -48,7 +48,7 @@ func (r *passthroughResolver) start() { r.cc.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: r.target.Endpoint}}}) } -func (*passthroughResolver) ResolveNow(o resolver.ResolveNowOption) {} +func (*passthroughResolver) ResolveNow(o resolver.ResolveNowOptions) {} func (*passthroughResolver) Close() {} diff --git a/vendor/google.golang.org/grpc/internal/transport/handler_server.go b/vendor/google.golang.org/grpc/internal/transport/handler_server.go index 78f9ddc3d3..fbf01d5fe7 100644 --- a/vendor/google.golang.org/grpc/internal/transport/handler_server.go +++ b/vendor/google.golang.org/grpc/internal/transport/handler_server.go @@ -227,7 +227,9 @@ func (ht *serverHandlerTransport) WriteStatus(s *Stream, st *status.Status) erro if err == nil { // transport has not been closed if ht.stats != nil { - ht.stats.HandleRPC(s.Context(), &stats.OutTrailer{}) + ht.stats.HandleRPC(s.Context(), &stats.OutTrailer{ + Trailer: s.trailer.Copy(), + }) } } ht.Close() @@ -289,7 +291,9 @@ func (ht *serverHandlerTransport) WriteHeader(s *Stream, md metadata.MD) error { if err == nil { if ht.stats != nil { - ht.stats.HandleRPC(s.Context(), &stats.OutHeader{}) + ht.stats.HandleRPC(s.Context(), &stats.OutHeader{ + Header: md.Copy(), + }) } } return err diff --git a/vendor/google.golang.org/grpc/internal/transport/http2_client.go b/vendor/google.golang.org/grpc/internal/transport/http2_client.go index 294661a3f3..e18935653c 100644 --- a/vendor/google.golang.org/grpc/internal/transport/http2_client.go +++ b/vendor/google.golang.org/grpc/internal/transport/http2_client.go @@ -45,9 +45,14 @@ import ( "google.golang.org/grpc/status" ) +// clientConnectionCounter counts the number of connections a client has +// initiated (equal to the number of http2Clients created). Must be accessed +// atomically. +var clientConnectionCounter uint64 + // http2Client implements the ClientTransport interface with HTTP2. type http2Client struct { - lastRead int64 // keep this field 64-bit aligned + lastRead int64 // Keep this field 64-bit aligned. Accessed atomically. ctx context.Context cancel context.CancelFunc ctxDone <-chan struct{} // Cache the ctx.Done() chan. @@ -126,6 +131,8 @@ type http2Client struct { onClose func() bufferPool *bufferPool + + connectionID uint64 } func dial(ctx context.Context, fn func(context.Context, string) (net.Conn, error), addr string) (net.Conn, error) { @@ -329,6 +336,8 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts Conne } } + t.connectionID = atomic.AddUint64(&clientConnectionCounter, 1) + if err := t.framer.writer.Flush(); err != nil { return nil, err } @@ -424,6 +433,7 @@ func (t *http2Client) createHeaderFields(ctx context.Context, callHdr *CallHdr) if callHdr.SendCompress != "" { headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-encoding", Value: callHdr.SendCompress}) + headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-accept-encoding", Value: callHdr.SendCompress}) } if dl, ok := ctx.Deadline(); ok { // Send out timeout regardless its value. The server can detect timeout context by itself. @@ -669,12 +679,14 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Strea } } if t.statsHandler != nil { + header, _, _ := metadata.FromOutgoingContextRaw(ctx) outHeader := &stats.OutHeader{ Client: true, FullMethod: callHdr.Method, RemoteAddr: t.remoteAddr, LocalAddr: t.localAddr, Compression: callHdr.SendCompress, + Header: header.Copy(), } t.statsHandler.HandleRPC(s.ctx, outHeader) } @@ -1177,12 +1189,14 @@ func (t *http2Client) operateHeaders(frame *http2.MetaHeadersFrame) { inHeader := &stats.InHeader{ Client: true, WireLength: int(frame.Header().Length), + Header: s.header.Copy(), } t.statsHandler.HandleRPC(s.ctx, inHeader) } else { inTrailer := &stats.InTrailer{ Client: true, WireLength: int(frame.Header().Length), + Trailer: s.trailer.Copy(), } t.statsHandler.HandleRPC(s.ctx, inTrailer) } @@ -1369,7 +1383,6 @@ func (t *http2Client) keepalive() { // acked). sleepDuration := minTime(t.kp.Time, timeoutLeft) timeoutLeft -= sleepDuration - prevNano = lastRead timer.Reset(sleepDuration) case <-t.ctx.Done(): if !timer.Stop() { diff --git a/vendor/google.golang.org/grpc/internal/transport/http2_server.go b/vendor/google.golang.org/grpc/internal/transport/http2_server.go index 0760383646..8b04b0392a 100644 --- a/vendor/google.golang.org/grpc/internal/transport/http2_server.go +++ b/vendor/google.golang.org/grpc/internal/transport/http2_server.go @@ -62,8 +62,13 @@ var ( statusRawProto = internal.StatusRawProto.(func(*status.Status) *spb.Status) ) +// serverConnectionCounter counts the number of connections a server has seen +// (equal to the number of http2Servers created). Must be accessed atomically. +var serverConnectionCounter uint64 + // http2Server implements the ServerTransport interface with HTTP2. type http2Server struct { + lastRead int64 // Keep this field 64-bit aligned. Accessed atomically. ctx context.Context done chan struct{} conn net.Conn @@ -83,12 +88,8 @@ type http2Server struct { controlBuf *controlBuffer fc *trInFlow stats stats.Handler - // Flag to keep track of reading activity on transport. - // 1 is true and 0 is false. - activity uint32 // Accessed atomically. // Keepalive and max-age parameters for the server. kp keepalive.ServerParameters - // Keepalive enforcement policy. kep keepalive.EnforcementPolicy // The time instance last ping was received. @@ -124,6 +125,8 @@ type http2Server struct { channelzID int64 // channelz unique identification number czData *channelzData bufferPool *bufferPool + + connectionID uint64 } // newHTTP2Server constructs a ServerTransport based on HTTP2. ConnectionError is @@ -253,6 +256,9 @@ func newHTTP2Server(conn net.Conn, config *ServerConfig) (_ ServerTransport, err if channelz.IsOn() { t.channelzID = channelz.RegisterNormalSocket(t, config.ChannelzParentID, fmt.Sprintf("%s -> %s", t.remoteAddr, t.localAddr)) } + + t.connectionID = atomic.AddUint64(&serverConnectionCounter, 1) + t.framer.writer.Flush() defer func() { @@ -277,7 +283,7 @@ func newHTTP2Server(conn net.Conn, config *ServerConfig) (_ ServerTransport, err if err != nil { return nil, connectionErrorf(false, err, "transport: http2Server.HandleStreams failed to read initial settings frame: %v", err) } - atomic.StoreUint32(&t.activity, 1) + atomic.StoreInt64(&t.lastRead, time.Now().UnixNano()) sf, ok := frame.(*http2.SettingsFrame) if !ok { return nil, connectionErrorf(false, nil, "transport: http2Server.HandleStreams saw invalid preface type %T from client", frame) @@ -416,6 +422,7 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( LocalAddr: t.localAddr, Compression: s.recvCompress, WireLength: int(frame.Header().Length), + Header: metadata.MD(state.data.mdata).Copy(), } t.stats.HandleRPC(s.ctx, inHeader) } @@ -449,7 +456,7 @@ func (t *http2Server) HandleStreams(handle func(*Stream), traceCtx func(context. for { t.controlBuf.throttle() frame, err := t.framer.fr.ReadFrame() - atomic.StoreUint32(&t.activity, 1) + atomic.StoreInt64(&t.lastRead, time.Now().UnixNano()) if err != nil { if se, ok := err.(http2.StreamError); ok { warningf("transport: http2Server.HandleStreams encountered http2.StreamError: %v", se) @@ -808,7 +815,9 @@ func (t *http2Server) writeHeaderLocked(s *Stream) error { if t.stats != nil { // Note: WireLength is not set in outHeader. // TODO(mmukhi): Revisit this later, if needed. - outHeader := &stats.OutHeader{} + outHeader := &stats.OutHeader{ + Header: s.header.Copy(), + } t.stats.HandleRPC(s.Context(), outHeader) } return nil @@ -871,7 +880,9 @@ func (t *http2Server) WriteStatus(s *Stream, st *status.Status) error { rst := s.getState() == streamActive t.finishStream(s, rst, http2.ErrCodeNo, trailingHeader, true) if t.stats != nil { - t.stats.HandleRPC(s.Context(), &stats.OutTrailer{}) + t.stats.HandleRPC(s.Context(), &stats.OutTrailer{ + Trailer: s.trailer.Copy(), + }) } return nil } @@ -932,32 +943,35 @@ func (t *http2Server) Write(s *Stream, hdr []byte, data []byte, opts *Options) e // after an additional duration of keepalive.Timeout. func (t *http2Server) keepalive() { p := &ping{} - var pingSent bool - maxIdle := time.NewTimer(t.kp.MaxConnectionIdle) - maxAge := time.NewTimer(t.kp.MaxConnectionAge) - keepalive := time.NewTimer(t.kp.Time) - // NOTE: All exit paths of this function should reset their - // respective timers. A failure to do so will cause the - // following clean-up to deadlock and eventually leak. + // True iff a ping has been sent, and no data has been received since then. + outstandingPing := false + // Amount of time remaining before which we should receive an ACK for the + // last sent ping. + kpTimeoutLeft := time.Duration(0) + // Records the last value of t.lastRead before we go block on the timer. + // This is required to check for read activity since then. + prevNano := time.Now().UnixNano() + // Initialize the different timers to their default values. + idleTimer := time.NewTimer(t.kp.MaxConnectionIdle) + ageTimer := time.NewTimer(t.kp.MaxConnectionAge) + kpTimer := time.NewTimer(t.kp.Time) defer func() { - if !maxIdle.Stop() { - <-maxIdle.C - } - if !maxAge.Stop() { - <-maxAge.C - } - if !keepalive.Stop() { - <-keepalive.C - } + // We need to drain the underlying channel in these timers after a call + // to Stop(), only if we are interested in resetting them. Clearly we + // are not interested in resetting them here. + idleTimer.Stop() + ageTimer.Stop() + kpTimer.Stop() }() + for { select { - case <-maxIdle.C: + case <-idleTimer.C: t.mu.Lock() idle := t.idle if idle.IsZero() { // The connection is non-idle. t.mu.Unlock() - maxIdle.Reset(t.kp.MaxConnectionIdle) + idleTimer.Reset(t.kp.MaxConnectionIdle) continue } val := t.kp.MaxConnectionIdle - time.Since(idle) @@ -966,43 +980,51 @@ func (t *http2Server) keepalive() { // The connection has been idle for a duration of keepalive.MaxConnectionIdle or more. // Gracefully close the connection. t.drain(http2.ErrCodeNo, []byte{}) - // Resetting the timer so that the clean-up doesn't deadlock. - maxIdle.Reset(infinity) return } - maxIdle.Reset(val) - case <-maxAge.C: + idleTimer.Reset(val) + case <-ageTimer.C: t.drain(http2.ErrCodeNo, []byte{}) - maxAge.Reset(t.kp.MaxConnectionAgeGrace) + ageTimer.Reset(t.kp.MaxConnectionAgeGrace) select { - case <-maxAge.C: + case <-ageTimer.C: // Close the connection after grace period. infof("transport: closing server transport due to maximum connection age.") t.Close() - // Resetting the timer so that the clean-up doesn't deadlock. - maxAge.Reset(infinity) case <-t.done: } return - case <-keepalive.C: - if atomic.CompareAndSwapUint32(&t.activity, 1, 0) { - pingSent = false - keepalive.Reset(t.kp.Time) + case <-kpTimer.C: + lastRead := atomic.LoadInt64(&t.lastRead) + if lastRead > prevNano { + // There has been read activity since the last time we were + // here. Setup the timer to fire at kp.Time seconds from + // lastRead time and continue. + outstandingPing = false + kpTimer.Reset(time.Duration(lastRead) + t.kp.Time - time.Duration(time.Now().UnixNano())) + prevNano = lastRead continue } - if pingSent { + if outstandingPing && kpTimeoutLeft <= 0 { infof("transport: closing server transport due to idleness.") t.Close() - // Resetting the timer so that the clean-up doesn't deadlock. - keepalive.Reset(infinity) return } - pingSent = true - if channelz.IsOn() { - atomic.AddInt64(&t.czData.kpCount, 1) + if !outstandingPing { + if channelz.IsOn() { + atomic.AddInt64(&t.czData.kpCount, 1) + } + t.controlBuf.put(p) + kpTimeoutLeft = t.kp.Timeout + outstandingPing = true } - t.controlBuf.put(p) - keepalive.Reset(t.kp.Timeout) + // The amount of time to sleep here is the minimum of kp.Time and + // timeoutLeft. This will ensure that we wait only for kp.Time + // before sending out the next ping (for cases where the ping is + // acked). + sleepDuration := minTime(t.kp.Time, kpTimeoutLeft) + kpTimeoutLeft -= sleepDuration + kpTimer.Reset(sleepDuration) case <-t.done: return } diff --git a/vendor/google.golang.org/grpc/internal/transport/transport.go b/vendor/google.golang.org/grpc/internal/transport/transport.go index bfab940bd0..a30da9eb32 100644 --- a/vendor/google.golang.org/grpc/internal/transport/transport.go +++ b/vendor/google.golang.org/grpc/internal/transport/transport.go @@ -73,10 +73,11 @@ type recvMsg struct { } // recvBuffer is an unbounded channel of recvMsg structs. -// Note recvBuffer differs from controlBuffer only in that recvBuffer -// holds a channel of only recvMsg structs instead of objects implementing "item" interface. -// recvBuffer is written to much more often than -// controlBuffer and using strict recvMsg structs helps avoid allocation in "recvBuffer.put" +// +// Note: recvBuffer differs from buffer.Unbounded only in the fact that it +// holds a channel of recvMsg structs instead of objects implementing "item" +// interface. recvBuffer is written to much more often and using strict recvMsg +// structs helps avoid allocation in "recvBuffer.put" type recvBuffer struct { c chan recvMsg mu sync.Mutex diff --git a/vendor/google.golang.org/grpc/picker_wrapper.go b/vendor/google.golang.org/grpc/picker_wrapper.go index 45baa2ae13..00447894f0 100644 --- a/vendor/google.golang.org/grpc/picker_wrapper.go +++ b/vendor/google.golang.org/grpc/picker_wrapper.go @@ -20,6 +20,7 @@ package grpc import ( "context" + "fmt" "io" "sync" @@ -31,49 +32,78 @@ import ( "google.golang.org/grpc/status" ) +// v2PickerWrapper wraps a balancer.Picker while providing the +// balancer.V2Picker API. It requires a pickerWrapper to generate errors +// including the latest connectionError. To be deleted when balancer.Picker is +// updated to the balancer.V2Picker API. +type v2PickerWrapper struct { + picker balancer.Picker + connErr *connErr +} + +func (v *v2PickerWrapper) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + sc, done, err := v.picker.Pick(info.Ctx, info) + if err != nil { + if err == balancer.ErrTransientFailure { + return balancer.PickResult{}, balancer.TransientFailureError(fmt.Errorf("%v, latest connection error: %v", err, v.connErr.connectionError())) + } + return balancer.PickResult{}, err + } + return balancer.PickResult{SubConn: sc, Done: done}, nil +} + // pickerWrapper is a wrapper of balancer.Picker. It blocks on certain pick // actions and unblock when there's a picker update. type pickerWrapper struct { mu sync.Mutex done bool blockingCh chan struct{} - picker balancer.Picker + picker balancer.V2Picker - // The latest connection happened. - connErrMu sync.Mutex - connErr error + // The latest connection error. TODO: remove when V1 picker is deprecated; + // balancer should be responsible for providing the error. + *connErr } -func newPickerWrapper() *pickerWrapper { - bp := &pickerWrapper{blockingCh: make(chan struct{})} - return bp +type connErr struct { + mu sync.Mutex + err error } -func (bp *pickerWrapper) updateConnectionError(err error) { - bp.connErrMu.Lock() - bp.connErr = err - bp.connErrMu.Unlock() +func (c *connErr) updateConnectionError(err error) { + c.mu.Lock() + c.err = err + c.mu.Unlock() } -func (bp *pickerWrapper) connectionError() error { - bp.connErrMu.Lock() - err := bp.connErr - bp.connErrMu.Unlock() +func (c *connErr) connectionError() error { + c.mu.Lock() + err := c.err + c.mu.Unlock() return err } +func newPickerWrapper() *pickerWrapper { + return &pickerWrapper{blockingCh: make(chan struct{}), connErr: &connErr{}} +} + // updatePicker is called by UpdateBalancerState. It unblocks all blocked pick. -func (bp *pickerWrapper) updatePicker(p balancer.Picker) { - bp.mu.Lock() - if bp.done { - bp.mu.Unlock() +func (pw *pickerWrapper) updatePicker(p balancer.Picker) { + pw.updatePickerV2(&v2PickerWrapper{picker: p, connErr: pw.connErr}) +} + +// updatePicker is called by UpdateBalancerState. It unblocks all blocked pick. +func (pw *pickerWrapper) updatePickerV2(p balancer.V2Picker) { + pw.mu.Lock() + if pw.done { + pw.mu.Unlock() return } - bp.picker = p - // bp.blockingCh should never be nil. - close(bp.blockingCh) - bp.blockingCh = make(chan struct{}) - bp.mu.Unlock() + pw.picker = p + // pw.blockingCh should never be nil. + close(pw.blockingCh) + pw.blockingCh = make(chan struct{}) + pw.mu.Unlock() } func doneChannelzWrapper(acw *acBalancerWrapper, done func(balancer.DoneInfo)) func(balancer.DoneInfo) { @@ -100,83 +130,85 @@ func doneChannelzWrapper(acw *acBalancerWrapper, done func(balancer.DoneInfo)) f // - the current picker returns other errors and failfast is false. // - the subConn returned by the current picker is not READY // When one of these situations happens, pick blocks until the picker gets updated. -func (bp *pickerWrapper) pick(ctx context.Context, failfast bool, opts balancer.PickOptions) (transport.ClientTransport, func(balancer.DoneInfo), error) { +func (pw *pickerWrapper) pick(ctx context.Context, failfast bool, info balancer.PickInfo) (transport.ClientTransport, func(balancer.DoneInfo), error) { var ch chan struct{} + var lastPickErr error for { - bp.mu.Lock() - if bp.done { - bp.mu.Unlock() + pw.mu.Lock() + if pw.done { + pw.mu.Unlock() return nil, nil, ErrClientConnClosing } - if bp.picker == nil { - ch = bp.blockingCh + if pw.picker == nil { + ch = pw.blockingCh } - if ch == bp.blockingCh { + if ch == pw.blockingCh { // This could happen when either: - // - bp.picker is nil (the previous if condition), or + // - pw.picker is nil (the previous if condition), or // - has called pick on the current picker. - bp.mu.Unlock() + pw.mu.Unlock() select { case <-ctx.Done(): - if connectionErr := bp.connectionError(); connectionErr != nil { - switch ctx.Err() { - case context.DeadlineExceeded: - return nil, nil, status.Errorf(codes.DeadlineExceeded, "latest connection error: %v", connectionErr) - case context.Canceled: - return nil, nil, status.Errorf(codes.Canceled, "latest connection error: %v", connectionErr) - } + var errStr string + if lastPickErr != nil { + errStr = "latest balancer error: " + lastPickErr.Error() + } else if connectionErr := pw.connectionError(); connectionErr != nil { + errStr = "latest connection error: " + connectionErr.Error() + } else { + errStr = ctx.Err().Error() + } + switch ctx.Err() { + case context.DeadlineExceeded: + return nil, nil, status.Error(codes.DeadlineExceeded, errStr) + case context.Canceled: + return nil, nil, status.Error(codes.Canceled, errStr) } - return nil, nil, ctx.Err() case <-ch: } continue } - ch = bp.blockingCh - p := bp.picker - bp.mu.Unlock() + ch = pw.blockingCh + p := pw.picker + pw.mu.Unlock() - subConn, done, err := p.Pick(ctx, opts) + pickResult, err := p.Pick(info) if err != nil { - switch err { - case balancer.ErrNoSubConnAvailable: + if err == balancer.ErrNoSubConnAvailable { continue - case balancer.ErrTransientFailure: + } + if tfe, ok := err.(interface{ IsTransientFailure() bool }); ok && tfe.IsTransientFailure() { if !failfast { + lastPickErr = err continue } - return nil, nil, status.Errorf(codes.Unavailable, "%v, latest connection error: %v", err, bp.connectionError()) - case context.DeadlineExceeded: - return nil, nil, status.Error(codes.DeadlineExceeded, err.Error()) - case context.Canceled: - return nil, nil, status.Error(codes.Canceled, err.Error()) - default: - if _, ok := status.FromError(err); ok { - return nil, nil, err - } - // err is some other error. - return nil, nil, status.Error(codes.Unknown, err.Error()) + return nil, nil, status.Error(codes.Unavailable, err.Error()) } + if _, ok := status.FromError(err); ok { + return nil, nil, err + } + // err is some other error. + return nil, nil, status.Error(codes.Unknown, err.Error()) } - acw, ok := subConn.(*acBalancerWrapper) + acw, ok := pickResult.SubConn.(*acBalancerWrapper) if !ok { grpclog.Error("subconn returned from pick is not *acBalancerWrapper") continue } if t, ok := acw.getAddrConn().getReadyTransport(); ok { if channelz.IsOn() { - return t, doneChannelzWrapper(acw, done), nil + return t, doneChannelzWrapper(acw, pickResult.Done), nil } - return t, done, nil + return t, pickResult.Done, nil } - if done != nil { + if pickResult.Done != nil { // Calling done with nil error, no bytes sent and no bytes received. // DoneInfo with default value works. - done(balancer.DoneInfo{}) + pickResult.Done(balancer.DoneInfo{}) } grpclog.Infof("blockingPicker: the picked transport is not ready, loop back to repick") // If ok == false, ac.state is not READY. @@ -186,12 +218,12 @@ func (bp *pickerWrapper) pick(ctx context.Context, failfast bool, opts balancer. } } -func (bp *pickerWrapper) close() { - bp.mu.Lock() - defer bp.mu.Unlock() - if bp.done { +func (pw *pickerWrapper) close() { + pw.mu.Lock() + defer pw.mu.Unlock() + if pw.done { return } - bp.done = true - close(bp.blockingCh) + pw.done = true + close(pw.blockingCh) } diff --git a/vendor/google.golang.org/grpc/pickfirst.go b/vendor/google.golang.org/grpc/pickfirst.go index ed05b02ed9..c43dac9ad8 100644 --- a/vendor/google.golang.org/grpc/pickfirst.go +++ b/vendor/google.golang.org/grpc/pickfirst.go @@ -19,12 +19,14 @@ package grpc import ( - "context" + "errors" "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/resolver" + "google.golang.org/grpc/status" ) // PickFirstBalancerName is the name of the pick_first balancer. @@ -45,35 +47,67 @@ func (*pickfirstBuilder) Name() string { } type pickfirstBalancer struct { - cc balancer.ClientConn - sc balancer.SubConn + state connectivity.State + cc balancer.ClientConn + sc balancer.SubConn } +var _ balancer.V2Balancer = &pickfirstBalancer{} // Assert we implement v2 + func (b *pickfirstBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) { if err != nil { - if grpclog.V(2) { - grpclog.Infof("pickfirstBalancer: HandleResolvedAddrs called with error %v", err) - } + b.ResolverError(err) return } + b.UpdateClientConnState(balancer.ClientConnState{ResolverState: resolver.State{Addresses: addrs}}) // Ignore error +} + +func (b *pickfirstBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) { + b.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: s}) +} + +func (b *pickfirstBalancer) ResolverError(err error) { + switch b.state { + case connectivity.TransientFailure, connectivity.Idle, connectivity.Connecting: + // Set a failing picker if we don't have a good picker. + b.cc.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: status.Errorf(codes.Unavailable, "name resolver error: %v", err)}}, + ) + } + if grpclog.V(2) { + grpclog.Infof("pickfirstBalancer: ResolverError called with error %v", err) + } +} + +func (b *pickfirstBalancer) UpdateClientConnState(cs balancer.ClientConnState) error { + if len(cs.ResolverState.Addresses) == 0 { + b.ResolverError(errors.New("produced zero addresses")) + return balancer.ErrBadResolverState + } if b.sc == nil { - b.sc, err = b.cc.NewSubConn(addrs, balancer.NewSubConnOptions{}) + var err error + b.sc, err = b.cc.NewSubConn(cs.ResolverState.Addresses, balancer.NewSubConnOptions{}) if err != nil { - //TODO(yuxuanli): why not change the cc state to Idle? if grpclog.V(2) { grpclog.Errorf("pickfirstBalancer: failed to NewSubConn: %v", err) } - return + b.state = connectivity.TransientFailure + b.cc.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: status.Errorf(codes.Unavailable, "error creating connection: %v", err)}}, + ) + return balancer.ErrBadResolverState } - b.cc.UpdateBalancerState(connectivity.Idle, &picker{sc: b.sc}) + b.state = connectivity.Idle + b.cc.UpdateState(balancer.State{ConnectivityState: connectivity.Idle, Picker: &picker{result: balancer.PickResult{SubConn: b.sc}}}) b.sc.Connect() } else { - b.sc.UpdateAddresses(addrs) + b.sc.UpdateAddresses(cs.ResolverState.Addresses) b.sc.Connect() } + return nil } -func (b *pickfirstBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) { +func (b *pickfirstBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) { if grpclog.V(2) { grpclog.Infof("pickfirstBalancer: HandleSubConnStateChange: %p, %v", sc, s) } @@ -83,18 +117,28 @@ func (b *pickfirstBalancer) HandleSubConnStateChange(sc balancer.SubConn, s conn } return } - if s == connectivity.Shutdown { + b.state = s.ConnectivityState + if s.ConnectivityState == connectivity.Shutdown { b.sc = nil return } - switch s { + switch s.ConnectivityState { case connectivity.Ready, connectivity.Idle: - b.cc.UpdateBalancerState(s, &picker{sc: sc}) + b.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: &picker{result: balancer.PickResult{SubConn: sc}}}) case connectivity.Connecting: - b.cc.UpdateBalancerState(s, &picker{err: balancer.ErrNoSubConnAvailable}) + b.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: &picker{err: balancer.ErrNoSubConnAvailable}}) case connectivity.TransientFailure: - b.cc.UpdateBalancerState(s, &picker{err: balancer.ErrTransientFailure}) + err := balancer.ErrTransientFailure + // TODO: this can be unconditional after the V1 API is removed, as + // SubConnState will always contain a connection error. + if s.ConnectionError != nil { + err = balancer.TransientFailureError(s.ConnectionError) + } + b.cc.UpdateState(balancer.State{ + ConnectivityState: s.ConnectivityState, + Picker: &picker{err: err}, + }) } } @@ -102,15 +146,12 @@ func (b *pickfirstBalancer) Close() { } type picker struct { - err error - sc balancer.SubConn + result balancer.PickResult + err error } -func (p *picker) Pick(ctx context.Context, opts balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) { - if p.err != nil { - return nil, nil, p.err - } - return p.sc, nil, nil +func (p *picker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + return p.result, p.err } func init() { diff --git a/vendor/google.golang.org/grpc/resolver/resolver.go b/vendor/google.golang.org/grpc/resolver/resolver.go index 4c5423ba0f..03567d7be7 100644 --- a/vendor/google.golang.org/grpc/resolver/resolver.go +++ b/vendor/google.golang.org/grpc/resolver/resolver.go @@ -24,6 +24,7 @@ import ( "context" "net" + "google.golang.org/grpc/attributes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/serviceconfig" ) @@ -73,12 +74,18 @@ func GetDefaultScheme() string { } // AddressType indicates the address type returned by name resolution. +// +// Deprecated: use Attributes in Address instead. type AddressType uint8 const ( // Backend indicates the address is for a backend server. + // + // Deprecated: use Attributes in Address instead. Backend AddressType = iota // GRPCLB indicates the address is for a grpclb load balancer. + // + // Deprecated: use Attributes in Address instead. GRPCLB ) @@ -87,8 +94,7 @@ const ( type Address struct { // Addr is the server address on which a connection will be established. Addr string - // Type is the type of this address. - Type AddressType + // ServerName is the name of this address. // If non-empty, the ServerName is used as the transport certification authority for // the address, instead of the hostname from the Dial target string. In most cases, @@ -101,14 +107,31 @@ type Address struct { // is insecure to populate it with data from untrusted inputs since untrusted // values could be used to bypass the authority checks performed by TLS. ServerName string + + // Attributes contains arbitrary data about this address intended for + // consumption by the load balancing policy. + Attributes *attributes.Attributes + + // Type is the type of this address. + // + // Deprecated: use Attributes instead. + Type AddressType + // Metadata is the information associated with Addr, which may be used // to make load balancing decision. + // + // Deprecated: use Attributes instead. Metadata interface{} } -// BuildOption includes additional information for the builder to create +// BuildOption is a type alias of BuildOptions for legacy reasons. +// +// Deprecated: use BuildOptions instead. +type BuildOption = BuildOptions + +// BuildOptions includes additional information for the builder to create // the resolver. -type BuildOption struct { +type BuildOptions struct { // DisableServiceConfig indicates whether a resolver implementation should // fetch service config data. DisableServiceConfig bool @@ -141,6 +164,10 @@ type State struct { // config. If it is nil, it indicates no service config is present or the // resolver does not provide service configs. ServiceConfig *serviceconfig.ParseResult + + // Attributes contains arbitrary data about the resolver intended for + // consumption by the load balancing policy. + Attributes *attributes.Attributes } // ClientConn contains the callbacks for resolver to notify any updates @@ -202,14 +229,19 @@ type Builder interface { // // gRPC dial calls Build synchronously, and fails if the returned error is // not nil. - Build(target Target, cc ClientConn, opts BuildOption) (Resolver, error) + Build(target Target, cc ClientConn, opts BuildOptions) (Resolver, error) // Scheme returns the scheme supported by this resolver. // Scheme is defined at https://github.com/grpc/grpc/blob/master/doc/naming.md. Scheme() string } -// ResolveNowOption includes additional information for ResolveNow. -type ResolveNowOption struct{} +// ResolveNowOption is a type alias of ResolveNowOptions for legacy reasons. +// +// Deprecated: use ResolveNowOptions instead. +type ResolveNowOption = ResolveNowOptions + +// ResolveNowOptions includes additional information for ResolveNow. +type ResolveNowOptions struct{} // Resolver watches for the updates on the specified target. // Updates include address updates and service config updates. @@ -218,7 +250,7 @@ type Resolver interface { // again. It's just a hint, resolver can ignore this if it's not necessary. // // It could be called multiple times concurrently. - ResolveNow(ResolveNowOption) + ResolveNow(ResolveNowOptions) // Close closes the resolver. Close() } diff --git a/vendor/google.golang.org/grpc/resolver_conn_wrapper.go b/vendor/google.golang.org/grpc/resolver_conn_wrapper.go index 7dcefcfa0f..89ba9fa3e0 100644 --- a/vendor/google.golang.org/grpc/resolver_conn_wrapper.go +++ b/vendor/google.golang.org/grpc/resolver_conn_wrapper.go @@ -92,7 +92,7 @@ func newCCResolverWrapper(cc *ClientConn) (*ccResolverWrapper, error) { if creds := cc.dopts.copts.TransportCredentials; creds != nil { credsClone = creds.Clone() } - rbo := resolver.BuildOption{ + rbo := resolver.BuildOptions{ DisableServiceConfig: cc.dopts.disableServiceConfig, DialCreds: credsClone, CredsBundle: cc.dopts.copts.CredsBundle, @@ -105,15 +105,15 @@ func newCCResolverWrapper(cc *ClientConn) (*ccResolverWrapper, error) { // rb.Build-->ccr.ReportError-->ccr.poll-->ccr.resolveNow, would end up // accessing ccr.resolver which is being assigned here. ccr.resolverMu.Lock() + defer ccr.resolverMu.Unlock() ccr.resolver, err = rb.Build(cc.parsedTarget, ccr, rbo) if err != nil { return nil, err } - ccr.resolverMu.Unlock() return ccr, nil } -func (ccr *ccResolverWrapper) resolveNow(o resolver.ResolveNowOption) { +func (ccr *ccResolverWrapper) resolveNow(o resolver.ResolveNowOptions) { ccr.resolverMu.Lock() if !ccr.done.HasFired() { ccr.resolver.ResolveNow(o) @@ -149,7 +149,7 @@ func (ccr *ccResolverWrapper) poll(err error) { ccr.polling = p go func() { for i := 0; ; i++ { - ccr.resolveNow(resolver.ResolveNowOption{}) + ccr.resolveNow(resolver.ResolveNowOptions{}) t := time.NewTimer(ccr.cc.dopts.resolveNowBackoff(i)) select { case <-p: @@ -217,6 +217,10 @@ func (ccr *ccResolverWrapper) NewServiceConfig(sc string) { return } grpclog.Infof("ccResolverWrapper: got new service config: %v", sc) + if ccr.cc.dopts.disableServiceConfig { + grpclog.Infof("Service config lookups disabled; ignoring config") + return + } scpr := parseServiceConfig(sc) if scpr.Err != nil { grpclog.Warningf("ccResolverWrapper: error parsing service config: %v", scpr.Err) diff --git a/vendor/google.golang.org/grpc/server.go b/vendor/google.golang.org/grpc/server.go index e54083d850..0d75cb109a 100644 --- a/vendor/google.golang.org/grpc/server.go +++ b/vendor/google.golang.org/grpc/server.go @@ -344,8 +344,8 @@ func StatsHandler(h stats.Handler) ServerOption { // unknown service handler. The provided method is a bidi-streaming RPC service // handler that will be invoked instead of returning the "unimplemented" gRPC // error whenever a request is received for an unregistered service or method. -// The handling function has full access to the Context of the request and the -// stream, and the invocation bypasses interceptors. +// The handling function and stream interceptor (if set) have full access to +// the ServerStream, including its Context. func UnknownServiceHandler(streamHandler StreamHandler) ServerOption { return newFuncServerOption(func(o *serverOptions) { o.unknownStreamDesc = &StreamDesc{ @@ -865,41 +865,58 @@ func (s *Server) sendResponse(t transport.ServerTransport, stream *transport.Str } func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport.Stream, srv *service, md *MethodDesc, trInfo *traceInfo) (err error) { - if channelz.IsOn() { - s.incrCallsStarted() - defer func() { - if err != nil && err != io.EOF { - s.incrCallsFailed() - } else { - s.incrCallsSucceeded() - } - }() - } sh := s.opts.statsHandler - if sh != nil { - beginTime := time.Now() - begin := &stats.Begin{ - BeginTime: beginTime, + if sh != nil || trInfo != nil || channelz.IsOn() { + if channelz.IsOn() { + s.incrCallsStarted() } - sh.HandleRPC(stream.Context(), begin) - defer func() { - end := &stats.End{ + var statsBegin *stats.Begin + if sh != nil { + beginTime := time.Now() + statsBegin = &stats.Begin{ BeginTime: beginTime, - EndTime: time.Now(), } - if err != nil && err != io.EOF { - end.Error = toRPCErr(err) - } - sh.HandleRPC(stream.Context(), end) - }() - } - if trInfo != nil { - defer trInfo.tr.Finish() - trInfo.tr.LazyLog(&trInfo.firstLine, false) + sh.HandleRPC(stream.Context(), statsBegin) + } + if trInfo != nil { + trInfo.tr.LazyLog(&trInfo.firstLine, false) + } + // The deferred error handling for tracing, stats handler and channelz are + // combined into one function to reduce stack usage -- a defer takes ~56-64 + // bytes on the stack, so overflowing the stack will require a stack + // re-allocation, which is expensive. + // + // To maintain behavior similar to separate deferred statements, statements + // should be executed in the reverse order. That is, tracing first, stats + // handler second, and channelz last. Note that panics *within* defers will + // lead to different behavior, but that's an acceptable compromise; that + // would be undefined behavior territory anyway. defer func() { - if err != nil && err != io.EOF { - trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true) - trInfo.tr.SetError() + if trInfo != nil { + if err != nil && err != io.EOF { + trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true) + trInfo.tr.SetError() + } + trInfo.tr.Finish() + } + + if sh != nil { + end := &stats.End{ + BeginTime: statsBegin.BeginTime, + EndTime: time.Now(), + } + if err != nil && err != io.EOF { + end.Error = toRPCErr(err) + } + sh.HandleRPC(stream.Context(), end) + } + + if channelz.IsOn() { + if err != nil && err != io.EOF { + s.incrCallsFailed() + } else { + s.incrCallsSucceeded() + } } }() } @@ -1099,31 +1116,15 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transport.Stream, srv *service, sd *StreamDesc, trInfo *traceInfo) (err error) { if channelz.IsOn() { s.incrCallsStarted() - defer func() { - if err != nil && err != io.EOF { - s.incrCallsFailed() - } else { - s.incrCallsSucceeded() - } - }() } sh := s.opts.statsHandler + var statsBegin *stats.Begin if sh != nil { beginTime := time.Now() - begin := &stats.Begin{ + statsBegin = &stats.Begin{ BeginTime: beginTime, } - sh.HandleRPC(stream.Context(), begin) - defer func() { - end := &stats.End{ - BeginTime: beginTime, - EndTime: time.Now(), - } - if err != nil && err != io.EOF { - end.Error = toRPCErr(err) - } - sh.HandleRPC(stream.Context(), end) - }() + sh.HandleRPC(stream.Context(), statsBegin) } ctx := NewContextWithServerTransportStream(stream.Context(), stream) ss := &serverStream{ @@ -1138,6 +1139,41 @@ func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transp statsHandler: sh, } + if sh != nil || trInfo != nil || channelz.IsOn() { + // See comment in processUnaryRPC on defers. + defer func() { + if trInfo != nil { + ss.mu.Lock() + if err != nil && err != io.EOF { + ss.trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true) + ss.trInfo.tr.SetError() + } + ss.trInfo.tr.Finish() + ss.trInfo.tr = nil + ss.mu.Unlock() + } + + if sh != nil { + end := &stats.End{ + BeginTime: statsBegin.BeginTime, + EndTime: time.Now(), + } + if err != nil && err != io.EOF { + end.Error = toRPCErr(err) + } + sh.HandleRPC(stream.Context(), end) + } + + if channelz.IsOn() { + if err != nil && err != io.EOF { + s.incrCallsFailed() + } else { + s.incrCallsSucceeded() + } + } + }() + } + ss.binlog = binarylog.GetMethodLogger(stream.Method()) if ss.binlog != nil { md, _ := metadata.FromIncomingContext(ctx) @@ -1191,16 +1227,6 @@ func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transp if trInfo != nil { trInfo.tr.LazyLog(&trInfo.firstLine, false) - defer func() { - ss.mu.Lock() - if err != nil && err != io.EOF { - ss.trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true) - ss.trInfo.tr.SetError() - } - ss.trInfo.tr.Finish() - ss.trInfo.tr = nil - ss.mu.Unlock() - }() } var appErr error var server interface{} diff --git a/vendor/google.golang.org/grpc/stats/stats.go b/vendor/google.golang.org/grpc/stats/stats.go index f3f593c844..9e22c393f1 100644 --- a/vendor/google.golang.org/grpc/stats/stats.go +++ b/vendor/google.golang.org/grpc/stats/stats.go @@ -91,6 +91,8 @@ type InHeader struct { LocalAddr net.Addr // Compression is the compression algorithm used for the RPC. Compression string + // Header contains the header metadata received. + Header metadata.MD } // IsClient indicates if the stats information is from client side. @@ -104,6 +106,9 @@ type InTrailer struct { Client bool // WireLength is the wire length of trailer. WireLength int + // Trailer contains the trailer metadata received from the server. This + // field is only valid if this InTrailer is from the client side. + Trailer metadata.MD } // IsClient indicates if the stats information is from client side. @@ -146,6 +151,8 @@ type OutHeader struct { LocalAddr net.Addr // Compression is the compression algorithm used for the RPC. Compression string + // Header contains the header metadata sent. + Header metadata.MD } // IsClient indicates if this stats information is from client side. @@ -159,6 +166,9 @@ type OutTrailer struct { Client bool // WireLength is the wire length of trailer. WireLength int + // Trailer contains the trailer metadata sent to the client. This + // field is only valid if this OutTrailer is from the server side. + Trailer metadata.MD } // IsClient indicates if this stats information is from client side. @@ -176,6 +186,7 @@ type End struct { EndTime time.Time // Trailer contains the trailer metadata received from the server. This // field is only valid if this End is from the client side. + // Deprecated: use Trailer in InTrailer instead. Trailer metadata.MD // Error is the error the RPC ended with. It is an error generated from // status.Status and can be converted back to status.Status using diff --git a/vendor/google.golang.org/grpc/version.go b/vendor/google.golang.org/grpc/version.go index 40af09639c..1d3b043ec4 100644 --- a/vendor/google.golang.org/grpc/version.go +++ b/vendor/google.golang.org/grpc/version.go @@ -19,4 +19,4 @@ package grpc // Version is the current grpc version. -const Version = "1.25.1" +const Version = "1.26.0" diff --git a/vendor/google.golang.org/grpc/vet.sh b/vendor/google.golang.org/grpc/vet.sh index f324be509a..798921acc8 100644 --- a/vendor/google.golang.org/grpc/vet.sh +++ b/vendor/google.golang.org/grpc/vet.sh @@ -92,10 +92,12 @@ go list -f {{.Dir}} ./... | xargs go run test/go_vet/vet.go # - gofmt, goimports, golint (with exceptions for generated code), go vet. gofmt -s -d -l . 2>&1 | fail_on_output -goimports -l . 2>&1 | (! grep -vE "(_mock|\.pb)\.go") | fail_on_output +goimports -l . 2>&1 | (! grep -vE "(_mock|\.pb)\.go") golint ./... 2>&1 | (! grep -vE "(_mock|\.pb)\.go:") go vet -all . +misspell -error . + # - Check that generated proto files are up to date. if [[ -z "${VET_SKIP_PROTO}" ]]; then PATH="/home/travis/bin:${PATH}" make proto && \ @@ -111,32 +113,47 @@ if go help mod >& /dev/null; then fi # - Collection of static analysis checks -# TODO(dfawley): don't use deprecated functions in examples. -staticcheck -go 1.9 -checks 'inherit,-ST1015' -ignore ' -google.golang.org/grpc/balancer.go:SA1019 -google.golang.org/grpc/balancer/grpclb/grpclb_remote_balancer.go:SA1019 -google.golang.org/grpc/balancer/grpclb/grpclb_test.go:SA1019 -google.golang.org/grpc/balancer/roundrobin/roundrobin_test.go:SA1019 -google.golang.org/grpc/xds/internal/balancer/edsbalancer/balancergroup.go:SA1019 -google.golang.org/grpc/xds/internal/resolver/xds_resolver.go:SA1019 -google.golang.org/grpc/xds/internal/balancer/xds.go:SA1019 -google.golang.org/grpc/xds/internal/balancer/xds_client.go:SA1019 -google.golang.org/grpc/balancer_conn_wrappers.go:SA1019 -google.golang.org/grpc/balancer_test.go:SA1019 -google.golang.org/grpc/benchmark/benchmain/main.go:SA1019 -google.golang.org/grpc/benchmark/worker/benchmark_client.go:SA1019 -google.golang.org/grpc/clientconn.go:S1024 -google.golang.org/grpc/clientconn_state_transition_test.go:SA1019 -google.golang.org/grpc/clientconn_test.go:SA1019 -google.golang.org/grpc/examples/features/debugging/client/main.go:SA1019 -google.golang.org/grpc/examples/features/load_balancing/client/main.go:SA1019 -google.golang.org/grpc/internal/transport/handler_server.go:SA1019 -google.golang.org/grpc/internal/transport/handler_server_test.go:SA1019 -google.golang.org/grpc/internal/resolver/dns/dns_resolver.go:SA1019 -google.golang.org/grpc/stats/stats_test.go:SA1019 -google.golang.org/grpc/test/balancer_test.go:SA1019 -google.golang.org/grpc/test/channelz_test.go:SA1019 -google.golang.org/grpc/test/end2end_test.go:SA1019 -google.golang.org/grpc/test/healthcheck_test.go:SA1019 -' ./... -misspell -error . +# +# TODO(dfawley): don't use deprecated functions in examples or first-party +# plugins. +SC_OUT="$(mktemp)" +staticcheck -go 1.9 -checks 'inherit,-ST1015' ./... > "${SC_OUT}" || true +# Error if anything other than deprecation warnings are printed. +(! grep -v "is deprecated:.*SA1019" "${SC_OUT}") +# Only ignore the following deprecated types/fields/functions. +(! grep -Fv '.HandleResolvedAddrs +.HandleSubConnStateChange +.HeaderMap +.NewAddress +.NewServiceConfig +.Metadata is deprecated: use Attributes +.Type is deprecated: use Attributes +.UpdateBalancerState +balancer.Picker +grpc.CallCustomCodec +grpc.Code +grpc.Compressor +grpc.Decompressor +grpc.MaxMsgSize +grpc.MethodConfig +grpc.NewGZIPCompressor +grpc.NewGZIPDecompressor +grpc.RPCCompressor +grpc.RPCDecompressor +grpc.RoundRobin +grpc.ServiceConfig +grpc.WithBalancer +grpc.WithBalancerName +grpc.WithCompressor +grpc.WithDecompressor +grpc.WithDialer +grpc.WithMaxMsgSize +grpc.WithServiceConfig +grpc.WithTimeout +http.CloseNotifier +naming.Resolver +naming.Update +naming.Watcher +resolver.Backend +resolver.GRPCLB' "${SC_OUT}" +) diff --git a/vendor/gopkg.in/yaml.v2/scannerc.go b/vendor/gopkg.in/yaml.v2/scannerc.go index b33bdbaece..0b9bb6030a 100644 --- a/vendor/gopkg.in/yaml.v2/scannerc.go +++ b/vendor/gopkg.in/yaml.v2/scannerc.go @@ -626,32 +626,18 @@ func trace(args ...interface{}) func() { func yaml_parser_fetch_more_tokens(parser *yaml_parser_t) bool { // While we need more tokens to fetch, do it. for { - // Check if we really need to fetch more tokens. - need_more_tokens := false - - if parser.tokens_head == len(parser.tokens) { - // Queue is empty. - need_more_tokens = true - } else { - // Check if any potential simple key may occupy the head position. - for i := len(parser.simple_keys) - 1; i >= 0; i-- { - simple_key := &parser.simple_keys[i] - if simple_key.token_number < parser.tokens_parsed { - break - } - if valid, ok := yaml_simple_key_is_valid(parser, simple_key); !ok { - return false - } else if valid && simple_key.token_number == parser.tokens_parsed { - need_more_tokens = true - break - } + if parser.tokens_head != len(parser.tokens) { + // If queue is non-empty, check if any potential simple key may + // occupy the head position. + head_tok_idx, ok := parser.simple_keys_by_tok[parser.tokens_parsed] + if !ok { + break + } else if valid, ok := yaml_simple_key_is_valid(parser, &parser.simple_keys[head_tok_idx]); !ok { + return false + } else if !valid { + break } } - - // We are finished. - if !need_more_tokens { - break - } // Fetch the next token. if !yaml_parser_fetch_next_token(parser) { return false @@ -883,6 +869,7 @@ func yaml_parser_save_simple_key(parser *yaml_parser_t) bool { return false } parser.simple_keys[len(parser.simple_keys)-1] = simple_key + parser.simple_keys_by_tok[simple_key.token_number] = len(parser.simple_keys) - 1 } return true } @@ -897,9 +884,10 @@ func yaml_parser_remove_simple_key(parser *yaml_parser_t) bool { "while scanning a simple key", parser.simple_keys[i].mark, "could not find expected ':'") } + // Remove the key from the stack. + parser.simple_keys[i].possible = false + delete(parser.simple_keys_by_tok, parser.simple_keys[i].token_number) } - // Remove the key from the stack. - parser.simple_keys[i].possible = false return true } @@ -930,7 +918,9 @@ func yaml_parser_increase_flow_level(parser *yaml_parser_t) bool { func yaml_parser_decrease_flow_level(parser *yaml_parser_t) bool { if parser.flow_level > 0 { parser.flow_level-- - parser.simple_keys = parser.simple_keys[:len(parser.simple_keys)-1] + last := len(parser.simple_keys) - 1 + delete(parser.simple_keys_by_tok, parser.simple_keys[last].token_number) + parser.simple_keys = parser.simple_keys[:last] } return true } @@ -1007,6 +997,8 @@ func yaml_parser_fetch_stream_start(parser *yaml_parser_t) bool { // Initialize the simple key stack. parser.simple_keys = append(parser.simple_keys, yaml_simple_key_t{}) + parser.simple_keys_by_tok = make(map[int]int) + // A simple key is allowed at the beginning of the stream. parser.simple_key_allowed = true @@ -1310,6 +1302,7 @@ func yaml_parser_fetch_value(parser *yaml_parser_t) bool { // Remove the simple key. simple_key.possible = false + delete(parser.simple_keys_by_tok, simple_key.token_number) // A simple key cannot follow another simple key. parser.simple_key_allowed = false diff --git a/vendor/gopkg.in/yaml.v2/yamlh.go b/vendor/gopkg.in/yaml.v2/yamlh.go index e25cee563b..f6a9c8e34b 100644 --- a/vendor/gopkg.in/yaml.v2/yamlh.go +++ b/vendor/gopkg.in/yaml.v2/yamlh.go @@ -579,6 +579,7 @@ type yaml_parser_t struct { simple_key_allowed bool // May a simple key occur at the current position? simple_keys []yaml_simple_key_t // The stack of simple keys. + simple_keys_by_tok map[int]int // possible simple_key indexes indexed by token_number // Parser stuff diff --git a/vendor/modules.txt b/vendor/modules.txt index 0816f6dda6..7c6ef07abe 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -57,7 +57,7 @@ github.com/armon/go-metrics github.com/armon/go-metrics/prometheus # github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a github.com/asaskevich/govalidator -# github.com/aws/aws-sdk-go v1.25.48 +# github.com/aws/aws-sdk-go v1.27.0 github.com/aws/aws-sdk-go/aws github.com/aws/aws-sdk-go/aws/arn github.com/aws/aws-sdk-go/aws/awserr @@ -110,7 +110,7 @@ github.com/beorn7/perks/quantile github.com/blang/semver # github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b github.com/bradfitz/gomemcache/memcache -# github.com/cenkalti/backoff v1.0.0 +# github.com/cenkalti/backoff v2.2.1+incompatible github.com/cenkalti/backoff # github.com/cespare/xxhash v1.1.0 github.com/cespare/xxhash @@ -118,7 +118,7 @@ github.com/cespare/xxhash github.com/cespare/xxhash/v2 # github.com/coreos/go-semver v0.3.0 github.com/coreos/go-semver/semver -# github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d +# github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf github.com/coreos/go-systemd/journal # github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f github.com/coreos/pkg/capnslog @@ -137,10 +137,10 @@ github.com/facette/natsort # github.com/fsouza/fake-gcs-server v1.7.0 github.com/fsouza/fake-gcs-server/fakestorage github.com/fsouza/fake-gcs-server/internal/backend -# github.com/go-kit/kit v0.9.0 +# github.com/go-kit/kit v0.10.0 github.com/go-kit/kit/log github.com/go-kit/kit/log/level -# github.com/go-logfmt/logfmt v0.4.0 +# github.com/go-logfmt/logfmt v0.5.0 github.com/go-logfmt/logfmt # github.com/go-openapi/analysis v0.19.4 github.com/go-openapi/analysis @@ -197,7 +197,7 @@ github.com/golang-migrate/migrate/v4/source/file # github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 github.com/golang/groupcache/lru github.com/golang/groupcache/singleflight -# github.com/golang/protobuf v1.3.2 +# github.com/golang/protobuf v1.3.3 github.com/golang/protobuf/descriptor github.com/golang/protobuf/jsonpb github.com/golang/protobuf/proto @@ -250,7 +250,7 @@ github.com/gophercloud/gophercloud/openstack/identity/v2/tokens github.com/gophercloud/gophercloud/openstack/identity/v3/tokens github.com/gophercloud/gophercloud/openstack/utils github.com/gophercloud/gophercloud/pagination -# github.com/gorilla/mux v1.7.1 +# github.com/gorilla/mux v1.7.3 github.com/gorilla/mux # github.com/gorilla/websocket v1.4.0 github.com/gorilla/websocket @@ -308,8 +308,6 @@ github.com/jstemmer/go-junit-report/parser github.com/julienschmidt/httprouter # github.com/konsorten/go-windows-terminal-sequences v1.0.2 github.com/konsorten/go-windows-terminal-sequences -# github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 -github.com/kr/logfmt # github.com/lann/builder v0.0.0-20150808151131-f22ce00fd939 github.com/lann/builder # github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 @@ -420,10 +418,13 @@ github.com/prometheus/client_model/go github.com/prometheus/common/config github.com/prometheus/common/expfmt github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg +github.com/prometheus/common/log github.com/prometheus/common/model github.com/prometheus/common/route github.com/prometheus/common/version -# github.com/prometheus/procfs v0.0.8 +# github.com/prometheus/node_exporter v1.0.0-rc.0.0.20200417100735-fa4edd700ebc +github.com/prometheus/node_exporter/https +# github.com/prometheus/procfs v0.0.11 github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs github.com/prometheus/procfs/internal/util @@ -568,7 +569,7 @@ github.com/uber/jaeger-client-go/utils # github.com/uber/jaeger-lib v2.2.0+incompatible github.com/uber/jaeger-lib/metrics github.com/uber/jaeger-lib/metrics/prometheus -# github.com/weaveworks/common v0.0.0-20200310113808-2708ba4e60a4 => /Users/annanay/Desktop/git/go/src/github.com/weaveworks/common +# github.com/weaveworks/common v0.0.0-20200422083114-5790e7482ff8 github.com/weaveworks/common/aws github.com/weaveworks/common/errors github.com/weaveworks/common/grpc @@ -589,15 +590,17 @@ github.com/weaveworks/promrus github.com/xiang90/probing # go.etcd.io/bbolt v1.3.3 go.etcd.io/bbolt -# go.etcd.io/etcd v0.0.0-20190709142735-eb7dd97135a5 +# go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 go.etcd.io/etcd/auth go.etcd.io/etcd/auth/authpb go.etcd.io/etcd/client go.etcd.io/etcd/clientv3 go.etcd.io/etcd/clientv3/balancer +go.etcd.io/etcd/clientv3/balancer/connectivity go.etcd.io/etcd/clientv3/balancer/picker go.etcd.io/etcd/clientv3/balancer/resolver/endpoint go.etcd.io/etcd/clientv3/concurrency +go.etcd.io/etcd/clientv3/credentials go.etcd.io/etcd/embed go.etcd.io/etcd/etcdserver go.etcd.io/etcd/etcdserver/api @@ -652,11 +655,13 @@ go.etcd.io/etcd/pkg/schedule go.etcd.io/etcd/pkg/srv go.etcd.io/etcd/pkg/systemd go.etcd.io/etcd/pkg/tlsutil +go.etcd.io/etcd/pkg/traceutil go.etcd.io/etcd/pkg/transport go.etcd.io/etcd/pkg/types go.etcd.io/etcd/pkg/wait go.etcd.io/etcd/proxy/grpcproxy/adapter go.etcd.io/etcd/raft +go.etcd.io/etcd/raft/confchange go.etcd.io/etcd/raft/quorum go.etcd.io/etcd/raft/raftpb go.etcd.io/etcd/raft/tracker @@ -688,11 +693,13 @@ go.opencensus.io/trace go.opencensus.io/trace/internal go.opencensus.io/trace/propagation go.opencensus.io/trace/tracestate -# go.uber.org/atomic v1.5.0 +# go.uber.org/atomic v1.5.1 go.uber.org/atomic -# go.uber.org/multierr v1.1.0 +# go.uber.org/multierr v1.4.0 go.uber.org/multierr -# go.uber.org/zap v1.10.0 +# go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee +go.uber.org/tools/update-license +# go.uber.org/zap v1.13.0 go.uber.org/zap go.uber.org/zap/buffer go.uber.org/zap/internal/bufferpool @@ -710,7 +717,7 @@ golang.org/x/crypto/ssh/terminal # golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136 golang.org/x/exp/apidiff golang.org/x/exp/cmd/apidiff -# golang.org/x/lint v0.0.0-20190930215403-16217165b5de +# golang.org/x/lint v0.0.0-20200130185559-910be7a94367 golang.org/x/lint golang.org/x/lint/golint # golang.org/x/mod v0.2.0 @@ -741,11 +748,12 @@ golang.org/x/oauth2/jws golang.org/x/oauth2/jwt # golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e golang.org/x/sync/errgroup -# golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 +# golang.org/x/sys v0.0.0-20200217220822-9197077df867 golang.org/x/sys/cpu golang.org/x/sys/unix golang.org/x/sys/windows golang.org/x/sys/windows/registry +golang.org/x/sys/windows/svc/eventlog # golang.org/x/text v0.3.2 golang.org/x/text/secure/bidirule golang.org/x/text/transform @@ -814,8 +822,9 @@ google.golang.org/genproto/googleapis/rpc/code google.golang.org/genproto/googleapis/rpc/status google.golang.org/genproto/googleapis/type/expr google.golang.org/genproto/protobuf/field_mask -# google.golang.org/grpc v1.25.1 +# google.golang.org/grpc v1.26.0 google.golang.org/grpc +google.golang.org/grpc/attributes google.golang.org/grpc/backoff google.golang.org/grpc/balancer google.golang.org/grpc/balancer/base @@ -875,7 +884,7 @@ gopkg.in/fsnotify/fsnotify.v1 gopkg.in/inf.v0 # gopkg.in/ini.v1 v1.51.0 gopkg.in/ini.v1 -# gopkg.in/yaml.v2 v2.2.7 +# gopkg.in/yaml.v2 v2.2.8 gopkg.in/yaml.v2 # gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2 gopkg.in/yaml.v3 From 8a71ce910b77351b63d0634394f6367a7320cc98 Mon Sep 17 00:00:00 2001 From: Annanay Date: Thu, 23 Apr 2020 14:16:09 +0530 Subject: [PATCH 05/29] Fix build, add docs Signed-off-by: Annanay --- CHANGELOG.md | 1 + docs/configuration/config-file-reference.md | 50 +++++++++ docs/production/tls.md | 108 ++++++++++++++++++++ pkg/alertmanager/storage.go | 5 +- pkg/util/grpcclient/grpcclient.go | 6 +- pkg/util/httpclient/httpclient.go | 6 +- 6 files changed, 168 insertions(+), 8 deletions(-) create mode 100644 docs/production/tls.md diff --git a/CHANGELOG.md b/CHANGELOG.md index a971aff8ff..3b94497ea5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ * [CHANGE] Experimental TSDB: Modified default values for `compactor.deletion-delay` option from 48h to 12h and `-experimental.tsdb.bucket-store.ignore-deletion-marks-delay` from 24h to 6h. #2414 * [CHANGE] Experimental WAL: Default value of `-ingester.checkpoint-enabled` changed to `true`. #2416 * [FEATURE] Ruler: The `-ruler.evaluation-delay` flag was added to allow users to configure a default evaluation delay for all rules in cortex. The default value is 0 which is the current behavior. #2423 +* [FEATURE] TLS support for both HTTP and GRPC clients to enable secure communication between cortex components. #2502 * [ENHANCEMENT] Experimental TSDB: sample ingestion errors are now reported via existing `cortex_discarded_samples_total` metric. #2370 * [ENHANCEMENT] Failures on samples at distributors and ingesters return the first validation error as opposed to the last. #2383 * [ENHANCEMENT] Experimental TSDB: Added `cortex_querier_blocks_meta_synced`, which reflects current state of synced blocks over all tenants. #2392 diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index daa628ec78..52dd32f6f2 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -183,6 +183,12 @@ The `server_config` configures the HTTP and gRPC server of the launched service( # CLI flag: -server.grpc-conn-limit [grpc_listen_conn_limit: | default = 0] +# TLS Config for the HTTP Server +[http_tls_config: ] + +# TLS Config for the GRPC Server +[grpc_tls_config: ] + # Register the intrumentation handlers (/metrics etc). # CLI flag: -server.register-instrumentation [register_instrumentation: | default = true] @@ -2001,6 +2007,50 @@ The `memberlist_config` configures the Gossip memberlist. [packet_write_timeout: | default = 5s] ``` +### `http_server_tls_config` + +The `http_server_tls_config` configures TLS parameters for the server. + +```yaml +# Path to the TLS Cert for the HTTP Server +# CLI flag: -server.http-tls-cert-path +[tlsCertPath: | default = ] + +# Path to the TLS Key for the HTTP Server +# CLI flag: -server.http-tls-key-path +[tlsKeyPath: | default = ] + +# Client authentication type +# CLI flag: -server.http-tls-client-auth +[clientAuth: | default = "NoClientCert" ] + +# Path to the Client CA for the HTTP Server +# CLI flag: -server.http-tls-ca-path +[clientCA: | default = ] +``` + +### `grpc_server_tls_config` + +The `grpc_server_tls_config` configures TLS parameters for the server. + +```yaml +# Path to the TLS Cert for the GRPC Server +# CLI flag: -server.grpc-tls-cert-path +[tlsCertPath: | default = ] + +# Path to the TLS Key for the GRPC Server +# CLI flag: -server.grpc-tls-key-path +[tlsKeyPath: | default = ] + +# Client authentication type +# CLI flag: -server.grpc-tls-client-auth +[clientAuth: | default = "NoClientCert" ] + +# Path to the Client CA for the GRPC Server +# CLI flag: -server.grpc-tls-ca-path +[clientCA: | default = ] +``` + ### `limits_config` The `limits_config` configures default and per-tenant limits imposed by Cortex services (ie. distributor, ingester, ...). diff --git a/docs/production/tls.md b/docs/production/tls.md new file mode 100644 index 0000000000..9d208e6109 --- /dev/null +++ b/docs/production/tls.md @@ -0,0 +1,108 @@ +--- +title: "Securing communication between cortex components with TLS" +linkTitle: "Securing communication between cortex components with TLS" +weight: 5 +slug: tls +--- + +Cortex is a distributed system with significant traffic between its services. +To allow for secure communication, Cortex supports TLS between all its +components. This guide describes the process of setting up TLS. + +### Generation of certs to configure TLS + +The first step to securing inter-service communication in Cortex with TLS is +generating certificates. A Certifying Authority (CA) will be used for this +purpose which should be private to the organization, as any certificates signed +by this CA will have permissions to communicate with the cluster. + +We will use the following script to generate self signed certs for the cluster: + +``` +# Refer: https://github.com/joe-elliott/cert-exporter/blob/69d3d7230378325a1de4fa313432d3d6ced4a518/test/files/genCerts.sh + +# keys +openssl genrsa -out root.key +openssl genrsa -out client.key +openssl genrsa -out server.key + +# root cert / certifying authority +openssl req -x509 -new -nodes -key root.key -subj "/C=US/ST=KY/O=Org/CN=root" -sha256 -days 100000 -out root.crt + +# csrs - certificate signing requests +openssl req -new -sha256 -key client.key -subj "/C=US/ST=KY/O=Org/CN=client" -out client.csr +openssl req -new -sha256 -key server.key -subj "/C=US/ST=KY/O=Org/CN=localhost" -out server.csr + +# certificates +openssl x509 -req -in client.csr -CA root.crt -CAkey root.key -CAcreateserial -out client.crt -days 100000 -sha256 +openssl x509 -req -in server.csr -CA root.crt -CAkey root.key -CAcreateserial -out server.crt -days 100000 -sha256 +``` + +Note that the above script generates certificates that are valid for 100000 days. +This can be changed by adjusting the `-days` option in the above commands. +It is recommended that the certs be replaced atleast once every 2 years. + +The above script generates keys `client.key, server.key` and certs +`client.crt, server.crt` for both the client and server. The CA cert is +generated as `root.crt`. + +### Load certs into the HTTP/GRPC server/client + +Every HTTP/GRPC link between Cortex components supports TLS configuration +through the following config parameters: + +#### Server flags + +``` + # Path to the TLS Cert for the HTTP Server + -server.http-tls-cert-path=/path/to/server.crt + + # Path to the TLS Key for the HTTP Server + -server.http-tls-key-path=/path/to/server.key + + # Type of Client Auth for the HTTP Server + -server.http-tls-client-auth="RequireAndVerifyClientCert" + + # Path to the Client CA Cert for the HTTP Server + -server.http-tls-ca-path="/path/to/root.crt" + + # Path to the TLS Cert for the GRPC Server + -server.grpc-tls-cert-path=/path/to/server.crt + + # Path to the TLS Key for the GRPC Server + -server.grpc-tls-key-path=/path/to/server.key + + # Type of Client Auth for the GRPC Server + -server.grpc-tls-client-auth="RequireAndVerifyClientCert" + + # Path to the Client CA Cert for the GRPC Server + -server.grpc-tls-ca-path=/path/to/root.crt +``` + +#### Client flags + +Client flags are component specific. + +For an HTTP client in the alertmanager: +``` + # Path to the TLS Cert for the HTTP Client + -alertmanager.client.http-tls-cert-path=/path/to/client.crt + + # Path to the TLS Key for the HTTP Client + -alertmanager.client.http-tls-key-path=/path/to/client.key + + # Path to the TLS CA for the HTTP Client + -alertmanager.client.http-tls-ca-path=/path/to/root.crt +``` + +For a GRPC client in the ruler: +``` + # Path to the TLS Cert for the GRPC Client + -ruler.grpc-tls-cert-path=/path/to/client.crt + + # Path to the TLS Key for the GRPC Client + -ruler.grpc-tls-key-path=/path/to/client.key + + # Path to the TLS CA for the GRPC Client + -ruler.grpc-tls-ca-path=/path/to/root.crt +``` \ No newline at end of file diff --git a/pkg/alertmanager/storage.go b/pkg/alertmanager/storage.go index 7a94b67611..021521934f 100644 --- a/pkg/alertmanager/storage.go +++ b/pkg/alertmanager/storage.go @@ -9,6 +9,7 @@ import ( "github.com/cortexproject/cortex/pkg/alertmanager/alerts/configdb" "github.com/cortexproject/cortex/pkg/alertmanager/alerts/local" "github.com/cortexproject/cortex/pkg/configs/client" + "github.com/cortexproject/cortex/pkg/util/httpclient" ) // AlertStore stores and configures users rule configs @@ -19,14 +20,14 @@ type AlertStore interface { // AlertStoreConfig configures the alertmanager backend type AlertStoreConfig struct { Type string `yaml:"type"` - ConfigDB client.Config `yaml:"configdb"` + ConfigDB httpclient.Config `yaml:"configdb"` Local local.StoreConfig `yaml:"local"` } // RegisterFlags registers flags. func (cfg *AlertStoreConfig) RegisterFlags(f *flag.FlagSet) { cfg.Local.RegisterFlags(f) - cfg.ConfigDB.RegisterFlagsWithPrefix("alertmanager.", f) + cfg.ConfigDB.RegisterFlagsWithPrefix("alertmanager.client", f) f.StringVar(&cfg.Type, "alertmanager.storage.type", "configdb", "Type of backend to use to store alertmanager configs. Supported values are: \"configdb\", \"local\".") } diff --git a/pkg/util/grpcclient/grpcclient.go b/pkg/util/grpcclient/grpcclient.go index 401bef59da..b93cf1df11 100644 --- a/pkg/util/grpcclient/grpcclient.go +++ b/pkg/util/grpcclient/grpcclient.go @@ -46,9 +46,9 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { cfg.BackoffConfig.RegisterFlags(prefix, f) - f.StringVar(&cfg.TLSCertPath, prefix+".tls-cert-path", "", "gRPC TLS cert path.") - f.StringVar(&cfg.TLSKeyPath, prefix+".tls-key-path", "", "gRPC TLS key path.") - f.StringVar(&cfg.TLSCAPath, prefix+".tls-ca-path", "", "gRPC TLS CA path.") + f.StringVar(&cfg.TLSCertPath, prefix+".grpc-tls-cert-path", "", "TLS cert path for the GRPC client") + f.StringVar(&cfg.TLSKeyPath, prefix+".grpc-tls-key-path", "", "TLS key path for the GRPC client") + f.StringVar(&cfg.TLSCAPath, prefix+".grpc-tls-ca-path", "", "TLS CA path for the GRPC client") } // CallOptions returns the config in terms of CallOptions. diff --git a/pkg/util/httpclient/httpclient.go b/pkg/util/httpclient/httpclient.go index b2cb3cc5c6..ccf3925271 100644 --- a/pkg/util/httpclient/httpclient.go +++ b/pkg/util/httpclient/httpclient.go @@ -39,9 +39,9 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { f.Var(&cfg.HTTPEndpoint, prefix+".http-endpoint", "Endpoint to connect to") f.DurationVar(&cfg.HTTPClientTimeout, prefix+".http-client-timeout", 5*time.Second, "Timeout for connecting to the endpoint.") - f.StringVar(&cfg.TLSCertPath, prefix+".tls-cert-path", "", "HTTP TLS cert path.") - f.StringVar(&cfg.TLSKeyPath, prefix+".tls-key-path", "", "HTTP TLS key path.") - f.StringVar(&cfg.TLSCAPath, prefix+".tls-ca-path", "", "HTTP TLS CA path.") + f.StringVar(&cfg.TLSCertPath, prefix+".http-tls-cert-path", "", "TLS cert path for the HTTP client") + f.StringVar(&cfg.TLSKeyPath, prefix+".http-tls-key-path", "", "TLS key path for the HTTP client") + f.StringVar(&cfg.TLSCAPath, prefix+".http-tls-ca-path", "", "TLS CA path for the HTTP client") } // GetTLSConfig initialises tls.Config from config options. From bab4c82cface7502415c5557189aab1815f41503 Mon Sep 17 00:00:00 2001 From: Annanay Date: Thu, 23 Apr 2020 16:56:00 +0530 Subject: [PATCH 06/29] Fix tests Signed-off-by: Annanay --- cmd/query-tee/proxy_test.go | 6 +++++- pkg/util/grpcclient/grpcclient.go | 2 ++ tools/doc-generator/main.go | 4 ++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/cmd/query-tee/proxy_test.go b/cmd/query-tee/proxy_test.go index e7a02db99f..429c2e0d19 100644 --- a/cmd/query-tee/proxy_test.go +++ b/cmd/query-tee/proxy_test.go @@ -13,6 +13,8 @@ import ( "github.com/go-kit/kit/log" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/cortexproject/cortex/pkg/util/httpclient" ) func Test_NewProxy(t *testing.T) { @@ -150,7 +152,9 @@ func Test_Proxy_RequestsForwarding(t *testing.T) { PreferredBackend: strconv.Itoa(testData.preferredBackendIdx), ServerServicePort: 0, ServerMetricsPort: 0, - BackendReadTimeout: time.Second, + ClientConfig: httpclient.Config{ + HTTPClientTimeout: time.Second, + }, } p, err := NewProxy(cfg, log.NewNopLogger(), nil) diff --git a/pkg/util/grpcclient/grpcclient.go b/pkg/util/grpcclient/grpcclient.go index b93cf1df11..21d9fff44a 100644 --- a/pkg/util/grpcclient/grpcclient.go +++ b/pkg/util/grpcclient/grpcclient.go @@ -94,6 +94,8 @@ func (cfg *Config) DialOption(unaryClientInterceptors []grpc.UnaryClientIntercep } opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))) } + } else { + opts = append(opts, grpc.WithInsecure()) } return append(opts, grpc.WithDefaultCallOptions(cfg.CallOptions()...), diff --git a/tools/doc-generator/main.go b/tools/doc-generator/main.go index c5055e4b08..bb24acfd46 100644 --- a/tools/doc-generator/main.go +++ b/tools/doc-generator/main.go @@ -18,7 +18,6 @@ import ( "github.com/cortexproject/cortex/pkg/chunk/storage" "github.com/cortexproject/cortex/pkg/compactor" "github.com/cortexproject/cortex/pkg/configs" - config_client "github.com/cortexproject/cortex/pkg/configs/client" "github.com/cortexproject/cortex/pkg/cortex" "github.com/cortexproject/cortex/pkg/distributor" "github.com/cortexproject/cortex/pkg/flusher" @@ -33,6 +32,7 @@ import ( "github.com/cortexproject/cortex/pkg/ruler" "github.com/cortexproject/cortex/pkg/storage/tsdb" "github.com/cortexproject/cortex/pkg/storegateway" + "github.com/cortexproject/cortex/pkg/util/httpclient" "github.com/cortexproject/cortex/pkg/util/validation" ) @@ -162,7 +162,7 @@ var ( }, { name: "configstore_config", - structType: reflect.TypeOf(config_client.Config{}), + structType: reflect.TypeOf(httpclient.Config{}), desc: "The configstore_config configures the config database storing rules and alerts, and is used by the Cortex alertmanager.", }, { From 8fee6197d7f72339351fbd122ceb5710b4491f2a Mon Sep 17 00:00:00 2001 From: Annanay Date: Thu, 23 Apr 2020 18:04:40 +0530 Subject: [PATCH 07/29] Fix lint, add tls to store-gw-client Signed-off-by: Annanay --- cmd/query-tee/proxy_test.go | 8 ++++---- pkg/querier/blocks_store_balanced_set.go | 5 +++-- pkg/querier/blocks_store_balanced_set_test.go | 3 ++- pkg/querier/blocks_store_queryable.go | 4 ++-- pkg/querier/blocks_store_replicated_set.go | 5 +++-- pkg/querier/blocks_store_replicated_set_test.go | 3 ++- pkg/querier/querier.go | 5 ++++- pkg/querier/store_gateway_client.go | 12 +----------- 8 files changed, 21 insertions(+), 24 deletions(-) diff --git a/cmd/query-tee/proxy_test.go b/cmd/query-tee/proxy_test.go index 429c2e0d19..9a590e9af9 100644 --- a/cmd/query-tee/proxy_test.go +++ b/cmd/query-tee/proxy_test.go @@ -148,10 +148,10 @@ func Test_Proxy_RequestsForwarding(t *testing.T) { // Start the proxy. cfg := Config{ - BackendEndpoints: strings.Join(backendURLs, ","), - PreferredBackend: strconv.Itoa(testData.preferredBackendIdx), - ServerServicePort: 0, - ServerMetricsPort: 0, + BackendEndpoints: strings.Join(backendURLs, ","), + PreferredBackend: strconv.Itoa(testData.preferredBackendIdx), + ServerServicePort: 0, + ServerMetricsPort: 0, ClientConfig: httpclient.Config{ HTTPClientTimeout: time.Second, }, diff --git a/pkg/querier/blocks_store_balanced_set.go b/pkg/querier/blocks_store_balanced_set.go index ff54044a1d..ed79e5a7e6 100644 --- a/pkg/querier/blocks_store_balanced_set.go +++ b/pkg/querier/blocks_store_balanced_set.go @@ -16,6 +16,7 @@ import ( "github.com/cortexproject/cortex/pkg/ring/client" "github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb" + "github.com/cortexproject/cortex/pkg/util/grpcclient" "github.com/cortexproject/cortex/pkg/util/services" ) @@ -29,7 +30,7 @@ type blocksStoreBalancedSet struct { dnsProvider *dns.Provider } -func newBlocksStoreBalancedSet(serviceAddresses []string, logger log.Logger, reg prometheus.Registerer) *blocksStoreBalancedSet { +func newBlocksStoreBalancedSet(serviceAddresses []string, clientCfg grpcclient.Config, logger log.Logger, reg prometheus.Registerer) *blocksStoreBalancedSet { const dnsResolveInterval = 10 * time.Second dnsProviderReg := extprom.WrapRegistererWithPrefix("cortex_storegateway_client_", reg) @@ -37,7 +38,7 @@ func newBlocksStoreBalancedSet(serviceAddresses []string, logger log.Logger, reg s := &blocksStoreBalancedSet{ serviceAddresses: serviceAddresses, dnsProvider: dns.NewProvider(logger, dnsProviderReg, dns.GolangResolverType), - clientsPool: newStoreGatewayClientPool(nil, logger, reg), + clientsPool: newStoreGatewayClientPool(nil, clientCfg, logger, reg), } s.Service = services.NewTimerService(dnsResolveInterval, s.starting, s.resolve, nil) diff --git a/pkg/querier/blocks_store_balanced_set_test.go b/pkg/querier/blocks_store_balanced_set_test.go index 3afe3c4213..5e79bebe20 100644 --- a/pkg/querier/blocks_store_balanced_set_test.go +++ b/pkg/querier/blocks_store_balanced_set_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/cortexproject/cortex/pkg/util/grpcclient" "github.com/cortexproject/cortex/pkg/util/services" ) @@ -20,7 +21,7 @@ func TestBlocksStoreBalancedSet_GetClientsFor(t *testing.T) { ctx := context.Background() reg := prometheus.NewPedanticRegistry() - s := newBlocksStoreBalancedSet(serviceAddrs, log.NewNopLogger(), reg) + s := newBlocksStoreBalancedSet(serviceAddrs, grpcclient.Config{},log.NewNopLogger(), reg) require.NoError(t, services.StartAndAwaitRunning(ctx, s)) defer services.StopAndAwaitTerminated(ctx, s) //nolint:errcheck diff --git a/pkg/querier/blocks_store_queryable.go b/pkg/querier/blocks_store_queryable.go index 68b2f3f877..d6ead4240e 100644 --- a/pkg/querier/blocks_store_queryable.go +++ b/pkg/querier/blocks_store_queryable.go @@ -126,7 +126,7 @@ func NewBlocksStoreQueryableFromConfig(querierCfg Config, gatewayCfg storegatewa reg.MustRegister(storesRing) } - stores, err = newBlocksStoreReplicationSet(storesRing, logger, reg) + stores, err = newBlocksStoreReplicationSet(storesRing, querierCfg.StoreGatewayGRPCClientConfig, logger, reg) if err != nil { return nil, errors.Wrap(err, "failed to create store set") } @@ -135,7 +135,7 @@ func NewBlocksStoreQueryableFromConfig(querierCfg Config, gatewayCfg storegatewa return nil, errNoStoreGatewayAddress } - stores = newBlocksStoreBalancedSet(querierCfg.GetStoreGatewayAddresses(), logger, reg) + stores = newBlocksStoreBalancedSet(querierCfg.GetStoreGatewayAddresses(), querierCfg.StoreGatewayGRPCClientConfig, logger, reg) } return NewBlocksStoreQueryable(stores, scanner, reg) diff --git a/pkg/querier/blocks_store_replicated_set.go b/pkg/querier/blocks_store_replicated_set.go index 992a28ac4f..d939b5a8fe 100644 --- a/pkg/querier/blocks_store_replicated_set.go +++ b/pkg/querier/blocks_store_replicated_set.go @@ -12,6 +12,7 @@ import ( "github.com/cortexproject/cortex/pkg/ring/client" cortex_tsdb "github.com/cortexproject/cortex/pkg/storage/tsdb" "github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb" + "github.com/cortexproject/cortex/pkg/util/grpcclient" "github.com/cortexproject/cortex/pkg/util/services" ) @@ -28,10 +29,10 @@ type blocksStoreReplicationSet struct { subservicesWatcher *services.FailureWatcher } -func newBlocksStoreReplicationSet(storesRing *ring.Ring, logger log.Logger, reg prometheus.Registerer) (*blocksStoreReplicationSet, error) { +func newBlocksStoreReplicationSet(storesRing *ring.Ring, clientCfg grpcclient.Config, logger log.Logger, reg prometheus.Registerer) (*blocksStoreReplicationSet, error) { s := &blocksStoreReplicationSet{ storesRing: storesRing, - clientsPool: newStoreGatewayClientPool(storesRing, logger, reg), + clientsPool: newStoreGatewayClientPool(storesRing, clientCfg, logger, reg), } var err error diff --git a/pkg/querier/blocks_store_replicated_set_test.go b/pkg/querier/blocks_store_replicated_set_test.go index e6dee1578c..7d35ad0006 100644 --- a/pkg/querier/blocks_store_replicated_set_test.go +++ b/pkg/querier/blocks_store_replicated_set_test.go @@ -22,6 +22,7 @@ import ( "github.com/cortexproject/cortex/pkg/storegateway" "github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb" "github.com/cortexproject/cortex/pkg/util/flagext" + "github.com/cortexproject/cortex/pkg/util/grpcclient" "github.com/cortexproject/cortex/pkg/util/services" "github.com/cortexproject/cortex/pkg/util/test" ) @@ -175,7 +176,7 @@ func TestBlocksStoreReplicationSet_GetClientsFor(t *testing.T) { require.NoError(t, err) reg := prometheus.NewPedanticRegistry() - s, err := newBlocksStoreReplicationSet(r, log.NewNopLogger(), reg) + s, err := newBlocksStoreReplicationSet(r, grpcclient.Config{},log.NewNopLogger(), reg) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(ctx, s)) defer services.StopAndAwaitTerminated(ctx, s) //nolint:errcheck diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index 3e2b3373a7..61bbf71b76 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -8,6 +8,7 @@ import ( "time" "github.com/cortexproject/cortex/pkg/chunk/purger" + "github.com/cortexproject/cortex/pkg/util/grpcclient" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" @@ -51,7 +52,8 @@ type Config struct { ActiveQueryTrackerDir string `yaml:"active_query_tracker_dir"` // Blocks storage only. - StoreGatewayAddresses string `yaml:"store_gateway_addresses"` + StoreGatewayAddresses string `yaml:"store_gateway_addresses"` + StoreGatewayGRPCClientConfig grpcclient.Config `yaml:"store_gateway_client_config"` } var ( @@ -60,6 +62,7 @@ var ( // RegisterFlags adds the flags required to config this to the given FlagSet. func (cfg *Config) RegisterFlags(f *flag.FlagSet) { + cfg.StoreGatewayGRPCClientConfig.RegisterFlagsWithPrefix("querier.store-gateway-client", f) f.IntVar(&cfg.MaxConcurrent, "querier.max-concurrent", 20, "The maximum number of concurrent queries.") f.DurationVar(&cfg.Timeout, "querier.timeout", 2*time.Minute, "The timeout for a query.") if f.Lookup("promql.lookback-delta") == nil { diff --git a/pkg/querier/store_gateway_client.go b/pkg/querier/store_gateway_client.go index 00c90c817c..4d4758d203 100644 --- a/pkg/querier/store_gateway_client.go +++ b/pkg/querier/store_gateway_client.go @@ -58,17 +58,7 @@ func (c *storeGatewayClient) String() string { return c.conn.Target() } -func newStoreGatewayClientPool(r ring.ReadRing, logger log.Logger, reg prometheus.Registerer) *client.Pool { - // We prefer sane defaults instead of exposing further config options. - clientCfg := grpcclient.Config{ - MaxRecvMsgSize: 100 << 20, - MaxSendMsgSize: 16 << 20, - UseGzipCompression: false, - RateLimit: 0, - RateLimitBurst: 0, - BackoffOnRatelimits: false, - } - +func newStoreGatewayClientPool(r ring.ReadRing, clientCfg grpcclient.Config, logger log.Logger, reg prometheus.Registerer) *client.Pool { poolCfg := client.PoolConfig{ CheckInterval: time.Minute, HealthCheckEnabled: true, From d24a44d1092a2f8221c86cbc65d16bc5f44febce Mon Sep 17 00:00:00 2001 From: Annanay Date: Thu, 23 Apr 2020 20:26:31 +0530 Subject: [PATCH 08/29] Rename config parameters Signed-off-by: Annanay --- cmd/query-tee/proxy.go | 2 +- cmd/query-tee/proxy_backend.go | 16 ++++++++-------- cmd/query-tee/proxy_test.go | 2 +- pkg/configs/client/client.go | 6 +++--- pkg/configs/client/configs_test.go | 2 +- pkg/util/httpclient/httpclient.go | 14 ++++++++------ 6 files changed, 22 insertions(+), 20 deletions(-) diff --git a/cmd/query-tee/proxy.go b/cmd/query-tee/proxy.go index 0735594480..5baedf1da2 100644 --- a/cmd/query-tee/proxy.go +++ b/cmd/query-tee/proxy.go @@ -71,7 +71,7 @@ func NewProxy(cfg Config, logger log.Logger, registerer prometheus.Registerer) ( } clientConfig := cfg.ClientConfig - clientConfig.HTTPEndpoint = flagext.URLValue{URL: u} + clientConfig.Endpoint = flagext.URLValue{URL: u} p.backends = append(p.backends, NewProxyBackend(name, &clientConfig, preferred)) } diff --git a/cmd/query-tee/proxy_backend.go b/cmd/query-tee/proxy_backend.go index d053109cde..6c38b20c71 100644 --- a/cmd/query-tee/proxy_backend.go +++ b/cmd/query-tee/proxy_backend.go @@ -29,12 +29,12 @@ func NewProxyBackend(name string, config *httpclient.Config, preferred bool) *Pr roundTripper := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, + Timeout: config.ClientTimeout, KeepAlive: 30 * time.Second, }).DialContext, MaxIdleConns: 100, MaxIdleConnsPerHost: 100, // see https://github.com/golang/go/issues/13801 - IdleConnTimeout: 90 * time.Second, + IdleConnTimeout: config.BackendReadTimeout, } if tlsConfig := config.GetTLSConfig(); tlsConfig != nil { roundTripper.TLSClientConfig = tlsConfig @@ -69,19 +69,19 @@ func (b *ProxyBackend) createBackendRequest(orig *http.Request) (*http.Request, } // Replace the endpoint with the backend one. - req.URL.Scheme = b.clientConfig.HTTPEndpoint.Scheme - req.URL.Host = b.clientConfig.HTTPEndpoint.Host + req.URL.Scheme = b.clientConfig.Endpoint.Scheme + req.URL.Host = b.clientConfig.Endpoint.Host // Prepend the endpoint path to the request path. - req.URL.Path = path.Join(b.clientConfig.HTTPEndpoint.Path, req.URL.Path) + req.URL.Path = path.Join(b.clientConfig.Endpoint.Path, req.URL.Path) // Replace the auth: // - If the endpoint has user and password, use it. // - If the endpoint has user only, keep it and use the request password (if any). // - If the endpoint has no user and no password, use the request auth (if any). clientUser, clientPass, clientAuth := orig.BasicAuth() - endpointUser := b.clientConfig.HTTPEndpoint.User.Username() - endpointPass, _ := b.clientConfig.HTTPEndpoint.User.Password() + endpointUser := b.clientConfig.Endpoint.User.Username() + endpointPass, _ := b.clientConfig.Endpoint.User.Password() if endpointUser != "" && endpointPass != "" { req.SetBasicAuth(endpointUser, endpointPass) @@ -96,7 +96,7 @@ func (b *ProxyBackend) createBackendRequest(orig *http.Request) (*http.Request, func (b *ProxyBackend) doBackendRequest(req *http.Request) (int, []byte, error) { // Honor the read timeout. - ctx, cancel := context.WithTimeout(context.Background(), b.clientConfig.HTTPClientTimeout) + ctx, cancel := context.WithTimeout(context.Background(), b.clientConfig.ClientTimeout) defer cancel() // Execute the request. diff --git a/cmd/query-tee/proxy_test.go b/cmd/query-tee/proxy_test.go index 9a590e9af9..d5f85ad6a3 100644 --- a/cmd/query-tee/proxy_test.go +++ b/cmd/query-tee/proxy_test.go @@ -153,7 +153,7 @@ func Test_Proxy_RequestsForwarding(t *testing.T) { ServerServicePort: 0, ServerMetricsPort: 0, ClientConfig: httpclient.Config{ - HTTPClientTimeout: time.Second, + ClientTimeout: time.Second, }, } diff --git a/pkg/configs/client/client.go b/pkg/configs/client/client.go index 3d1c9b89d8..1224515a31 100644 --- a/pkg/configs/client/client.go +++ b/pkg/configs/client/client.go @@ -52,7 +52,7 @@ func (c ConfigDBClient) GetRules(ctx context.Context, since userconfig.ID) (map[ if since != 0 { suffix = fmt.Sprintf("?since=%d", since) } - endpoint := fmt.Sprintf("%s/private/api/prom/configs/rules%s", c.Config.HTTPEndpoint.URL.String(), suffix) + endpoint := fmt.Sprintf("%s/private/api/prom/configs/rules%s", c.Config.Endpoint.URL.String(), suffix) var response *ConfigsResponse err := instrument.CollectedRequest(ctx, "GetRules", configsRequestDuration, instrument.ErrorCode, func(ctx context.Context) error { var err error @@ -78,7 +78,7 @@ func (c ConfigDBClient) GetAlerts(ctx context.Context, since userconfig.ID) (*Co if since != 0 { suffix = fmt.Sprintf("?since=%d", since) } - endpoint := fmt.Sprintf("%s/private/api/prom/configs/alertmanager%s", c.Config.HTTPEndpoint.URL.String(), suffix) + endpoint := fmt.Sprintf("%s/private/api/prom/configs/alertmanager%s", c.Config.Endpoint.URL.String(), suffix) var response *ConfigsResponse err := instrument.CollectedRequest(ctx, "GetAlerts", configsRequestDuration, instrument.ErrorCode, func(ctx context.Context) error { var err error @@ -94,7 +94,7 @@ func doRequest(endpoint string, clientOpts httpclient.Config, since userconfig.I return nil, err } - client := &http.Client{Timeout: clientOpts.HTTPClientTimeout} + client := &http.Client{Timeout: clientOpts.ClientTimeout} if tlsConfig := clientOpts.GetTLSConfig(); tlsConfig != nil { client.Transport = &http.Transport{TLSClientConfig: tlsConfig} } diff --git a/pkg/configs/client/configs_test.go b/pkg/configs/client/configs_test.go index 3982b79d93..62443c9980 100644 --- a/pkg/configs/client/configs_test.go +++ b/pkg/configs/client/configs_test.go @@ -36,7 +36,7 @@ func TestDoRequest(t *testing.T) { })) defer server.Close() - resp, err := doRequest(server.URL, httpclient.Config{HTTPClientTimeout: 1 * time.Second}, 0) + resp, err := doRequest(server.URL, httpclient.Config{ClientTimeout: 1 * time.Second}, 0) assert.Nil(t, err) expected := ConfigsResponse{Configs: map[string]userconfig.View{ diff --git a/pkg/util/httpclient/httpclient.go b/pkg/util/httpclient/httpclient.go index ccf3925271..53fc6365db 100644 --- a/pkg/util/httpclient/httpclient.go +++ b/pkg/util/httpclient/httpclient.go @@ -14,8 +14,9 @@ import ( // Config is the config for the HTTP client type Config struct { - HTTPEndpoint flagext.URLValue `yaml:"http_endpoint"` - HTTPClientTimeout time.Duration `yaml:"http_client_timeout"` + Endpoint flagext.URLValue `yaml:"endpoint"` + ClientTimeout time.Duration `yaml:"client_timeout"` + BackendReadTimeout time.Duration `yaml:"backend_read_timeout"` TLSCertPath string `yaml:"tls_cert_path"` TLSKeyPath string `yaml:"tls_key_path"` @@ -29,15 +30,16 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { func ConfigFromURLAndTimeout(u *url.URL, timeout time.Duration) *Config { return &Config{ - HTTPEndpoint: flagext.URLValue{URL: u}, - HTTPClientTimeout: timeout, + Endpoint: flagext.URLValue{URL: u}, + ClientTimeout: timeout, } } // RegisterFlagsWithPrefix registers flags with prefix. func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { - f.Var(&cfg.HTTPEndpoint, prefix+".http-endpoint", "Endpoint to connect to") - f.DurationVar(&cfg.HTTPClientTimeout, prefix+".http-client-timeout", 5*time.Second, "Timeout for connecting to the endpoint.") + f.Var(&cfg.Endpoint, prefix+".http-endpoint", "Endpoint to connect to") + f.DurationVar(&cfg.ClientTimeout, prefix+".http-client-timeout", 5*time.Second, "Timeout for connecting to the endpoint") + f.DurationVar(&cfg.BackendReadTimeout, prefix+".http-backend-read-timeout", 90*time.Second, "Timeout for reading from backend") f.StringVar(&cfg.TLSCertPath, prefix+".http-tls-cert-path", "", "TLS cert path for the HTTP client") f.StringVar(&cfg.TLSKeyPath, prefix+".http-tls-key-path", "", "TLS key path for the HTTP client") From 485f8ba3cd16c3b12de2eef4c1de517130f4400d Mon Sep 17 00:00:00 2001 From: Annanay Date: Thu, 23 Apr 2020 20:31:40 +0530 Subject: [PATCH 09/29] Lint Signed-off-by: Annanay --- pkg/util/httpclient/httpclient.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/util/httpclient/httpclient.go b/pkg/util/httpclient/httpclient.go index 53fc6365db..a49108c98e 100644 --- a/pkg/util/httpclient/httpclient.go +++ b/pkg/util/httpclient/httpclient.go @@ -8,8 +8,9 @@ import ( "net/url" "time" - "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/prometheus/common/log" + + "github.com/cortexproject/cortex/pkg/util/flagext" ) // Config is the config for the HTTP client From 5561a0f85c41d1a25686c4b7fbf9726329be3920 Mon Sep 17 00:00:00 2001 From: Annanay Date: Tue, 28 Apr 2020 12:12:15 +0530 Subject: [PATCH 10/29] Nit fix Signed-off-by: Annanay --- pkg/configs/client/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/configs/client/client.go b/pkg/configs/client/client.go index 1224515a31..d7b0b8b084 100644 --- a/pkg/configs/client/client.go +++ b/pkg/configs/client/client.go @@ -52,7 +52,7 @@ func (c ConfigDBClient) GetRules(ctx context.Context, since userconfig.ID) (map[ if since != 0 { suffix = fmt.Sprintf("?since=%d", since) } - endpoint := fmt.Sprintf("%s/private/api/prom/configs/rules%s", c.Config.Endpoint.URL.String(), suffix) + endpoint := fmt.Sprintf("%s/private/api/prom/configs/rules%s", c.Config.Endpoint.String(), suffix) var response *ConfigsResponse err := instrument.CollectedRequest(ctx, "GetRules", configsRequestDuration, instrument.ErrorCode, func(ctx context.Context) error { var err error From 548ddfa302a0a25464f61ffaa9016359e0155275 Mon Sep 17 00:00:00 2001 From: Annanay Date: Wed, 29 Apr 2020 15:44:41 +0530 Subject: [PATCH 11/29] Checkpoint Signed-off-by: Annanay --- integration/certs/genCerts.sh | 37 +++++++++++++++++++++++++++++++++++ integration/configs.go | 3 +++ pkg/ruler/ruler.go | 2 +- 3 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 integration/certs/genCerts.sh diff --git a/integration/certs/genCerts.sh b/integration/certs/genCerts.sh new file mode 100644 index 0000000000..455f6398c2 --- /dev/null +++ b/integration/certs/genCerts.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Copied from https://github.com/joe-elliott/cert-exporter/blob/5ce49ebf6bfcdcb178d31145ae2a460f3b348cf5/test/files/genCerts.sh +# Copyright [2020] [cert-exporter authors] +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +certFolder=$1 +days=$2 + +pushd $certFolder + +# keys +openssl genrsa -out root.key +openssl genrsa -out client.key +openssl genrsa -out server.key + +# root cert +openssl req -x509 -new -nodes -key root.key -subj "/C=US/ST=KY/O=Org/CN=root" -sha256 -days $days -out root.crt + +# csrs +openssl req -new -sha256 -key client.key -subj "/C=US/ST=KY/O=Org/CN=client" -out client.csr +openssl req -new -sha256 -key server.key -subj "/C=US/ST=KY/O=Org/CN=localhost" -out server.csr + +openssl x509 -req -in client.csr -CA root.crt -CAkey root.key -CAcreateserial -out client.crt -days $days -sha256 +openssl x509 -req -in server.csr -CA root.crt -CAkey root.key -CAcreateserial -out server.crt -days $days -sha256 + +popd diff --git a/integration/configs.go b/integration/configs.go index aab608fb7e..df8b43c346 100644 --- a/integration/configs.go +++ b/integration/configs.go @@ -52,6 +52,9 @@ var ( } RulerConfigs = map[string]string{ + "--ruler.client.tls-cert-path": "certs/client.crt", + "--ruler.client.tls-key-path": "certs/client.key", + "--ruler.client.tls-ca-path": "certs/root.crt", "-ruler.enable-sharding": "false", "-ruler.poll-interval": "2s", "-experimental.ruler.enable-api": "true", diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 280c4ca76d..34c4359845 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -106,7 +106,7 @@ func (cfg *Config) Validate() error { // RegisterFlags adds the flags required to config this to the given FlagSet func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - cfg.GRPCClientConfig.RegisterFlagsWithPrefix("ruler", f) + cfg.GRPCClientConfig.RegisterFlagsWithPrefix("ruler.client", f) cfg.StoreConfig.RegisterFlags(f) cfg.Ring.RegisterFlags(f) From 1a185b52ed5019bd519d8dbf76de00962c09ead2 Mon Sep 17 00:00:00 2001 From: Annanay Date: Wed, 29 Apr 2020 20:39:06 +0530 Subject: [PATCH 12/29] Checkpoint Signed-off-by: Annanay --- integration/configs.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/integration/configs.go b/integration/configs.go index df8b43c346..739bbedd65 100644 --- a/integration/configs.go +++ b/integration/configs.go @@ -46,15 +46,18 @@ var ( cortexSchemaConfigYaml = buildSchemaConfigWith([]storeConfig{{From: "2019-03-20", IndexStore: "aws-dynamo"}}) AlertmanagerFlags = map[string]string{ + "-server.tls-cert-path": filepath.Join(e2e.ContainerSharedDir, "certs/server.crt"), + "-server.tls-key-path": filepath.Join(e2e.ContainerSharedDir, "certs/server.key"), + "-server.tls-ca-path": filepath.Join(e2e.ContainerSharedDir, "certs/root.crt"), "-alertmanager.storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), "-alertmanager.storage.type": "local", "-alertmanager.web.external-url": "http://localhost/api/prom", } RulerConfigs = map[string]string{ - "--ruler.client.tls-cert-path": "certs/client.crt", - "--ruler.client.tls-key-path": "certs/client.key", - "--ruler.client.tls-ca-path": "certs/root.crt", + "--ruler.client.tls-cert-path": filepath.Join(e2e.ContainerSharedDir, "certs/client.crt"), + "--ruler.client.tls-key-path": filepath.Join(e2e.ContainerSharedDir, "certs/client.key"), + "--ruler.client.tls-ca-path": filepath.Join(e2e.ContainerSharedDir, "certs/root.crt"), "-ruler.enable-sharding": "false", "-ruler.poll-interval": "2s", "-experimental.ruler.enable-api": "true", From 1ccefbf9d99992b5a80d35c8ff1dbcbe245a8188 Mon Sep 17 00:00:00 2001 From: Annanay Date: Thu, 30 Apr 2020 13:06:56 +0530 Subject: [PATCH 13/29] Checkpoint Signed-off-by: Annanay --- integration/configs.go | 6 ------ integration/util.go | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/integration/configs.go b/integration/configs.go index 739bbedd65..aab608fb7e 100644 --- a/integration/configs.go +++ b/integration/configs.go @@ -46,18 +46,12 @@ var ( cortexSchemaConfigYaml = buildSchemaConfigWith([]storeConfig{{From: "2019-03-20", IndexStore: "aws-dynamo"}}) AlertmanagerFlags = map[string]string{ - "-server.tls-cert-path": filepath.Join(e2e.ContainerSharedDir, "certs/server.crt"), - "-server.tls-key-path": filepath.Join(e2e.ContainerSharedDir, "certs/server.key"), - "-server.tls-ca-path": filepath.Join(e2e.ContainerSharedDir, "certs/root.crt"), "-alertmanager.storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), "-alertmanager.storage.type": "local", "-alertmanager.web.external-url": "http://localhost/api/prom", } RulerConfigs = map[string]string{ - "--ruler.client.tls-cert-path": filepath.Join(e2e.ContainerSharedDir, "certs/client.crt"), - "--ruler.client.tls-key-path": filepath.Join(e2e.ContainerSharedDir, "certs/client.key"), - "--ruler.client.tls-ca-path": filepath.Join(e2e.ContainerSharedDir, "certs/root.crt"), "-ruler.enable-sharding": "false", "-ruler.poll-interval": "2s", "-experimental.ruler.enable-api": "true", diff --git a/integration/util.go b/integration/util.go index 0b5fc39b7b..95f1415eda 100644 --- a/integration/util.go +++ b/integration/util.go @@ -51,3 +51,19 @@ func copyFileToSharedDir(s *e2e.Scenario, src, dst string) error { return writeFileToSharedDir(s, dst, content) } + +func generateClientTLSConfig(prefix string) map[string]string { + return map[string]string{ + "-" + prefix + ".tls-cert-path": filepath.Join(e2e.ContainerSharedDir, "certs/client.crt"), + "-" + prefix + ".tls-key-path": filepath.Join(e2e.ContainerSharedDir, "certs/client.key"), + "-" + prefix + ".tls-ca-path": filepath.Join(e2e.ContainerSharedDir, "certs/root.crt"), + } +} + +func generateServerTLSConfig() map[string]string { + return map[string]string{ + "-server.tls-cert-path": filepath.Join(e2e.ContainerSharedDir, "certs/server.crt"), + "-server.tls-key-path": filepath.Join(e2e.ContainerSharedDir, "certs/server.key"), + "-server.tls-ca-path": filepath.Join(e2e.ContainerSharedDir, "certs/root.crt"), + } +} From 0a8c4eb56091f742cf992c772b8fa61035b4e813 Mon Sep 17 00:00:00 2001 From: Annanay Date: Mon, 4 May 2020 16:40:29 +0530 Subject: [PATCH 14/29] Add integration tests for TLS Signed-off-by: Annanay --- integration/configs.go | 12 +++++++++--- integration/query_frontend_test.go | 18 ++++++++++++++---- integration/util.go | 23 ++++++++++++++--------- 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/integration/configs.go b/integration/configs.go index aab608fb7e..2695c89b8e 100644 --- a/integration/configs.go +++ b/integration/configs.go @@ -22,16 +22,22 @@ const ( cortexConfigFile = "config.yaml" cortexSchemaConfigFile = "schema.yaml" blocksStorageEngine = "tsdb" + certsFolder = "integration/certs/" + clientCertFile = "certs/client.crt" + clientKeyFile = "certs/client.key" + rootCertFile = "certs/root.crt" + serverCertFile = "certs/server.crt" + serverKeyFile = "certs/server.key" storeConfigTemplate = ` - from: {{.From}} store: {{.IndexStore}} schema: v9 index: prefix: cortex_ - period: 168h + period: 168h chunks: prefix: cortex_chunks_ - period: 168h + period: 168h ` cortexAlertmanagerUserConfigYaml = `route: @@ -120,7 +126,7 @@ storage: table_manager: poll_interval: 1m - retention_period: 168h + retention_period: 168h schema: {{.SchemaConfig}} diff --git a/integration/query_frontend_test.go b/integration/query_frontend_test.go index cac4e67adf..6319900801 100644 --- a/integration/query_frontend_test.go +++ b/integration/query_frontend_test.go @@ -3,6 +3,7 @@ package main import ( + "exec" "fmt" "sync" "testing" @@ -92,6 +93,15 @@ func runQueryFrontendTest(t *testing.T, setup queryFrontendSetup) { configFile, flags := setup(t, s) + // setup tls + cmd := exec.Command("bash", "certs/genCerts.sh", "certs", "1") + require.NoError(t, cmd.Run()) + require.NoError(t, copyFileToSharedDir(s, clientCertFile, clientCertFile)) + require.NoError(t, copyFileToSharedDir(s, clientKeyFile, clientKeyFile)) + require.NoError(t, copyFileToSharedDir(s, rootCertFile, rootCertFile)) + require.NoError(t, copyFileToSharedDir(s, serverCertFile, serverCertFile)) + require.NoError(t, copyFileToSharedDir(s, serverKeyFile, serverKeyFile)) + flags = mergeFlags(flags, map[string]string{ "-querier.cache-results": "true", "-querier.split-queries-by-interval": "24h", @@ -99,9 +109,9 @@ func runQueryFrontendTest(t *testing.T, setup queryFrontendSetup) { }) // Start Cortex components. - queryFrontend := e2ecortex.NewQueryFrontendWithConfigFile("query-frontend", configFile, flags, "") - ingester := e2ecortex.NewIngesterWithConfigFile("ingester", consul.NetworkHTTPEndpoint(), configFile, flags, "") - distributor := e2ecortex.NewDistributorWithConfigFile("distributor", consul.NetworkHTTPEndpoint(), configFile, flags, "") + queryFrontend := e2ecortex.NewQueryFrontendWithConfigFile("query-frontend", configFile, mergeFlags(flags, GetServerTLSFlags()), "") + ingester := e2ecortex.NewIngesterWithConfigFile("ingester", consul.NetworkHTTPEndpoint(), configFile, mergeFlags(flags, GetServerTLSFlags()), "") + distributor := e2ecortex.NewDistributorWithConfigFile("distributor", consul.NetworkHTTPEndpoint(), configFile, mergeFlags(flags, GetClientTLSFlagsWithPrefix("ingester.client")), "") require.NoError(t, s.StartAndWaitReady(queryFrontend, distributor, ingester)) // Check if we're discovering memcache or not. @@ -112,7 +122,7 @@ func runQueryFrontendTest(t *testing.T, setup queryFrontendSetup) { // able to get the query-frontend network endpoint. querier := e2ecortex.NewQuerierWithConfigFile("querier", consul.NetworkHTTPEndpoint(), configFile, mergeFlags(flags, map[string]string{ "-querier.frontend-address": queryFrontend.NetworkGRPCEndpoint(), - }), "") + }, GetClientTLSFlagsWithPrefix("querier.frontend-client"), GetClientTLSFlagsWithPrefix("ingester.client")), "") require.NoError(t, s.StartAndWaitReady(querier)) // Wait until both the distributor and querier have updated the ring. diff --git a/integration/util.go b/integration/util.go index 95f1415eda..6ccba1384a 100644 --- a/integration/util.go +++ b/integration/util.go @@ -14,7 +14,7 @@ import ( ) var ( - // Expose some utilities form the framework so that we don't have to prefix them + // Expose some utilities from the framework so that we don't have to prefix them // with the package name in tests. mergeFlags = e2e.MergeFlags newDynamoClient = e2edb.NewDynamoClient @@ -52,18 +52,23 @@ func copyFileToSharedDir(s *e2e.Scenario, src, dst string) error { return writeFileToSharedDir(s, dst, content) } -func generateClientTLSConfig(prefix string) map[string]string { +// GetServerTLSFlags generates generic TLS flags for a server +func GetServerTLSFlags() map[string]string { return map[string]string{ - "-" + prefix + ".tls-cert-path": filepath.Join(e2e.ContainerSharedDir, "certs/client.crt"), - "-" + prefix + ".tls-key-path": filepath.Join(e2e.ContainerSharedDir, "certs/client.key"), - "-" + prefix + ".tls-ca-path": filepath.Join(e2e.ContainerSharedDir, "certs/root.crt"), + "-server.http-tls-cert-path": filepath.Join(e2e.ContainerSharedDir, "certs/server.crt"), + "-server.http-tls-key-path": filepath.Join(e2e.ContainerSharedDir, "certs/server.key"), + "-server.http-tls-ca-path": filepath.Join(e2e.ContainerSharedDir, "certs/root.crt"), + "-server.grpc-tls-cert-path": filepath.Join(e2e.ContainerSharedDir, "certs/server.crt"), + "-server.grpc-tls-key-path": filepath.Join(e2e.ContainerSharedDir, "certs/server.key"), + "-server.grpc-tls-ca-path": filepath.Join(e2e.ContainerSharedDir, "certs/root.crt"), } } -func generateServerTLSConfig() map[string]string { +// GetClientTLSFlagsWithPrefix generates generic TLS flags for a client +func GetClientTLSFlagsWithPrefix(prefix string) map[string]string { return map[string]string{ - "-server.tls-cert-path": filepath.Join(e2e.ContainerSharedDir, "certs/server.crt"), - "-server.tls-key-path": filepath.Join(e2e.ContainerSharedDir, "certs/server.key"), - "-server.tls-ca-path": filepath.Join(e2e.ContainerSharedDir, "certs/root.crt"), + "-" + prefix + ".tls-cert-path": filepath.Join(e2e.ContainerSharedDir, "certs/client.crt"), + "-" + prefix + ".tls-key-path": filepath.Join(e2e.ContainerSharedDir, "certs/client.key"), + "-" + prefix + ".tls-ca-path": filepath.Join(e2e.ContainerSharedDir, "certs/root.crt"), } } From d28a3d642e02d06a7478a45b7e32a59349f6b64d Mon Sep 17 00:00:00 2001 From: Annanay Date: Mon, 4 May 2020 16:50:32 +0530 Subject: [PATCH 15/29] Correct package names, fix config file reference Signed-off-by: Annanay --- docs/configuration/config-file-reference.md | 44 --------------------- integration/query_frontend_test.go | 2 +- 2 files changed, 1 insertion(+), 45 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 3fb69ebe21..7972f9b852 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -2165,50 +2165,6 @@ The `memberlist_config` configures the Gossip memberlist. [packet_write_timeout: | default = 5s] ``` -### `http_server_tls_config` - -The `http_server_tls_config` configures TLS parameters for the server. - -```yaml -# Path to the TLS Cert for the HTTP Server -# CLI flag: -server.http-tls-cert-path -[tlsCertPath: | default = ] - -# Path to the TLS Key for the HTTP Server -# CLI flag: -server.http-tls-key-path -[tlsKeyPath: | default = ] - -# Client authentication type -# CLI flag: -server.http-tls-client-auth -[clientAuth: | default = "NoClientCert" ] - -# Path to the Client CA for the HTTP Server -# CLI flag: -server.http-tls-ca-path -[clientCA: | default = ] -``` - -### `grpc_server_tls_config` - -The `grpc_server_tls_config` configures TLS parameters for the server. - -```yaml -# Path to the TLS Cert for the GRPC Server -# CLI flag: -server.grpc-tls-cert-path -[tlsCertPath: | default = ] - -# Path to the TLS Key for the GRPC Server -# CLI flag: -server.grpc-tls-key-path -[tlsKeyPath: | default = ] - -# Client authentication type -# CLI flag: -server.grpc-tls-client-auth -[clientAuth: | default = "NoClientCert" ] - -# Path to the Client CA for the GRPC Server -# CLI flag: -server.grpc-tls-ca-path -[clientCA: | default = ] -``` - ### `limits_config` The `limits_config` configures default and per-tenant limits imposed by Cortex services (ie. distributor, ingester, ...). diff --git a/integration/query_frontend_test.go b/integration/query_frontend_test.go index 6319900801..f533b1cfc4 100644 --- a/integration/query_frontend_test.go +++ b/integration/query_frontend_test.go @@ -3,7 +3,7 @@ package main import ( - "exec" + "os/exec" "fmt" "sync" "testing" From 480308e8ff52362948aecf81aa9d9a3f301e22f0 Mon Sep 17 00:00:00 2001 From: Annanay Date: Wed, 6 May 2020 17:06:37 +0530 Subject: [PATCH 16/29] Fix cert paths Signed-off-by: Annanay --- integration/query_frontend_test.go | 16 ++++++++++------ integration/util.go | 18 +++++++++--------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/integration/query_frontend_test.go b/integration/query_frontend_test.go index f533b1cfc4..8d2f4a5660 100644 --- a/integration/query_frontend_test.go +++ b/integration/query_frontend_test.go @@ -3,8 +3,8 @@ package main import ( - "os/exec" "fmt" + "os/exec" "sync" "testing" "time" @@ -20,6 +20,10 @@ import ( "github.com/cortexproject/cortex/integration/e2ecortex" ) +const ( + integrationHomeFolder = "integration/" +) + type queryFrontendSetup func(t *testing.T, s *e2e.Scenario) (configFile string, flags map[string]string) func TestQueryFrontendWithBlocksStorageViaFlags(t *testing.T) { @@ -96,11 +100,11 @@ func runQueryFrontendTest(t *testing.T, setup queryFrontendSetup) { // setup tls cmd := exec.Command("bash", "certs/genCerts.sh", "certs", "1") require.NoError(t, cmd.Run()) - require.NoError(t, copyFileToSharedDir(s, clientCertFile, clientCertFile)) - require.NoError(t, copyFileToSharedDir(s, clientKeyFile, clientKeyFile)) - require.NoError(t, copyFileToSharedDir(s, rootCertFile, rootCertFile)) - require.NoError(t, copyFileToSharedDir(s, serverCertFile, serverCertFile)) - require.NoError(t, copyFileToSharedDir(s, serverKeyFile, serverKeyFile)) + require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+clientCertFile, clientCertFile)) + require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+clientKeyFile, clientKeyFile)) + require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+rootCertFile, rootCertFile)) + require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+serverCertFile, serverCertFile)) + require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+serverKeyFile, serverKeyFile)) flags = mergeFlags(flags, map[string]string{ "-querier.cache-results": "true", diff --git a/integration/util.go b/integration/util.go index 6ccba1384a..c97dcd8a1b 100644 --- a/integration/util.go +++ b/integration/util.go @@ -55,20 +55,20 @@ func copyFileToSharedDir(s *e2e.Scenario, src, dst string) error { // GetServerTLSFlags generates generic TLS flags for a server func GetServerTLSFlags() map[string]string { return map[string]string{ - "-server.http-tls-cert-path": filepath.Join(e2e.ContainerSharedDir, "certs/server.crt"), - "-server.http-tls-key-path": filepath.Join(e2e.ContainerSharedDir, "certs/server.key"), - "-server.http-tls-ca-path": filepath.Join(e2e.ContainerSharedDir, "certs/root.crt"), - "-server.grpc-tls-cert-path": filepath.Join(e2e.ContainerSharedDir, "certs/server.crt"), - "-server.grpc-tls-key-path": filepath.Join(e2e.ContainerSharedDir, "certs/server.key"), - "-server.grpc-tls-ca-path": filepath.Join(e2e.ContainerSharedDir, "certs/root.crt"), + "-server.http-tls-cert-path": filepath.Join(e2e.ContainerSharedDir, serverCertFile), + "-server.http-tls-key-path": filepath.Join(e2e.ContainerSharedDir, serverKeyFile), + "-server.http-tls-ca-path": filepath.Join(e2e.ContainerSharedDir, rootCertFile), + "-server.grpc-tls-cert-path": filepath.Join(e2e.ContainerSharedDir, serverCertFile), + "-server.grpc-tls-key-path": filepath.Join(e2e.ContainerSharedDir, serverKeyFile), + "-server.grpc-tls-ca-path": filepath.Join(e2e.ContainerSharedDir, rootCertFile), } } // GetClientTLSFlagsWithPrefix generates generic TLS flags for a client func GetClientTLSFlagsWithPrefix(prefix string) map[string]string { return map[string]string{ - "-" + prefix + ".tls-cert-path": filepath.Join(e2e.ContainerSharedDir, "certs/client.crt"), - "-" + prefix + ".tls-key-path": filepath.Join(e2e.ContainerSharedDir, "certs/client.key"), - "-" + prefix + ".tls-ca-path": filepath.Join(e2e.ContainerSharedDir, "certs/root.crt"), + "-" + prefix + ".tls-cert-path": filepath.Join(e2e.ContainerSharedDir, clientCertFile), + "-" + prefix + ".tls-key-path": filepath.Join(e2e.ContainerSharedDir, clientKeyFile), + "-" + prefix + ".tls-ca-path": filepath.Join(e2e.ContainerSharedDir, rootCertFile), } } From ca7a6d9718af0ab80017dc0a1a29cb74682a50cd Mon Sep 17 00:00:00 2001 From: Annanay Date: Thu, 7 May 2020 16:46:01 +0530 Subject: [PATCH 17/29] Fix lint, add sample tls config file Signed-off-by: Annanay --- cmd/query-tee/proxy.go | 3 +- cmd/query-tee/proxy_backend_test.go | 3 +- .../single-process-config-blocks-tls.yaml | 100 ++++++++++++++++++ integration/configs.go | 1 - pkg/configs/client/client.go | 5 +- pkg/querier/blocks_store_balanced_set_test.go | 2 +- 6 files changed, 107 insertions(+), 7 deletions(-) create mode 100644 docs/configuration/single-process-config-blocks-tls.yaml diff --git a/cmd/query-tee/proxy.go b/cmd/query-tee/proxy.go index 5baedf1da2..2d7e7bfa85 100644 --- a/cmd/query-tee/proxy.go +++ b/cmd/query-tee/proxy.go @@ -11,12 +11,13 @@ import ( "sync" "time" - "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/gorilla/mux" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + + "github.com/cortexproject/cortex/pkg/util/flagext" ) var ( diff --git a/cmd/query-tee/proxy_backend_test.go b/cmd/query-tee/proxy_backend_test.go index 3a3297cca4..fadf0362d6 100644 --- a/cmd/query-tee/proxy_backend_test.go +++ b/cmd/query-tee/proxy_backend_test.go @@ -7,9 +7,10 @@ import ( "testing" "time" - "github.com/cortexproject/cortex/pkg/util/httpclient" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/cortexproject/cortex/pkg/util/httpclient" ) func Test_ProxyBackend_createBackendRequest_HTTPBasicAuthentication(t *testing.T) { diff --git a/docs/configuration/single-process-config-blocks-tls.yaml b/docs/configuration/single-process-config-blocks-tls.yaml new file mode 100644 index 0000000000..3a5b9214c3 --- /dev/null +++ b/docs/configuration/single-process-config-blocks-tls.yaml @@ -0,0 +1,100 @@ + +# Configuration for running Cortex in single-process mode. +# This should not be used in production. It is only for getting started +# and development. + +# Disable the requirement that every request to Cortex has a +# X-Scope-OrgID header. `fake` will be substituted in instead. +auth_enabled: false + +server: + http_listen_port: 9009 + + # Configure the server to allow messages up to 100MB. + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + grpc_server_max_concurrent_streams: 1000 + grpc_tls_config: + cert_file: "server.crt" + key_file: "server.key" + client_auth_type: "RequireAndVerifyClientCert" + client_ca_file: "root.crt" + + +distributor: + shard_by_all_labels: true + pool: + health_check_ingesters: true + +ingester_client: + grpc_client_config: + # Configure the client to allow messages up to 100MB. + max_recv_msg_size: 104857600 + max_send_msg_size: 104857600 + use_gzip_compression: true + tls_cert_path: "client.crt" + tls_key_path: "client.key" + tls_ca_path: "root.crt" + +ingester: + # Disable blocks transfers on ingesters shutdown or rollout. + max_transfer_retries: 0 + + lifecycler: + # The address to advertise for this ingester. Will be autodiscovered by + # looking up address on eth0 or en0; can be specified if this fails. + # address: 127.0.0.1 + + # We want to start immediately and flush on shutdown. + join_after: 0 + min_ready_duration: 0s + final_sleep: 0s + num_tokens: 512 + + # Use an in memory ring store, so we don't need to launch a Consul. + ring: + kvstore: + store: inmemory + replication_factor: 1 + +storage: + engine: tsdb + +tsdb: + dir: /tmp/cortex/tsdb + bucket_store: + sync_dir: /tmp/cortex/tsdb-sync + + # You can choose between local storage and Amazon S3, Google GCS and Azure storage. Each option requires additional configuration + # as shown below. All options can be configured via flags as well which might be handy for secret inputs. + backend: s3 # s3, gcs, azure or filesystem are valid options + s3: + bucket_name: cortex + endpoint: s3.dualstack.us-east-1.amazonaws.com + # Configure your S3 credentials below. + # secret_access_key: "TODO" + # access_key_id: "TODO" +# gcs: +# bucket_name: cortex +# service_account: # if empty or omitted Cortex will use your default service account as per Google's fallback logic +# azure: +# account_name: +# account_key: +# container_name: +# endpoint_suffix: +# max_retries: # Number of retries for recoverable errors (defaults to 20) +# filesystem: +# dir: ./data/tsdb + +compactor: + data_dir: /tmp/cortex/compactor + sharding_ring: + kvstore: + store: inmemory + +frontend_worker: + match_max_concurrent: true + grpc_client_config: + tls_cert_path: "client.crt" + tls_key_path: "client.key" + tls_ca_path: "root.crt" diff --git a/integration/configs.go b/integration/configs.go index 2695c89b8e..90ce590072 100644 --- a/integration/configs.go +++ b/integration/configs.go @@ -22,7 +22,6 @@ const ( cortexConfigFile = "config.yaml" cortexSchemaConfigFile = "schema.yaml" blocksStorageEngine = "tsdb" - certsFolder = "integration/certs/" clientCertFile = "certs/client.crt" clientKeyFile = "certs/client.key" rootCertFile = "certs/root.crt" diff --git a/pkg/configs/client/client.go b/pkg/configs/client/client.go index d7b0b8b084..bcec7bdadc 100644 --- a/pkg/configs/client/client.go +++ b/pkg/configs/client/client.go @@ -6,15 +6,14 @@ import ( "fmt" "net/http" - "github.com/cortexproject/cortex/pkg/configs/userconfig" - "github.com/cortexproject/cortex/pkg/util/httpclient" - "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/weaveworks/common/instrument" + "github.com/cortexproject/cortex/pkg/configs/userconfig" "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/httpclient" ) var configsRequestDuration = instrument.NewHistogramCollector(promauto.NewHistogramVec(prometheus.HistogramOpts{ diff --git a/pkg/querier/blocks_store_balanced_set_test.go b/pkg/querier/blocks_store_balanced_set_test.go index 5e79bebe20..19b43f84ff 100644 --- a/pkg/querier/blocks_store_balanced_set_test.go +++ b/pkg/querier/blocks_store_balanced_set_test.go @@ -21,7 +21,7 @@ func TestBlocksStoreBalancedSet_GetClientsFor(t *testing.T) { ctx := context.Background() reg := prometheus.NewPedanticRegistry() - s := newBlocksStoreBalancedSet(serviceAddrs, grpcclient.Config{},log.NewNopLogger(), reg) + s := newBlocksStoreBalancedSet(serviceAddrs, grpcclient.Config{}, log.NewNopLogger(), reg) require.NoError(t, services.StartAndAwaitRunning(ctx, s)) defer services.StopAndAwaitTerminated(ctx, s) //nolint:errcheck From db64cd46663d3ef513275794f337e671c60646b0 Mon Sep 17 00:00:00 2001 From: Annanay Date: Thu, 7 May 2020 18:03:54 +0530 Subject: [PATCH 18/29] Crash quickly if certs are bad Signed-off-by: Annanay --- pkg/chunk/gcp/bigtable_index_client.go | 12 ++++++++++-- pkg/chunk/gcp/bigtable_object_client.go | 6 +++++- pkg/chunk/gcp/table_client.go | 6 +++++- pkg/ingester/client/client.go | 7 +++++-- pkg/querier/frontend/worker.go | 5 ++++- pkg/querier/store_gateway_client.go | 5 ++++- pkg/ruler/ruler.go | 6 +++++- pkg/util/grpcclient/grpcclient.go | 12 +++++++----- 8 files changed, 45 insertions(+), 14 deletions(-) diff --git a/pkg/chunk/gcp/bigtable_index_client.go b/pkg/chunk/gcp/bigtable_index_client.go index f5822bdc5a..33ef61d34a 100644 --- a/pkg/chunk/gcp/bigtable_index_client.go +++ b/pkg/chunk/gcp/bigtable_index_client.go @@ -69,7 +69,11 @@ type storageClientV1 struct { // NewStorageClientV1 returns a new v1 StorageClient. func NewStorageClientV1(ctx context.Context, cfg Config, schemaCfg chunk.SchemaConfig) (chunk.IndexClient, error) { - opts := toOptions(cfg.GRPCClientConfig.DialOption(bigtableInstrumentation())) + dialOpts, err := cfg.GRPCClientConfig.DialOption(bigtableInstrumentation()) + if err != nil { + return nil, err + } + opts := toOptions(dialOpts) client, err := bigtable.NewClient(ctx, cfg.Project, cfg.Instance, opts...) if err != nil { return nil, err @@ -93,7 +97,11 @@ func newStorageClientV1(cfg Config, schemaCfg chunk.SchemaConfig, client *bigtab // NewStorageClientColumnKey returns a new v2 StorageClient. func NewStorageClientColumnKey(ctx context.Context, cfg Config, schemaCfg chunk.SchemaConfig) (chunk.IndexClient, error) { - opts := toOptions(cfg.GRPCClientConfig.DialOption(bigtableInstrumentation())) + dialOpts, err := cfg.GRPCClientConfig.DialOption(bigtableInstrumentation()) + if err != nil { + return nil, err + } + opts := toOptions(dialOpts) client, err := bigtable.NewClient(ctx, cfg.Project, cfg.Instance, opts...) if err != nil { return nil, err diff --git a/pkg/chunk/gcp/bigtable_object_client.go b/pkg/chunk/gcp/bigtable_object_client.go index 46fbe2c2da..6982b12f18 100644 --- a/pkg/chunk/gcp/bigtable_object_client.go +++ b/pkg/chunk/gcp/bigtable_object_client.go @@ -22,7 +22,11 @@ type bigtableObjectClient struct { // NewBigtableObjectClient makes a new chunk.Client that stores chunks in // Bigtable. func NewBigtableObjectClient(ctx context.Context, cfg Config, schemaCfg chunk.SchemaConfig) (chunk.Client, error) { - opts := toOptions(cfg.GRPCClientConfig.DialOption(bigtableInstrumentation())) + dialOpts, err := cfg.GRPCClientConfig.DialOption(bigtableInstrumentation()) + if err != nil { + return nil, err + } + opts := toOptions(dialOpts) client, err := bigtable.NewClient(ctx, cfg.Project, cfg.Instance, opts...) if err != nil { return nil, err diff --git a/pkg/chunk/gcp/table_client.go b/pkg/chunk/gcp/table_client.go index 725021f554..409f8ce42d 100644 --- a/pkg/chunk/gcp/table_client.go +++ b/pkg/chunk/gcp/table_client.go @@ -24,7 +24,11 @@ type tableClient struct { // NewTableClient returns a new TableClient. func NewTableClient(ctx context.Context, cfg Config) (chunk.TableClient, error) { - opts := toOptions(cfg.GRPCClientConfig.DialOption(bigtableInstrumentation())) + dialOpts, err := cfg.GRPCClientConfig.DialOption(bigtableInstrumentation()) + if err != nil { + return nil, err + } + opts := toOptions(dialOpts) client, err := bigtable.NewAdminClient(ctx, cfg.Project, cfg.Instance, opts...) if err != nil { return nil, err diff --git a/pkg/ingester/client/client.go b/pkg/ingester/client/client.go index 3636dbcd64..cc2de4e1e4 100644 --- a/pkg/ingester/client/client.go +++ b/pkg/ingester/client/client.go @@ -34,8 +34,11 @@ type closableHealthAndIngesterClient struct { // MakeIngesterClient makes a new IngesterClient func MakeIngesterClient(addr string, cfg Config) (HealthAndIngesterClient, error) { - opts := cfg.GRPCClientConfig.DialOption(grpcclient.Instrument(ingesterClientRequestDuration)) - conn, err := grpc.Dial(addr, opts...) + dialOpts, err := cfg.GRPCClientConfig.DialOption(grpcclient.Instrument(ingesterClientRequestDuration)) + if err != nil { + return nil, err + } + conn, err := grpc.Dial(addr, dialOpts...) if err != nil { return nil, err } diff --git a/pkg/querier/frontend/worker.go b/pkg/querier/frontend/worker.go index 7723dc82a2..1a65507074 100644 --- a/pkg/querier/frontend/worker.go +++ b/pkg/querier/frontend/worker.go @@ -138,7 +138,10 @@ func (w *worker) watchDNSLoop(servCtx context.Context) error { } func (w *worker) connect(ctx context.Context, address string) (FrontendClient, error) { - opts := w.cfg.GRPCClientConfig.DialOption([]grpc.UnaryClientInterceptor{middleware.ClientUserHeaderInterceptor}, nil) + opts, err := w.cfg.GRPCClientConfig.DialOption([]grpc.UnaryClientInterceptor{middleware.ClientUserHeaderInterceptor}, nil) + if err != nil { + return nil, err + } conn, err := grpc.DialContext(ctx, address, opts...) if err != nil { diff --git a/pkg/querier/store_gateway_client.go b/pkg/querier/store_gateway_client.go index 98efc0f934..19114530ea 100644 --- a/pkg/querier/store_gateway_client.go +++ b/pkg/querier/store_gateway_client.go @@ -30,7 +30,10 @@ func newStoreGatewayClientFactory(cfg grpcclient.Config, reg prometheus.Register } func dialStoreGatewayClient(cfg grpcclient.Config, addr string, requestDuration *prometheus.HistogramVec) (*storeGatewayClient, error) { - opts := cfg.DialOption(grpcclient.Instrument(requestDuration)) + opts, err := cfg.DialOption(grpcclient.Instrument(requestDuration)) + if err != nil { + return nil, err + } conn, err := grpc.Dial(addr, opts...) if err != nil { return nil, errors.Wrapf(err, "failed to dial store-gateway %s", addr) diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 7c4984e532..f8dd8afae4 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -626,7 +626,11 @@ func (r *Ruler) getShardedRules(ctx context.Context) ([]*GroupStateDesc, error) rgs := []*GroupStateDesc{} for _, rlr := range rulers.Ingesters { - conn, err := grpc.Dial(rlr.Addr, r.cfg.GRPCClientConfig.DialOption(nil, nil)...) + dialOpts, err := r.cfg.GRPCClientConfig.DialOption(nil, nil) + if err != nil { + return nil, err + } + conn, err := grpc.Dial(rlr.Addr, dialOpts...) if err != nil { return nil, err } diff --git a/pkg/util/grpcclient/grpcclient.go b/pkg/util/grpcclient/grpcclient.go index 21d9fff44a..6b61aa769b 100644 --- a/pkg/util/grpcclient/grpcclient.go +++ b/pkg/util/grpcclient/grpcclient.go @@ -6,8 +6,8 @@ import ( "flag" "io/ioutil" + "github.com/go-kit/kit/log/level" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" - "github.com/prometheus/common/log" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -63,7 +63,7 @@ func (cfg *Config) CallOptions() []grpc.CallOption { } // DialOption returns the config as a grpc.DialOptions. -func (cfg *Config) DialOption(unaryClientInterceptors []grpc.UnaryClientInterceptor, streamClientInterceptors []grpc.StreamClientInterceptor) []grpc.DialOption { +func (cfg *Config) DialOption(unaryClientInterceptors []grpc.UnaryClientInterceptor, streamClientInterceptors []grpc.StreamClientInterceptor) ([]grpc.DialOption, error) { if cfg.BackoffOnRatelimits { unaryClientInterceptors = append([]grpc.UnaryClientInterceptor{NewBackoffRetry(cfg.BackoffConfig)}, unaryClientInterceptors...) } @@ -76,12 +76,14 @@ func (cfg *Config) DialOption(unaryClientInterceptors []grpc.UnaryClientIntercep if cfg.TLSCertPath != "" && cfg.TLSKeyPath != "" && cfg.TLSCAPath != "" { clientCert, err := tls.LoadX509KeyPair(cfg.TLSCertPath, cfg.TLSKeyPath) if err != nil { - log.Warnf("error loading cert %s or key %s, tls disabled", cfg.TLSCertPath, cfg.TLSKeyPath) + level.Error(util.Logger).Log("msg", "error loading certs", "error", err) + return nil, err } var caCertPool *x509.CertPool caCert, err := ioutil.ReadFile(cfg.TLSCAPath) if err != nil { - log.Warnf("error loading ca cert %s, tls disabled", cfg.TLSCAPath) + level.Error(util.Logger).Log("msg", "error loading ca cert", "error", err) + return nil, err } else { caCertPool = x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) @@ -101,5 +103,5 @@ func (cfg *Config) DialOption(unaryClientInterceptors []grpc.UnaryClientIntercep return append(opts, grpc.WithDefaultCallOptions(cfg.CallOptions()...), grpc.WithUnaryInterceptor(grpc_middleware.ChainUnaryClient(unaryClientInterceptors...)), grpc.WithStreamInterceptor(grpc_middleware.ChainStreamClient(streamClientInterceptors...)), - ) + ), nil } From ee48ed3ab42cbe222c528e7df59a103f80905f1c Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Fri, 8 May 2020 17:12:59 +0200 Subject: [PATCH 19/29] Fixed linter and doc generation Signed-off-by: Marco Pracucci --- docs/configuration/config-file-reference.md | 170 +++++++++++++++++- .../blocks_store_replicated_set_test.go | 2 +- pkg/ruler/storage.go | 1 + 3 files changed, 163 insertions(+), 10 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 7972f9b852..a9a74f1ac1 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -646,6 +646,56 @@ The `querier_config` configures the Cortex querier. # instances form a ring and addresses are picked from the ring). # CLI flag: -experimental.querier.store-gateway-addresses [store_gateway_addresses: | default = ""] + +store_gateway_client_config: + # gRPC client max receive message size (bytes). + # CLI flag: -querier.store-gateway-client.grpc-max-recv-msg-size + [max_recv_msg_size: | default = 104857600] + + # gRPC client max send message size (bytes). + # CLI flag: -querier.store-gateway-client.grpc-max-send-msg-size + [max_send_msg_size: | default = 16777216] + + # Use compression when sending messages. + # CLI flag: -querier.store-gateway-client.grpc-use-gzip-compression + [use_gzip_compression: | default = false] + + # Rate limit for gRPC client; 0 means disabled. + # CLI flag: -querier.store-gateway-client.grpc-client-rate-limit + [rate_limit: | default = 0] + + # Rate limit burst for gRPC client. + # CLI flag: -querier.store-gateway-client.grpc-client-rate-limit-burst + [rate_limit_burst: | default = 0] + + # Enable backoff and retry when we hit ratelimits. + # CLI flag: -querier.store-gateway-client.backoff-on-ratelimits + [backoff_on_ratelimits: | default = false] + + backoff_config: + # Minimum delay when backing off. + # CLI flag: -querier.store-gateway-client.backoff-min-period + [min_period: | default = 100ms] + + # Maximum delay when backing off. + # CLI flag: -querier.store-gateway-client.backoff-max-period + [max_period: | default = 10s] + + # Number of times to backoff and retry before failing. + # CLI flag: -querier.store-gateway-client.backoff-retries + [max_retries: | default = 10] + + # TLS cert path for the GRPC client + # CLI flag: -querier.store-gateway-client.grpc-tls-cert-path + [tls_cert_path: | default = ""] + + # TLS key path for the GRPC client + # CLI flag: -querier.store-gateway-client.grpc-tls-key-path + [tls_key_path: | default = ""] + + # TLS CA path for the GRPC client + # CLI flag: -querier.store-gateway-client.grpc-tls-ca-path + [tls_ca_path: | default = ""] ``` ### `query_frontend_config` @@ -757,6 +807,56 @@ The `ruler_config` configures the Cortex ruler. # CLI flag: -ruler.external.url [external_url: | default = ] +grpc_client_config: + # gRPC client max receive message size (bytes). + # CLI flag: -ruler.client.grpc-max-recv-msg-size + [max_recv_msg_size: | default = 104857600] + + # gRPC client max send message size (bytes). + # CLI flag: -ruler.client.grpc-max-send-msg-size + [max_send_msg_size: | default = 16777216] + + # Use compression when sending messages. + # CLI flag: -ruler.client.grpc-use-gzip-compression + [use_gzip_compression: | default = false] + + # Rate limit for gRPC client; 0 means disabled. + # CLI flag: -ruler.client.grpc-client-rate-limit + [rate_limit: | default = 0] + + # Rate limit burst for gRPC client. + # CLI flag: -ruler.client.grpc-client-rate-limit-burst + [rate_limit_burst: | default = 0] + + # Enable backoff and retry when we hit ratelimits. + # CLI flag: -ruler.client.backoff-on-ratelimits + [backoff_on_ratelimits: | default = false] + + backoff_config: + # Minimum delay when backing off. + # CLI flag: -ruler.client.backoff-min-period + [min_period: | default = 100ms] + + # Maximum delay when backing off. + # CLI flag: -ruler.client.backoff-max-period + [max_period: | default = 10s] + + # Number of times to backoff and retry before failing. + # CLI flag: -ruler.client.backoff-retries + [max_retries: | default = 10] + + # TLS cert path for the GRPC client + # CLI flag: -ruler.client.grpc-tls-cert-path + [tls_cert_path: | default = ""] + + # TLS key path for the GRPC client + # CLI flag: -ruler.client.grpc-tls-key-path + [tls_key_path: | default = ""] + + # TLS CA path for the GRPC client + # CLI flag: -ruler.client.grpc-tls-ca-path + [tls_ca_path: | default = ""] + # How frequently to evaluate rules # CLI flag: -ruler.evaluation-interval [evaluation_interval: | default = 1m] @@ -777,7 +877,7 @@ storage: # The configstore_config configures the config database storing rules and # alerts, and is used by the Cortex alertmanager. - # The CLI flags prefix for this block config is: ruler + # The CLI flags prefix for this block config is: ruler. [configdb: ] azure: @@ -1064,7 +1164,7 @@ storage: # The configstore_config configures the config database storing rules and # alerts, and is used by the Cortex alertmanager. - # The CLI flags prefix for this block config is: alertmanager + # The CLI flags prefix for this block config is: alertmanager.client [configdb: ] local: @@ -1584,6 +1684,18 @@ bigtable: # CLI flag: -bigtable.backoff-retries [max_retries: | default = 10] + # TLS cert path for the GRPC client + # CLI flag: -bigtable.grpc-tls-cert-path + [tls_cert_path: | default = ""] + + # TLS key path for the GRPC client + # CLI flag: -bigtable.grpc-tls-key-path + [tls_key_path: | default = ""] + + # TLS CA path for the GRPC client + # CLI flag: -bigtable.grpc-tls-ca-path + [tls_ca_path: | default = ""] + # If enabled, once a tables info is fetched, it is cached. # CLI flag: -bigtable.table-cache.enabled [table_cache_enabled: | default = true] @@ -1960,6 +2072,18 @@ grpc_client_config: # Number of times to backoff and retry before failing. # CLI flag: -ingester.client.backoff-retries [max_retries: | default = 10] + + # TLS cert path for the GRPC client + # CLI flag: -ingester.client.grpc-tls-cert-path + [tls_cert_path: | default = ""] + + # TLS key path for the GRPC client + # CLI flag: -ingester.client.grpc-tls-key-path + [tls_key_path: | default = ""] + + # TLS CA path for the GRPC client + # CLI flag: -ingester.client.grpc-tls-ca-path + [tls_ca_path: | default = ""] ``` ### `frontend_worker_config` @@ -2021,6 +2145,18 @@ grpc_client_config: # Number of times to backoff and retry before failing. # CLI flag: -querier.frontend-client.backoff-retries [max_retries: | default = 10] + + # TLS cert path for the GRPC client + # CLI flag: -querier.frontend-client.grpc-tls-cert-path + [tls_cert_path: | default = ""] + + # TLS key path for the GRPC client + # CLI flag: -querier.frontend-client.grpc-tls-key-path + [tls_key_path: | default = ""] + + # TLS CA path for the GRPC client + # CLI flag: -querier.frontend-client.grpc-tls-ca-path + [tls_ca_path: | default = ""] ``` ### `etcd_config` @@ -2513,19 +2649,35 @@ api: The `configstore_config` configures the config database storing rules and alerts, and is used by the Cortex alertmanager. The supported CLI flags `` used to reference this config block are: -- `alertmanager` -- `ruler` +- `alertmanager.client` +- `ruler.`   ```yaml -# URL of configs API server. -# CLI flag: -.configs.url -[configs_api_url: | default = ] +# Endpoint to connect to +# CLI flag: -.http-endpoint +[endpoint: | default = ] -# Timeout for requests to Weave Cloud configs service. -# CLI flag: -.configs.client-timeout +# Timeout for connecting to the endpoint +# CLI flag: -.http-client-timeout [client_timeout: | default = 5s] + +# Timeout for reading from backend +# CLI flag: -.http-backend-read-timeout +[backend_read_timeout: | default = 1m30s] + +# TLS cert path for the HTTP client +# CLI flag: -.http-tls-cert-path +[tls_cert_path: | default = ""] + +# TLS key path for the HTTP client +# CLI flag: -.http-tls-key-path +[tls_key_path: | default = ""] + +# TLS CA path for the HTTP client +# CLI flag: -.http-tls-ca-path +[tls_ca_path: | default = ""] ``` ### `tsdb_config` diff --git a/pkg/querier/blocks_store_replicated_set_test.go b/pkg/querier/blocks_store_replicated_set_test.go index f52100e40b..53e265b6bb 100644 --- a/pkg/querier/blocks_store_replicated_set_test.go +++ b/pkg/querier/blocks_store_replicated_set_test.go @@ -176,7 +176,7 @@ func TestBlocksStoreReplicationSet_GetClientsFor(t *testing.T) { require.NoError(t, err) reg := prometheus.NewPedanticRegistry() - s, err := newBlocksStoreReplicationSet(r, grpcclient.Config{},log.NewNopLogger(), reg) + s, err := newBlocksStoreReplicationSet(r, grpcclient.Config{}, log.NewNopLogger(), reg) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(ctx, s)) defer services.StopAndAwaitTerminated(ctx, s) //nolint:errcheck diff --git a/pkg/ruler/storage.go b/pkg/ruler/storage.go index 353f775ac6..643d28b2da 100644 --- a/pkg/ruler/storage.go +++ b/pkg/ruler/storage.go @@ -34,6 +34,7 @@ type RuleStoreConfig struct { // RegisterFlags registers flags. func (cfg *RuleStoreConfig) RegisterFlags(f *flag.FlagSet) { + cfg.ConfigDB.RegisterFlagsWithPrefix("ruler.", f) cfg.Azure.RegisterFlagsWithPrefix("ruler.storage.", f) cfg.GCS.RegisterFlagsWithPrefix("ruler.storage.", f) cfg.S3.RegisterFlagsWithPrefix("ruler.storage.", f) From b9325bd17e52717b211a622f88b916c2e1c04f70 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Fri, 8 May 2020 17:43:49 +0200 Subject: [PATCH 20/29] Cleaned white noise Signed-off-by: Marco Pracucci --- docs/production/tls.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/production/tls.md b/docs/production/tls.md index 9d208e6109..652ec8fb77 100644 --- a/docs/production/tls.md +++ b/docs/production/tls.md @@ -5,16 +5,16 @@ weight: 5 slug: tls --- -Cortex is a distributed system with significant traffic between its services. -To allow for secure communication, Cortex supports TLS between all its +Cortex is a distributed system with significant traffic between its services. +To allow for secure communication, Cortex supports TLS between all its components. This guide describes the process of setting up TLS. ### Generation of certs to configure TLS -The first step to securing inter-service communication in Cortex with TLS is -generating certificates. A Certifying Authority (CA) will be used for this -purpose which should be private to the organization, as any certificates signed -by this CA will have permissions to communicate with the cluster. +The first step to securing inter-service communication in Cortex with TLS is +generating certificates. A Certifying Authority (CA) will be used for this +purpose which should be private to the organization, as any certificates signed +by this CA will have permissions to communicate with the cluster. We will use the following script to generate self signed certs for the cluster: @@ -42,13 +42,13 @@ Note that the above script generates certificates that are valid for 100000 days This can be changed by adjusting the `-days` option in the above commands. It is recommended that the certs be replaced atleast once every 2 years. -The above script generates keys `client.key, server.key` and certs -`client.crt, server.crt` for both the client and server. The CA cert is -generated as `root.crt`. +The above script generates keys `client.key, server.key` and certs +`client.crt, server.crt` for both the client and server. The CA cert is +generated as `root.crt`. ### Load certs into the HTTP/GRPC server/client -Every HTTP/GRPC link between Cortex components supports TLS configuration +Every HTTP/GRPC link between Cortex components supports TLS configuration through the following config parameters: #### Server flags @@ -81,7 +81,7 @@ through the following config parameters: #### Client flags -Client flags are component specific. +Client flags are component specific. For an HTTP client in the alertmanager: ``` From af4935d9a48a79f541ef978a2ad7a04472bde727 Mon Sep 17 00:00:00 2001 From: Annanay Date: Mon, 11 May 2020 17:20:09 +0530 Subject: [PATCH 21/29] Address review comments Signed-off-by: Annanay --- cmd/query-tee/main.go | 16 +- cmd/query-tee/proxy.go | 6 +- cmd/query-tee/proxy_backend.go | 56 ++++--- cmd/query-tee/proxy_backend_test.go | 4 +- cmd/query-tee/proxy_test.go | 14 +- docs/configuration/config-file-reference.md | 156 +++++++------------- pkg/alertmanager/storage.go | 5 +- pkg/configs/client/client.go | 58 ++++++-- pkg/configs/client/configs_test.go | 3 +- pkg/querier/querier.go | 2 +- pkg/ruler/ruler.go | 10 +- pkg/ruler/storage.go | 5 +- pkg/util/grpcclient/grpcclient.go | 44 +----- pkg/util/httpclient/httpclient.go | 75 ---------- pkg/util/tls/tls.go | 69 +++++++++ tools/doc-generator/main.go | 4 +- 16 files changed, 231 insertions(+), 296 deletions(-) delete mode 100644 pkg/util/httpclient/httpclient.go create mode 100644 pkg/util/tls/tls.go diff --git a/cmd/query-tee/main.go b/cmd/query-tee/main.go index 46c0d25c5d..1167dcede4 100644 --- a/cmd/query-tee/main.go +++ b/cmd/query-tee/main.go @@ -3,6 +3,7 @@ package main import ( "flag" "os" + "time" "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" @@ -10,16 +11,15 @@ import ( "github.com/weaveworks/common/server" "github.com/cortexproject/cortex/pkg/util" - "github.com/cortexproject/cortex/pkg/util/httpclient" ) type Config struct { - ServerServicePort int - ServerMetricsPort int - BackendEndpoints string - PreferredBackend string - ClientConfig httpclient.Config - LogLevel logging.Level + ServerServicePort int + ServerMetricsPort int + BackendEndpoints string + PreferredBackend string + BackendReadTimeout time.Duration + LogLevel logging.Level } func main() { @@ -29,7 +29,7 @@ func main() { flag.IntVar(&cfg.ServerMetricsPort, "server.metrics-port", 9900, "The port where metrics are exposed.") flag.StringVar(&cfg.BackendEndpoints, "backend.endpoints", "", "Comma separated list of backend endpoints to query.") flag.StringVar(&cfg.PreferredBackend, "backend.preferred", "", "The hostname of the preferred backend when selecting the response to send back to the client.") - cfg.ClientConfig.RegisterFlags(flag.CommandLine) + flag.DurationVar(&cfg.BackendReadTimeout, "backend.read-timeout", 90*time.Second, "The timeout when reading the response from a backend.") cfg.LogLevel.RegisterFlags(flag.CommandLine) flag.Parse() diff --git a/cmd/query-tee/proxy.go b/cmd/query-tee/proxy.go index 2d7e7bfa85..9987d0cae8 100644 --- a/cmd/query-tee/proxy.go +++ b/cmd/query-tee/proxy.go @@ -16,8 +16,6 @@ import ( "github.com/gorilla/mux" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - - "github.com/cortexproject/cortex/pkg/util/flagext" ) var ( @@ -71,9 +69,7 @@ func NewProxy(cfg Config, logger log.Logger, registerer prometheus.Registerer) ( preferred = preferredIdx == idx } - clientConfig := cfg.ClientConfig - clientConfig.Endpoint = flagext.URLValue{URL: u} - p.backends = append(p.backends, NewProxyBackend(name, &clientConfig, preferred)) + p.backends = append(p.backends, NewProxyBackend(name, u, cfg.BackendReadTimeout, preferred)) } // At least 1 backend is required diff --git a/cmd/query-tee/proxy_backend.go b/cmd/query-tee/proxy_backend.go index 6c38b20c71..86dee7d5a0 100644 --- a/cmd/query-tee/proxy_backend.go +++ b/cmd/query-tee/proxy_backend.go @@ -5,19 +5,19 @@ import ( "io/ioutil" "net" "net/http" + "net/url" "path" "time" "github.com/pkg/errors" - - "github.com/cortexproject/cortex/pkg/util/httpclient" ) // ProxyBackend holds the information of a single backend. type ProxyBackend struct { - name string - clientConfig *httpclient.Config - client *http.Client + name string + endpoint *url.URL + client *http.Client + timeout time.Duration // Whether this is the preferred backend from which picking up // the response and sending it back to the client. @@ -25,30 +25,26 @@ type ProxyBackend struct { } // NewProxyBackend makes a new ProxyBackend -func NewProxyBackend(name string, config *httpclient.Config, preferred bool) *ProxyBackend { - roundTripper := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: config.ClientTimeout, - KeepAlive: 30 * time.Second, - }).DialContext, - MaxIdleConns: 100, - MaxIdleConnsPerHost: 100, // see https://github.com/golang/go/issues/13801 - IdleConnTimeout: config.BackendReadTimeout, - } - if tlsConfig := config.GetTLSConfig(); tlsConfig != nil { - roundTripper.TLSClientConfig = tlsConfig - } - +func NewProxyBackend(name string, endpoint *url.URL, timeout time.Duration, preferred bool) *ProxyBackend { return &ProxyBackend{ - name: name, - clientConfig: config, - preferred: preferred, + name: name, + endpoint: endpoint, + timeout: timeout, + preferred: preferred, client: &http.Client{ CheckRedirect: func(_ *http.Request, _ []*http.Request) error { return errors.New("the query-tee proxy does not follow redirects") }, - Transport: roundTripper, + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, // see https://github.com/golang/go/issues/13801 + IdleConnTimeout: 90 * time.Second, + }, }, } } @@ -69,19 +65,19 @@ func (b *ProxyBackend) createBackendRequest(orig *http.Request) (*http.Request, } // Replace the endpoint with the backend one. - req.URL.Scheme = b.clientConfig.Endpoint.Scheme - req.URL.Host = b.clientConfig.Endpoint.Host + req.URL.Scheme = b.endpoint.Scheme + req.URL.Host = b.endpoint.Host // Prepend the endpoint path to the request path. - req.URL.Path = path.Join(b.clientConfig.Endpoint.Path, req.URL.Path) + req.URL.Path = path.Join(b.endpoint.Path, req.URL.Path) // Replace the auth: // - If the endpoint has user and password, use it. // - If the endpoint has user only, keep it and use the request password (if any). // - If the endpoint has no user and no password, use the request auth (if any). clientUser, clientPass, clientAuth := orig.BasicAuth() - endpointUser := b.clientConfig.Endpoint.User.Username() - endpointPass, _ := b.clientConfig.Endpoint.User.Password() + endpointUser := b.endpoint.User.Username() + endpointPass, _ := b.endpoint.User.Password() if endpointUser != "" && endpointPass != "" { req.SetBasicAuth(endpointUser, endpointPass) @@ -96,7 +92,7 @@ func (b *ProxyBackend) createBackendRequest(orig *http.Request) (*http.Request, func (b *ProxyBackend) doBackendRequest(req *http.Request) (int, []byte, error) { // Honor the read timeout. - ctx, cancel := context.WithTimeout(context.Background(), b.clientConfig.ClientTimeout) + ctx, cancel := context.WithTimeout(context.Background(), b.timeout) defer cancel() // Execute the request. diff --git a/cmd/query-tee/proxy_backend_test.go b/cmd/query-tee/proxy_backend_test.go index fadf0362d6..d4f50a3d4a 100644 --- a/cmd/query-tee/proxy_backend_test.go +++ b/cmd/query-tee/proxy_backend_test.go @@ -9,8 +9,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "github.com/cortexproject/cortex/pkg/util/httpclient" ) func Test_ProxyBackend_createBackendRequest_HTTPBasicAuthentication(t *testing.T) { @@ -63,7 +61,7 @@ func Test_ProxyBackend_createBackendRequest_HTTPBasicAuthentication(t *testing.T orig := httptest.NewRequest("GET", "/test", nil) orig.SetBasicAuth(testData.clientUser, testData.clientPass) - b := NewProxyBackend("test", httpclient.ConfigFromURLAndTimeout(u, time.Second), false) + b := NewProxyBackend("test", u, time.Second, false) r, err := b.createBackendRequest(orig) require.NoError(t, err) diff --git a/cmd/query-tee/proxy_test.go b/cmd/query-tee/proxy_test.go index d5f85ad6a3..e7a02db99f 100644 --- a/cmd/query-tee/proxy_test.go +++ b/cmd/query-tee/proxy_test.go @@ -13,8 +13,6 @@ import ( "github.com/go-kit/kit/log" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "github.com/cortexproject/cortex/pkg/util/httpclient" ) func Test_NewProxy(t *testing.T) { @@ -148,13 +146,11 @@ func Test_Proxy_RequestsForwarding(t *testing.T) { // Start the proxy. cfg := Config{ - BackendEndpoints: strings.Join(backendURLs, ","), - PreferredBackend: strconv.Itoa(testData.preferredBackendIdx), - ServerServicePort: 0, - ServerMetricsPort: 0, - ClientConfig: httpclient.Config{ - ClientTimeout: time.Second, - }, + BackendEndpoints: strings.Join(backendURLs, ","), + PreferredBackend: strconv.Itoa(testData.preferredBackendIdx), + ServerServicePort: 0, + ServerMetricsPort: 0, + BackendReadTimeout: time.Second, } p, err := NewProxy(cfg, log.NewNopLogger(), nil) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index a9a74f1ac1..146efdf65a 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -649,52 +649,52 @@ The `querier_config` configures the Cortex querier. store_gateway_client_config: # gRPC client max receive message size (bytes). - # CLI flag: -querier.store-gateway-client.grpc-max-recv-msg-size + # CLI flag: -experimental.querier.store-gateway-client.grpc-max-recv-msg-size [max_recv_msg_size: | default = 104857600] # gRPC client max send message size (bytes). - # CLI flag: -querier.store-gateway-client.grpc-max-send-msg-size + # CLI flag: -experimental.querier.store-gateway-client.grpc-max-send-msg-size [max_send_msg_size: | default = 16777216] # Use compression when sending messages. - # CLI flag: -querier.store-gateway-client.grpc-use-gzip-compression + # CLI flag: -experimental.querier.store-gateway-client.grpc-use-gzip-compression [use_gzip_compression: | default = false] # Rate limit for gRPC client; 0 means disabled. - # CLI flag: -querier.store-gateway-client.grpc-client-rate-limit + # CLI flag: -experimental.querier.store-gateway-client.grpc-client-rate-limit [rate_limit: | default = 0] # Rate limit burst for gRPC client. - # CLI flag: -querier.store-gateway-client.grpc-client-rate-limit-burst + # CLI flag: -experimental.querier.store-gateway-client.grpc-client-rate-limit-burst [rate_limit_burst: | default = 0] # Enable backoff and retry when we hit ratelimits. - # CLI flag: -querier.store-gateway-client.backoff-on-ratelimits + # CLI flag: -experimental.querier.store-gateway-client.backoff-on-ratelimits [backoff_on_ratelimits: | default = false] backoff_config: # Minimum delay when backing off. - # CLI flag: -querier.store-gateway-client.backoff-min-period + # CLI flag: -experimental.querier.store-gateway-client.backoff-min-period [min_period: | default = 100ms] # Maximum delay when backing off. - # CLI flag: -querier.store-gateway-client.backoff-max-period + # CLI flag: -experimental.querier.store-gateway-client.backoff-max-period [max_period: | default = 10s] # Number of times to backoff and retry before failing. - # CLI flag: -querier.store-gateway-client.backoff-retries + # CLI flag: -experimental.querier.store-gateway-client.backoff-retries [max_retries: | default = 10] - # TLS cert path for the GRPC client - # CLI flag: -querier.store-gateway-client.grpc-tls-cert-path + # TLS cert path for the client + # CLI flag: -experimental.querier.store-gateway-client.tls-cert-path [tls_cert_path: | default = ""] - # TLS key path for the GRPC client - # CLI flag: -querier.store-gateway-client.grpc-tls-key-path + # TLS key path for the client + # CLI flag: -experimental.querier.store-gateway-client.tls-key-path [tls_key_path: | default = ""] - # TLS CA path for the GRPC client - # CLI flag: -querier.store-gateway-client.grpc-tls-ca-path + # TLS CA path for the client + # CLI flag: -experimental.querier.store-gateway-client.tls-ca-path [tls_ca_path: | default = ""] ``` @@ -807,55 +807,17 @@ The `ruler_config` configures the Cortex ruler. # CLI flag: -ruler.external.url [external_url: | default = ] -grpc_client_config: - # gRPC client max receive message size (bytes). - # CLI flag: -ruler.client.grpc-max-recv-msg-size - [max_recv_msg_size: | default = 104857600] - - # gRPC client max send message size (bytes). - # CLI flag: -ruler.client.grpc-max-send-msg-size - [max_send_msg_size: | default = 16777216] - - # Use compression when sending messages. - # CLI flag: -ruler.client.grpc-use-gzip-compression - [use_gzip_compression: | default = false] - - # Rate limit for gRPC client; 0 means disabled. - # CLI flag: -ruler.client.grpc-client-rate-limit - [rate_limit: | default = 0] - - # Rate limit burst for gRPC client. - # CLI flag: -ruler.client.grpc-client-rate-limit-burst - [rate_limit_burst: | default = 0] - - # Enable backoff and retry when we hit ratelimits. - # CLI flag: -ruler.client.backoff-on-ratelimits - [backoff_on_ratelimits: | default = false] - - backoff_config: - # Minimum delay when backing off. - # CLI flag: -ruler.client.backoff-min-period - [min_period: | default = 100ms] - - # Maximum delay when backing off. - # CLI flag: -ruler.client.backoff-max-period - [max_period: | default = 10s] - - # Number of times to backoff and retry before failing. - # CLI flag: -ruler.client.backoff-retries - [max_retries: | default = 10] - - # TLS cert path for the GRPC client - # CLI flag: -ruler.client.grpc-tls-cert-path - [tls_cert_path: | default = ""] +# TLS cert path for the client +# CLI flag: -ruler.client..tls-cert-path +[tls_cert_path: | default = ""] - # TLS key path for the GRPC client - # CLI flag: -ruler.client.grpc-tls-key-path - [tls_key_path: | default = ""] +# TLS key path for the client +# CLI flag: -ruler.client..tls-key-path +[tls_key_path: | default = ""] - # TLS CA path for the GRPC client - # CLI flag: -ruler.client.grpc-tls-ca-path - [tls_ca_path: | default = ""] +# TLS CA path for the client +# CLI flag: -ruler.client..tls-ca-path +[tls_ca_path: | default = ""] # How frequently to evaluate rules # CLI flag: -ruler.evaluation-interval @@ -877,7 +839,7 @@ storage: # The configstore_config configures the config database storing rules and # alerts, and is used by the Cortex alertmanager. - # The CLI flags prefix for this block config is: ruler. + # The CLI flags prefix for this block config is: ruler [configdb: ] azure: @@ -1164,7 +1126,7 @@ storage: # The configstore_config configures the config database storing rules and # alerts, and is used by the Cortex alertmanager. - # The CLI flags prefix for this block config is: alertmanager.client + # The CLI flags prefix for this block config is: alertmanager [configdb: ] local: @@ -1684,16 +1646,16 @@ bigtable: # CLI flag: -bigtable.backoff-retries [max_retries: | default = 10] - # TLS cert path for the GRPC client - # CLI flag: -bigtable.grpc-tls-cert-path + # TLS cert path for the client + # CLI flag: -bigtable.tls-cert-path [tls_cert_path: | default = ""] - # TLS key path for the GRPC client - # CLI flag: -bigtable.grpc-tls-key-path + # TLS key path for the client + # CLI flag: -bigtable.tls-key-path [tls_key_path: | default = ""] - # TLS CA path for the GRPC client - # CLI flag: -bigtable.grpc-tls-ca-path + # TLS CA path for the client + # CLI flag: -bigtable.tls-ca-path [tls_ca_path: | default = ""] # If enabled, once a tables info is fetched, it is cached. @@ -2073,16 +2035,16 @@ grpc_client_config: # CLI flag: -ingester.client.backoff-retries [max_retries: | default = 10] - # TLS cert path for the GRPC client - # CLI flag: -ingester.client.grpc-tls-cert-path + # TLS cert path for the client + # CLI flag: -ingester.client.tls-cert-path [tls_cert_path: | default = ""] - # TLS key path for the GRPC client - # CLI flag: -ingester.client.grpc-tls-key-path + # TLS key path for the client + # CLI flag: -ingester.client.tls-key-path [tls_key_path: | default = ""] - # TLS CA path for the GRPC client - # CLI flag: -ingester.client.grpc-tls-ca-path + # TLS CA path for the client + # CLI flag: -ingester.client.tls-ca-path [tls_ca_path: | default = ""] ``` @@ -2146,16 +2108,16 @@ grpc_client_config: # CLI flag: -querier.frontend-client.backoff-retries [max_retries: | default = 10] - # TLS cert path for the GRPC client - # CLI flag: -querier.frontend-client.grpc-tls-cert-path + # TLS cert path for the client + # CLI flag: -querier.frontend-client.tls-cert-path [tls_cert_path: | default = ""] - # TLS key path for the GRPC client - # CLI flag: -querier.frontend-client.grpc-tls-key-path + # TLS key path for the client + # CLI flag: -querier.frontend-client.tls-key-path [tls_key_path: | default = ""] - # TLS CA path for the GRPC client - # CLI flag: -querier.frontend-client.grpc-tls-ca-path + # TLS CA path for the client + # CLI flag: -querier.frontend-client.tls-ca-path [tls_ca_path: | default = ""] ``` @@ -2649,34 +2611,30 @@ api: The `configstore_config` configures the config database storing rules and alerts, and is used by the Cortex alertmanager. The supported CLI flags `` used to reference this config block are: -- `alertmanager.client` -- `ruler.` +- `alertmanager` +- `ruler`   ```yaml -# Endpoint to connect to -# CLI flag: -.http-endpoint -[endpoint: | default = ] +# URL of configs API server. +# CLI flag: -.configs.url +[configs_api_url: | default = ] -# Timeout for connecting to the endpoint -# CLI flag: -.http-client-timeout +# Timeout for requests to Weave Cloud configs service. +# CLI flag: -.configs.client-timeout [client_timeout: | default = 5s] -# Timeout for reading from backend -# CLI flag: -.http-backend-read-timeout -[backend_read_timeout: | default = 1m30s] - -# TLS cert path for the HTTP client -# CLI flag: -.http-tls-cert-path +# TLS cert path for the client +# CLI flag: -.configs.tls-cert-path [tls_cert_path: | default = ""] -# TLS key path for the HTTP client -# CLI flag: -.http-tls-key-path +# TLS key path for the client +# CLI flag: -.configs.tls-key-path [tls_key_path: | default = ""] -# TLS CA path for the HTTP client -# CLI flag: -.http-tls-ca-path +# TLS CA path for the client +# CLI flag: -.configs.tls-ca-path [tls_ca_path: | default = ""] ``` diff --git a/pkg/alertmanager/storage.go b/pkg/alertmanager/storage.go index 021521934f..7a94b67611 100644 --- a/pkg/alertmanager/storage.go +++ b/pkg/alertmanager/storage.go @@ -9,7 +9,6 @@ import ( "github.com/cortexproject/cortex/pkg/alertmanager/alerts/configdb" "github.com/cortexproject/cortex/pkg/alertmanager/alerts/local" "github.com/cortexproject/cortex/pkg/configs/client" - "github.com/cortexproject/cortex/pkg/util/httpclient" ) // AlertStore stores and configures users rule configs @@ -20,14 +19,14 @@ type AlertStore interface { // AlertStoreConfig configures the alertmanager backend type AlertStoreConfig struct { Type string `yaml:"type"` - ConfigDB httpclient.Config `yaml:"configdb"` + ConfigDB client.Config `yaml:"configdb"` Local local.StoreConfig `yaml:"local"` } // RegisterFlags registers flags. func (cfg *AlertStoreConfig) RegisterFlags(f *flag.FlagSet) { cfg.Local.RegisterFlags(f) - cfg.ConfigDB.RegisterFlagsWithPrefix("alertmanager.client", f) + cfg.ConfigDB.RegisterFlagsWithPrefix("alertmanager.", f) f.StringVar(&cfg.Type, "alertmanager.storage.type", "configdb", "Type of backend to use to store alertmanager configs. Supported values are: \"configdb\", \"local\".") } diff --git a/pkg/configs/client/client.go b/pkg/configs/client/client.go index bcec7bdadc..7cc82fe922 100644 --- a/pkg/configs/client/client.go +++ b/pkg/configs/client/client.go @@ -2,9 +2,13 @@ package client import ( "context" + "crypto/tls" "encoding/json" + "flag" "fmt" "net/http" + "net/url" + "time" "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" @@ -13,9 +17,24 @@ import ( "github.com/cortexproject/cortex/pkg/configs/userconfig" "github.com/cortexproject/cortex/pkg/util" - "github.com/cortexproject/cortex/pkg/util/httpclient" + "github.com/cortexproject/cortex/pkg/util/flagext" + tls_cfg "github.com/cortexproject/cortex/pkg/util/tls" ) +// Config says where we can find the ruler userconfig. +type Config struct { + ConfigsAPIURL flagext.URLValue `yaml:"configs_api_url"` + ClientTimeout time.Duration `yaml:"client_timeout"` // HTTP timeout duration for requests made to the Weave Cloud configs service. + TLSStruct tls_cfg.TLSStruct `yaml:",inline"` +} + +// RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet +func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { + f.Var(&cfg.ConfigsAPIURL, prefix+"configs.url", "URL of configs API server.") + f.DurationVar(&cfg.ClientTimeout, prefix+"configs.client-timeout", 5*time.Second, "Timeout for requests to Weave Cloud configs service.") + cfg.TLSStruct.RegisterFlagsWithPrefix(prefix+"configs", f) +} + var configsRequestDuration = instrument.NewHistogramCollector(promauto.NewHistogramVec(prometheus.HistogramOpts{ Namespace: "cortex", Name: "configs_request_duration_seconds", @@ -34,15 +53,28 @@ type Client interface { } // New creates a new ConfigClient. -func New(cfg httpclient.Config) (*ConfigDBClient, error) { - return &ConfigDBClient{ - Config: cfg, - }, nil +func New(cfg Config) (*ConfigDBClient, error) { + client := &ConfigDBClient{ + URL: cfg.ConfigsAPIURL.URL, + Timeout: cfg.ClientTimeout, + } + + tlsConfig, err := cfg.TLSStruct.GetTLSConfig() + if err != nil { + return nil, err + } + + if tlsConfig != nil { + client.TLSConfig = tlsConfig + } + return client, nil } // ConfigDBClient allows retrieving recording and alerting rules from the configs server. type ConfigDBClient struct { - Config httpclient.Config + URL *url.URL + Timeout time.Duration + TLSConfig *tls.Config } // GetRules implements Client @@ -51,11 +83,11 @@ func (c ConfigDBClient) GetRules(ctx context.Context, since userconfig.ID) (map[ if since != 0 { suffix = fmt.Sprintf("?since=%d", since) } - endpoint := fmt.Sprintf("%s/private/api/prom/configs/rules%s", c.Config.Endpoint.String(), suffix) + endpoint := fmt.Sprintf("%s/private/api/prom/configs/rules%s", c.URL.String(), suffix) var response *ConfigsResponse err := instrument.CollectedRequest(ctx, "GetRules", configsRequestDuration, instrument.ErrorCode, func(ctx context.Context) error { var err error - response, err = doRequest(endpoint, c.Config, since) + response, err = doRequest(endpoint, c.Timeout, c.TLSConfig, since) return err }) if err != nil { @@ -77,24 +109,24 @@ func (c ConfigDBClient) GetAlerts(ctx context.Context, since userconfig.ID) (*Co if since != 0 { suffix = fmt.Sprintf("?since=%d", since) } - endpoint := fmt.Sprintf("%s/private/api/prom/configs/alertmanager%s", c.Config.Endpoint.URL.String(), suffix) + endpoint := fmt.Sprintf("%s/private/api/prom/configs/alertmanager%s", c.URL.String(), suffix) var response *ConfigsResponse err := instrument.CollectedRequest(ctx, "GetAlerts", configsRequestDuration, instrument.ErrorCode, func(ctx context.Context) error { var err error - response, err = doRequest(endpoint, c.Config, since) + response, err = doRequest(endpoint, c.Timeout, c.TLSConfig, since) return err }) return response, err } -func doRequest(endpoint string, clientOpts httpclient.Config, since userconfig.ID) (*ConfigsResponse, error) { +func doRequest(endpoint string, timeout time.Duration, tlsConfig *tls.Config, since userconfig.ID) (*ConfigsResponse, error) { req, err := http.NewRequest("GET", endpoint, nil) if err != nil { return nil, err } - client := &http.Client{Timeout: clientOpts.ClientTimeout} - if tlsConfig := clientOpts.GetTLSConfig(); tlsConfig != nil { + client := &http.Client{Timeout: timeout} + if tlsConfig != nil { client.Transport = &http.Transport{TLSClientConfig: tlsConfig} } diff --git a/pkg/configs/client/configs_test.go b/pkg/configs/client/configs_test.go index 62443c9980..86b0427c7e 100644 --- a/pkg/configs/client/configs_test.go +++ b/pkg/configs/client/configs_test.go @@ -7,7 +7,6 @@ import ( "time" "github.com/cortexproject/cortex/pkg/configs/userconfig" - "github.com/cortexproject/cortex/pkg/util/httpclient" "github.com/stretchr/testify/require" @@ -36,7 +35,7 @@ func TestDoRequest(t *testing.T) { })) defer server.Close() - resp, err := doRequest(server.URL, httpclient.Config{ClientTimeout: 1 * time.Second}, 0) + resp, err := doRequest(server.URL, 1*time.Second, 0) assert.Nil(t, err) expected := ConfigsResponse{Configs: map[string]userconfig.View{ diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index 61bbf71b76..cb7285b323 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -62,7 +62,7 @@ var ( // RegisterFlags adds the flags required to config this to the given FlagSet. func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - cfg.StoreGatewayGRPCClientConfig.RegisterFlagsWithPrefix("querier.store-gateway-client", f) + cfg.StoreGatewayGRPCClientConfig.RegisterFlagsWithPrefix("experimental.querier.store-gateway-client", f) f.IntVar(&cfg.MaxConcurrent, "querier.max-concurrent", 20, "The maximum number of concurrent queries.") f.DurationVar(&cfg.Timeout, "querier.timeout", 2*time.Minute, "The timeout for a query.") if f.Lookup("promql.lookback-delta") == nil { diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index f8dd8afae4..b25c4e22f2 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -34,8 +34,8 @@ import ( store "github.com/cortexproject/cortex/pkg/ruler/rules" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/flagext" - "github.com/cortexproject/cortex/pkg/util/grpcclient" "github.com/cortexproject/cortex/pkg/util/services" + tls_cfg "github.com/cortexproject/cortex/pkg/util/tls" ) var ( @@ -60,8 +60,8 @@ var ( type Config struct { // This is used for template expansion in alerts; must be a valid URL. ExternalURL flagext.URLValue `yaml:"external_url"` - // Config parameters for the GRPC Client - GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config"` + // TLS parameters for the GRPC Client + ClientTLSConfig tls_cfg.TLSStruct `yaml:",inline"` // How frequently to evaluate rules by default. EvaluationInterval time.Duration `yaml:"evaluation_interval"` // Delay the evaluation of all rules by a set interval to give a buffer @@ -106,7 +106,7 @@ func (cfg *Config) Validate() error { // RegisterFlags adds the flags required to config this to the given FlagSet func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - cfg.GRPCClientConfig.RegisterFlagsWithPrefix("ruler.client", f) + cfg.ClientTLSConfig.RegisterFlagsWithPrefix("ruler.client.", f) cfg.StoreConfig.RegisterFlags(f) cfg.Ring.RegisterFlags(f) @@ -626,7 +626,7 @@ func (r *Ruler) getShardedRules(ctx context.Context) ([]*GroupStateDesc, error) rgs := []*GroupStateDesc{} for _, rlr := range rulers.Ingesters { - dialOpts, err := r.cfg.GRPCClientConfig.DialOption(nil, nil) + dialOpts, err := r.cfg.ClientTLSConfig.GetGRPCDialOptions() if err != nil { return nil, err } diff --git a/pkg/ruler/storage.go b/pkg/ruler/storage.go index 643d28b2da..e9da966591 100644 --- a/pkg/ruler/storage.go +++ b/pkg/ruler/storage.go @@ -15,13 +15,12 @@ import ( "github.com/cortexproject/cortex/pkg/configs/client" "github.com/cortexproject/cortex/pkg/ruler/rules" "github.com/cortexproject/cortex/pkg/ruler/rules/objectclient" - "github.com/cortexproject/cortex/pkg/util/httpclient" ) // RuleStoreConfig conigures a rule store type RuleStoreConfig struct { - Type string `yaml:"type"` - ConfigDB httpclient.Config `yaml:"configdb"` + Type string `yaml:"type"` + ConfigDB client.Config `yaml:"configdb"` // Object Storage Configs Azure azure.BlobStorageConfig `yaml:"azure"` diff --git a/pkg/util/grpcclient/grpcclient.go b/pkg/util/grpcclient/grpcclient.go index 6b61aa769b..27e1159447 100644 --- a/pkg/util/grpcclient/grpcclient.go +++ b/pkg/util/grpcclient/grpcclient.go @@ -1,17 +1,13 @@ package grpcclient import ( - "crypto/tls" - "crypto/x509" "flag" - "io/ioutil" - "github.com/go-kit/kit/log/level" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" "google.golang.org/grpc" - "google.golang.org/grpc/credentials" "github.com/cortexproject/cortex/pkg/util" + tls_cfg "github.com/cortexproject/cortex/pkg/util/tls" ) // Config for a gRPC client. @@ -25,9 +21,7 @@ type Config struct { BackoffOnRatelimits bool `yaml:"backoff_on_ratelimits"` BackoffConfig util.BackoffConfig `yaml:"backoff_config"` - TLSCertPath string `yaml:"tls_cert_path"` - TLSKeyPath string `yaml:"tls_key_path"` - TLSCAPath string `yaml:"tls_ca_path"` + TLSStruct tls_cfg.TLSStruct `yaml:",inline"` } // RegisterFlags registers flags. @@ -45,10 +39,7 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { f.BoolVar(&cfg.BackoffOnRatelimits, prefix+".backoff-on-ratelimits", false, "Enable backoff and retry when we hit ratelimits.") cfg.BackoffConfig.RegisterFlags(prefix, f) - - f.StringVar(&cfg.TLSCertPath, prefix+".grpc-tls-cert-path", "", "TLS cert path for the GRPC client") - f.StringVar(&cfg.TLSKeyPath, prefix+".grpc-tls-key-path", "", "TLS key path for the GRPC client") - f.StringVar(&cfg.TLSCAPath, prefix+".grpc-tls-ca-path", "", "TLS CA path for the GRPC client") + cfg.TLSStruct.RegisterFlagsWithPrefix(prefix, f) } // CallOptions returns the config in terms of CallOptions. @@ -72,32 +63,9 @@ func (cfg *Config) DialOption(unaryClientInterceptors []grpc.UnaryClientIntercep unaryClientInterceptors = append([]grpc.UnaryClientInterceptor{NewRateLimiter(cfg)}, unaryClientInterceptors...) } - var opts []grpc.DialOption - if cfg.TLSCertPath != "" && cfg.TLSKeyPath != "" && cfg.TLSCAPath != "" { - clientCert, err := tls.LoadX509KeyPair(cfg.TLSCertPath, cfg.TLSKeyPath) - if err != nil { - level.Error(util.Logger).Log("msg", "error loading certs", "error", err) - return nil, err - } - var caCertPool *x509.CertPool - caCert, err := ioutil.ReadFile(cfg.TLSCAPath) - if err != nil { - level.Error(util.Logger).Log("msg", "error loading ca cert", "error", err) - return nil, err - } else { - caCertPool = x509.NewCertPool() - caCertPool.AppendCertsFromPEM(caCert) - } - if len(clientCert.Certificate) > 0 && caCertPool != nil { - tlsConfig := &tls.Config{ - InsecureSkipVerify: true, - Certificates: []tls.Certificate{clientCert}, - RootCAs: caCertPool, - } - opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))) - } - } else { - opts = append(opts, grpc.WithInsecure()) + opts, err := cfg.TLSStruct.GetGRPCDialOptions() + if err != nil { + return nil, err } return append(opts, grpc.WithDefaultCallOptions(cfg.CallOptions()...), diff --git a/pkg/util/httpclient/httpclient.go b/pkg/util/httpclient/httpclient.go deleted file mode 100644 index a49108c98e..0000000000 --- a/pkg/util/httpclient/httpclient.go +++ /dev/null @@ -1,75 +0,0 @@ -package httpclient - -import ( - "crypto/tls" - "crypto/x509" - "flag" - "io/ioutil" - "net/url" - "time" - - "github.com/prometheus/common/log" - - "github.com/cortexproject/cortex/pkg/util/flagext" -) - -// Config is the config for the HTTP client -type Config struct { - Endpoint flagext.URLValue `yaml:"endpoint"` - ClientTimeout time.Duration `yaml:"client_timeout"` - BackendReadTimeout time.Duration `yaml:"backend_read_timeout"` - - TLSCertPath string `yaml:"tls_cert_path"` - TLSKeyPath string `yaml:"tls_key_path"` - TLSCAPath string `yaml:"tls_ca_path"` -} - -// RegisterFlags registers flags. -func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - cfg.RegisterFlagsWithPrefix("", f) -} - -func ConfigFromURLAndTimeout(u *url.URL, timeout time.Duration) *Config { - return &Config{ - Endpoint: flagext.URLValue{URL: u}, - ClientTimeout: timeout, - } -} - -// RegisterFlagsWithPrefix registers flags with prefix. -func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { - f.Var(&cfg.Endpoint, prefix+".http-endpoint", "Endpoint to connect to") - f.DurationVar(&cfg.ClientTimeout, prefix+".http-client-timeout", 5*time.Second, "Timeout for connecting to the endpoint") - f.DurationVar(&cfg.BackendReadTimeout, prefix+".http-backend-read-timeout", 90*time.Second, "Timeout for reading from backend") - - f.StringVar(&cfg.TLSCertPath, prefix+".http-tls-cert-path", "", "TLS cert path for the HTTP client") - f.StringVar(&cfg.TLSKeyPath, prefix+".http-tls-key-path", "", "TLS key path for the HTTP client") - f.StringVar(&cfg.TLSCAPath, prefix+".http-tls-ca-path", "", "TLS CA path for the HTTP client") -} - -// GetTLSConfig initialises tls.Config from config options. -func (cfg *Config) GetTLSConfig() *tls.Config { - if cfg.TLSCertPath != "" && cfg.TLSKeyPath != "" && cfg.TLSCAPath != "" { - clientCert, err := tls.LoadX509KeyPair(cfg.TLSCertPath, cfg.TLSKeyPath) - if err != nil { - log.Warnf("error loading cert %s or key %s, tls disabled", cfg.TLSCertPath, cfg.TLSKeyPath) - } - - var caCertPool *x509.CertPool - caCert, err := ioutil.ReadFile(cfg.TLSCAPath) - if err != nil { - log.Warnf("error loading ca cert %s, tls disabled", cfg.TLSCAPath) - } else { - caCertPool = x509.NewCertPool() - caCertPool.AppendCertsFromPEM(caCert) - } - if len(clientCert.Certificate) > 0 && caCertPool != nil { - return &tls.Config{ - InsecureSkipVerify: true, - Certificates: []tls.Certificate{clientCert}, - RootCAs: caCertPool, - } - } - } - return nil -} diff --git a/pkg/util/tls/tls.go b/pkg/util/tls/tls.go new file mode 100644 index 0000000000..e209cc579d --- /dev/null +++ b/pkg/util/tls/tls.go @@ -0,0 +1,69 @@ +package tls + +import ( + "crypto/tls" + "crypto/x509" + "flag" + "io/ioutil" + + "github.com/cortexproject/cortex/pkg/util" + "github.com/go-kit/kit/log/level" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" +) + +// TLSStruct is the config for client TLS. +type TLSStruct struct { + CertPath string `yaml:"tls_cert_path"` + KeyPath string `yaml:"tls_key_path"` + CAPath string `yaml:"tls_ca_path"` +} + +// RegisterFlagsWithPrefix registers flags with prefix. +func (cfg *TLSStruct) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { + f.StringVar(&cfg.CertPath, prefix+".tls-cert-path", "", "TLS cert path for the client") + f.StringVar(&cfg.KeyPath, prefix+".tls-key-path", "", "TLS key path for the client") + f.StringVar(&cfg.CAPath, prefix+".tls-ca-path", "", "TLS CA path for the client") +} + +// GetTLSConfig initialises tls.Config from config options +func (cfg *TLSStruct) GetTLSConfig() (*tls.Config, error) { + if cfg.CertPath != "" && cfg.KeyPath != "" && cfg.CAPath != "" { + clientCert, err := tls.LoadX509KeyPair(cfg.CertPath, cfg.KeyPath) + if err != nil { + level.Error(util.Logger).Log("msg", "error loading certs", "error", err) + return nil, err + } + + var caCertPool *x509.CertPool + caCert, err := ioutil.ReadFile(cfg.CAPath) + if err != nil { + level.Error(util.Logger).Log("msg", "error loading ca cert", "error", err) + return nil, err + } else { + caCertPool = x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + } + if len(clientCert.Certificate) > 0 && caCertPool != nil { + return &tls.Config{ + InsecureSkipVerify: true, + Certificates: []tls.Certificate{clientCert}, + RootCAs: caCertPool, + }, nil + } + } + return nil, nil +} + +// GetGRPCDialOptions creates GRPC DialOptions for TLS +func (cfg *TLSStruct) GetGRPCDialOptions() ([]grpc.DialOption, error) { + var opts []grpc.DialOption + if tlsConfig, err := cfg.GetTLSConfig(); err != nil { + return nil, err + } else if tlsConfig != nil { + opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))) + } else { + opts = append(opts, grpc.WithInsecure()) + } + return opts, nil +} diff --git a/tools/doc-generator/main.go b/tools/doc-generator/main.go index bb24acfd46..c5055e4b08 100644 --- a/tools/doc-generator/main.go +++ b/tools/doc-generator/main.go @@ -18,6 +18,7 @@ import ( "github.com/cortexproject/cortex/pkg/chunk/storage" "github.com/cortexproject/cortex/pkg/compactor" "github.com/cortexproject/cortex/pkg/configs" + config_client "github.com/cortexproject/cortex/pkg/configs/client" "github.com/cortexproject/cortex/pkg/cortex" "github.com/cortexproject/cortex/pkg/distributor" "github.com/cortexproject/cortex/pkg/flusher" @@ -32,7 +33,6 @@ import ( "github.com/cortexproject/cortex/pkg/ruler" "github.com/cortexproject/cortex/pkg/storage/tsdb" "github.com/cortexproject/cortex/pkg/storegateway" - "github.com/cortexproject/cortex/pkg/util/httpclient" "github.com/cortexproject/cortex/pkg/util/validation" ) @@ -162,7 +162,7 @@ var ( }, { name: "configstore_config", - structType: reflect.TypeOf(httpclient.Config{}), + structType: reflect.TypeOf(config_client.Config{}), desc: "The configstore_config configures the config database storing rules and alerts, and is used by the Cortex alertmanager.", }, { From 8f9f2e7c301c675dc87c48de36972505bbee6581 Mon Sep 17 00:00:00 2001 From: Annanay Date: Mon, 11 May 2020 18:28:29 +0530 Subject: [PATCH 22/29] Fix docs, flags Signed-off-by: Annanay --- docs/production/tls.md | 16 ++++++++-------- pkg/ruler/ruler.go | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/production/tls.md b/docs/production/tls.md index 652ec8fb77..77bf518025 100644 --- a/docs/production/tls.md +++ b/docs/production/tls.md @@ -83,26 +83,26 @@ through the following config parameters: Client flags are component specific. -For an HTTP client in the alertmanager: +For an HTTP client in the Alertmanager: ``` # Path to the TLS Cert for the HTTP Client - -alertmanager.client.http-tls-cert-path=/path/to/client.crt + -alertmanager.configs.tls-cert-path=/path/to/client.crt # Path to the TLS Key for the HTTP Client - -alertmanager.client.http-tls-key-path=/path/to/client.key + -alertmanager.configs.tls-key-path=/path/to/client.key # Path to the TLS CA for the HTTP Client - -alertmanager.client.http-tls-ca-path=/path/to/root.crt + -alertmanager.configs.tls-ca-path=/path/to/root.crt ``` -For a GRPC client in the ruler: +For a GRPC client in the Querier: ``` # Path to the TLS Cert for the GRPC Client - -ruler.grpc-tls-cert-path=/path/to/client.crt + -querier.frontend-client.tls-cert-path=/path/to/client.crt # Path to the TLS Key for the GRPC Client - -ruler.grpc-tls-key-path=/path/to/client.key + -querier.frontend-client.tls-key-path=/path/to/client.key # Path to the TLS CA for the GRPC Client - -ruler.grpc-tls-ca-path=/path/to/root.crt + -querier.frontend-client.tls-ca-path=/path/to/root.crt ``` \ No newline at end of file diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index b25c4e22f2..f5d9548c9b 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -106,7 +106,7 @@ func (cfg *Config) Validate() error { // RegisterFlags adds the flags required to config this to the given FlagSet func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - cfg.ClientTLSConfig.RegisterFlagsWithPrefix("ruler.client.", f) + cfg.ClientTLSConfig.RegisterFlagsWithPrefix("ruler.client", f) cfg.StoreConfig.RegisterFlags(f) cfg.Ring.RegisterFlags(f) From 668e98880537ff0874c50d4b0cfed084c55d065d Mon Sep 17 00:00:00 2001 From: Annanay Date: Mon, 11 May 2020 20:37:22 +0530 Subject: [PATCH 23/29] Fix test Signed-off-by: Annanay --- pkg/configs/client/configs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/configs/client/configs_test.go b/pkg/configs/client/configs_test.go index 86b0427c7e..7ada6b20d8 100644 --- a/pkg/configs/client/configs_test.go +++ b/pkg/configs/client/configs_test.go @@ -35,7 +35,7 @@ func TestDoRequest(t *testing.T) { })) defer server.Close() - resp, err := doRequest(server.URL, 1*time.Second, 0) + resp, err := doRequest(server.URL, 1*time.Second, nil, 0) assert.Nil(t, err) expected := ConfigsResponse{Configs: map[string]userconfig.View{ From ef761f55685843f9828d2121f9ee64b35a82186c Mon Sep 17 00:00:00 2001 From: Annanay Date: Mon, 11 May 2020 20:55:53 +0530 Subject: [PATCH 24/29] Fix lint, docs Signed-off-by: Annanay --- docs/configuration/config-file-reference.md | 6 +++--- pkg/configs/client/client.go | 6 +++--- pkg/ruler/ruler.go | 2 +- pkg/util/grpcclient/grpcclient.go | 2 +- pkg/util/tls/tls.go | 18 +++++++++--------- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 146efdf65a..032a9015a2 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -808,15 +808,15 @@ The `ruler_config` configures the Cortex ruler. [external_url: | default = ] # TLS cert path for the client -# CLI flag: -ruler.client..tls-cert-path +# CLI flag: -ruler.client.tls-cert-path [tls_cert_path: | default = ""] # TLS key path for the client -# CLI flag: -ruler.client..tls-key-path +# CLI flag: -ruler.client.tls-key-path [tls_key_path: | default = ""] # TLS CA path for the client -# CLI flag: -ruler.client..tls-ca-path +# CLI flag: -ruler.client.tls-ca-path [tls_ca_path: | default = ""] # How frequently to evaluate rules diff --git a/pkg/configs/client/client.go b/pkg/configs/client/client.go index 7cc82fe922..0f4c6a5c7b 100644 --- a/pkg/configs/client/client.go +++ b/pkg/configs/client/client.go @@ -23,9 +23,9 @@ import ( // Config says where we can find the ruler userconfig. type Config struct { - ConfigsAPIURL flagext.URLValue `yaml:"configs_api_url"` - ClientTimeout time.Duration `yaml:"client_timeout"` // HTTP timeout duration for requests made to the Weave Cloud configs service. - TLSStruct tls_cfg.TLSStruct `yaml:",inline"` + ConfigsAPIURL flagext.URLValue `yaml:"configs_api_url"` + ClientTimeout time.Duration `yaml:"client_timeout"` // HTTP timeout duration for requests made to the Weave Cloud configs service. + TLSStruct tls_cfg.ClientConfig `yaml:",inline"` } // RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index f5d9548c9b..fade05bf59 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -61,7 +61,7 @@ type Config struct { // This is used for template expansion in alerts; must be a valid URL. ExternalURL flagext.URLValue `yaml:"external_url"` // TLS parameters for the GRPC Client - ClientTLSConfig tls_cfg.TLSStruct `yaml:",inline"` + ClientTLSConfig tls_cfg.ClientConfig `yaml:",inline"` // How frequently to evaluate rules by default. EvaluationInterval time.Duration `yaml:"evaluation_interval"` // Delay the evaluation of all rules by a set interval to give a buffer diff --git a/pkg/util/grpcclient/grpcclient.go b/pkg/util/grpcclient/grpcclient.go index 27e1159447..93f1fd5d6c 100644 --- a/pkg/util/grpcclient/grpcclient.go +++ b/pkg/util/grpcclient/grpcclient.go @@ -21,7 +21,7 @@ type Config struct { BackoffOnRatelimits bool `yaml:"backoff_on_ratelimits"` BackoffConfig util.BackoffConfig `yaml:"backoff_config"` - TLSStruct tls_cfg.TLSStruct `yaml:",inline"` + TLSStruct tls_cfg.ClientConfig `yaml:",inline"` } // RegisterFlags registers flags. diff --git a/pkg/util/tls/tls.go b/pkg/util/tls/tls.go index e209cc579d..eea1b4ec6f 100644 --- a/pkg/util/tls/tls.go +++ b/pkg/util/tls/tls.go @@ -6,28 +6,29 @@ import ( "flag" "io/ioutil" - "github.com/cortexproject/cortex/pkg/util" "github.com/go-kit/kit/log/level" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + + "github.com/cortexproject/cortex/pkg/util" ) -// TLSStruct is the config for client TLS. -type TLSStruct struct { +// ClientConfig is the config for client TLS. +type ClientConfig struct { CertPath string `yaml:"tls_cert_path"` KeyPath string `yaml:"tls_key_path"` CAPath string `yaml:"tls_ca_path"` } // RegisterFlagsWithPrefix registers flags with prefix. -func (cfg *TLSStruct) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { +func (cfg *ClientConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { f.StringVar(&cfg.CertPath, prefix+".tls-cert-path", "", "TLS cert path for the client") f.StringVar(&cfg.KeyPath, prefix+".tls-key-path", "", "TLS key path for the client") f.StringVar(&cfg.CAPath, prefix+".tls-ca-path", "", "TLS CA path for the client") } // GetTLSConfig initialises tls.Config from config options -func (cfg *TLSStruct) GetTLSConfig() (*tls.Config, error) { +func (cfg *ClientConfig) GetTLSConfig() (*tls.Config, error) { if cfg.CertPath != "" && cfg.KeyPath != "" && cfg.CAPath != "" { clientCert, err := tls.LoadX509KeyPair(cfg.CertPath, cfg.KeyPath) if err != nil { @@ -40,10 +41,9 @@ func (cfg *TLSStruct) GetTLSConfig() (*tls.Config, error) { if err != nil { level.Error(util.Logger).Log("msg", "error loading ca cert", "error", err) return nil, err - } else { - caCertPool = x509.NewCertPool() - caCertPool.AppendCertsFromPEM(caCert) } + caCertPool = x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) if len(clientCert.Certificate) > 0 && caCertPool != nil { return &tls.Config{ InsecureSkipVerify: true, @@ -56,7 +56,7 @@ func (cfg *TLSStruct) GetTLSConfig() (*tls.Config, error) { } // GetGRPCDialOptions creates GRPC DialOptions for TLS -func (cfg *TLSStruct) GetGRPCDialOptions() ([]grpc.DialOption, error) { +func (cfg *ClientConfig) GetGRPCDialOptions() ([]grpc.DialOption, error) { var opts []grpc.DialOption if tlsConfig, err := cfg.GetTLSConfig(); err != nil { return nil, err From 24919ee8c643a19e77531adf6fa39ab61b764417 Mon Sep 17 00:00:00 2001 From: Annanay Date: Tue, 12 May 2020 02:13:31 +0530 Subject: [PATCH 25/29] Do not use TLS options with GCP clients Signed-off-by: Annanay --- pkg/chunk/gcp/bigtable_index_client.go | 12 ++---------- pkg/chunk/gcp/bigtable_object_client.go | 6 +----- pkg/chunk/gcp/table_client.go | 6 +----- pkg/ingester/client/client.go | 2 +- pkg/querier/frontend/worker.go | 2 +- pkg/querier/store_gateway_client.go | 2 +- pkg/util/grpcclient/grpcclient.go | 19 ++++++++++++++++++- 7 files changed, 25 insertions(+), 24 deletions(-) diff --git a/pkg/chunk/gcp/bigtable_index_client.go b/pkg/chunk/gcp/bigtable_index_client.go index 33ef61d34a..f5822bdc5a 100644 --- a/pkg/chunk/gcp/bigtable_index_client.go +++ b/pkg/chunk/gcp/bigtable_index_client.go @@ -69,11 +69,7 @@ type storageClientV1 struct { // NewStorageClientV1 returns a new v1 StorageClient. func NewStorageClientV1(ctx context.Context, cfg Config, schemaCfg chunk.SchemaConfig) (chunk.IndexClient, error) { - dialOpts, err := cfg.GRPCClientConfig.DialOption(bigtableInstrumentation()) - if err != nil { - return nil, err - } - opts := toOptions(dialOpts) + opts := toOptions(cfg.GRPCClientConfig.DialOption(bigtableInstrumentation())) client, err := bigtable.NewClient(ctx, cfg.Project, cfg.Instance, opts...) if err != nil { return nil, err @@ -97,11 +93,7 @@ func newStorageClientV1(cfg Config, schemaCfg chunk.SchemaConfig, client *bigtab // NewStorageClientColumnKey returns a new v2 StorageClient. func NewStorageClientColumnKey(ctx context.Context, cfg Config, schemaCfg chunk.SchemaConfig) (chunk.IndexClient, error) { - dialOpts, err := cfg.GRPCClientConfig.DialOption(bigtableInstrumentation()) - if err != nil { - return nil, err - } - opts := toOptions(dialOpts) + opts := toOptions(cfg.GRPCClientConfig.DialOption(bigtableInstrumentation())) client, err := bigtable.NewClient(ctx, cfg.Project, cfg.Instance, opts...) if err != nil { return nil, err diff --git a/pkg/chunk/gcp/bigtable_object_client.go b/pkg/chunk/gcp/bigtable_object_client.go index 6982b12f18..46fbe2c2da 100644 --- a/pkg/chunk/gcp/bigtable_object_client.go +++ b/pkg/chunk/gcp/bigtable_object_client.go @@ -22,11 +22,7 @@ type bigtableObjectClient struct { // NewBigtableObjectClient makes a new chunk.Client that stores chunks in // Bigtable. func NewBigtableObjectClient(ctx context.Context, cfg Config, schemaCfg chunk.SchemaConfig) (chunk.Client, error) { - dialOpts, err := cfg.GRPCClientConfig.DialOption(bigtableInstrumentation()) - if err != nil { - return nil, err - } - opts := toOptions(dialOpts) + opts := toOptions(cfg.GRPCClientConfig.DialOption(bigtableInstrumentation())) client, err := bigtable.NewClient(ctx, cfg.Project, cfg.Instance, opts...) if err != nil { return nil, err diff --git a/pkg/chunk/gcp/table_client.go b/pkg/chunk/gcp/table_client.go index 409f8ce42d..725021f554 100644 --- a/pkg/chunk/gcp/table_client.go +++ b/pkg/chunk/gcp/table_client.go @@ -24,11 +24,7 @@ type tableClient struct { // NewTableClient returns a new TableClient. func NewTableClient(ctx context.Context, cfg Config) (chunk.TableClient, error) { - dialOpts, err := cfg.GRPCClientConfig.DialOption(bigtableInstrumentation()) - if err != nil { - return nil, err - } - opts := toOptions(dialOpts) + opts := toOptions(cfg.GRPCClientConfig.DialOption(bigtableInstrumentation())) client, err := bigtable.NewAdminClient(ctx, cfg.Project, cfg.Instance, opts...) if err != nil { return nil, err diff --git a/pkg/ingester/client/client.go b/pkg/ingester/client/client.go index cc2de4e1e4..40917e759d 100644 --- a/pkg/ingester/client/client.go +++ b/pkg/ingester/client/client.go @@ -34,7 +34,7 @@ type closableHealthAndIngesterClient struct { // MakeIngesterClient makes a new IngesterClient func MakeIngesterClient(addr string, cfg Config) (HealthAndIngesterClient, error) { - dialOpts, err := cfg.GRPCClientConfig.DialOption(grpcclient.Instrument(ingesterClientRequestDuration)) + dialOpts, err := cfg.GRPCClientConfig.DialOptionWithTLS(grpcclient.Instrument(ingesterClientRequestDuration)) if err != nil { return nil, err } diff --git a/pkg/querier/frontend/worker.go b/pkg/querier/frontend/worker.go index 1a65507074..4d469376b5 100644 --- a/pkg/querier/frontend/worker.go +++ b/pkg/querier/frontend/worker.go @@ -138,7 +138,7 @@ func (w *worker) watchDNSLoop(servCtx context.Context) error { } func (w *worker) connect(ctx context.Context, address string) (FrontendClient, error) { - opts, err := w.cfg.GRPCClientConfig.DialOption([]grpc.UnaryClientInterceptor{middleware.ClientUserHeaderInterceptor}, nil) + opts, err := w.cfg.GRPCClientConfig.DialOptionWithTLS([]grpc.UnaryClientInterceptor{middleware.ClientUserHeaderInterceptor}, nil) if err != nil { return nil, err } diff --git a/pkg/querier/store_gateway_client.go b/pkg/querier/store_gateway_client.go index 19114530ea..baa3cd469d 100644 --- a/pkg/querier/store_gateway_client.go +++ b/pkg/querier/store_gateway_client.go @@ -30,7 +30,7 @@ func newStoreGatewayClientFactory(cfg grpcclient.Config, reg prometheus.Register } func dialStoreGatewayClient(cfg grpcclient.Config, addr string, requestDuration *prometheus.HistogramVec) (*storeGatewayClient, error) { - opts, err := cfg.DialOption(grpcclient.Instrument(requestDuration)) + opts, err := cfg.DialOptionWithTLS(grpcclient.Instrument(requestDuration)) if err != nil { return nil, err } diff --git a/pkg/util/grpcclient/grpcclient.go b/pkg/util/grpcclient/grpcclient.go index 93f1fd5d6c..152133170a 100644 --- a/pkg/util/grpcclient/grpcclient.go +++ b/pkg/util/grpcclient/grpcclient.go @@ -54,7 +54,24 @@ func (cfg *Config) CallOptions() []grpc.CallOption { } // DialOption returns the config as a grpc.DialOptions. -func (cfg *Config) DialOption(unaryClientInterceptors []grpc.UnaryClientInterceptor, streamClientInterceptors []grpc.StreamClientInterceptor) ([]grpc.DialOption, error) { +func (cfg *Config) DialOption(unaryClientInterceptors []grpc.UnaryClientInterceptor, streamClientInterceptors []grpc.StreamClientInterceptor) []grpc.DialOption { + if cfg.BackoffOnRatelimits { + unaryClientInterceptors = append([]grpc.UnaryClientInterceptor{NewBackoffRetry(cfg.BackoffConfig)}, unaryClientInterceptors...) + } + + if cfg.RateLimit > 0 { + unaryClientInterceptors = append([]grpc.UnaryClientInterceptor{NewRateLimiter(cfg)}, unaryClientInterceptors...) + } + + return []grpc.DialOption{ + grpc.WithDefaultCallOptions(cfg.CallOptions()...), + grpc.WithUnaryInterceptor(grpc_middleware.ChainUnaryClient(unaryClientInterceptors...)), + grpc.WithStreamInterceptor(grpc_middleware.ChainStreamClient(streamClientInterceptors...)), + } +} + +// DialOptionWithTLS returns the config as a grpc.DialOptions +func (cfg *Config) DialOptionWithTLS(unaryClientInterceptors []grpc.UnaryClientInterceptor, streamClientInterceptors []grpc.StreamClientInterceptor) ([]grpc.DialOption, error) { if cfg.BackoffOnRatelimits { unaryClientInterceptors = append([]grpc.UnaryClientInterceptor{NewBackoffRetry(cfg.BackoffConfig)}, unaryClientInterceptors...) } From 9939303cb13f97915479b17e3499231d083c99ee Mon Sep 17 00:00:00 2001 From: Annanay Date: Tue, 12 May 2020 11:16:49 +0530 Subject: [PATCH 26/29] Add client auth type, go mod tidy/vendor Signed-off-by: Annanay --- integration/util.go | 14 +- .../common/log/eventlog_formatter.go | 89 ----- .../github.com/prometheus/common/log/log.go | 368 ------------------ .../prometheus/common/log/syslog_formatter.go | 126 ------ .../x/sys/windows/svc/eventlog/install.go | 80 ---- .../x/sys/windows/svc/eventlog/log.go | 70 ---- vendor/modules.txt | 2 - 7 files changed, 8 insertions(+), 741 deletions(-) delete mode 100644 vendor/github.com/prometheus/common/log/eventlog_formatter.go delete mode 100644 vendor/github.com/prometheus/common/log/log.go delete mode 100644 vendor/github.com/prometheus/common/log/syslog_formatter.go delete mode 100644 vendor/golang.org/x/sys/windows/svc/eventlog/install.go delete mode 100644 vendor/golang.org/x/sys/windows/svc/eventlog/log.go diff --git a/integration/util.go b/integration/util.go index c97dcd8a1b..a4705fd880 100644 --- a/integration/util.go +++ b/integration/util.go @@ -55,12 +55,14 @@ func copyFileToSharedDir(s *e2e.Scenario, src, dst string) error { // GetServerTLSFlags generates generic TLS flags for a server func GetServerTLSFlags() map[string]string { return map[string]string{ - "-server.http-tls-cert-path": filepath.Join(e2e.ContainerSharedDir, serverCertFile), - "-server.http-tls-key-path": filepath.Join(e2e.ContainerSharedDir, serverKeyFile), - "-server.http-tls-ca-path": filepath.Join(e2e.ContainerSharedDir, rootCertFile), - "-server.grpc-tls-cert-path": filepath.Join(e2e.ContainerSharedDir, serverCertFile), - "-server.grpc-tls-key-path": filepath.Join(e2e.ContainerSharedDir, serverKeyFile), - "-server.grpc-tls-ca-path": filepath.Join(e2e.ContainerSharedDir, rootCertFile), + "-server.http-tls-cert-path": filepath.Join(e2e.ContainerSharedDir, serverCertFile), + "-server.http-tls-key-path": filepath.Join(e2e.ContainerSharedDir, serverKeyFile), + "-server.http-tls-client-auth": "RequireAndVerifyClientCert", + "-server.http-tls-ca-path": filepath.Join(e2e.ContainerSharedDir, rootCertFile), + "-server.grpc-tls-cert-path": filepath.Join(e2e.ContainerSharedDir, serverCertFile), + "-server.grpc-tls-key-path": filepath.Join(e2e.ContainerSharedDir, serverKeyFile), + "-server.grpc-tls-client-auth": "RequireAndVerifyClientCert", + "-server.grpc-tls-ca-path": filepath.Join(e2e.ContainerSharedDir, rootCertFile), } } diff --git a/vendor/github.com/prometheus/common/log/eventlog_formatter.go b/vendor/github.com/prometheus/common/log/eventlog_formatter.go deleted file mode 100644 index bcf68e6f24..0000000000 --- a/vendor/github.com/prometheus/common/log/eventlog_formatter.go +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2015 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build windows - -package log - -import ( - "fmt" - "os" - - "golang.org/x/sys/windows/svc/eventlog" - - "github.com/sirupsen/logrus" -) - -func init() { - setEventlogFormatter = func(l logger, name string, debugAsInfo bool) error { - if name == "" { - return fmt.Errorf("missing name parameter") - } - - fmter, err := newEventlogger(name, debugAsInfo, l.entry.Logger.Formatter) - if err != nil { - fmt.Fprintf(os.Stderr, "error creating eventlog formatter: %v\n", err) - l.Errorf("can't connect logger to eventlog: %v", err) - return err - } - l.entry.Logger.Formatter = fmter - return nil - } -} - -type eventlogger struct { - log *eventlog.Log - debugAsInfo bool - wrap logrus.Formatter -} - -func newEventlogger(name string, debugAsInfo bool, fmter logrus.Formatter) (*eventlogger, error) { - logHandle, err := eventlog.Open(name) - if err != nil { - return nil, err - } - return &eventlogger{log: logHandle, debugAsInfo: debugAsInfo, wrap: fmter}, nil -} - -func (s *eventlogger) Format(e *logrus.Entry) ([]byte, error) { - data, err := s.wrap.Format(e) - if err != nil { - fmt.Fprintf(os.Stderr, "eventlogger: can't format entry: %v\n", err) - return data, err - } - - switch e.Level { - case logrus.PanicLevel: - fallthrough - case logrus.FatalLevel: - fallthrough - case logrus.ErrorLevel: - err = s.log.Error(102, e.Message) - case logrus.WarnLevel: - err = s.log.Warning(101, e.Message) - case logrus.InfoLevel: - err = s.log.Info(100, e.Message) - case logrus.DebugLevel: - if s.debugAsInfo { - err = s.log.Info(100, e.Message) - } - default: - err = s.log.Info(100, e.Message) - } - - if err != nil { - fmt.Fprintf(os.Stderr, "eventlogger: can't send log to eventlog: %v\n", err) - } - - return data, err -} diff --git a/vendor/github.com/prometheus/common/log/log.go b/vendor/github.com/prometheus/common/log/log.go deleted file mode 100644 index b6adbce133..0000000000 --- a/vendor/github.com/prometheus/common/log/log.go +++ /dev/null @@ -1,368 +0,0 @@ -// Copyright 2015 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package log implements logging via logrus. -// -// Deprecated: This package has been replaced with github.com/prometheus/common/promlog. - -package log - -import ( - "fmt" - "io" - "io/ioutil" - "log" - "net/url" - "os" - "runtime" - "strconv" - "strings" - - "github.com/sirupsen/logrus" - "gopkg.in/alecthomas/kingpin.v2" -) - -// setSyslogFormatter is nil if the target architecture does not support syslog. -var setSyslogFormatter func(logger, string, string) error - -// setEventlogFormatter is nil if the target OS does not support Eventlog (i.e., is not Windows). -var setEventlogFormatter func(logger, string, bool) error - -func setJSONFormatter() { - origLogger.Formatter = &logrus.JSONFormatter{} -} - -type loggerSettings struct { - level string - format string -} - -func (s *loggerSettings) apply(ctx *kingpin.ParseContext) error { - err := baseLogger.SetLevel(s.level) - if err != nil { - return err - } - err = baseLogger.SetFormat(s.format) - return err -} - -// AddFlags adds the flags used by this package to the Kingpin application. -// To use the default Kingpin application, call AddFlags(kingpin.CommandLine) -func AddFlags(a *kingpin.Application) { - s := loggerSettings{} - a.Flag("log.level", "Only log messages with the given severity or above. Valid levels: [debug, info, warn, error, fatal]"). - Default(origLogger.Level.String()). - StringVar(&s.level) - defaultFormat := url.URL{Scheme: "logger", Opaque: "stderr"} - a.Flag("log.format", `Set the log target and format. Example: "logger:syslog?appname=bob&local=7" or "logger:stdout?json=true"`). - Default(defaultFormat.String()). - StringVar(&s.format) - a.Action(s.apply) -} - -// Logger is the interface for loggers used in the Prometheus components. -type Logger interface { - Debug(...interface{}) - Debugln(...interface{}) - Debugf(string, ...interface{}) - - Info(...interface{}) - Infoln(...interface{}) - Infof(string, ...interface{}) - - Warn(...interface{}) - Warnln(...interface{}) - Warnf(string, ...interface{}) - - Error(...interface{}) - Errorln(...interface{}) - Errorf(string, ...interface{}) - - Fatal(...interface{}) - Fatalln(...interface{}) - Fatalf(string, ...interface{}) - - With(key string, value interface{}) Logger - - SetFormat(string) error - SetLevel(string) error -} - -type logger struct { - entry *logrus.Entry -} - -func (l logger) With(key string, value interface{}) Logger { - return logger{l.entry.WithField(key, value)} -} - -// Debug logs a message at level Debug on the standard logger. -func (l logger) Debug(args ...interface{}) { - l.sourced().Debug(args...) -} - -// Debug logs a message at level Debug on the standard logger. -func (l logger) Debugln(args ...interface{}) { - l.sourced().Debugln(args...) -} - -// Debugf logs a message at level Debug on the standard logger. -func (l logger) Debugf(format string, args ...interface{}) { - l.sourced().Debugf(format, args...) -} - -// Info logs a message at level Info on the standard logger. -func (l logger) Info(args ...interface{}) { - l.sourced().Info(args...) -} - -// Info logs a message at level Info on the standard logger. -func (l logger) Infoln(args ...interface{}) { - l.sourced().Infoln(args...) -} - -// Infof logs a message at level Info on the standard logger. -func (l logger) Infof(format string, args ...interface{}) { - l.sourced().Infof(format, args...) -} - -// Warn logs a message at level Warn on the standard logger. -func (l logger) Warn(args ...interface{}) { - l.sourced().Warn(args...) -} - -// Warn logs a message at level Warn on the standard logger. -func (l logger) Warnln(args ...interface{}) { - l.sourced().Warnln(args...) -} - -// Warnf logs a message at level Warn on the standard logger. -func (l logger) Warnf(format string, args ...interface{}) { - l.sourced().Warnf(format, args...) -} - -// Error logs a message at level Error on the standard logger. -func (l logger) Error(args ...interface{}) { - l.sourced().Error(args...) -} - -// Error logs a message at level Error on the standard logger. -func (l logger) Errorln(args ...interface{}) { - l.sourced().Errorln(args...) -} - -// Errorf logs a message at level Error on the standard logger. -func (l logger) Errorf(format string, args ...interface{}) { - l.sourced().Errorf(format, args...) -} - -// Fatal logs a message at level Fatal on the standard logger. -func (l logger) Fatal(args ...interface{}) { - l.sourced().Fatal(args...) -} - -// Fatal logs a message at level Fatal on the standard logger. -func (l logger) Fatalln(args ...interface{}) { - l.sourced().Fatalln(args...) -} - -// Fatalf logs a message at level Fatal on the standard logger. -func (l logger) Fatalf(format string, args ...interface{}) { - l.sourced().Fatalf(format, args...) -} - -func (l logger) SetLevel(level string) error { - lvl, err := logrus.ParseLevel(level) - if err != nil { - return err - } - - l.entry.Logger.Level = lvl - return nil -} - -func (l logger) SetFormat(format string) error { - u, err := url.Parse(format) - if err != nil { - return err - } - if u.Scheme != "logger" { - return fmt.Errorf("invalid scheme %s", u.Scheme) - } - jsonq := u.Query().Get("json") - if jsonq == "true" { - setJSONFormatter() - } - - switch u.Opaque { - case "syslog": - if setSyslogFormatter == nil { - return fmt.Errorf("system does not support syslog") - } - appname := u.Query().Get("appname") - facility := u.Query().Get("local") - return setSyslogFormatter(l, appname, facility) - case "eventlog": - if setEventlogFormatter == nil { - return fmt.Errorf("system does not support eventlog") - } - name := u.Query().Get("name") - debugAsInfo := false - debugAsInfoRaw := u.Query().Get("debugAsInfo") - if parsedDebugAsInfo, err := strconv.ParseBool(debugAsInfoRaw); err == nil { - debugAsInfo = parsedDebugAsInfo - } - return setEventlogFormatter(l, name, debugAsInfo) - case "stdout": - l.entry.Logger.Out = os.Stdout - case "stderr": - l.entry.Logger.Out = os.Stderr - default: - return fmt.Errorf("unsupported logger %q", u.Opaque) - } - return nil -} - -// sourced adds a source field to the logger that contains -// the file name and line where the logging happened. -func (l logger) sourced() *logrus.Entry { - _, file, line, ok := runtime.Caller(2) - if !ok { - file = "" - line = 1 - } else { - slash := strings.LastIndex(file, "/") - file = file[slash+1:] - } - return l.entry.WithField("source", fmt.Sprintf("%s:%d", file, line)) -} - -var origLogger = logrus.New() -var baseLogger = logger{entry: logrus.NewEntry(origLogger)} - -// Base returns the default Logger logging to -func Base() Logger { - return baseLogger -} - -// NewLogger returns a new Logger logging to out. -func NewLogger(w io.Writer) Logger { - l := logrus.New() - l.Out = w - return logger{entry: logrus.NewEntry(l)} -} - -// NewNopLogger returns a logger that discards all log messages. -func NewNopLogger() Logger { - l := logrus.New() - l.Out = ioutil.Discard - return logger{entry: logrus.NewEntry(l)} -} - -// With adds a field to the logger. -func With(key string, value interface{}) Logger { - return baseLogger.With(key, value) -} - -// Debug logs a message at level Debug on the standard logger. -func Debug(args ...interface{}) { - baseLogger.sourced().Debug(args...) -} - -// Debugln logs a message at level Debug on the standard logger. -func Debugln(args ...interface{}) { - baseLogger.sourced().Debugln(args...) -} - -// Debugf logs a message at level Debug on the standard logger. -func Debugf(format string, args ...interface{}) { - baseLogger.sourced().Debugf(format, args...) -} - -// Info logs a message at level Info on the standard logger. -func Info(args ...interface{}) { - baseLogger.sourced().Info(args...) -} - -// Infoln logs a message at level Info on the standard logger. -func Infoln(args ...interface{}) { - baseLogger.sourced().Infoln(args...) -} - -// Infof logs a message at level Info on the standard logger. -func Infof(format string, args ...interface{}) { - baseLogger.sourced().Infof(format, args...) -} - -// Warn logs a message at level Warn on the standard logger. -func Warn(args ...interface{}) { - baseLogger.sourced().Warn(args...) -} - -// Warnln logs a message at level Warn on the standard logger. -func Warnln(args ...interface{}) { - baseLogger.sourced().Warnln(args...) -} - -// Warnf logs a message at level Warn on the standard logger. -func Warnf(format string, args ...interface{}) { - baseLogger.sourced().Warnf(format, args...) -} - -// Error logs a message at level Error on the standard logger. -func Error(args ...interface{}) { - baseLogger.sourced().Error(args...) -} - -// Errorln logs a message at level Error on the standard logger. -func Errorln(args ...interface{}) { - baseLogger.sourced().Errorln(args...) -} - -// Errorf logs a message at level Error on the standard logger. -func Errorf(format string, args ...interface{}) { - baseLogger.sourced().Errorf(format, args...) -} - -// Fatal logs a message at level Fatal on the standard logger. -func Fatal(args ...interface{}) { - baseLogger.sourced().Fatal(args...) -} - -// Fatalln logs a message at level Fatal on the standard logger. -func Fatalln(args ...interface{}) { - baseLogger.sourced().Fatalln(args...) -} - -// Fatalf logs a message at level Fatal on the standard logger. -func Fatalf(format string, args ...interface{}) { - baseLogger.sourced().Fatalf(format, args...) -} - -// AddHook adds hook to Prometheus' original logger. -func AddHook(hook logrus.Hook) { - origLogger.Hooks.Add(hook) -} - -type errorLogWriter struct{} - -func (errorLogWriter) Write(b []byte) (int, error) { - baseLogger.sourced().Error(string(b)) - return len(b), nil -} - -// NewErrorLogger returns a log.Logger that is meant to be used -// in the ErrorLog field of an http.Server to log HTTP server errors. -func NewErrorLogger() *log.Logger { - return log.New(&errorLogWriter{}, "", 0) -} diff --git a/vendor/github.com/prometheus/common/log/syslog_formatter.go b/vendor/github.com/prometheus/common/log/syslog_formatter.go deleted file mode 100644 index f882f2f848..0000000000 --- a/vendor/github.com/prometheus/common/log/syslog_formatter.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2015 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build !windows,!nacl,!plan9 - -package log - -import ( - "fmt" - "log/syslog" - "os" - - "github.com/sirupsen/logrus" -) - -var _ logrus.Formatter = (*syslogger)(nil) - -func init() { - setSyslogFormatter = func(l logger, appname, local string) error { - if appname == "" { - return fmt.Errorf("missing appname parameter") - } - if local == "" { - return fmt.Errorf("missing local parameter") - } - - fmter, err := newSyslogger(appname, local, l.entry.Logger.Formatter) - if err != nil { - fmt.Fprintf(os.Stderr, "error creating syslog formatter: %v\n", err) - l.entry.Errorf("can't connect logger to syslog: %v", err) - return err - } - l.entry.Logger.Formatter = fmter - return nil - } -} - -var prefixTag []byte - -type syslogger struct { - wrap logrus.Formatter - out *syslog.Writer -} - -func newSyslogger(appname string, facility string, fmter logrus.Formatter) (*syslogger, error) { - priority, err := getFacility(facility) - if err != nil { - return nil, err - } - out, err := syslog.New(priority, appname) - _, isJSON := fmter.(*logrus.JSONFormatter) - if isJSON { - // add cee tag to json formatted syslogs - prefixTag = []byte("@cee:") - } - return &syslogger{ - out: out, - wrap: fmter, - }, err -} - -func getFacility(facility string) (syslog.Priority, error) { - switch facility { - case "0": - return syslog.LOG_LOCAL0, nil - case "1": - return syslog.LOG_LOCAL1, nil - case "2": - return syslog.LOG_LOCAL2, nil - case "3": - return syslog.LOG_LOCAL3, nil - case "4": - return syslog.LOG_LOCAL4, nil - case "5": - return syslog.LOG_LOCAL5, nil - case "6": - return syslog.LOG_LOCAL6, nil - case "7": - return syslog.LOG_LOCAL7, nil - } - return syslog.LOG_LOCAL0, fmt.Errorf("invalid local(%s) for syslog", facility) -} - -func (s *syslogger) Format(e *logrus.Entry) ([]byte, error) { - data, err := s.wrap.Format(e) - if err != nil { - fmt.Fprintf(os.Stderr, "syslogger: can't format entry: %v\n", err) - return data, err - } - // only append tag to data sent to syslog (line), not to what - // is returned - line := string(append(prefixTag, data...)) - - switch e.Level { - case logrus.PanicLevel: - err = s.out.Crit(line) - case logrus.FatalLevel: - err = s.out.Crit(line) - case logrus.ErrorLevel: - err = s.out.Err(line) - case logrus.WarnLevel: - err = s.out.Warning(line) - case logrus.InfoLevel: - err = s.out.Info(line) - case logrus.DebugLevel: - err = s.out.Debug(line) - default: - err = s.out.Notice(line) - } - - if err != nil { - fmt.Fprintf(os.Stderr, "syslogger: can't send log to syslog: %v\n", err) - } - - return data, err -} diff --git a/vendor/golang.org/x/sys/windows/svc/eventlog/install.go b/vendor/golang.org/x/sys/windows/svc/eventlog/install.go deleted file mode 100644 index c76a3760a4..0000000000 --- a/vendor/golang.org/x/sys/windows/svc/eventlog/install.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -package eventlog - -import ( - "errors" - - "golang.org/x/sys/windows" - "golang.org/x/sys/windows/registry" -) - -const ( - // Log levels. - Info = windows.EVENTLOG_INFORMATION_TYPE - Warning = windows.EVENTLOG_WARNING_TYPE - Error = windows.EVENTLOG_ERROR_TYPE -) - -const addKeyName = `SYSTEM\CurrentControlSet\Services\EventLog\Application` - -// Install modifies PC registry to allow logging with an event source src. -// It adds all required keys and values to the event log registry key. -// Install uses msgFile as the event message file. If useExpandKey is true, -// the event message file is installed as REG_EXPAND_SZ value, -// otherwise as REG_SZ. Use bitwise of log.Error, log.Warning and -// log.Info to specify events supported by the new event source. -func Install(src, msgFile string, useExpandKey bool, eventsSupported uint32) error { - appkey, err := registry.OpenKey(registry.LOCAL_MACHINE, addKeyName, registry.CREATE_SUB_KEY) - if err != nil { - return err - } - defer appkey.Close() - - sk, alreadyExist, err := registry.CreateKey(appkey, src, registry.SET_VALUE) - if err != nil { - return err - } - defer sk.Close() - if alreadyExist { - return errors.New(addKeyName + `\` + src + " registry key already exists") - } - - err = sk.SetDWordValue("CustomSource", 1) - if err != nil { - return err - } - if useExpandKey { - err = sk.SetExpandStringValue("EventMessageFile", msgFile) - } else { - err = sk.SetStringValue("EventMessageFile", msgFile) - } - if err != nil { - return err - } - err = sk.SetDWordValue("TypesSupported", eventsSupported) - if err != nil { - return err - } - return nil -} - -// InstallAsEventCreate is the same as Install, but uses -// %SystemRoot%\System32\EventCreate.exe as the event message file. -func InstallAsEventCreate(src string, eventsSupported uint32) error { - return Install(src, "%SystemRoot%\\System32\\EventCreate.exe", true, eventsSupported) -} - -// Remove deletes all registry elements installed by the correspondent Install. -func Remove(src string) error { - appkey, err := registry.OpenKey(registry.LOCAL_MACHINE, addKeyName, registry.SET_VALUE) - if err != nil { - return err - } - defer appkey.Close() - return registry.DeleteKey(appkey, src) -} diff --git a/vendor/golang.org/x/sys/windows/svc/eventlog/log.go b/vendor/golang.org/x/sys/windows/svc/eventlog/log.go deleted file mode 100644 index 46e5153d02..0000000000 --- a/vendor/golang.org/x/sys/windows/svc/eventlog/log.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows - -// Package eventlog implements access to Windows event log. -// -package eventlog - -import ( - "errors" - "syscall" - - "golang.org/x/sys/windows" -) - -// Log provides access to the system log. -type Log struct { - Handle windows.Handle -} - -// Open retrieves a handle to the specified event log. -func Open(source string) (*Log, error) { - return OpenRemote("", source) -} - -// OpenRemote does the same as Open, but on different computer host. -func OpenRemote(host, source string) (*Log, error) { - if source == "" { - return nil, errors.New("Specify event log source") - } - var s *uint16 - if host != "" { - s = syscall.StringToUTF16Ptr(host) - } - h, err := windows.RegisterEventSource(s, syscall.StringToUTF16Ptr(source)) - if err != nil { - return nil, err - } - return &Log{Handle: h}, nil -} - -// Close closes event log l. -func (l *Log) Close() error { - return windows.DeregisterEventSource(l.Handle) -} - -func (l *Log) report(etype uint16, eid uint32, msg string) error { - ss := []*uint16{syscall.StringToUTF16Ptr(msg)} - return windows.ReportEvent(l.Handle, etype, 0, eid, 0, 1, 0, &ss[0], nil) -} - -// Info writes an information event msg with event id eid to the end of event log l. -// When EventCreate.exe is used, eid must be between 1 and 1000. -func (l *Log) Info(eid uint32, msg string) error { - return l.report(windows.EVENTLOG_INFORMATION_TYPE, eid, msg) -} - -// Warning writes an warning event msg with event id eid to the end of event log l. -// When EventCreate.exe is used, eid must be between 1 and 1000. -func (l *Log) Warning(eid uint32, msg string) error { - return l.report(windows.EVENTLOG_WARNING_TYPE, eid, msg) -} - -// Error writes an error event msg with event id eid to the end of event log l. -// When EventCreate.exe is used, eid must be between 1 and 1000. -func (l *Log) Error(eid uint32, msg string) error { - return l.report(windows.EVENTLOG_ERROR_TYPE, eid, msg) -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 03d15df923..77dd4e9ccd 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -423,7 +423,6 @@ github.com/prometheus/client_model/go github.com/prometheus/common/config github.com/prometheus/common/expfmt github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg -github.com/prometheus/common/log github.com/prometheus/common/model github.com/prometheus/common/route github.com/prometheus/common/version @@ -759,7 +758,6 @@ golang.org/x/sys/cpu golang.org/x/sys/unix golang.org/x/sys/windows golang.org/x/sys/windows/registry -golang.org/x/sys/windows/svc/eventlog # golang.org/x/text v0.3.2 golang.org/x/text/secure/bidirule golang.org/x/text/transform From 6eb93311b13ce4142e3b291530c32430d34fdbd5 Mon Sep 17 00:00:00 2001 From: Annanay Date: Wed, 13 May 2020 18:23:20 +0530 Subject: [PATCH 27/29] Address comments Signed-off-by: Annanay --- docs/configuration/config-file-reference.md | 70 +++---------------- docs/production/tls.md | 6 +- integration/configs.go | 2 +- integration/query_frontend_test.go | 10 +-- integration/util.go | 12 ++-- pkg/configs/client/client.go | 6 +- pkg/ingester/client/client.go | 4 +- pkg/querier/blocks_store_balanced_set.go | 6 +- pkg/querier/blocks_store_balanced_set_test.go | 4 +- pkg/querier/blocks_store_queryable.go | 4 +- pkg/querier/blocks_store_replicated_set.go | 6 +- .../blocks_store_replicated_set_test.go | 4 +- pkg/querier/frontend/worker.go | 4 +- .../frontend/worker_frontend_manager.go | 8 +-- .../frontend/worker_frontend_manager_test.go | 6 +- pkg/querier/frontend/worker_test.go | 2 +- pkg/querier/querier.go | 8 +-- pkg/querier/store_gateway_client.go | 22 ++++-- pkg/querier/store_gateway_client_test.go | 4 +- pkg/ruler/ruler.go | 2 +- pkg/util/grpcclient/grpcclient.go | 32 ++++----- pkg/util/tls/tls.go | 19 ++--- 22 files changed, 98 insertions(+), 143 deletions(-) diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 032a9015a2..b318fb8ee9 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -647,44 +647,7 @@ The `querier_config` configures the Cortex querier. # CLI flag: -experimental.querier.store-gateway-addresses [store_gateway_addresses: | default = ""] -store_gateway_client_config: - # gRPC client max receive message size (bytes). - # CLI flag: -experimental.querier.store-gateway-client.grpc-max-recv-msg-size - [max_recv_msg_size: | default = 104857600] - - # gRPC client max send message size (bytes). - # CLI flag: -experimental.querier.store-gateway-client.grpc-max-send-msg-size - [max_send_msg_size: | default = 16777216] - - # Use compression when sending messages. - # CLI flag: -experimental.querier.store-gateway-client.grpc-use-gzip-compression - [use_gzip_compression: | default = false] - - # Rate limit for gRPC client; 0 means disabled. - # CLI flag: -experimental.querier.store-gateway-client.grpc-client-rate-limit - [rate_limit: | default = 0] - - # Rate limit burst for gRPC client. - # CLI flag: -experimental.querier.store-gateway-client.grpc-client-rate-limit-burst - [rate_limit_burst: | default = 0] - - # Enable backoff and retry when we hit ratelimits. - # CLI flag: -experimental.querier.store-gateway-client.backoff-on-ratelimits - [backoff_on_ratelimits: | default = false] - - backoff_config: - # Minimum delay when backing off. - # CLI flag: -experimental.querier.store-gateway-client.backoff-min-period - [min_period: | default = 100ms] - - # Maximum delay when backing off. - # CLI flag: -experimental.querier.store-gateway-client.backoff-max-period - [max_period: | default = 10s] - - # Number of times to backoff and retry before failing. - # CLI flag: -experimental.querier.store-gateway-client.backoff-retries - [max_retries: | default = 10] - +store_gateway_client: # TLS cert path for the client # CLI flag: -experimental.querier.store-gateway-client.tls-cert-path [tls_cert_path: | default = ""] @@ -807,17 +770,18 @@ The `ruler_config` configures the Cortex ruler. # CLI flag: -ruler.external.url [external_url: | default = ] -# TLS cert path for the client -# CLI flag: -ruler.client.tls-cert-path -[tls_cert_path: | default = ""] +ruler_client: + # TLS cert path for the client + # CLI flag: -ruler.client.tls-cert-path + [tls_cert_path: | default = ""] -# TLS key path for the client -# CLI flag: -ruler.client.tls-key-path -[tls_key_path: | default = ""] + # TLS key path for the client + # CLI flag: -ruler.client.tls-key-path + [tls_key_path: | default = ""] -# TLS CA path for the client -# CLI flag: -ruler.client.tls-ca-path -[tls_ca_path: | default = ""] + # TLS CA path for the client + # CLI flag: -ruler.client.tls-ca-path + [tls_ca_path: | default = ""] # How frequently to evaluate rules # CLI flag: -ruler.evaluation-interval @@ -1646,18 +1610,6 @@ bigtable: # CLI flag: -bigtable.backoff-retries [max_retries: | default = 10] - # TLS cert path for the client - # CLI flag: -bigtable.tls-cert-path - [tls_cert_path: | default = ""] - - # TLS key path for the client - # CLI flag: -bigtable.tls-key-path - [tls_key_path: | default = ""] - - # TLS CA path for the client - # CLI flag: -bigtable.tls-ca-path - [tls_ca_path: | default = ""] - # If enabled, once a tables info is fetched, it is cached. # CLI flag: -bigtable.table-cache.enabled [table_cache_enabled: | default = true] diff --git a/docs/production/tls.md b/docs/production/tls.md index 77bf518025..ccbb2e0093 100644 --- a/docs/production/tls.md +++ b/docs/production/tls.md @@ -1,6 +1,6 @@ --- -title: "Securing communication between cortex components with TLS" -linkTitle: "Securing communication between cortex components with TLS" +title: "Securing communication between Cortex components with TLS" +linkTitle: "Securing communication between Cortex components with TLS" weight: 5 slug: tls --- @@ -19,7 +19,7 @@ by this CA will have permissions to communicate with the cluster. We will use the following script to generate self signed certs for the cluster: ``` -# Refer: https://github.com/joe-elliott/cert-exporter/blob/69d3d7230378325a1de4fa313432d3d6ced4a518/test/files/genCerts.sh +# Refer: github.com/cortexproject/cortex/integration/certs/genCerts.sh # keys openssl genrsa -out root.key diff --git a/integration/configs.go b/integration/configs.go index 90ce590072..162512aa82 100644 --- a/integration/configs.go +++ b/integration/configs.go @@ -24,7 +24,7 @@ const ( blocksStorageEngine = "tsdb" clientCertFile = "certs/client.crt" clientKeyFile = "certs/client.key" - rootCertFile = "certs/root.crt" + caCertFile = "certs/root.crt" serverCertFile = "certs/server.crt" serverKeyFile = "certs/server.key" storeConfigTemplate = ` diff --git a/integration/query_frontend_test.go b/integration/query_frontend_test.go index 8d2f4a5660..ce14d4b81f 100644 --- a/integration/query_frontend_test.go +++ b/integration/query_frontend_test.go @@ -102,7 +102,7 @@ func runQueryFrontendTest(t *testing.T, setup queryFrontendSetup) { require.NoError(t, cmd.Run()) require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+clientCertFile, clientCertFile)) require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+clientKeyFile, clientKeyFile)) - require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+rootCertFile, rootCertFile)) + require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+caCertFile, caCertFile)) require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+serverCertFile, serverCertFile)) require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+serverKeyFile, serverKeyFile)) @@ -113,9 +113,9 @@ func runQueryFrontendTest(t *testing.T, setup queryFrontendSetup) { }) // Start Cortex components. - queryFrontend := e2ecortex.NewQueryFrontendWithConfigFile("query-frontend", configFile, mergeFlags(flags, GetServerTLSFlags()), "") - ingester := e2ecortex.NewIngesterWithConfigFile("ingester", consul.NetworkHTTPEndpoint(), configFile, mergeFlags(flags, GetServerTLSFlags()), "") - distributor := e2ecortex.NewDistributorWithConfigFile("distributor", consul.NetworkHTTPEndpoint(), configFile, mergeFlags(flags, GetClientTLSFlagsWithPrefix("ingester.client")), "") + queryFrontend := e2ecortex.NewQueryFrontendWithConfigFile("query-frontend", configFile, mergeFlags(flags, getServerTLSFlags()), "") + ingester := e2ecortex.NewIngesterWithConfigFile("ingester", consul.NetworkHTTPEndpoint(), configFile, mergeFlags(flags, getServerTLSFlags()), "") + distributor := e2ecortex.NewDistributorWithConfigFile("distributor", consul.NetworkHTTPEndpoint(), configFile, mergeFlags(flags, getClientTLSFlagsWithPrefix("ingester.client")), "") require.NoError(t, s.StartAndWaitReady(queryFrontend, distributor, ingester)) // Check if we're discovering memcache or not. @@ -126,7 +126,7 @@ func runQueryFrontendTest(t *testing.T, setup queryFrontendSetup) { // able to get the query-frontend network endpoint. querier := e2ecortex.NewQuerierWithConfigFile("querier", consul.NetworkHTTPEndpoint(), configFile, mergeFlags(flags, map[string]string{ "-querier.frontend-address": queryFrontend.NetworkGRPCEndpoint(), - }, GetClientTLSFlagsWithPrefix("querier.frontend-client"), GetClientTLSFlagsWithPrefix("ingester.client")), "") + }, getClientTLSFlagsWithPrefix("querier.frontend-client"), getClientTLSFlagsWithPrefix("ingester.client")), "") require.NoError(t, s.StartAndWaitReady(querier)) // Wait until both the distributor and querier have updated the ring. diff --git a/integration/util.go b/integration/util.go index a4705fd880..738ad17f9c 100644 --- a/integration/util.go +++ b/integration/util.go @@ -52,25 +52,23 @@ func copyFileToSharedDir(s *e2e.Scenario, src, dst string) error { return writeFileToSharedDir(s, dst, content) } -// GetServerTLSFlags generates generic TLS flags for a server -func GetServerTLSFlags() map[string]string { +func getServerTLSFlags() map[string]string { return map[string]string{ "-server.http-tls-cert-path": filepath.Join(e2e.ContainerSharedDir, serverCertFile), "-server.http-tls-key-path": filepath.Join(e2e.ContainerSharedDir, serverKeyFile), "-server.http-tls-client-auth": "RequireAndVerifyClientCert", - "-server.http-tls-ca-path": filepath.Join(e2e.ContainerSharedDir, rootCertFile), + "-server.http-tls-ca-path": filepath.Join(e2e.ContainerSharedDir, caCertFile), "-server.grpc-tls-cert-path": filepath.Join(e2e.ContainerSharedDir, serverCertFile), "-server.grpc-tls-key-path": filepath.Join(e2e.ContainerSharedDir, serverKeyFile), "-server.grpc-tls-client-auth": "RequireAndVerifyClientCert", - "-server.grpc-tls-ca-path": filepath.Join(e2e.ContainerSharedDir, rootCertFile), + "-server.grpc-tls-ca-path": filepath.Join(e2e.ContainerSharedDir, caCertFile), } } -// GetClientTLSFlagsWithPrefix generates generic TLS flags for a client -func GetClientTLSFlagsWithPrefix(prefix string) map[string]string { +func getClientTLSFlagsWithPrefix(prefix string) map[string]string { return map[string]string{ "-" + prefix + ".tls-cert-path": filepath.Join(e2e.ContainerSharedDir, clientCertFile), "-" + prefix + ".tls-key-path": filepath.Join(e2e.ContainerSharedDir, clientKeyFile), - "-" + prefix + ".tls-ca-path": filepath.Join(e2e.ContainerSharedDir, rootCertFile), + "-" + prefix + ".tls-ca-path": filepath.Join(e2e.ContainerSharedDir, caCertFile), } } diff --git a/pkg/configs/client/client.go b/pkg/configs/client/client.go index 0f4c6a5c7b..05343c37c8 100644 --- a/pkg/configs/client/client.go +++ b/pkg/configs/client/client.go @@ -25,14 +25,14 @@ import ( type Config struct { ConfigsAPIURL flagext.URLValue `yaml:"configs_api_url"` ClientTimeout time.Duration `yaml:"client_timeout"` // HTTP timeout duration for requests made to the Weave Cloud configs service. - TLSStruct tls_cfg.ClientConfig `yaml:",inline"` + TLS tls_cfg.ClientConfig `yaml:",inline"` } // RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { f.Var(&cfg.ConfigsAPIURL, prefix+"configs.url", "URL of configs API server.") f.DurationVar(&cfg.ClientTimeout, prefix+"configs.client-timeout", 5*time.Second, "Timeout for requests to Weave Cloud configs service.") - cfg.TLSStruct.RegisterFlagsWithPrefix(prefix+"configs", f) + cfg.TLS.RegisterFlagsWithPrefix(prefix+"configs", f) } var configsRequestDuration = instrument.NewHistogramCollector(promauto.NewHistogramVec(prometheus.HistogramOpts{ @@ -59,7 +59,7 @@ func New(cfg Config) (*ConfigDBClient, error) { Timeout: cfg.ClientTimeout, } - tlsConfig, err := cfg.TLSStruct.GetTLSConfig() + tlsConfig, err := cfg.TLS.GetTLSConfig() if err != nil { return nil, err } diff --git a/pkg/ingester/client/client.go b/pkg/ingester/client/client.go index 40917e759d..8c5d3b54d3 100644 --- a/pkg/ingester/client/client.go +++ b/pkg/ingester/client/client.go @@ -34,7 +34,7 @@ type closableHealthAndIngesterClient struct { // MakeIngesterClient makes a new IngesterClient func MakeIngesterClient(addr string, cfg Config) (HealthAndIngesterClient, error) { - dialOpts, err := cfg.GRPCClientConfig.DialOptionWithTLS(grpcclient.Instrument(ingesterClientRequestDuration)) + dialOpts, err := cfg.GRPCClientConfig.DialOption(grpcclient.Instrument(ingesterClientRequestDuration)) if err != nil { return nil, err } @@ -55,7 +55,7 @@ func (c *closableHealthAndIngesterClient) Close() error { // Config is the configuration struct for the ingester client type Config struct { - GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config"` + GRPCClientConfig grpcclient.ConfigWithTLS `yaml:"grpc_client_config"` } // RegisterFlags registers configuration settings used by the ingester client config. diff --git a/pkg/querier/blocks_store_balanced_set.go b/pkg/querier/blocks_store_balanced_set.go index ed79e5a7e6..4be29bb17a 100644 --- a/pkg/querier/blocks_store_balanced_set.go +++ b/pkg/querier/blocks_store_balanced_set.go @@ -16,8 +16,8 @@ import ( "github.com/cortexproject/cortex/pkg/ring/client" "github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb" - "github.com/cortexproject/cortex/pkg/util/grpcclient" "github.com/cortexproject/cortex/pkg/util/services" + "github.com/cortexproject/cortex/pkg/util/tls" ) // BlocksStoreSet implementation used when the blocks are not sharded in the store-gateway @@ -30,7 +30,7 @@ type blocksStoreBalancedSet struct { dnsProvider *dns.Provider } -func newBlocksStoreBalancedSet(serviceAddresses []string, clientCfg grpcclient.Config, logger log.Logger, reg prometheus.Registerer) *blocksStoreBalancedSet { +func newBlocksStoreBalancedSet(serviceAddresses []string, tlsCfg tls.ClientConfig, logger log.Logger, reg prometheus.Registerer) *blocksStoreBalancedSet { const dnsResolveInterval = 10 * time.Second dnsProviderReg := extprom.WrapRegistererWithPrefix("cortex_storegateway_client_", reg) @@ -38,7 +38,7 @@ func newBlocksStoreBalancedSet(serviceAddresses []string, clientCfg grpcclient.C s := &blocksStoreBalancedSet{ serviceAddresses: serviceAddresses, dnsProvider: dns.NewProvider(logger, dnsProviderReg, dns.GolangResolverType), - clientsPool: newStoreGatewayClientPool(nil, clientCfg, logger, reg), + clientsPool: newStoreGatewayClientPool(nil, tlsCfg, logger, reg), } s.Service = services.NewTimerService(dnsResolveInterval, s.starting, s.resolve, nil) diff --git a/pkg/querier/blocks_store_balanced_set_test.go b/pkg/querier/blocks_store_balanced_set_test.go index 19b43f84ff..c27c4779cc 100644 --- a/pkg/querier/blocks_store_balanced_set_test.go +++ b/pkg/querier/blocks_store_balanced_set_test.go @@ -11,8 +11,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/cortexproject/cortex/pkg/util/grpcclient" "github.com/cortexproject/cortex/pkg/util/services" + "github.com/cortexproject/cortex/pkg/util/tls" ) func TestBlocksStoreBalancedSet_GetClientsFor(t *testing.T) { @@ -21,7 +21,7 @@ func TestBlocksStoreBalancedSet_GetClientsFor(t *testing.T) { ctx := context.Background() reg := prometheus.NewPedanticRegistry() - s := newBlocksStoreBalancedSet(serviceAddrs, grpcclient.Config{}, log.NewNopLogger(), reg) + s := newBlocksStoreBalancedSet(serviceAddrs, tls.ClientConfig{}, log.NewNopLogger(), reg) require.NoError(t, services.StartAndAwaitRunning(ctx, s)) defer services.StopAndAwaitTerminated(ctx, s) //nolint:errcheck diff --git a/pkg/querier/blocks_store_queryable.go b/pkg/querier/blocks_store_queryable.go index d6ead4240e..108e3d592f 100644 --- a/pkg/querier/blocks_store_queryable.go +++ b/pkg/querier/blocks_store_queryable.go @@ -126,7 +126,7 @@ func NewBlocksStoreQueryableFromConfig(querierCfg Config, gatewayCfg storegatewa reg.MustRegister(storesRing) } - stores, err = newBlocksStoreReplicationSet(storesRing, querierCfg.StoreGatewayGRPCClientConfig, logger, reg) + stores, err = newBlocksStoreReplicationSet(storesRing, querierCfg.StoreGatewayClient, logger, reg) if err != nil { return nil, errors.Wrap(err, "failed to create store set") } @@ -135,7 +135,7 @@ func NewBlocksStoreQueryableFromConfig(querierCfg Config, gatewayCfg storegatewa return nil, errNoStoreGatewayAddress } - stores = newBlocksStoreBalancedSet(querierCfg.GetStoreGatewayAddresses(), querierCfg.StoreGatewayGRPCClientConfig, logger, reg) + stores = newBlocksStoreBalancedSet(querierCfg.GetStoreGatewayAddresses(), querierCfg.StoreGatewayClient, logger, reg) } return NewBlocksStoreQueryable(stores, scanner, reg) diff --git a/pkg/querier/blocks_store_replicated_set.go b/pkg/querier/blocks_store_replicated_set.go index ac6e9109c8..6e4e5723e8 100644 --- a/pkg/querier/blocks_store_replicated_set.go +++ b/pkg/querier/blocks_store_replicated_set.go @@ -12,8 +12,8 @@ import ( "github.com/cortexproject/cortex/pkg/ring/client" cortex_tsdb "github.com/cortexproject/cortex/pkg/storage/tsdb" "github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb" - "github.com/cortexproject/cortex/pkg/util/grpcclient" "github.com/cortexproject/cortex/pkg/util/services" + "github.com/cortexproject/cortex/pkg/util/tls" ) // BlocksStoreSet implementation used when the blocks are sharded and replicated across @@ -29,10 +29,10 @@ type blocksStoreReplicationSet struct { subservicesWatcher *services.FailureWatcher } -func newBlocksStoreReplicationSet(storesRing *ring.Ring, clientCfg grpcclient.Config, logger log.Logger, reg prometheus.Registerer) (*blocksStoreReplicationSet, error) { +func newBlocksStoreReplicationSet(storesRing *ring.Ring, tlsCfg tls.ClientConfig, logger log.Logger, reg prometheus.Registerer) (*blocksStoreReplicationSet, error) { s := &blocksStoreReplicationSet{ storesRing: storesRing, - clientsPool: newStoreGatewayClientPool(client.NewRingServiceDiscovery(storesRing), clientCfg, logger, reg), + clientsPool: newStoreGatewayClientPool(client.NewRingServiceDiscovery(storesRing), tlsCfg, logger, reg), } var err error diff --git a/pkg/querier/blocks_store_replicated_set_test.go b/pkg/querier/blocks_store_replicated_set_test.go index 53e265b6bb..353d8f1bfd 100644 --- a/pkg/querier/blocks_store_replicated_set_test.go +++ b/pkg/querier/blocks_store_replicated_set_test.go @@ -22,9 +22,9 @@ import ( "github.com/cortexproject/cortex/pkg/storegateway" "github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb" "github.com/cortexproject/cortex/pkg/util/flagext" - "github.com/cortexproject/cortex/pkg/util/grpcclient" "github.com/cortexproject/cortex/pkg/util/services" "github.com/cortexproject/cortex/pkg/util/test" + "github.com/cortexproject/cortex/pkg/util/tls" ) func Test_findSmallestInstanceSet(t *testing.T) { @@ -176,7 +176,7 @@ func TestBlocksStoreReplicationSet_GetClientsFor(t *testing.T) { require.NoError(t, err) reg := prometheus.NewPedanticRegistry() - s, err := newBlocksStoreReplicationSet(r, grpcclient.Config{}, log.NewNopLogger(), reg) + s, err := newBlocksStoreReplicationSet(r, tls.ClientConfig{}, log.NewNopLogger(), reg) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(ctx, s)) defer services.StopAndAwaitTerminated(ctx, s) //nolint:errcheck diff --git a/pkg/querier/frontend/worker.go b/pkg/querier/frontend/worker.go index 4d469376b5..fe99ad284d 100644 --- a/pkg/querier/frontend/worker.go +++ b/pkg/querier/frontend/worker.go @@ -27,7 +27,7 @@ type WorkerConfig struct { MatchMaxConcurrency bool `yaml:"match_max_concurrent"` DNSLookupDuration time.Duration `yaml:"dns_lookup_duration"` - GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config"` + GRPCClientConfig grpcclient.ConfigWithTLS `yaml:"grpc_client_config"` } // RegisterFlags adds the flags required to config this to the given FlagSet. @@ -138,7 +138,7 @@ func (w *worker) watchDNSLoop(servCtx context.Context) error { } func (w *worker) connect(ctx context.Context, address string) (FrontendClient, error) { - opts, err := w.cfg.GRPCClientConfig.DialOptionWithTLS([]grpc.UnaryClientInterceptor{middleware.ClientUserHeaderInterceptor}, nil) + opts, err := w.cfg.GRPCClientConfig.DialOption([]grpc.UnaryClientInterceptor{middleware.ClientUserHeaderInterceptor}, nil) if err != nil { return nil, err } diff --git a/pkg/querier/frontend/worker_frontend_manager.go b/pkg/querier/frontend/worker_frontend_manager.go index 2af3ce883d..f3ea40de4c 100644 --- a/pkg/querier/frontend/worker_frontend_manager.go +++ b/pkg/querier/frontend/worker_frontend_manager.go @@ -27,7 +27,7 @@ var ( type frontendManager struct { server *server.Server client FrontendClient - clientCfg grpcclient.Config + clientCfg grpcclient.ConfigWithTLS log log.Logger @@ -37,7 +37,7 @@ type frontendManager struct { currentProcessors *atomic.Int32 } -func newFrontendManager(serverCtx context.Context, log log.Logger, server *server.Server, client FrontendClient, clientCfg grpcclient.Config) *frontendManager { +func newFrontendManager(serverCtx context.Context, log log.Logger, server *server.Server, client FrontendClient, clientCfg grpcclient.ConfigWithTLS) *frontendManager { f := &frontendManager{ log: log, client: client, @@ -133,8 +133,8 @@ func (f *frontendManager) process(ctx context.Context, c Frontend_ProcessClient) } // Ensure responses that are too big are not retried. - if len(response.Body) >= f.clientCfg.MaxSendMsgSize { - errMsg := fmt.Sprintf("response larger than the max (%d vs %d)", len(response.Body), f.clientCfg.MaxSendMsgSize) + if len(response.Body) >= f.clientCfg.GRPC.MaxSendMsgSize { + errMsg := fmt.Sprintf("response larger than the max (%d vs %d)", len(response.Body), f.clientCfg.GRPC.MaxSendMsgSize) response = &httpgrpc.HTTPResponse{ Code: http.StatusRequestEntityTooLarge, Body: []byte(errMsg), diff --git a/pkg/querier/frontend/worker_frontend_manager_test.go b/pkg/querier/frontend/worker_frontend_manager_test.go index 9013136350..d409832b97 100644 --- a/pkg/querier/frontend/worker_frontend_manager_test.go +++ b/pkg/querier/frontend/worker_frontend_manager_test.go @@ -91,7 +91,7 @@ func TestConcurrency(t *testing.T) { } for _, tt := range tests { t.Run(fmt.Sprintf("Testing concurrency %v", tt.concurrency), func(t *testing.T) { - mgr := newFrontendManager(context.Background(), util.Logger, httpgrpc_server.NewServer(handler), &mockFrontendClient{}, grpcclient.Config{}) + mgr := newFrontendManager(context.Background(), util.Logger, httpgrpc_server.NewServer(handler), &mockFrontendClient{}, grpcclient.ConfigWithTLS{}) for _, c := range tt.concurrency { calls.Store(0) @@ -127,7 +127,7 @@ func TestRecvFailDoesntCancelProcess(t *testing.T) { failRecv: true, } - mgr := newFrontendManager(context.Background(), util.Logger, httpgrpc_server.NewServer(handler), client, grpcclient.Config{}) + mgr := newFrontendManager(context.Background(), util.Logger, httpgrpc_server.NewServer(handler), client, grpcclient.ConfigWithTLS{}) mgr.concurrentRequests(1) time.Sleep(50 * time.Millisecond) @@ -151,7 +151,7 @@ func TestServeCancelStopsProcess(t *testing.T) { } ctx, cancel := context.WithCancel(context.Background()) - mgr := newFrontendManager(ctx, util.Logger, httpgrpc_server.NewServer(handler), client, grpcclient.Config{MaxSendMsgSize: 100000}) + mgr := newFrontendManager(ctx, util.Logger, httpgrpc_server.NewServer(handler), client, grpcclient.ConfigWithTLS{GRPC: grpcclient.Config{MaxSendMsgSize: 100000}}) mgr.concurrentRequests(1) time.Sleep(50 * time.Millisecond) diff --git a/pkg/querier/frontend/worker_test.go b/pkg/querier/frontend/worker_test.go index ba3ee655a2..01ac2e3eb1 100644 --- a/pkg/querier/frontend/worker_test.go +++ b/pkg/querier/frontend/worker_test.go @@ -83,7 +83,7 @@ func TestResetConcurrency(t *testing.T) { } for i := 0; i < tt.numManagers; i++ { - w.managers[strconv.Itoa(i)] = newFrontendManager(context.Background(), util.Logger, httpgrpc_server.NewServer(handler), &mockFrontendClient{}, grpcclient.Config{}) + w.managers[strconv.Itoa(i)] = newFrontendManager(context.Background(), util.Logger, httpgrpc_server.NewServer(handler), &mockFrontendClient{}, grpcclient.ConfigWithTLS{}) } w.resetConcurrency() diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index cb7285b323..22ac39f754 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -8,7 +8,6 @@ import ( "time" "github.com/cortexproject/cortex/pkg/chunk/purger" - "github.com/cortexproject/cortex/pkg/util/grpcclient" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" @@ -24,6 +23,7 @@ import ( "github.com/cortexproject/cortex/pkg/querier/lazyquery" "github.com/cortexproject/cortex/pkg/querier/series" "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/tls" ) // Config contains the configuration require to create a querier @@ -52,8 +52,8 @@ type Config struct { ActiveQueryTrackerDir string `yaml:"active_query_tracker_dir"` // Blocks storage only. - StoreGatewayAddresses string `yaml:"store_gateway_addresses"` - StoreGatewayGRPCClientConfig grpcclient.Config `yaml:"store_gateway_client_config"` + StoreGatewayAddresses string `yaml:"store_gateway_addresses"` + StoreGatewayClient tls.ClientConfig `yaml:"store_gateway_client"` } var ( @@ -62,7 +62,7 @@ var ( // RegisterFlags adds the flags required to config this to the given FlagSet. func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - cfg.StoreGatewayGRPCClientConfig.RegisterFlagsWithPrefix("experimental.querier.store-gateway-client", f) + cfg.StoreGatewayClient.RegisterFlagsWithPrefix("experimental.querier.store-gateway-client", f) f.IntVar(&cfg.MaxConcurrent, "querier.max-concurrent", 20, "The maximum number of concurrent queries.") f.DurationVar(&cfg.Timeout, "querier.timeout", 2*time.Minute, "The timeout for a query.") if f.Lookup("promql.lookback-delta") == nil { diff --git a/pkg/querier/store_gateway_client.go b/pkg/querier/store_gateway_client.go index baa3cd469d..ba79804dd5 100644 --- a/pkg/querier/store_gateway_client.go +++ b/pkg/querier/store_gateway_client.go @@ -13,9 +13,10 @@ import ( "github.com/cortexproject/cortex/pkg/ring/client" "github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb" "github.com/cortexproject/cortex/pkg/util/grpcclient" + "github.com/cortexproject/cortex/pkg/util/tls" ) -func newStoreGatewayClientFactory(cfg grpcclient.Config, reg prometheus.Registerer) client.PoolFactory { +func newStoreGatewayClientFactory(clientCfg grpcclient.Config, tlsCfg tls.ClientConfig, reg prometheus.Registerer) client.PoolFactory { requestDuration := promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ Namespace: "cortex", Name: "storegateway_client_request_duration_seconds", @@ -25,15 +26,16 @@ func newStoreGatewayClientFactory(cfg grpcclient.Config, reg prometheus.Register }, []string{"operation", "status_code"}) return func(addr string) (client.PoolClient, error) { - return dialStoreGatewayClient(cfg, addr, requestDuration) + return dialStoreGatewayClient(clientCfg, tlsCfg, addr, requestDuration) } } -func dialStoreGatewayClient(cfg grpcclient.Config, addr string, requestDuration *prometheus.HistogramVec) (*storeGatewayClient, error) { - opts, err := cfg.DialOptionWithTLS(grpcclient.Instrument(requestDuration)) +func dialStoreGatewayClient(clientCfg grpcclient.Config, tlsCfg tls.ClientConfig, addr string, requestDuration *prometheus.HistogramVec) (*storeGatewayClient, error) { + opts, err := tlsCfg.GetGRPCDialOptions() if err != nil { return nil, err } + opts = append(opts, clientCfg.DialOption(grpcclient.Instrument(requestDuration))...) conn, err := grpc.Dial(addr, opts...) if err != nil { return nil, errors.Wrapf(err, "failed to dial store-gateway %s", addr) @@ -60,7 +62,15 @@ func (c *storeGatewayClient) String() string { return c.conn.Target() } -func newStoreGatewayClientPool(discovery client.PoolServiceDiscovery, clientCfg grpcclient.Config, logger log.Logger, reg prometheus.Registerer) *client.Pool { +func newStoreGatewayClientPool(discovery client.PoolServiceDiscovery, tlsCfg tls.ClientConfig, logger log.Logger, reg prometheus.Registerer) *client.Pool { + clientCfg := grpcclient.Config{ + MaxRecvMsgSize: 100 << 20, + MaxSendMsgSize: 16 << 20, + UseGzipCompression: false, + RateLimit: 0, + RateLimitBurst: 0, + BackoffOnRatelimits: false, + } poolCfg := client.PoolConfig{ CheckInterval: time.Minute, HealthCheckEnabled: true, @@ -74,5 +84,5 @@ func newStoreGatewayClientPool(discovery client.PoolServiceDiscovery, clientCfg ConstLabels: map[string]string{"client": "querier"}, }) - return client.NewPool("store-gateway", poolCfg, discovery, newStoreGatewayClientFactory(clientCfg, reg), clientsCount, logger) + return client.NewPool("store-gateway", poolCfg, discovery, newStoreGatewayClientFactory(clientCfg, tlsCfg, reg), clientsCount, logger) } diff --git a/pkg/querier/store_gateway_client_test.go b/pkg/querier/store_gateway_client_test.go index 6bb31a99e5..58d26f34a4 100644 --- a/pkg/querier/store_gateway_client_test.go +++ b/pkg/querier/store_gateway_client_test.go @@ -16,6 +16,7 @@ import ( "github.com/cortexproject/cortex/pkg/storegateway/storegatewaypb" "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/cortexproject/cortex/pkg/util/grpcclient" + "github.com/cortexproject/cortex/pkg/util/tls" ) func Test_newStoreGatewayClientFactory(t *testing.T) { @@ -36,10 +37,11 @@ func Test_newStoreGatewayClientFactory(t *testing.T) { // Create a client factory and query back the mocked service // with different clients. cfg := grpcclient.Config{} + tlsCfg := tls.ClientConfig{} flagext.DefaultValues(&cfg) reg := prometheus.NewPedanticRegistry() - factory := newStoreGatewayClientFactory(cfg, reg) + factory := newStoreGatewayClientFactory(cfg, tlsCfg, reg) for i := 0; i < 2; i++ { client, err := factory(listener.Addr().String()) diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index fade05bf59..c643aff742 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -61,7 +61,7 @@ type Config struct { // This is used for template expansion in alerts; must be a valid URL. ExternalURL flagext.URLValue `yaml:"external_url"` // TLS parameters for the GRPC Client - ClientTLSConfig tls_cfg.ClientConfig `yaml:",inline"` + ClientTLSConfig tls_cfg.ClientConfig `yaml:"ruler_client"` // How frequently to evaluate rules by default. EvaluationInterval time.Duration `yaml:"evaluation_interval"` // Delay the evaluation of all rules by a set interval to give a buffer diff --git a/pkg/util/grpcclient/grpcclient.go b/pkg/util/grpcclient/grpcclient.go index 152133170a..d921fafdf8 100644 --- a/pkg/util/grpcclient/grpcclient.go +++ b/pkg/util/grpcclient/grpcclient.go @@ -7,7 +7,7 @@ import ( "google.golang.org/grpc" "github.com/cortexproject/cortex/pkg/util" - tls_cfg "github.com/cortexproject/cortex/pkg/util/tls" + tls "github.com/cortexproject/cortex/pkg/util/tls" ) // Config for a gRPC client. @@ -20,8 +20,6 @@ type Config struct { BackoffOnRatelimits bool `yaml:"backoff_on_ratelimits"` BackoffConfig util.BackoffConfig `yaml:"backoff_config"` - - TLSStruct tls_cfg.ClientConfig `yaml:",inline"` } // RegisterFlags registers flags. @@ -39,7 +37,6 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { f.BoolVar(&cfg.BackoffOnRatelimits, prefix+".backoff-on-ratelimits", false, "Enable backoff and retry when we hit ratelimits.") cfg.BackoffConfig.RegisterFlags(prefix, f) - cfg.TLSStruct.RegisterFlagsWithPrefix(prefix, f) } // CallOptions returns the config in terms of CallOptions. @@ -70,23 +67,24 @@ func (cfg *Config) DialOption(unaryClientInterceptors []grpc.UnaryClientIntercep } } -// DialOptionWithTLS returns the config as a grpc.DialOptions -func (cfg *Config) DialOptionWithTLS(unaryClientInterceptors []grpc.UnaryClientInterceptor, streamClientInterceptors []grpc.StreamClientInterceptor) ([]grpc.DialOption, error) { - if cfg.BackoffOnRatelimits { - unaryClientInterceptors = append([]grpc.UnaryClientInterceptor{NewBackoffRetry(cfg.BackoffConfig)}, unaryClientInterceptors...) - } +// ConfigWithTLS is the config for a grpc client with tls +type ConfigWithTLS struct { + GRPC Config `yaml:",inline"` + TLS tls.ClientConfig `yaml:",inline"` +} - if cfg.RateLimit > 0 { - unaryClientInterceptors = append([]grpc.UnaryClientInterceptor{NewRateLimiter(cfg)}, unaryClientInterceptors...) - } +// RegisterFlagsWithPrefix registers flags with prefix. +func (cfg *ConfigWithTLS) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { + cfg.GRPC.RegisterFlagsWithPrefix(prefix, f) + cfg.TLS.RegisterFlagsWithPrefix(prefix, f) +} - opts, err := cfg.TLSStruct.GetGRPCDialOptions() +// DialOption returns the config as a grpc.DialOptions +func (cfg *ConfigWithTLS) DialOption(unaryClientInterceptors []grpc.UnaryClientInterceptor, streamClientInterceptors []grpc.StreamClientInterceptor) ([]grpc.DialOption, error) { + opts, err := cfg.TLS.GetGRPCDialOptions() if err != nil { return nil, err } - return append(opts, grpc.WithDefaultCallOptions(cfg.CallOptions()...), - grpc.WithUnaryInterceptor(grpc_middleware.ChainUnaryClient(unaryClientInterceptors...)), - grpc.WithStreamInterceptor(grpc_middleware.ChainStreamClient(streamClientInterceptors...)), - ), nil + return append(opts, cfg.GRPC.DialOption(unaryClientInterceptors, streamClientInterceptors)...), nil } diff --git a/pkg/util/tls/tls.go b/pkg/util/tls/tls.go index eea1b4ec6f..5030deab38 100644 --- a/pkg/util/tls/tls.go +++ b/pkg/util/tls/tls.go @@ -4,13 +4,11 @@ import ( "crypto/tls" "crypto/x509" "flag" + "fmt" "io/ioutil" - "github.com/go-kit/kit/log/level" "google.golang.org/grpc" "google.golang.org/grpc/credentials" - - "github.com/cortexproject/cortex/pkg/util" ) // ClientConfig is the config for client TLS. @@ -32,15 +30,13 @@ func (cfg *ClientConfig) GetTLSConfig() (*tls.Config, error) { if cfg.CertPath != "" && cfg.KeyPath != "" && cfg.CAPath != "" { clientCert, err := tls.LoadX509KeyPair(cfg.CertPath, cfg.KeyPath) if err != nil { - level.Error(util.Logger).Log("msg", "error loading certs", "error", err) - return nil, err + return nil, fmt.Errorf("failed to load TLS certs: %v", err) } var caCertPool *x509.CertPool caCert, err := ioutil.ReadFile(cfg.CAPath) if err != nil { - level.Error(util.Logger).Log("msg", "error loading ca cert", "error", err) - return nil, err + return nil, fmt.Errorf("error loading ca cert: %v", err) } caCertPool = x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) @@ -57,13 +53,12 @@ func (cfg *ClientConfig) GetTLSConfig() (*tls.Config, error) { // GetGRPCDialOptions creates GRPC DialOptions for TLS func (cfg *ClientConfig) GetGRPCDialOptions() ([]grpc.DialOption, error) { - var opts []grpc.DialOption if tlsConfig, err := cfg.GetTLSConfig(); err != nil { - return nil, err + return nil, fmt.Errorf("error creating grpc dial options: %v", err) } else if tlsConfig != nil { - opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))) + return []grpc.DialOption{grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))}, nil } else { - opts = append(opts, grpc.WithInsecure()) + return []grpc.DialOption{grpc.WithInsecure()}, nil } - return opts, nil + return nil, nil } From 5aa65d1e9284ac6ffa529597c4725f976570bf38 Mon Sep 17 00:00:00 2001 From: Annanay Date: Wed, 13 May 2020 20:54:15 +0530 Subject: [PATCH 28/29] Fix lint, add new integration test Signed-off-by: Annanay --- docs/production/tls.md | 5 +++- integration/e2ecortex/services.go | 2 +- integration/query_frontend_test.go | 41 ++++++++++++++++++++--------- integration/util.go | 4 --- pkg/querier/store_gateway_client.go | 1 + pkg/ruler/ruler.go | 4 +-- pkg/util/grpcclient/grpcclient.go | 2 +- pkg/util/tls/tls.go | 12 ++++----- 8 files changed, 42 insertions(+), 29 deletions(-) diff --git a/docs/production/tls.md b/docs/production/tls.md index ccbb2e0093..6c1b859713 100644 --- a/docs/production/tls.md +++ b/docs/production/tls.md @@ -105,4 +105,7 @@ For a GRPC client in the Querier: # Path to the TLS CA for the GRPC Client -querier.frontend-client.tls-ca-path=/path/to/root.crt -``` \ No newline at end of file +``` + +TLS can be configured in a similar fashion for other GRPC clients like the +ingester client. \ No newline at end of file diff --git a/integration/e2ecortex/services.go b/integration/e2ecortex/services.go index b0eed836bb..586c9e961d 100644 --- a/integration/e2ecortex/services.go +++ b/integration/e2ecortex/services.go @@ -72,7 +72,7 @@ func NewQuerierWithConfigFile(name, consulAddress, configFile string, flags map[ image, e2e.NewCommandWithoutEntrypoint("cortex", e2e.BuildArgs(e2e.MergeFlags(map[string]string{ "-target": "querier", - "-log.level": "warn", + "-log.level": "debug", "-distributor.replication-factor": "1", // Ingesters ring backend. "-ring.store": "consul", diff --git a/integration/query_frontend_test.go b/integration/query_frontend_test.go index ce14d4b81f..cccf8470f7 100644 --- a/integration/query_frontend_test.go +++ b/integration/query_frontend_test.go @@ -83,6 +83,30 @@ func TestQueryFrontendWithChunksStorageViaConfigFile(t *testing.T) { }) } +func TestQueryFrontendTLSWithBlocksStorageViaFlags(t *testing.T) { + runQueryFrontendTest(t, func(t *testing.T, s *e2e.Scenario) (configFile string, flags map[string]string) { + minio := e2edb.NewMinio(9000, BlocksStorageFlags["-experimental.tsdb.s3.bucket-name"]) + require.NoError(t, s.StartAndWaitReady(minio)) + + // setup tls + cmd := exec.Command("bash", "certs/genCerts.sh", "certs", "1") + require.NoError(t, cmd.Run()) + require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+clientCertFile, clientCertFile)) + require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+clientKeyFile, clientKeyFile)) + require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+caCertFile, caCertFile)) + require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+serverCertFile, serverCertFile)) + require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+serverKeyFile, serverKeyFile)) + + return "", mergeFlags( + BlocksStorageFlags, + getServerTLSFlags(), + getClientTLSFlagsWithPrefix("ingester.client"), + getClientTLSFlagsWithPrefix("querier.frontend-client"), + getClientTLSFlagsWithPrefix("ingester.client"), + ) + }) +} + func runQueryFrontendTest(t *testing.T, setup queryFrontendSetup) { const numUsers = 10 const numQueriesPerUser = 10 @@ -97,15 +121,6 @@ func runQueryFrontendTest(t *testing.T, setup queryFrontendSetup) { configFile, flags := setup(t, s) - // setup tls - cmd := exec.Command("bash", "certs/genCerts.sh", "certs", "1") - require.NoError(t, cmd.Run()) - require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+clientCertFile, clientCertFile)) - require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+clientKeyFile, clientKeyFile)) - require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+caCertFile, caCertFile)) - require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+serverCertFile, serverCertFile)) - require.NoError(t, copyFileToSharedDir(s, integrationHomeFolder+serverKeyFile, serverKeyFile)) - flags = mergeFlags(flags, map[string]string{ "-querier.cache-results": "true", "-querier.split-queries-by-interval": "24h", @@ -113,9 +128,9 @@ func runQueryFrontendTest(t *testing.T, setup queryFrontendSetup) { }) // Start Cortex components. - queryFrontend := e2ecortex.NewQueryFrontendWithConfigFile("query-frontend", configFile, mergeFlags(flags, getServerTLSFlags()), "") - ingester := e2ecortex.NewIngesterWithConfigFile("ingester", consul.NetworkHTTPEndpoint(), configFile, mergeFlags(flags, getServerTLSFlags()), "") - distributor := e2ecortex.NewDistributorWithConfigFile("distributor", consul.NetworkHTTPEndpoint(), configFile, mergeFlags(flags, getClientTLSFlagsWithPrefix("ingester.client")), "") + queryFrontend := e2ecortex.NewQueryFrontendWithConfigFile("query-frontend", configFile, flags, "") + ingester := e2ecortex.NewIngesterWithConfigFile("ingester", consul.NetworkHTTPEndpoint(), configFile, flags, "") + distributor := e2ecortex.NewDistributorWithConfigFile("distributor", consul.NetworkHTTPEndpoint(), configFile, flags, "") require.NoError(t, s.StartAndWaitReady(queryFrontend, distributor, ingester)) // Check if we're discovering memcache or not. @@ -126,7 +141,7 @@ func runQueryFrontendTest(t *testing.T, setup queryFrontendSetup) { // able to get the query-frontend network endpoint. querier := e2ecortex.NewQuerierWithConfigFile("querier", consul.NetworkHTTPEndpoint(), configFile, mergeFlags(flags, map[string]string{ "-querier.frontend-address": queryFrontend.NetworkGRPCEndpoint(), - }, getClientTLSFlagsWithPrefix("querier.frontend-client"), getClientTLSFlagsWithPrefix("ingester.client")), "") + }), "") require.NoError(t, s.StartAndWaitReady(querier)) // Wait until both the distributor and querier have updated the ring. diff --git a/integration/util.go b/integration/util.go index 738ad17f9c..893c516ec9 100644 --- a/integration/util.go +++ b/integration/util.go @@ -54,10 +54,6 @@ func copyFileToSharedDir(s *e2e.Scenario, src, dst string) error { func getServerTLSFlags() map[string]string { return map[string]string{ - "-server.http-tls-cert-path": filepath.Join(e2e.ContainerSharedDir, serverCertFile), - "-server.http-tls-key-path": filepath.Join(e2e.ContainerSharedDir, serverKeyFile), - "-server.http-tls-client-auth": "RequireAndVerifyClientCert", - "-server.http-tls-ca-path": filepath.Join(e2e.ContainerSharedDir, caCertFile), "-server.grpc-tls-cert-path": filepath.Join(e2e.ContainerSharedDir, serverCertFile), "-server.grpc-tls-key-path": filepath.Join(e2e.ContainerSharedDir, serverKeyFile), "-server.grpc-tls-client-auth": "RequireAndVerifyClientCert", diff --git a/pkg/querier/store_gateway_client.go b/pkg/querier/store_gateway_client.go index ba79804dd5..75f3c3bfe0 100644 --- a/pkg/querier/store_gateway_client.go +++ b/pkg/querier/store_gateway_client.go @@ -63,6 +63,7 @@ func (c *storeGatewayClient) String() string { } func newStoreGatewayClientPool(discovery client.PoolServiceDiscovery, tlsCfg tls.ClientConfig, logger log.Logger, reg prometheus.Registerer) *client.Pool { + // We prefer sane defaults instead of exposing further config options. clientCfg := grpcclient.Config{ MaxRecvMsgSize: 100 << 20, MaxSendMsgSize: 16 << 20, diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index c643aff742..83fcac2176 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -35,7 +35,7 @@ import ( "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/cortexproject/cortex/pkg/util/services" - tls_cfg "github.com/cortexproject/cortex/pkg/util/tls" + "github.com/cortexproject/cortex/pkg/util/tls" ) var ( @@ -61,7 +61,7 @@ type Config struct { // This is used for template expansion in alerts; must be a valid URL. ExternalURL flagext.URLValue `yaml:"external_url"` // TLS parameters for the GRPC Client - ClientTLSConfig tls_cfg.ClientConfig `yaml:"ruler_client"` + ClientTLSConfig tls.ClientConfig `yaml:"ruler_client"` // How frequently to evaluate rules by default. EvaluationInterval time.Duration `yaml:"evaluation_interval"` // Delay the evaluation of all rules by a set interval to give a buffer diff --git a/pkg/util/grpcclient/grpcclient.go b/pkg/util/grpcclient/grpcclient.go index d921fafdf8..f2ef20e9bd 100644 --- a/pkg/util/grpcclient/grpcclient.go +++ b/pkg/util/grpcclient/grpcclient.go @@ -7,7 +7,7 @@ import ( "google.golang.org/grpc" "github.com/cortexproject/cortex/pkg/util" - tls "github.com/cortexproject/cortex/pkg/util/tls" + "github.com/cortexproject/cortex/pkg/util/tls" ) // Config for a gRPC client. diff --git a/pkg/util/tls/tls.go b/pkg/util/tls/tls.go index 5030deab38..8b4c7d2129 100644 --- a/pkg/util/tls/tls.go +++ b/pkg/util/tls/tls.go @@ -4,9 +4,9 @@ import ( "crypto/tls" "crypto/x509" "flag" - "fmt" "io/ioutil" + "github.com/pkg/errors" "google.golang.org/grpc" "google.golang.org/grpc/credentials" ) @@ -30,13 +30,13 @@ func (cfg *ClientConfig) GetTLSConfig() (*tls.Config, error) { if cfg.CertPath != "" && cfg.KeyPath != "" && cfg.CAPath != "" { clientCert, err := tls.LoadX509KeyPair(cfg.CertPath, cfg.KeyPath) if err != nil { - return nil, fmt.Errorf("failed to load TLS certs: %v", err) + return nil, errors.Wrapf(err, "failed to load TLS certificate %s,%s", cfg.CertPath, cfg.KeyPath) } var caCertPool *x509.CertPool caCert, err := ioutil.ReadFile(cfg.CAPath) if err != nil { - return nil, fmt.Errorf("error loading ca cert: %v", err) + return nil, errors.Wrapf(err, "error loading ca cert: %s", cfg.CAPath) } caCertPool = x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) @@ -54,11 +54,9 @@ func (cfg *ClientConfig) GetTLSConfig() (*tls.Config, error) { // GetGRPCDialOptions creates GRPC DialOptions for TLS func (cfg *ClientConfig) GetGRPCDialOptions() ([]grpc.DialOption, error) { if tlsConfig, err := cfg.GetTLSConfig(); err != nil { - return nil, fmt.Errorf("error creating grpc dial options: %v", err) + return nil, errors.Wrap(err, "error creating grpc dial options") } else if tlsConfig != nil { return []grpc.DialOption{grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))}, nil - } else { - return []grpc.DialOption{grpc.WithInsecure()}, nil } - return nil, nil + return []grpc.DialOption{grpc.WithInsecure()}, nil } From ae5f6f7e9347418aaacfe1df4c4d2b7397f3380b Mon Sep 17 00:00:00 2001 From: Annanay Date: Wed, 13 May 2020 23:21:13 +0530 Subject: [PATCH 29/29] Revert logging level to warn, add CHANGELOG entry Signed-off-by: Annanay --- CHANGELOG.md | 1 + integration/e2ecortex/services.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 908b544b78..6c37ce5f64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ * [FEATURE] Experimental: Added a new object storage client for OpenStack Swift. #2440 * [FEATURE] Update in dependency `weaveworks/common`. TLS config options added to the Server. #2535 * [FEATURE] Experimental: Added support for `/api/v1/metadata` Prometheus-based endpoint. #2549 +* [FEATURE] TLS config options added for GRPC clients in Querier (Query-frontend client & Ingester client), Ruler, Store Gateway, as well as HTTP client in Config store client. #2502 * [ENHANCEMENT] Experimental TSDB: sample ingestion errors are now reported via existing `cortex_discarded_samples_total` metric. #2370 * [ENHANCEMENT] Failures on samples at distributors and ingesters return the first validation error as opposed to the last. #2383 * [ENHANCEMENT] Experimental TSDB: Added `cortex_querier_blocks_meta_synced`, which reflects current state of synced blocks over all tenants. #2392 diff --git a/integration/e2ecortex/services.go b/integration/e2ecortex/services.go index 586c9e961d..b0eed836bb 100644 --- a/integration/e2ecortex/services.go +++ b/integration/e2ecortex/services.go @@ -72,7 +72,7 @@ func NewQuerierWithConfigFile(name, consulAddress, configFile string, flags map[ image, e2e.NewCommandWithoutEntrypoint("cortex", e2e.BuildArgs(e2e.MergeFlags(map[string]string{ "-target": "querier", - "-log.level": "debug", + "-log.level": "warn", "-distributor.replication-factor": "1", // Ingesters ring backend. "-ring.store": "consul",