From 03fd703d46f5cdfc2c31bcf02b09a93fad656615 Mon Sep 17 00:00:00 2001 From: milinddethe15 Date: Fri, 13 Sep 2024 21:42:18 +0530 Subject: [PATCH 1/3] add roundtripper param in NewBucket Signed-off-by: milinddethe15 --- client/factory.go | 19 +++++++++-------- client/factory_test.go | 6 +++--- go.mod | 3 +++ go.sum | 2 ++ providers/azure/azure.go | 12 +++++++---- providers/azure/azure_test.go | 27 +++++++++++++++++++++++- providers/azure/helpers.go | 17 +++++++++++---- providers/cos/cos.go | 27 +++++++++++++++++------- providers/cos/cos_test.go | 31 ++++++++++++++++++++++++++++ providers/gcs/gcs.go | 12 ++++++----- providers/gcs/gcs_test.go | 25 +++++++++++++++++++++- providers/obs/obs.go | 28 ++++++++++++++++--------- providers/oci/oci.go | 26 ++++++++++++++--------- providers/oss/oss.go | 22 +++++++++++++------- providers/s3/s3.go | 20 ++++++++++-------- providers/s3/s3_e2e_test.go | 1 + providers/s3/s3_test.go | 39 ++++++++++++++++++++++++++++------- providers/swift/swift.go | 22 +++++++++++--------- providers/swift/swift_test.go | 24 +++++++++++++++++++++ 19 files changed, 275 insertions(+), 88 deletions(-) diff --git a/client/factory.go b/client/factory.go index 9e1adec9..cfecd67c 100644 --- a/client/factory.go +++ b/client/factory.go @@ -6,6 +6,7 @@ package client import ( "context" "fmt" + "net/http" "strings" "github.com/thanos-io/objstore" @@ -49,7 +50,7 @@ type BucketConfig struct { // NewBucket initializes and returns new object storage clients. // NOTE: confContentYaml can contain secrets. -func NewBucket(logger log.Logger, confContentYaml []byte, component string) (objstore.Bucket, error) { +func NewBucket(logger log.Logger, confContentYaml []byte, component string, rt http.RoundTripper) (objstore.Bucket, error) { level.Info(logger).Log("msg", "loading bucket configuration") bucketConf := &BucketConfig{} if err := yaml.UnmarshalStrict(confContentYaml, bucketConf); err != nil { @@ -64,25 +65,25 @@ func NewBucket(logger log.Logger, confContentYaml []byte, component string) (obj var bucket objstore.Bucket switch strings.ToUpper(string(bucketConf.Type)) { case string(GCS): - bucket, err = gcs.NewBucket(context.Background(), logger, config, component) + bucket, err = gcs.NewBucket(context.Background(), logger, config, component, rt) case string(S3): - bucket, err = s3.NewBucket(logger, config, component) + bucket, err = s3.NewBucket(logger, config, component, rt) case string(AZURE): - bucket, err = azure.NewBucket(logger, config, component) + bucket, err = azure.NewBucket(logger, config, component, rt) case string(SWIFT): - bucket, err = swift.NewContainer(logger, config) + bucket, err = swift.NewContainer(logger, config, rt) case string(COS): - bucket, err = cos.NewBucket(logger, config, component) + bucket, err = cos.NewBucket(logger, config, component, rt) case string(ALIYUNOSS): - bucket, err = oss.NewBucket(logger, config, component) + bucket, err = oss.NewBucket(logger, config, component, rt) case string(FILESYSTEM): bucket, err = filesystem.NewBucketFromConfig(config) case string(BOS): bucket, err = bos.NewBucket(logger, config, component) case string(OCI): - bucket, err = oci.NewBucket(logger, config) + bucket, err = oci.NewBucket(logger, config, rt) case string(OBS): - bucket, err = obs.NewBucket(logger, config) + bucket, err = obs.NewBucket(logger, config, rt) default: return nil, errors.Errorf("bucket with type %s is not supported", bucketConf.Type) } diff --git a/client/factory_test.go b/client/factory_test.go index 4a9cf879..cd008add 100644 --- a/client/factory_test.go +++ b/client/factory_test.go @@ -23,7 +23,7 @@ func ExampleBucket() { } // Create a new bucket. - bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example") + bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example", nil) if err != nil { panic(err) } @@ -46,7 +46,7 @@ func ExampleTracingBucketUsingOpenTracing() { //nolint:govet } // Create a new bucket. - bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example") + bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example", nil) if err != nil { panic(err) } @@ -72,7 +72,7 @@ func ExampleTracingBucketUsingOpenTelemetry() { //nolint:govet } // Create a new bucket. - bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example") + bucket, err := NewBucket(log.NewNopLogger(), confContentYaml, "example", nil) if err != nil { panic(err) } diff --git a/go.mod b/go.mod index 4ec29639..5a98916f 100644 --- a/go.mod +++ b/go.mod @@ -82,6 +82,7 @@ require ( github.com/mozillazg/go-httpheader v0.2.1 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect github.com/prometheus/procfs v0.11.1 // indirect github.com/rs/xid v1.5.0 // indirect @@ -99,6 +100,7 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( @@ -107,5 +109,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect + github.com/stretchr/testify v1.9.0 golang.org/x/crypto v0.24.0 // indirect ) diff --git a/go.sum b/go.sum index f2ecc7c5..f015ecc4 100644 --- a/go.sum +++ b/go.sum @@ -214,6 +214,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4O8IB2ozzxM= diff --git a/providers/azure/azure.go b/providers/azure/azure.go index 63bd2c1a..c2f3adb5 100644 --- a/providers/azure/azure.go +++ b/providers/azure/azure.go @@ -6,6 +6,7 @@ package azure import ( "context" "io" + "net/http" "os" "strings" "testing" @@ -145,7 +146,7 @@ type Bucket struct { } // NewBucket returns a new Bucket using the provided Azure config. -func NewBucket(logger log.Logger, azureConfig []byte, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, azureConfig []byte, component string, rt http.RoundTripper) (*Bucket, error) { level.Debug(logger).Log("msg", "creating new Azure bucket connection", "component", component) conf, err := parseConfig(azureConfig) if err != nil { @@ -154,11 +155,14 @@ func NewBucket(logger log.Logger, azureConfig []byte, component string) (*Bucket if conf.MSIResource != "" { level.Warn(logger).Log("msg", "The field msi_resource has been deprecated and should no longer be set") } - return NewBucketWithConfig(logger, conf, component) + return NewBucketWithConfig(logger, conf, component, rt) } // NewBucketWithConfig returns a new Bucket using the provided Azure config struct. -func NewBucketWithConfig(logger log.Logger, conf Config, component string) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, conf Config, component string, rt http.RoundTripper) (*Bucket, error) { + if rt != nil { + conf.HTTPConfig.Transport = rt + } if err := conf.validate(); err != nil { return nil, err } @@ -355,7 +359,7 @@ func NewTestBucket(t testing.TB, component string) (objstore.Bucket, func(), err if err != nil { return nil, nil, err } - bkt, err := NewBucket(log.NewNopLogger(), bc, component) + bkt, err := NewBucket(log.NewNopLogger(), bc, component, nil) if err != nil { t.Errorf("Cannot create Azure storage container:") return nil, nil, err diff --git a/providers/azure/azure_test.go b/providers/azure/azure_test.go index c49695ab..cc9c1317 100644 --- a/providers/azure/azure_test.go +++ b/providers/azure/azure_test.go @@ -4,10 +4,13 @@ package azure import ( + "net/http" "testing" "time" "github.com/efficientgo/core/testutil" + "github.com/go-kit/log" + "github.com/pkg/errors" "github.com/thanos-io/objstore/exthttp" ) @@ -20,7 +23,7 @@ type TestCase struct { } var validConfig = []byte(`storage_account: "myStorageAccount" -storage_account_key: "abc123" +storage_account_key: "bXlTdXBlclNlY3JldEtleTEyMyFAIw==" container: "MyContainer" endpoint: "blob.core.windows.net" reader_config: @@ -222,3 +225,25 @@ http_config: testutil.Ok(t, err) testutil.Equals(t, true, transport.TLSClientConfig.InsecureSkipVerify) } + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + cfg, err := parseConfig(validConfig) + testutil.Ok(t, err) + + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + + _, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", rt) + + // We expect an error from the RoundTripper + testutil.NotOk(t, err) + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/azure/helpers.go b/providers/azure/helpers.go index 846394a0..5ced695b 100644 --- a/providers/azure/helpers.go +++ b/providers/azure/helpers.go @@ -20,9 +20,17 @@ import ( const DirDelim = "/" func getContainerClient(conf Config) (*container.Client, error) { - dt, err := exthttp.DefaultTransport(conf.HTTPConfig) - if err != nil { - return nil, err + // Check if a roundtripper has been set in the config + // otherwise build the default transport. + var rt http.RoundTripper + if conf.HTTPConfig.Transport != nil { + rt = conf.HTTPConfig.Transport + } else { + var err error + rt, err = exthttp.DefaultTransport(conf.HTTPConfig) + if err != nil { + return nil, err + } } opt := &container.ClientOptions{ ClientOptions: azcore.ClientOptions{ @@ -35,7 +43,7 @@ func getContainerClient(conf Config) (*container.Client, error) { Telemetry: policy.TelemetryOptions{ ApplicationID: "Thanos", }, - Transport: &http.Client{Transport: dt}, + Transport: &http.Client{Transport: rt}, }, } @@ -65,6 +73,7 @@ func getContainerClient(conf Config) (*container.Client, error) { // Otherwise use a token credential var cred azcore.TokenCredential + var err error // Use Managed Identity Credential if a user assigned ID is set if conf.UserAssignedID != "" { diff --git a/providers/cos/cos.go b/providers/cos/cos.go index e518cae2..ce61ca87 100644 --- a/providers/cos/cos.go +++ b/providers/cos/cos.go @@ -95,7 +95,7 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket using the provided cos configuration. -func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { if logger == nil { logger = log.NewNopLogger() } @@ -104,12 +104,11 @@ func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error if err != nil { return nil, errors.Wrap(err, "parsing cos configuration") } - - return NewBucketWithConfig(logger, config, component) + return NewBucketWithConfig(logger, config, component, rt) } // NewBucketWithConfig returns a new Bucket using the provided cos config values. -func NewBucketWithConfig(logger log.Logger, config Config, component string) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, component string, rt http.RoundTripper) (*Bucket, error) { if err := config.validate(); err != nil { return nil, errors.Wrap(err, "validate cos configuration") } @@ -128,7 +127,21 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B } } b := &cos.BaseURL{BucketURL: bucketURL} - tpt, _ := exthttp.DefaultTransport(config.HTTPConfig) + if rt != nil { + config.HTTPConfig.Transport = rt + } + // Check if a roundtripper has been set in the config + // otherwise build the default transport. + var tpt http.RoundTripper + if config.HTTPConfig.Transport != nil { + tpt = config.HTTPConfig.Transport + } else { + var err error + tpt, err = exthttp.DefaultTransport(config.HTTPConfig) + if err != nil { + return nil, err + } + } client := cos.NewClient(b, &http.Client{ Transport: &cos.AuthorizationTransport{ SecretID: config.SecretId, @@ -485,7 +498,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test", nil) if err != nil { return nil, nil, err } @@ -506,7 +519,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test", nil) if err != nil { return nil, nil, err } diff --git a/providers/cos/cos_test.go b/providers/cos/cos_test.go index 6d4316f2..c0f713b1 100644 --- a/providers/cos/cos_test.go +++ b/providers/cos/cos_test.go @@ -4,10 +4,14 @@ package cos import ( + "context" + "net/http" "testing" "time" "github.com/efficientgo/core/testutil" + "github.com/go-kit/log" + "github.com/pkg/errors" "github.com/prometheus/common/model" "github.com/thanos-io/objstore/exthttp" @@ -137,3 +141,30 @@ func TestConfig_validate(t *testing.T) { }) } } + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + config := Config{ + Bucket: "bucket", + AppId: "123", + Region: "ap-beijing", + SecretId: "sid", + SecretKey: "skey", + } + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + + bkt, err := NewBucketWithConfig(log.NewNopLogger(), config, "test", rt) + testutil.Ok(t, err) + _, err = bkt.Get(context.Background(), "Test") + // We expect an error from the RoundTripper + testutil.NotOk(t, err) + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/gcs/gcs.go b/providers/gcs/gcs.go index 87e31f66..e07ca05b 100644 --- a/providers/gcs/gcs.go +++ b/providers/gcs/gcs.go @@ -76,20 +76,22 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket against the given bucket handle. -func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component string) (*Bucket, error) { +func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, err } - return NewBucketWithConfig(ctx, logger, config, component) + return NewBucketWithConfig(ctx, logger, config, component, rt) } // NewBucketWithConfig returns a new Bucket with gcs Config struct. -func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, component string) (*Bucket, error) { +func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, component string, rt http.RoundTripper) (*Bucket, error) { if gc.Bucket == "" { return nil, errors.New("missing Google Cloud Storage bucket name for stored blocks") } - + if rt != nil { + gc.HTTPConfig.Transport = rt + } var opts []option.ClientOption // If ServiceAccount is provided, use them in GCS client, otherwise fallback to Google default logic. @@ -312,7 +314,7 @@ func NewTestBucket(t testing.TB, project string) (objstore.Bucket, func(), error return nil, nil, err } - b, err := NewBucket(ctx, log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(ctx, log.NewNopLogger(), bc, "thanos-e2e-test", nil) if err != nil { return nil, nil, err } diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index d03c5dd9..76e2b99b 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -66,7 +66,7 @@ func TestNewBucketWithConfig_ShouldCreateGRPC(t *testing.T) { err = os.Setenv("STORAGE_EMULATOR_HOST_GRPC", svr.Addr) testutil.Ok(t, err) - bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket") + bkt, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", nil) testutil.Ok(t, err) // Check if the bucket is created. @@ -157,3 +157,26 @@ http_config: }) } } + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +// func TestNewBucketWithErrorRoundTripper(t *testing.T) { +// cfg := Config{ +// Bucket: "test-bucket", +// ServiceAccount: "", +// } +// // Create an error RoundTripper +// rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + +// // Create the bucket with the custom RoundTripper +// _, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", rt) +// testutil.NotOk(t, err) // Expect an error when using the error RoundTripper +// testutil.Assert(t, err.Error() == "RoundTripper error" || errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +// } diff --git a/providers/obs/obs.go b/providers/obs/obs.go index 7bd9666b..dcd342d5 100644 --- a/providers/obs/obs.go +++ b/providers/obs/obs.go @@ -7,6 +7,7 @@ import ( "context" "io" "math" + "net/http" "os" "strings" "testing" @@ -74,13 +75,12 @@ type Bucket struct { name string } -func NewBucket(logger log.Logger, conf []byte) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, rt http.RoundTripper) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, errors.Wrap(err, "parsing cos configuration") } - - return NewBucketWithConfig(logger, config) + return NewBucketWithConfig(logger, config, rt) } func parseConfig(conf []byte) (Config, error) { @@ -92,17 +92,25 @@ func parseConfig(conf []byte) (Config, error) { return config, nil } -func NewBucketWithConfig(logger log.Logger, config Config) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, rt http.RoundTripper) (*Bucket, error) { if err := config.validate(); err != nil { return nil, errors.Wrap(err, "validate obs config err") } - - rt, err := exthttp.DefaultTransport(config.HTTPConfig) - if err != nil { - return nil, errors.Wrap(err, "get http transport err") + if rt != nil { + config.HTTPConfig.Transport = rt + } + var tpt *http.Transport + if config.HTTPConfig.Transport != nil { + tpt = config.HTTPConfig.Transport.(*http.Transport) + } else { + var err error + tpt, err = exthttp.DefaultTransport(config.HTTPConfig) + if err != nil { + return nil, errors.Wrap(err, "get http transport err") + } } - client, err := obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(rt)) + client, err := obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(tpt)) if err != nil { return nil, errors.Wrap(err, "initialize obs client err") } @@ -369,7 +377,7 @@ func NewTestBucketFromConfig(t testing.TB, c Config, reuseBucket bool, location if err != nil { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc) + b, err := NewBucket(log.NewNopLogger(), bc, nil) if err != nil { return nil, nil, err } diff --git a/providers/oci/oci.go b/providers/oci/oci.go index 2db35461..e8f8e0c5 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -58,13 +58,14 @@ type HTTPConfig struct { ResponseHeaderTimeout model.Duration `yaml:"response_header_timeout"` InsecureSkipVerify bool `yaml:"insecure_skip_verify"` - TLSHandshakeTimeout model.Duration `yaml:"tls_handshake_timeout"` - ExpectContinueTimeout model.Duration `yaml:"expect_continue_timeout"` - MaxIdleConns int `yaml:"max_idle_conns"` - MaxIdleConnsPerHost int `yaml:"max_idle_conns_per_host"` - MaxConnsPerHost int `yaml:"max_conns_per_host"` - DisableCompression bool `yaml:"disable_compression"` - ClientTimeout time.Duration `yaml:"client_timeout"` + TLSHandshakeTimeout model.Duration `yaml:"tls_handshake_timeout"` + ExpectContinueTimeout model.Duration `yaml:"expect_continue_timeout"` + MaxIdleConns int `yaml:"max_idle_conns"` + MaxIdleConnsPerHost int `yaml:"max_idle_conns_per_host"` + MaxConnsPerHost int `yaml:"max_conns_per_host"` + DisableCompression bool `yaml:"disable_compression"` + ClientTimeout time.Duration `yaml:"client_timeout"` + Transport http.RoundTripper `yaml:"-"` } // Config stores the configuration for oci bucket. @@ -288,7 +289,7 @@ func (b *Bucket) deleteBucket(ctx context.Context) (err error) { } // NewBucket returns a new Bucket using the provided oci config values. -func NewBucket(logger log.Logger, ociConfig []byte) (*Bucket, error) { +func NewBucket(logger log.Logger, ociConfig []byte, rt http.RoundTripper) (*Bucket, error) { level.Debug(logger).Log("msg", "creating new oci bucket connection") var config = DefaultConfig var configurationProvider common.ConfigurationProvider @@ -335,8 +336,13 @@ func NewBucket(logger log.Logger, ociConfig []byte) (*Bucket, error) { return nil, errors.Wrapf(err, "unable to create ObjectStorage client with the given oci configurations") } + if rt != nil { + config.HTTPConfig.Transport = rt + } else { + config.HTTPConfig.Transport = CustomTransport(config) + } httpClient := http.Client{ - Transport: CustomTransport(config), + Transport: config.HTTPConfig.Transport, Timeout: config.HTTPConfig.ClientTimeout, } client.HTTPClient = &httpClient @@ -375,7 +381,7 @@ func NewTestBucket(t testing.TB) (objstore.Bucket, func(), error) { return nil, nil, err } - bkt, err := NewBucket(log.NewNopLogger(), ociConfig) + bkt, err := NewBucket(log.NewNopLogger(), ociConfig, nil) if err != nil { return nil, nil, err } diff --git a/providers/oss/oss.go b/providers/oss/oss.go index e7e3a648..572778d1 100644 --- a/providers/oss/oss.go +++ b/providers/oss/oss.go @@ -158,22 +158,30 @@ func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAt } // NewBucket returns a new Bucket using the provided oss config values. -func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { var config Config if err := yaml.Unmarshal(conf, &config); err != nil { return nil, errors.Wrap(err, "parse aliyun oss config file failed") } - - return NewBucketWithConfig(logger, config, component) + return NewBucketWithConfig(logger, config, component, rt) } // NewBucketWithConfig returns a new Bucket using the provided oss config struct. -func NewBucketWithConfig(logger log.Logger, config Config, component string) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, component string, rt http.RoundTripper) (*Bucket, error) { if err := validate(config); err != nil { return nil, err } - - client, err := alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret) + var client *alioss.Client + var err error + if rt != nil { + // custom RoundTripper + clientOption := func(client *alioss.Client) { + client.HTTPClient = &http.Client{Transport: rt} + } + client, err = alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret, clientOption) + } else { + client, err = alioss.New(config.Endpoint, config.AccessKeyID, config.AccessKeySecret) + } if err != nil { return nil, errors.Wrap(err, "create aliyun oss client failed") } @@ -274,7 +282,7 @@ func NewTestBucketFromConfig(t testing.TB, c Config, reuseBucket bool) (objstore return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-aliyun-oss-test") + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-aliyun-oss-test", nil) if err != nil { return nil, nil, err } diff --git a/providers/s3/s3.go b/providers/s3/s3.go index dad89e66..5fc5f188 100644 --- a/providers/s3/s3.go +++ b/providers/s3/s3.go @@ -176,13 +176,13 @@ func parseConfig(conf []byte) (Config, error) { } // NewBucket returns a new Bucket using the provided s3 config values. -func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { +func NewBucket(logger log.Logger, conf []byte, component string, rt http.RoundTripper) (*Bucket, error) { config, err := parseConfig(conf) if err != nil { return nil, err } - return NewBucketWithConfig(logger, config, component) + return NewBucketWithConfig(logger, config, component, rt) } type overrideSignerType struct { @@ -202,7 +202,7 @@ func (s *overrideSignerType) Retrieve() (credentials.Value, error) { } // NewBucketWithConfig returns a new Bucket using the provided s3 config values. -func NewBucketWithConfig(logger log.Logger, config Config, component string) (*Bucket, error) { +func NewBucketWithConfig(logger log.Logger, config Config, component string, rt http.RoundTripper) (*Bucket, error) { var chain []credentials.Provider // TODO(bwplotka): Don't do flags as they won't scale, use actual params like v2, v4 instead @@ -242,15 +242,17 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B }), } } - + if rt != nil { + config.HTTPConfig.Transport = rt + } // Check if a roundtripper has been set in the config // otherwise build the default transport. - var rt http.RoundTripper + var tpt http.RoundTripper if config.HTTPConfig.Transport != nil { - rt = config.HTTPConfig.Transport + tpt = config.HTTPConfig.Transport } else { var err error - rt, err = exthttp.DefaultTransport(config.HTTPConfig) + tpt, err = exthttp.DefaultTransport(config.HTTPConfig) if err != nil { return nil, err } @@ -260,7 +262,7 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string) (*B Creds: credentials.NewChainCredentials(chain), Secure: !config.Insecure, Region: config.Region, - Transport: rt, + Transport: tpt, BucketLookup: config.BucketLookupType.MinioType(), }) if err != nil { @@ -611,7 +613,7 @@ func NewTestBucketFromConfig(t testing.TB, location string, c Config, reuseBucke if err != nil { return nil, nil, err } - b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test") + b, err := NewBucket(log.NewNopLogger(), bc, "thanos-e2e-test", nil) if err != nil { return nil, nil, err } diff --git a/providers/s3/s3_e2e_test.go b/providers/s3/s3_e2e_test.go index 4b75a014..ac9ec261 100644 --- a/providers/s3/s3_e2e_test.go +++ b/providers/s3/s3_e2e_test.go @@ -37,6 +37,7 @@ func BenchmarkUpload(b *testing.B) { log.NewNopLogger(), e2ethanos.NewS3Config(bucket, m.Endpoint("https"), m.Dir()), "test-feed", + nil, ) testutil.Ok(b, err) diff --git a/providers/s3/s3_test.go b/providers/s3/s3_test.go index cdab39c3..049b330b 100644 --- a/providers/s3/s3_test.go +++ b/providers/s3/s3_test.go @@ -7,6 +7,7 @@ import ( "context" "encoding/base64" "encoding/json" + "errors" "io" "net/http" "net/http/httptest" @@ -324,7 +325,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { // Default config should return no SSE config. cfg := DefaultConfig cfg.Endpoint = endpoint - bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) sse, err := bkt.getServerSideEncryption(context.Background()) @@ -335,7 +336,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { cfg = DefaultConfig cfg.Endpoint = endpoint cfg.SSEConfig = SSEConfig{Type: SSES3} - bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) sse, err = bkt.getServerSideEncryption(context.Background()) @@ -351,7 +352,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { Type: SSEKMS, KMSKeyID: "key", } - bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) sse, err = bkt.getServerSideEncryption(context.Background()) @@ -375,7 +376,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { KMSKeyID: "key", KMSEncryptionContext: map[string]string{"foo": "bar"}, } - bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) sse, err = bkt.getServerSideEncryption(context.Background()) @@ -396,7 +397,7 @@ func TestBucket_getServerSideEncryption(t *testing.T) { override, err := encrypt.NewSSEKMS("test", nil) testutil.Ok(t, err) - bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err = NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) sse, err = bkt.getServerSideEncryption(context.WithValue(context.Background(), sseConfigKey, override)) @@ -423,7 +424,7 @@ func TestBucket_Get_ShouldReturnErrorIfServerTruncateResponse(t *testing.T) { cfg.AccessKey = "test" cfg.SecretKey = "test" - bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) reader, err := bkt.Get(context.Background(), "test") @@ -448,7 +449,7 @@ func TestParseConfig_CustomStorageClass(t *testing.T) { cfg.Endpoint = endpoint storageClass := "STANDARD_IA" cfg.PutUserMetadata[testCase.storageClassKey] = storageClass - bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) testutil.Equals(t, storageClass, bkt.storageClass) }) @@ -458,7 +459,29 @@ func TestParseConfig_CustomStorageClass(t *testing.T) { func TestParseConfig_DefaultStorageClassIsZero(t *testing.T) { cfg := DefaultConfig cfg.Endpoint = endpoint - bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test") + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", nil) testutil.Ok(t, err) testutil.Equals(t, "", bkt.storageClass) } + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + cfg := DefaultConfig + cfg.Endpoint = endpoint + cfg.Bucket = "test" + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + bkt, err := NewBucketWithConfig(log.NewNopLogger(), cfg, "test", rt) + testutil.Ok(t, err) + _, err = bkt.Get(context.Background(), "test") + // We expect an error from the RoundTripper + testutil.NotOk(t, err) + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/swift/swift.go b/providers/swift/swift.go index a8c56c55..f287f551 100644 --- a/providers/swift/swift.go +++ b/providers/swift/swift.go @@ -154,12 +154,12 @@ type Container struct { segmentsContainer string } -func NewContainer(logger log.Logger, conf []byte) (*Container, error) { +func NewContainer(logger log.Logger, conf []byte, rt http.RoundTripper) (*Container, error) { sc, err := parseConfig(conf) if err != nil { return nil, errors.Wrap(err, "parse config") } - return NewContainerFromConfig(logger, sc, false) + return NewContainerFromConfig(logger, sc, false, rt) } func ensureContainer(connection *swift.Connection, name string, createIfNotExist bool) error { @@ -178,22 +178,24 @@ func ensureContainer(connection *swift.Connection, name string, createIfNotExist return nil } -func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool) (*Container, error) { - +func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool, rt http.RoundTripper) (*Container, error) { + if rt != nil { + sc.HTTPConfig.Transport = rt + } // Check if a roundtripper has been set in the config // otherwise build the default transport. - var rt http.RoundTripper + var tpt http.RoundTripper if sc.HTTPConfig.Transport != nil { - rt = sc.HTTPConfig.Transport + tpt = sc.HTTPConfig.Transport } else { var err error - rt, err = exthttp.DefaultTransport(sc.HTTPConfig) + tpt, err = exthttp.DefaultTransport(sc.HTTPConfig) if err != nil { return nil, err } } - connection := connectionFromConfig(sc, rt) + connection := connectionFromConfig(sc, tpt) if err := connection.Authenticate(); err != nil { return nil, errors.Wrap(err, "authentication") } @@ -378,7 +380,7 @@ func NewTestContainer(t testing.TB) (objstore.Bucket, func(), error) { "needs to be manually cleared. This means that it is only useful to run one test in a time. This is due " + "to safety (accidentally pointing prod container for test) as well as swift not being fully strong consistent.") } - c, err := NewContainerFromConfig(log.NewNopLogger(), config, false) + c, err := NewContainerFromConfig(log.NewNopLogger(), config, false, nil) if err != nil { return nil, nil, errors.Wrap(err, "initializing new container") } @@ -392,7 +394,7 @@ func NewTestContainer(t testing.TB) (objstore.Bucket, func(), error) { } config.ContainerName = objstore.CreateTemporaryTestBucketName(t) config.SegmentContainerName = config.ContainerName - c, err := NewContainerFromConfig(log.NewNopLogger(), config, true) + c, err := NewContainerFromConfig(log.NewNopLogger(), config, true, nil) if err != nil { return nil, nil, errors.Wrap(err, "initializing new container") } diff --git a/providers/swift/swift_test.go b/providers/swift/swift_test.go index 656e7756..a203378c 100644 --- a/providers/swift/swift_test.go +++ b/providers/swift/swift_test.go @@ -4,10 +4,13 @@ package swift import ( + "net/http" "testing" "time" "github.com/efficientgo/core/testutil" + "github.com/go-kit/log" + "github.com/pkg/errors" "github.com/prometheus/common/model" ) @@ -64,3 +67,24 @@ http_config: testutil.Equals(t, false, cfg.HTTPConfig.InsecureSkipVerify) } + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + logger := log.NewNopLogger() + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + config := DefaultConfig + config.AuthUrl = "http://identity.something.com/v3" + _, err := NewContainerFromConfig(logger, &config, false, rt) + + // We expect an error from the RoundTripper + testutil.NotOk(t, err) + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} From 528b2c24299388f74c1a2c56f699fa39b8c95a23 Mon Sep 17 00:00:00 2001 From: milinddethe15 Date: Sun, 15 Sep 2024 05:26:56 +0530 Subject: [PATCH 2/3] add tests for custom rt Signed-off-by: milinddethe15 --- go.mod | 4 +-- go.sum | 5 ++-- providers/cos/cos_test.go | 2 +- providers/gcs/gcs_test.go | 34 ++++++++++++++--------- providers/obs/obs_test.go | 33 +++++++++++++++++++++++ providers/oci/oci_test.go | 57 +++++++++++++++++++++++++++++++++++++++ providers/oss/oss_test.go | 39 +++++++++++++++++++++++++++ 7 files changed, 154 insertions(+), 20 deletions(-) create mode 100644 providers/obs/obs_test.go create mode 100644 providers/oci/oci_test.go create mode 100644 providers/oss/oss_test.go diff --git a/go.mod b/go.mod index 5a98916f..1d10fb58 100644 --- a/go.mod +++ b/go.mod @@ -82,7 +82,6 @@ require ( github.com/mozillazg/go-httpheader v0.2.1 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect github.com/prometheus/procfs v0.11.1 // indirect github.com/rs/xid v1.5.0 // indirect @@ -100,7 +99,6 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( @@ -109,6 +107,6 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 github.com/kr/text v0.2.0 // indirect github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.9.0 // indirect golang.org/x/crypto v0.24.0 // indirect ) diff --git a/go.sum b/go.sum index f015ecc4..61b88553 100644 --- a/go.sum +++ b/go.sum @@ -204,16 +204,15 @@ github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= diff --git a/providers/cos/cos_test.go b/providers/cos/cos_test.go index c0f713b1..d67e2dfd 100644 --- a/providers/cos/cos_test.go +++ b/providers/cos/cos_test.go @@ -155,7 +155,7 @@ func TestNewBucketWithErrorRoundTripper(t *testing.T) { config := Config{ Bucket: "bucket", AppId: "123", - Region: "ap-beijing", + Region: "test", SecretId: "sid", SecretKey: "skey", } diff --git a/providers/gcs/gcs_test.go b/providers/gcs/gcs_test.go index 76e2b99b..120e3450 100644 --- a/providers/gcs/gcs_test.go +++ b/providers/gcs/gcs_test.go @@ -15,6 +15,7 @@ import ( "github.com/efficientgo/core/testutil" "github.com/fullstorydev/emulators/storage/gcsemu" "github.com/go-kit/log" + "github.com/pkg/errors" "github.com/prometheus/common/model" "google.golang.org/api/option" ) @@ -167,16 +168,23 @@ func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { return nil, ert.Err } -// func TestNewBucketWithErrorRoundTripper(t *testing.T) { -// cfg := Config{ -// Bucket: "test-bucket", -// ServiceAccount: "", -// } -// // Create an error RoundTripper -// rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} - -// // Create the bucket with the custom RoundTripper -// _, err := NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", rt) -// testutil.NotOk(t, err) // Expect an error when using the error RoundTripper -// testutil.Assert(t, err.Error() == "RoundTripper error" || errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) -// } +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + // Create an error RoundTripper + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + cfg := Config{ + Bucket: "test-bucket", + ServiceAccount: "", + UseGRPC: false, + } + svr, err := gcsemu.NewServer("127.0.0.1:0", gcsemu.Options{}) + testutil.Ok(t, err) + err = os.Setenv("STORAGE_EMULATOR_HOST", svr.Addr) + testutil.Ok(t, err) + err = os.Setenv("GCS_EMULATOR_HOST", svr.Addr) + testutil.Ok(t, err) + defer svr.Close() + + _, err = NewBucketWithConfig(context.Background(), log.NewNopLogger(), cfg, "test-bucket", rt) + testutil.NotOk(t, err) // Expect an error when using the error RoundTripper + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/obs/obs_test.go b/providers/obs/obs_test.go new file mode 100644 index 00000000..26563b3c --- /dev/null +++ b/providers/obs/obs_test.go @@ -0,0 +1,33 @@ +package obs + +import ( + "net/http" + "testing" + + "github.com/efficientgo/core/testutil" + "github.com/go-kit/log" + "github.com/pkg/errors" +) + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error + Transport *http.Transport +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + config := DefaultConfig + config.Endpoint = "https://test.com" + config.SecretKey = "test123" + config.AccessKey = "test123" + + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + _, err := NewBucketWithConfig(log.NewNopLogger(), config, rt) + // We expect an error from the RoundTripper + testutil.NotOk(t, err) + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/oci/oci_test.go b/providers/oci/oci_test.go new file mode 100644 index 00000000..87288cee --- /dev/null +++ b/providers/oci/oci_test.go @@ -0,0 +1,57 @@ +package oci + +import ( + "net/http" + "testing" + + "github.com/efficientgo/core/testutil" + "github.com/go-kit/log" + "github.com/pkg/errors" + "gopkg.in/yaml.v2" +) + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + + mockPrivateKey := `-----BEGIN RSA PRIVATE KEY----- +MIICXgIBAAKBgQDCFENGw33yGihy92pDjZQhl0C36rPJj+CvfSC8+q28hxA161QF +NUd13wuCTUcq0Qd2qsBe/2hFyc2DCJJg0h1L78+6Z4UMR7EOcpfdUE9Hf3m/hs+F +UR45uBJeDK1HSFHD8bHKD6kv8FPGfJTotc+2xjJwoYi+1hqp1fIekaxsyQIDAQAB +AoGBAJR8ZkCUvx5kzv+utdl7T5MnordT1TvoXXJGXK7ZZ+UuvMNUCdN2QPc4sBiA +QWvLw1cSKt5DsKZ8UETpYPy8pPYnnDEz2dDYiaew9+xEpubyeW2oH4Zx71wqBtOK +kqwrXa/pzdpiucRRjk6vE6YY7EBBs/g7uanVpGibOVAEsqH1AkEA7DkjVH28WDUg +f1nqvfn2Kj6CT7nIcE3jGJsZZ7zlZmBmHFDONMLUrXR/Zm3pR5m0tCmBqa5RK95u +412jt1dPIwJBANJT3v8pnkth48bQo/fKel6uEYyboRtA5/uHuHkZ6FQF7OUkGogc +mSJluOdc5t6hI1VsLn0QZEjQZMEOWr+wKSMCQQCC4kXJEsHAve77oP6HtG/IiEn7 +kpyUXRNvFsDE0czpJJBvL/aRFUJxuRK91jhjC68sA7NsKMGg5OXb5I5Jj36xAkEA +gIT7aFOYBFwGgQAQkWNKLvySgKbAZRTeLBacpHMuQdl1DfdntvAyqpAZ0lY0RKmW +G6aFKaqQfOXKCyWoUiVknQJAXrlgySFci/2ueKlIE1QqIiLSZ8V8OlpFLRnb1pzI +7U1yQXnTAEFYM560yJlzUpOb1V4cScGd365tiSMvxLOvTA== +-----END RSA PRIVATE KEY-----` + + config := DefaultConfig + config.Provider = "raw" + config.Tenancy = "test" + config.User = "test" + config.Region = "test" + config.Fingerprint = "123" + config.PrivateKey = mockPrivateKey + config.Passphrase = "123" + ociConfig, err := yaml.Marshal(config) + testutil.Ok(t, err) + + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + + _, err = NewBucket(log.NewNopLogger(), ociConfig, rt) + // We expect an error from the RoundTripper + testutil.NotOk(t, err) + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} diff --git a/providers/oss/oss_test.go b/providers/oss/oss_test.go new file mode 100644 index 00000000..23f174d5 --- /dev/null +++ b/providers/oss/oss_test.go @@ -0,0 +1,39 @@ +package oss + +import ( + "context" + "net/http" + "testing" + + "github.com/efficientgo/core/testutil" + "github.com/go-kit/log" + "github.com/pkg/errors" +) + +// ErrorRoundTripper is a custom RoundTripper that always returns an error +type ErrorRoundTripper struct { + Err error +} + +func (ert *ErrorRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { + return nil, ert.Err +} + +func TestNewBucketWithErrorRoundTripper(t *testing.T) { + + var config Config + config.Endpoint = "http://test.com/" + config.AccessKeyID = "123" + config.AccessKeySecret = "123" + config.Bucket = "test" + config.AccessKeySecret = "123" + + rt := &ErrorRoundTripper{Err: errors.New("RoundTripper error")} + + bkt, err := NewBucketWithConfig(log.NewNopLogger(), config, "test", rt) + // We expect an error from the RoundTripper + testutil.Ok(t, err) + _, err = bkt.Get(context.Background(), "test") + testutil.NotOk(t, err) + testutil.Assert(t, errors.Is(err, rt.Err), "Expected RoundTripper error, got: %v", err) +} From f2f0217f183f5d4a1db10a86cbb48919a61b5646 Mon Sep 17 00:00:00 2001 From: Milind Dethe <99114125+milinddethe15@users.noreply.github.com> Date: Sun, 15 Sep 2024 00:26:25 +0530 Subject: [PATCH 3/3] fix lint Signed-off-by: Milind Dethe <99114125+milinddethe15@users.noreply.github.com> --- providers/s3/s3_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/providers/s3/s3_test.go b/providers/s3/s3_test.go index 049b330b..e34e19bb 100644 --- a/providers/s3/s3_test.go +++ b/providers/s3/s3_test.go @@ -7,7 +7,6 @@ import ( "context" "encoding/base64" "encoding/json" - "errors" "io" "net/http" "net/http/httptest" @@ -17,7 +16,8 @@ import ( "github.com/efficientgo/core/testutil" "github.com/go-kit/log" "github.com/minio/minio-go/v7/pkg/encrypt" - + "github.com/pkg/errors" + "github.com/thanos-io/objstore/exthttp" )