From 9f671b505a8accbf4d63d29dd58b49f4b0447af9 Mon Sep 17 00:00:00 2001 From: Andy Asp <90626759+andyasp@users.noreply.github.com> Date: Thu, 21 Sep 2023 11:22:46 -0400 Subject: [PATCH] Update github.com/thanos-io/objstore (#6061) * Update github.com/thanos-io/objstore * Update pkg/storage/tsdb/block/global_markers_bucket_client.go * Update github.com/thanos-io/objstore again --- go.mod | 2 +- go.sum | 8 +-- pkg/storage/bucket/client_mock.go | 2 +- pkg/storage/bucket/delayed_bucket_client.go | 4 +- pkg/storage/bucket/prefixed_bucket_client.go | 6 +-- pkg/storage/bucket/sse_bucket_client.go | 6 +-- .../block/global_markers_bucket_client.go | 6 +-- .../thanos-io/objstore/CHANGELOG.md | 10 +++- .../github.com/thanos-io/objstore/README.md | 22 +++++++- vendor/github.com/thanos-io/objstore/inmem.go | 4 +- .../github.com/thanos-io/objstore/objstore.go | 52 ++++++++++++------- .../thanos-io/objstore/prefixed_bucket.go | 6 +-- .../objstore/providers/azure/azure.go | 36 ++++++++----- .../objstore/providers/azure/helpers.go | 10 ++++ .../providers/filesystem/filesystem.go | 4 +- .../thanos-io/objstore/providers/gcs/gcs.go | 19 +++++-- .../thanos-io/objstore/providers/s3/s3.go | 19 ++++--- .../objstore/providers/swift/swift.go | 6 +-- .../github.com/thanos-io/objstore/testing.go | 4 +- .../tracing/opentracing/opentracing.go | 4 +- vendor/modules.txt | 2 +- 21 files changed, 154 insertions(+), 78 deletions(-) diff --git a/go.mod b/go.mod index e4a3a18687c..9846123464e 100644 --- a/go.mod +++ b/go.mod @@ -71,7 +71,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus v0.84.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite v0.84.0 github.com/prometheus/procfs v0.11.1 - github.com/thanos-io/objstore v0.0.0-20230727115635-d0c43443ecda + github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed github.com/xlab/treeprint v1.2.0 go.opentelemetry.io/collector/pdata v1.0.0-rcv0014 go.opentelemetry.io/otel v1.17.0 diff --git a/go.sum b/go.sum index b3da9b95e16..b44c02b713e 100644 --- a/go.sum +++ b/go.sum @@ -1156,8 +1156,8 @@ github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e/go.mod github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU= -github.com/oracle/oci-go-sdk/v65 v65.13.0 h1:0+9ea5goYfhI3/MPfbIQU6yzHYWE6sCk6VuUepxk5Nk= -github.com/oracle/oci-go-sdk/v65 v65.13.0/go.mod h1:oyMrMa1vOzzKTmPN+kqrTR9y9kPA2tU1igN3NUSNTIE= +github.com/oracle/oci-go-sdk/v65 v65.41.1 h1:+lbosOyNiib3TGJDvLq1HwEAuFqkOjPJDIkyxM15WdQ= +github.com/oracle/oci-go-sdk/v65 v65.41.1/go.mod h1:MXMLMzHnnd9wlpgadPkdlkZ9YrwQmCOmbX5kjVEJodw= github.com/ovh/go-ovh v1.4.1 h1:VBGa5wMyQtTP7Zb+w97zRCh9sLtM/2YKRyy+MEJmWaM= github.com/ovh/go-ovh v1.4.1/go.mod h1:6bL6pPyUT7tBfI0pqOegJgRjgjuO+mOo+MyXd1EEC0M= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -1296,8 +1296,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4O8IB2ozzxM= github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw= -github.com/thanos-io/objstore v0.0.0-20230727115635-d0c43443ecda h1:DtxaU/a7QRPiUhwtPrZFlS81y+9Mgny4KoLq65cu04U= -github.com/thanos-io/objstore v0.0.0-20230727115635-d0c43443ecda/go.mod h1:IS7Z25+0KaknyU2P5PTP/5hwY6Yr/FzbInF88Yd5auU= +github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed h1:iWQdY3S6DpWjelVvKKSKgS7LeLkhK4VaEnQfphB9ZXA= +github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed/go.mod h1:oJ82xgcBDzGJrEgUsjlTj6n01+ZWUMMUR8BlZzX5xDE= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= diff --git a/pkg/storage/bucket/client_mock.go b/pkg/storage/bucket/client_mock.go index 52d71cd67f2..f51bc2b39c1 100644 --- a/pkg/storage/bucket/client_mock.go +++ b/pkg/storage/bucket/client_mock.go @@ -145,7 +145,7 @@ func (m *ClientMock) IsObjNotFoundErr(err error) bool { return errors.Is(err, ErrObjectDoesNotExist) } -func (m *ClientMock) IsCustomerManagedKeyError(_ error) bool { +func (m *ClientMock) IsAccessDeniedErr(_ error) bool { return false } diff --git a/pkg/storage/bucket/delayed_bucket_client.go b/pkg/storage/bucket/delayed_bucket_client.go index 77e31cc1bd5..95662f92059 100644 --- a/pkg/storage/bucket/delayed_bucket_client.go +++ b/pkg/storage/bucket/delayed_bucket_client.go @@ -81,8 +81,8 @@ func (m *DelayedBucketClient) IsObjNotFoundErr(err error) bool { return m.wrapped.IsObjNotFoundErr(err) } -func (m *DelayedBucketClient) IsCustomerManagedKeyError(err error) bool { - return m.wrapped.IsCustomerManagedKeyError(err) +func (m *DelayedBucketClient) IsAccessDeniedErr(err error) bool { + return m.wrapped.IsAccessDeniedErr(err) } func (m *DelayedBucketClient) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error) { diff --git a/pkg/storage/bucket/prefixed_bucket_client.go b/pkg/storage/bucket/prefixed_bucket_client.go index a5d8a05c113..05843ddd9a9 100644 --- a/pkg/storage/bucket/prefixed_bucket_client.go +++ b/pkg/storage/bucket/prefixed_bucket_client.go @@ -78,9 +78,9 @@ func (b *PrefixedBucketClient) IsObjNotFoundErr(err error) bool { return b.bucket.IsObjNotFoundErr(err) } -// IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. -func (b *PrefixedBucketClient) IsCustomerManagedKeyError(err error) bool { - return b.bucket.IsCustomerManagedKeyError(err) +// IsAccessDeniedErr returns true if access to an operation is denied +func (b *PrefixedBucketClient) IsAccessDeniedErr(err error) bool { + return b.bucket.IsAccessDeniedErr(err) } // Attributes returns attributes of the specified object. diff --git a/pkg/storage/bucket/sse_bucket_client.go b/pkg/storage/bucket/sse_bucket_client.go index 469a0119e84..851474f1d23 100644 --- a/pkg/storage/bucket/sse_bucket_client.go +++ b/pkg/storage/bucket/sse_bucket_client.go @@ -124,9 +124,9 @@ func (b *SSEBucketClient) IsObjNotFoundErr(err error) bool { return b.bucket.IsObjNotFoundErr(err) } -// IsCustomerManagedKeyError implements objstore.Bucket. -func (b *SSEBucketClient) IsCustomerManagedKeyError(err error) bool { - return b.bucket.IsCustomerManagedKeyError(err) +// IsAccessDeniedErr implements objstore.Bucket. +func (b *SSEBucketClient) IsAccessDeniedErr(err error) bool { + return b.bucket.IsAccessDeniedErr(err) } // Attributes implements objstore.Bucket. diff --git a/pkg/storage/tsdb/block/global_markers_bucket_client.go b/pkg/storage/tsdb/block/global_markers_bucket_client.go index ce97311dd75..cdbeef34c88 100644 --- a/pkg/storage/tsdb/block/global_markers_bucket_client.go +++ b/pkg/storage/tsdb/block/global_markers_bucket_client.go @@ -131,9 +131,9 @@ func (b *globalMarkersBucket) IsObjNotFoundErr(err error) bool { return b.parent.IsObjNotFoundErr(err) } -// IsCustomerManagedKeyError implements objstore.Bucket. -func (b *globalMarkersBucket) IsCustomerManagedKeyError(err error) bool { - return b.parent.IsCustomerManagedKeyError(err) +// IsAccessDeniedErr implements objstore.Bucket. +func (b *globalMarkersBucket) IsAccessDeniedErr(err error) bool { + return b.parent.IsAccessDeniedErr(err) } // Attributes implements objstore.Bucket. diff --git a/vendor/github.com/thanos-io/objstore/CHANGELOG.md b/vendor/github.com/thanos-io/objstore/CHANGELOG.md index 564faa91860..0dc65033d0a 100644 --- a/vendor/github.com/thanos-io/objstore/CHANGELOG.md +++ b/vendor/github.com/thanos-io/objstore/CHANGELOG.md @@ -14,6 +14,8 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#33](https://github.com/thanos-io/objstore/pull/33) Tracing: Add `ContextWithTracer()` to inject the tracer into the context. - [#34](https://github.com/thanos-io/objstore/pull/34) Fix ignored options when creating shared credential Azure client. - [#62](https://github.com/thanos-io/objstore/pull/62) S3: Fix ignored context cancellation in `Iter` method. +- [#77](https://github.com/thanos-io/objstore/pull/77) Fix buckets wrapped with metrics from being unable to determine object sizes in `Upload`. +- [#78](https://github.com/thanos-io/objstore/pull/78) S3: Fix possible concurrent modification of the PutUserMetadata map. ### Added - [#15](https://github.com/thanos-io/objstore/pull/15) Add Oracle Cloud Infrastructure Object Storage Bucket support. @@ -27,11 +29,17 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#61](https://github.com/thanos-io/objstore/pull/61) Add OpenTelemetry TracingBucket. > This also changes the behaviour of `client.NewBucket`. Now it returns, uninstrumented and untraced bucket. You can combine `objstore.WrapWithMetrics` and `tracing/{opentelemetry,opentracing}.WrapWithTraces` to have old behavior. +- [#69](https://github.com/thanos-io/objstore/pull/69) [#66](https://github.com/thanos-io/objstore/pull/66) Add `objstore_bucket_operation_transferred_bytes` that counts the number of total bytes read from the bucket operation Get/GetRange and also counts the number of total bytes written to the bucket operation Upload. +- [#64](https://github.com/thanos-io/objstore/pull/64) OCI: OKE Workload Identity support. +- [#73](https://github.com/thanos-io/objstore/pull/73) Аdded file path to erros from DownloadFile +- [#51](https://github.com/thanos-io/objstore/pull/51) Azure: Support using connection string authentication. +- [#76](https://github.com/thanos-io/objstore/pull/76) GCS: Query for object names only in `Iter` to possibly improve performance when listing objects. ### Changed - [#38](https://github.com/thanos-io/objstore/pull/38) *: Upgrade minio-go version to `v7.0.45`. - [#39](https://github.com/thanos-io/objstore/pull/39) COS: Upgrade cos sdk version to `v0.7.40`. - [#35](https://github.com/thanos-io/objstore/pull/35) Azure: Update Azure SDK and fix breaking changes. - [#65](https://github.com/thanos-io/objstore/pull/65) *: Upgrade minio-go version to `v7.0.61`. - +- [#70](https://github.com/thanos-io/objstore/pull/70) GCS: Update cloud.google.com/go/storage version to `v1.27.0`. +- [#71](https://github.com/thanos-io/objstore/pull/71) Replace method `IsCustomerManagedKeyError` for a more generic `IsAccessDeniedErr` on the bucket interface. ### Removed diff --git a/vendor/github.com/thanos-io/objstore/README.md b/vendor/github.com/thanos-io/objstore/README.md index c738889478b..2a1552714cd 100644 --- a/vendor/github.com/thanos-io/objstore/README.md +++ b/vendor/github.com/thanos-io/objstore/README.md @@ -88,7 +88,8 @@ type BucketReader interface { // IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations. IsObjNotFoundErr(err error) bool - // IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. + // IsAccessDeniedErr returns true if access to object is denied. + IsAccessDeniedErr(err error) bool ``` Those interfaces represent the object storage operations your code can use from `objstore` clients. @@ -418,6 +419,7 @@ type: AZURE config: storage_account: "" storage_account_key: "" + storage_connection_string: "" container: "" endpoint: "" user_assigned_id: "" @@ -453,6 +455,8 @@ If `msi_resource` is used, authentication is done via system-assigned managed id If `user_assigned_id` is used, authentication is done via user-assigned managed identity. When using `user_assigned_id` the `msi_resource` defaults to `https://.` +If `storage_connection_string` is set, the values of `storage_account` and `endpoint` values will not be used. Use this method over `storage_account_key` if you need to authenticate via a SAS token. + The generic `max_retries` will be used as value for the `pipeline_config`'s `max_tries` and `reader_config`'s `max_retry_requests`. For more control, `max_retries` could be ignored (0) and one could set specific retry values. ##### OpenStack Swift @@ -578,7 +582,7 @@ prefix: "" ### Oracle Cloud Infrastructure Object Storage -To configure Oracle Cloud Infrastructure (OCI) Object Storage as Thanos Object Store, you need to provide appropriate authentication credentials to your OCI tenancy. The OCI object storage client implementation for Thanos supports either the default keypair or instance principal authentication. +To configure Oracle Cloud Infrastructure (OCI) Object Storage as a Thanos Object Store, you need to provide appropriate authentication credentials to your OCI tenancy. The OCI object storage client implementation for Thanos supports default keypair, instance principal, and OKE workload identity authentication. #### API Signing Key @@ -642,6 +646,20 @@ config: You can also include any of the optional configuration just like the example in `Default Provider`. +#### OKE Workload Identity Provider + +For Example: + +```yaml +type: OCI +config: + provider: "oke-workload-identity" + bucket: "" + region: "" +``` + +The `bucket` and `region` fields are required. The `region` field identifies the bucket region. + ##### HuaweiCloud OBS To use HuaweiCloud OBS as an object store, you should apply for a HuaweiCloud Account to create an object storage bucket at first. More details: [HuaweiCloud OBS](https://support.huaweicloud.com/obs/index.html) diff --git a/vendor/github.com/thanos-io/objstore/inmem.go b/vendor/github.com/thanos-io/objstore/inmem.go index aee4aec6cfa..3f6f35e94e7 100644 --- a/vendor/github.com/thanos-io/objstore/inmem.go +++ b/vendor/github.com/thanos-io/objstore/inmem.go @@ -207,8 +207,8 @@ func (b *InMemBucket) IsObjNotFoundErr(err error) bool { return errors.Is(err, errNotFound) } -// IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. -func (b *InMemBucket) IsCustomerManagedKeyError(_ error) bool { +// IsAccessDeniedErr returns true if access to object is denied. +func (b *InMemBucket) IsAccessDeniedErr(err error) bool { return false } diff --git a/vendor/github.com/thanos-io/objstore/objstore.go b/vendor/github.com/thanos-io/objstore/objstore.go index 2a0fe4dc5b0..20723704df5 100644 --- a/vendor/github.com/thanos-io/objstore/objstore.go +++ b/vendor/github.com/thanos-io/objstore/objstore.go @@ -85,8 +85,8 @@ type BucketReader interface { // IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations. IsObjNotFoundErr(err error) bool - // IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. - IsCustomerManagedKeyError(err error) bool + // IsAccessDeniedErr returns true if acces to object is denied. + IsAccessDeniedErr(err error) bool // Attributes returns information about the specified object. Attributes(ctx context.Context, name string) (ObjectAttributes, error) @@ -315,7 +315,7 @@ func DownloadFile(ctx context.Context, logger log.Logger, bkt BucketReader, src, f, err := os.Create(dst) if err != nil { - return errors.Wrap(err, "create file") + return errors.Wrapf(err, "create file %s", dst) } defer func() { if err != nil { @@ -327,7 +327,7 @@ func DownloadFile(ctx context.Context, logger log.Logger, bkt BucketReader, src, defer logerrcapture.Do(logger, f.Close, "close block's output file") if _, err = io.Copy(f, rc); err != nil { - return errors.Wrap(err, "copy object to file") + return errors.Wrapf(err, "copy object to file %s", src) } return nil } @@ -438,10 +438,11 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu Buckets: []float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}, }, []string{"operation"}), - lastSuccessfulUploadTime: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ - Name: "objstore_bucket_last_successful_upload_time", - Help: "Second timestamp of the last successful upload to the bucket.", - }, []string{"bucket"}), + lastSuccessfulUploadTime: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ + Name: "objstore_bucket_last_successful_upload_time", + Help: "Second timestamp of the last successful upload to the bucket.", + ConstLabels: prometheus.Labels{"bucket": name}, + }), } for _, op := range []string{ OpIter, @@ -457,15 +458,15 @@ func WrapWithMetrics(b Bucket, reg prometheus.Registerer, name string) *metricBu bkt.opsDuration.WithLabelValues(op) bkt.opsFetchedBytes.WithLabelValues(op) } - // fetched bytes only relevant for get and getrange + + // fetched bytes only relevant for get, getrange and upload for _, op := range []string{ OpGet, OpGetRange, - // TODO: Add uploads + OpUpload, } { bkt.opsTransferredBytes.WithLabelValues(op) } - bkt.lastSuccessfulUploadTime.WithLabelValues(b.Name()) return bkt } @@ -479,7 +480,7 @@ type metricBucket struct { opsFetchedBytes *prometheus.CounterVec opsTransferredBytes *prometheus.HistogramVec opsDuration *prometheus.HistogramVec - lastSuccessfulUploadTime *prometheus.GaugeVec + lastSuccessfulUploadTime prometheus.Gauge } func (b *metricBucket) WithExpectedErrs(fn IsOpFailureExpectedFunc) Bucket { @@ -592,15 +593,25 @@ func (b *metricBucket) Upload(ctx context.Context, name string, r io.Reader) err const op = OpUpload b.ops.WithLabelValues(op).Inc() - start := time.Now() - if err := b.bkt.Upload(ctx, name, r); err != nil { + trc := newTimingReadCloser( + NopCloserWithSize(r), + op, + b.opsDuration, + b.opsFailures, + b.isOpFailureExpected, + nil, + b.opsTransferredBytes, + ) + defer trc.Close() + err := b.bkt.Upload(ctx, name, trc) + if err != nil { if !b.isOpFailureExpected(err) && ctx.Err() != context.Canceled { b.opsFailures.WithLabelValues(op).Inc() } return err } - b.lastSuccessfulUploadTime.WithLabelValues(b.bkt.Name()).SetToCurrentTime() - b.opsDuration.WithLabelValues(op).Observe(time.Since(start).Seconds()) + b.lastSuccessfulUploadTime.SetToCurrentTime() + return nil } @@ -624,8 +635,8 @@ func (b *metricBucket) IsObjNotFoundErr(err error) bool { return b.bkt.IsObjNotFoundErr(err) } -func (b *metricBucket) IsCustomerManagedKeyError(err error) bool { - return b.bkt.IsCustomerManagedKeyError(err) +func (b *metricBucket) IsAccessDeniedErr(err error) bool { + return b.bkt.IsAccessDeniedErr(err) } func (b *metricBucket) Close() error { @@ -692,7 +703,10 @@ func (rc *timingReadCloser) Close() error { func (rc *timingReadCloser) Read(b []byte) (n int, err error) { n, err = rc.ReadCloser.Read(b) - rc.fetchedBytes.WithLabelValues(rc.op).Add(float64(n)) + if rc.fetchedBytes != nil { + rc.fetchedBytes.WithLabelValues(rc.op).Add(float64(n)) + } + rc.readBytes += int64(n) // Report metric just once. if !rc.alreadyGotErr && err != nil && err != io.EOF { diff --git a/vendor/github.com/thanos-io/objstore/prefixed_bucket.go b/vendor/github.com/thanos-io/objstore/prefixed_bucket.go index 41448011729..f2b71434683 100644 --- a/vendor/github.com/thanos-io/objstore/prefixed_bucket.go +++ b/vendor/github.com/thanos-io/objstore/prefixed_bucket.go @@ -74,9 +74,9 @@ func (p *PrefixedBucket) IsObjNotFoundErr(err error) bool { return p.bkt.IsObjNotFoundErr(err) } -// IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. -func (p *PrefixedBucket) IsCustomerManagedKeyError(err error) bool { - return p.bkt.IsCustomerManagedKeyError(err) +// IsAccessDeniedErr returns true if access to object is denied. +func (p *PrefixedBucket) IsAccessDeniedErr(err error) bool { + return p.bkt.IsAccessDeniedErr(err) } // Attributes returns information about the specified object. diff --git a/vendor/github.com/thanos-io/objstore/providers/azure/azure.go b/vendor/github.com/thanos-io/objstore/providers/azure/azure.go index a5f41ed1769..a72f30528ec 100644 --- a/vendor/github.com/thanos-io/objstore/providers/azure/azure.go +++ b/vendor/github.com/thanos-io/objstore/providers/azure/azure.go @@ -44,15 +44,16 @@ var DefaultConfig = Config{ // Config Azure storage configuration. type Config struct { - StorageAccountName string `yaml:"storage_account"` - StorageAccountKey string `yaml:"storage_account_key"` - ContainerName string `yaml:"container"` - Endpoint string `yaml:"endpoint"` - UserAssignedID string `yaml:"user_assigned_id"` - MaxRetries int `yaml:"max_retries"` - ReaderConfig ReaderConfig `yaml:"reader_config"` - PipelineConfig PipelineConfig `yaml:"pipeline_config"` - HTTPConfig exthttp.HTTPConfig `yaml:"http_config"` + StorageAccountName string `yaml:"storage_account"` + StorageAccountKey string `yaml:"storage_account_key"` + StorageConnectionString string `yaml:"storage_connection_string"` + ContainerName string `yaml:"container"` + Endpoint string `yaml:"endpoint"` + UserAssignedID string `yaml:"user_assigned_id"` + MaxRetries int `yaml:"max_retries"` + ReaderConfig ReaderConfig `yaml:"reader_config"` + PipelineConfig PipelineConfig `yaml:"pipeline_config"` + HTTPConfig exthttp.HTTPConfig `yaml:"http_config"` // Deprecated: Is automatically set by the Azure SDK. MSIResource string `yaml:"msi_resource"` @@ -76,6 +77,14 @@ func (conf *Config) validate() error { errMsg = append(errMsg, "user_assigned_id cannot be set when using storage_account_key authentication") } + if conf.UserAssignedID != "" && conf.StorageConnectionString != "" { + errMsg = append(errMsg, "user_assigned_id cannot be set when using storage_connection_string authentication") + } + + if conf.StorageAccountKey != "" && conf.StorageConnectionString != "" { + errMsg = append(errMsg, "storage_account_key and storage_connection_string cannot both be set") + } + if conf.StorageAccountName == "" { errMsg = append(errMsg, "storage_account_name is required but not configured") } @@ -235,9 +244,12 @@ func (b *Bucket) IsObjNotFoundErr(err error) bool { return bloberror.HasCode(err, bloberror.BlobNotFound) || bloberror.HasCode(err, bloberror.InvalidURI) } -// IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. -func (b *Bucket) IsCustomerManagedKeyError(_ error) bool { - return false +// IsAccessDeniedErr returns true if access to object is denied. +func (b *Bucket) IsAccessDeniedErr(err error) bool { + if err == nil { + return false + } + return bloberror.HasCode(err, bloberror.AuthorizationPermissionMismatch) || bloberror.HasCode(err, bloberror.InsufficientAccountPermissions) } func (b *Bucket) getBlobReader(ctx context.Context, name string, httpRange blob.HTTPRange) (io.ReadCloser, error) { diff --git a/vendor/github.com/thanos-io/objstore/providers/azure/helpers.go b/vendor/github.com/thanos-io/objstore/providers/azure/helpers.go index b76154d6c2c..7b4a5fbe51f 100644 --- a/vendor/github.com/thanos-io/objstore/providers/azure/helpers.go +++ b/vendor/github.com/thanos-io/objstore/providers/azure/helpers.go @@ -38,6 +38,16 @@ func getContainerClient(conf Config) (*container.Client, error) { Transport: &http.Client{Transport: dt}, }, } + + // Use connection string if set + if conf.StorageConnectionString != "" { + containerClient, err := container.NewClientFromConnectionString(conf.StorageConnectionString, conf.ContainerName, opt) + if err != nil { + return nil, err + } + return containerClient, nil + } + containerURL := fmt.Sprintf("https://%s.%s/%s", conf.StorageAccountName, conf.Endpoint, conf.ContainerName) // Use shared keys if set diff --git a/vendor/github.com/thanos-io/objstore/providers/filesystem/filesystem.go b/vendor/github.com/thanos-io/objstore/providers/filesystem/filesystem.go index 8ccd33b10f6..21c70485054 100644 --- a/vendor/github.com/thanos-io/objstore/providers/filesystem/filesystem.go +++ b/vendor/github.com/thanos-io/objstore/providers/filesystem/filesystem.go @@ -258,8 +258,8 @@ func (b *Bucket) IsObjNotFoundErr(err error) bool { return os.IsNotExist(errors.Cause(err)) } -// IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. -func (b *Bucket) IsCustomerManagedKeyError(_ error) bool { +// IsAccessDeniedErr returns true if access to object is denied. +func (b *Bucket) IsAccessDeniedErr(_ error) bool { return false } diff --git a/vendor/github.com/thanos-io/objstore/providers/gcs/gcs.go b/vendor/github.com/thanos-io/objstore/providers/gcs/gcs.go index 8b107c83d82..ad305d6e150 100644 --- a/vendor/github.com/thanos-io/objstore/providers/gcs/gcs.go +++ b/vendor/github.com/thanos-io/objstore/providers/gcs/gcs.go @@ -19,6 +19,8 @@ import ( "golang.org/x/oauth2/google" "google.golang.org/api/iterator" "google.golang.org/api/option" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "gopkg.in/yaml.v2" "github.com/thanos-io/objstore" @@ -106,10 +108,16 @@ func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opt delimiter = "" } - it := b.bkt.Objects(ctx, &storage.Query{ + query := &storage.Query{ Prefix: dir, Delimiter: delimiter, - }) + } + err := query.SetAttrSelection([]string{"Name"}) + if err != nil { + return err + } + + it := b.bkt.Objects(ctx, query) for { select { case <-ctx.Done(): @@ -188,8 +196,11 @@ func (b *Bucket) IsObjNotFoundErr(err error) bool { return errors.Is(err, storage.ErrObjectNotExist) } -// IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. -func (b *Bucket) IsCustomerManagedKeyError(_ error) bool { +// IsAccessDeniedErr returns true if access to object is denied. +func (b *Bucket) IsAccessDeniedErr(err error) bool { + if s, ok := status.FromError(err); ok && s.Code() == codes.PermissionDenied { + return true + } return false } diff --git a/vendor/github.com/thanos-io/objstore/providers/s3/s3.go b/vendor/github.com/thanos-io/objstore/providers/s3/s3.go index 337bd0d8127..83e3a2de768 100644 --- a/vendor/github.com/thanos-io/objstore/providers/s3/s3.go +++ b/vendor/github.com/thanos-io/objstore/providers/s3/s3.go @@ -98,9 +98,6 @@ const ( // Storage class header. amzStorageClass = "X-Amz-Storage-Class" - - // amzKmsKeyAccessDeniedErrorMessage is the error message returned by s3 when the permissions to the KMS key is revoked. - amzKmsKeyAccessDeniedErrorMessage = "The ciphertext refers to a customer master key that does not exist, does not exist in this region, or you are not allowed to access." ) var DefaultConfig = Config{ @@ -495,6 +492,13 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { if size < int64(partSize) { partSize = 0 } + + // Cloning map since minio may modify it + userMetadata := make(map[string]string, len(b.putUserMetadata)) + for k, v := range b.putUserMetadata { + userMetadata[k] = v + } + if _, err := b.client.PutObject( ctx, b.name, @@ -504,7 +508,7 @@ func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error { minio.PutObjectOptions{ PartSize: partSize, ServerSideEncryption: sse, - UserMetadata: b.putUserMetadata, + UserMetadata: userMetadata, StorageClass: b.storageClass, // 4 is what minio-go have as the default. To be certain we do micro benchmark before any changes we // ensure we pin this number to four. @@ -541,10 +545,9 @@ func (b *Bucket) IsObjNotFoundErr(err error) bool { return minio.ToErrorResponse(errors.Cause(err)).Code == "NoSuchKey" } -// IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. -func (b *Bucket) IsCustomerManagedKeyError(err error) bool { - errResponse := minio.ToErrorResponse(errors.Cause(err)) - return errResponse.Code == "AccessDenied" && errResponse.Message == amzKmsKeyAccessDeniedErrorMessage +// IsAccessDeniedErr returns true if access to object is denied. +func (b *Bucket) IsAccessDeniedErr(err error) bool { + return minio.ToErrorResponse(errors.Cause(err)).Code == "AccessDenied" } func (b *Bucket) Close() error { return nil } diff --git a/vendor/github.com/thanos-io/objstore/providers/swift/swift.go b/vendor/github.com/thanos-io/objstore/providers/swift/swift.go index c24d03fd2d5..9bfa3cf8510 100644 --- a/vendor/github.com/thanos-io/objstore/providers/swift/swift.go +++ b/vendor/github.com/thanos-io/objstore/providers/swift/swift.go @@ -290,9 +290,9 @@ func (c *Container) IsObjNotFoundErr(err error) bool { return errors.Is(err, swift.ObjectNotFound) } -// IsCustomerManagedKeyError returns true if the permissions for key used to encrypt the object was revoked. -func (b *Container) IsCustomerManagedKeyError(_ error) bool { - return false +// IsAccessDeniedErr returns true if access to object is denied. +func (c *Container) IsAccessDeniedErr(err error) bool { + return errors.Is(err, swift.Forbidden) } // Upload writes the contents of the reader as an object into the container. diff --git a/vendor/github.com/thanos-io/objstore/testing.go b/vendor/github.com/thanos-io/objstore/testing.go index 4e41b278825..b8e3744cb89 100644 --- a/vendor/github.com/thanos-io/objstore/testing.go +++ b/vendor/github.com/thanos-io/objstore/testing.go @@ -309,6 +309,6 @@ func (d *delayingBucket) IsObjNotFoundErr(err error) bool { return d.bkt.IsObjNotFoundErr(err) } -func (d *delayingBucket) IsCustomerManagedKeyError(err error) bool { - return d.bkt.IsCustomerManagedKeyError(err) +func (d *delayingBucket) IsAccessDeniedErr(err error) bool { + return d.bkt.IsAccessDeniedErr(err) } diff --git a/vendor/github.com/thanos-io/objstore/tracing/opentracing/opentracing.go b/vendor/github.com/thanos-io/objstore/tracing/opentracing/opentracing.go index 8174afb142a..8b99e304ddf 100644 --- a/vendor/github.com/thanos-io/objstore/tracing/opentracing/opentracing.go +++ b/vendor/github.com/thanos-io/objstore/tracing/opentracing/opentracing.go @@ -124,8 +124,8 @@ func (t TracingBucket) IsObjNotFoundErr(err error) bool { return t.bkt.IsObjNotFoundErr(err) } -func (t TracingBucket) IsCustomerManagedKeyError(err error) bool { - return t.bkt.IsCustomerManagedKeyError(err) +func (t TracingBucket) IsAccessDeniedErr(err error) bool { + return t.bkt.IsAccessDeniedErr(err) } func (t TracingBucket) WithExpectedErrs(expectedFunc objstore.IsOpFailureExpectedFunc) objstore.Bucket { diff --git a/vendor/modules.txt b/vendor/modules.txt index 7bdf2535f53..4d92315a46c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1015,7 +1015,7 @@ github.com/stretchr/objx github.com/stretchr/testify/assert github.com/stretchr/testify/mock github.com/stretchr/testify/require -# github.com/thanos-io/objstore v0.0.0-20230727115635-d0c43443ecda +# github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed ## explicit; go 1.18 github.com/thanos-io/objstore github.com/thanos-io/objstore/exthttp