From 92a115b1c4add603827770658527dee13f13a745 Mon Sep 17 00:00:00 2001 From: Martin Chodur Date: Sun, 7 Jun 2020 23:08:51 +0200 Subject: [PATCH] feat swift: use ncs/swift and support large files for OpenStack Swift Signed-off-by: Martin Chodur --- CHANGELOG.md | 3 + docs/storage.md | 18 +- go.mod | 2 +- go.sum | 2 + pkg/objstore/swift/swift.go | 456 +++++++++++++++++-------------- pkg/objstore/swift/swift_test.go | 19 -- scripts/cfggen/main.go | 2 +- 7 files changed, 278 insertions(+), 224 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6496d6b18d0..f1789087b40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,9 @@ We use *breaking* word for marking changes that are not backward compatible (rel - [#2667](https://github.com/thanos-io/thanos/pull/2667) Store: removed support to the legacy `index.cache.json`. The hidden flag `--store.disable-index-header` was removed. - [#2667](https://github.com/thanos-io/thanos/pull/2667) Compact: the deprecated flag `--index.generate-missing-cache-file` and the metric `thanos_compact_generated_index_total` were removed. - [2603](https://github.com/thanos-io/thanos/pull/2603) Store/Querier: Significantly optimize cases where StoreAPIs or blocks returns exact overlapping chunks (e.g Store GW and sidecar or brute force Store Gateway HA). +- [#TBA](https://github.com/thanos-io/thanos/pull/TBA) Swift: Switched to a new library [ncw/swift]() providing large objects support. + By default, segments will be uploaded to the same container directory `segments/` if the file is bigger than `1GB`. + To change the defaults see [the docs](./docs/storage.md#openstack-swift). ## [v0.13.0](https://github.com/thanos-io/thanos/releases) - IN PROGRESS diff --git a/docs/storage.md b/docs/storage.md index d95d7311d37..7d7972a63e0 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -58,7 +58,7 @@ Current object storage client implementations: | [Google Cloud Storage](./storage.md#gcs) | Stable (production usage) | yes | @bwplotka | | [AWS/S3](./storage.md#s3) | Stable (production usage) | yes | @bwplotka | | [Azure Storage Account](./storage.md#azure) | Stable (production usage) | no | @vglafirov | -| [OpenStack Swift](./storage.md#openstack-swift) | Beta (working PoCs, testing usage) | no | @sudhi-vm | +| [OpenStack Swift](./storage.md#openstack-swift) | Beta (testing usage) | no | @FUSAKLA | | [Tencent COS](./storage.md#tencent-cos) | Beta (testing usage) | no | @jojohappy | | [AliYun OSS](./storage.md#aliyun-oss) | Beta (testing usage) | no | @shaulboozhiao,@wujinhu | | [Local Filesystem](./storage.md#filesystem) | Beta (testing usage) | yes | @bwplotka | @@ -297,16 +297,25 @@ config: ### OpenStack Swift -Thanos uses [gophercloud](http://gophercloud.io/) client to upload Prometheus data into [OpenStack Swift](https://docs.openstack.org/swift/latest/). +Thanos uses [ncw/swift](https://github.com/ncw/swift) client to upload Prometheus data into [OpenStack Swift](https://docs.openstack.org/swift/latest/). Below is an example configuration file for thanos to use OpenStack swift container as an object store. Note that if the `name` of a user, project or tenant is used one must also specify its domain by ID or name. Various examples for OpenStack authentication can be found in the [official documentation](https://developer.openstack.org/api-ref/identity/v3/index.html?expanded=password-authentication-with-scoped-authorization-detail#password-authentication-with-unscoped-authorization). +By default, OpenStack Swift has a limit for maximum file size of 5 GiB. Thanos index files are often larger than that. +To resolve this issue, Thanos uses [Static Large Objects (SLO)](https://docs.openstack.org/swift/latest/overview_large_objects.html) +which are uploaded as segments. These are by default put into the `segments` directory of the same container. +Default limit for using SLO is 1 GiB which is also maximal size of the segment. +If you don't want to use the same container for the segments +(best practise is to use `_segments` to avoid polluting listing of the container objects) +you can use the `large_file_segments_container_name` option to override the default and put the segments to other container. + [embedmd]:# (flags/config_bucket_swift.txt yaml) ```yaml type: SWIFT config: + auth_version: 0 auth_url: "" username: "" user_domain_name: "" @@ -321,6 +330,11 @@ config: project_domain_name: "" region_name: "" container_name: "" + large_object_chunk_size: 1073741824 + large_object_segments_container_name: "" + retries: 3 + connect_timeout: 10s + timeout: 5m ``` ### Tencent COS diff --git a/go.mod b/go.mod index 7839e578ece..601fcd7801d 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,6 @@ require ( github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e github.com/golang/snappy v0.0.1 github.com/googleapis/gax-go v2.0.2+incompatible - github.com/gophercloud/gophercloud v0.10.0 github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/hashicorp/golang-lru v0.5.4 @@ -34,6 +33,7 @@ require ( github.com/minio/minio-go/v6 v6.0.56 github.com/mozillazg/go-cos v0.13.0 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f + github.com/ncw/swift v1.0.52 github.com/oklog/run v1.1.0 github.com/oklog/ulid v1.3.1 github.com/olekukonko/tablewriter v0.0.2 diff --git a/go.sum b/go.sum index 7f8411cf91a..3380353eb41 100644 --- a/go.sum +++ b/go.sum @@ -747,6 +747,8 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/ncw/swift v1.0.52 h1:ACF3JufDGgeKp/9mrDgQlEgS8kRYC4XKcuzj/8EJjQU= +github.com/ncw/swift v1.0.52/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= diff --git a/pkg/objstore/swift/swift.go b/pkg/objstore/swift/swift.go index c11715ce4fa..adcfbbce776 100644 --- a/pkg/objstore/swift/swift.go +++ b/pkg/objstore/swift/swift.go @@ -8,75 +8,193 @@ import ( "context" "fmt" "io" + "io/ioutil" "os" + "strconv" "strings" "testing" + "time" + + "github.com/go-kit/kit/log/level" + + "github.com/thanos-io/thanos/pkg/runutil" "github.com/go-kit/kit/log" - "github.com/gophercloud/gophercloud" - "github.com/gophercloud/gophercloud/openstack" - "github.com/gophercloud/gophercloud/openstack/objectstorage/v1/containers" - "github.com/gophercloud/gophercloud/openstack/objectstorage/v1/objects" - "github.com/gophercloud/gophercloud/pagination" + "github.com/ncw/swift" "github.com/pkg/errors" "github.com/thanos-io/thanos/pkg/objstore" "gopkg.in/yaml.v2" ) // DirDelim is the delimiter used to model a directory structure in an object store bucket. -const DirDelim = "/" - -type SwiftConfig struct { - AuthUrl string `yaml:"auth_url"` - Username string `yaml:"username"` - UserDomainName string `yaml:"user_domain_name"` - UserDomainID string `yaml:"user_domain_id"` - UserId string `yaml:"user_id"` - Password string `yaml:"password"` - DomainId string `yaml:"domain_id"` - DomainName string `yaml:"domain_name"` - ProjectID string `yaml:"project_id"` - ProjectName string `yaml:"project_name"` - ProjectDomainID string `yaml:"project_domain_id"` - ProjectDomainName string `yaml:"project_domain_name"` - RegionName string `yaml:"region_name"` - ContainerName string `yaml:"container_name"` +const DirDelim = '/' + +var DefaultConfig = Config{ + ChunkSize: 1024 * 1024 * 1024, + Retries: 3, + ConnectTimeout: "10s", + Timeout: "5m", +} + +type Config struct { + AuthVersion int `yaml:"auth_version"` + AuthUrl string `yaml:"auth_url"` + Username string `yaml:"username"` + UserDomainName string `yaml:"user_domain_name"` + UserDomainID string `yaml:"user_domain_id"` + UserId string `yaml:"user_id"` + Password string `yaml:"password"` + DomainId string `yaml:"domain_id"` + DomainName string `yaml:"domain_name"` + ProjectID string `yaml:"project_id"` + ProjectName string `yaml:"project_name"` + ProjectDomainID string `yaml:"project_domain_id"` + ProjectDomainName string `yaml:"project_domain_name"` + RegionName string `yaml:"region_name"` + ContainerName string `yaml:"container_name"` + ChunkSize int64 `yaml:"large_object_chunk_size"` + SegmentContainerName string `yaml:"large_object_segments_container_name"` + Retries int `yaml:"retries"` + ConnectTimeout string `yaml:"connect_timeout"` + Timeout string `yaml:"timeout"` +} + +func parseConfig(conf []byte) (*Config, error) { + sc := DefaultConfig + err := yaml.UnmarshalStrict(conf, &sc) + return &sc, err +} + +func configFromEnv() (*Config, error) { + c := swift.Connection{} + if err := c.ApplyEnvironment(); err != nil { + return nil, err + } + + config := Config{ + AuthVersion: c.AuthVersion, + AuthUrl: c.AuthUrl, + Password: c.ApiKey, + Username: c.UserName, + UserId: c.UserId, + DomainId: c.DomainId, + DomainName: c.Domain, + ProjectID: c.TenantId, + ProjectName: c.Tenant, + ProjectDomainID: c.TenantDomainId, + ProjectDomainName: c.TenantDomain, + RegionName: c.Region, + ContainerName: os.Getenv("OS_CONTAINER_NAME"), + SegmentContainerName: os.Getenv("SWIFT_SEGMENTS_CONTAINER_NAME"), + Retries: c.Retries, + ConnectTimeout: c.ConnectTimeout.String(), + Timeout: c.Timeout.String(), + } + if os.Getenv("SWIFT_CHUNK_SIZE") != "" { + var err error + config.ChunkSize, err = strconv.ParseInt(os.Getenv("SWIFT_CHUNK_SIZE"), 10, 64) + if err != nil { + return nil, errors.Wrap(err, "swift parsing chunk size") + } + } + return &config, nil +} + +func connectionFromConfig(sc *Config) (*swift.Connection, error) { + connection := swift.Connection{ + Domain: sc.DomainName, + DomainId: sc.DomainId, + UserName: sc.Username, + UserId: sc.UserId, + ApiKey: sc.Password, + AuthUrl: sc.AuthUrl, + Retries: sc.Retries, + Region: sc.RegionName, + AuthVersion: sc.AuthVersion, + Tenant: sc.ProjectName, + TenantId: sc.ProjectID, + TenantDomain: sc.ProjectDomainName, + TenantDomainId: sc.ProjectDomainID, + } + if sc.ConnectTimeout != "" { + connectTimeout, err := time.ParseDuration(sc.ConnectTimeout) + if err != nil { + return nil, errors.Wrap(err, "swift parsing connectionTimeout") + } + connection.ConnectTimeout = connectTimeout + } + if sc.Timeout != "" { + timeout, err := time.ParseDuration(sc.Timeout) + if err != nil { + return nil, errors.Wrap(err, "swift parsing timeout") + } + connection.Timeout = timeout + } + return &connection, nil } type Container struct { - logger log.Logger - client *gophercloud.ServiceClient - name string + logger log.Logger + name string + connection *swift.Connection + chunkSize int64 + segmentsContainer string } func NewContainer(logger log.Logger, conf []byte) (*Container, error) { sc, err := parseConfig(conf) if err != nil { - return nil, err + return nil, errors.Wrap(err, "swift parse configs") } + return NewContainerFromConfig(logger, sc, false) +} - authOpts, err := authOptsFromConfig(sc) - if err != nil { - return nil, err +func ensureContainer(connection *swift.Connection, name string, createIfNotExist bool) (string, error) { + if _, _, err := connection.Container(name); err != nil { + if err != swift.ContainerNotFound { + return "", errors.Wrapf(err, "swift verify container %s", name) + } + if !createIfNotExist { + return "", fmt.Errorf("unable to find the expected container %s", name) + } + var newContainer swift.Container + if err = connection.ContainerCreate(name, swift.Headers{}); err != nil { + return "", errors.Wrapf(err, "create container %s", name) + } + return newContainer.Name, nil } + return "", nil +} - provider, err := openstack.AuthenticatedClient(authOpts) +func NewContainerFromConfig(logger log.Logger, sc *Config, createContainer bool) (*Container, error) { + connection, err := connectionFromConfig(sc) if err != nil { - return nil, err + return nil, errors.Wrap(err, "swift load config") } - client, err := openstack.NewObjectStorageV1(provider, gophercloud.EndpointOpts{ - Region: sc.RegionName, - }) - if err != nil { + if err := connection.Authenticate(); err != nil { + return nil, errors.Wrap(err, "swift authentication") + } + + if _, err := ensureContainer(connection, sc.ContainerName, createContainer); err != nil { return nil, err } + if sc.SegmentContainerName == "" { + sc.SegmentContainerName = sc.ContainerName + } else { + if _, err := ensureContainer(connection, sc.SegmentContainerName, createContainer); err != nil { + return nil, err + } + } - return &Container{ - logger: logger, - client: client, - name: sc.ContainerName, - }, nil + container := Container{ + logger: logger, + name: sc.ContainerName, + connection: connection, + chunkSize: sc.ChunkSize, + segmentsContainer: sc.SegmentContainerName, + } + return &container, nil } // Name returns the container name for swift. @@ -87,206 +205,138 @@ func (c *Container) Name() string { // Iter calls f for each entry in the given directory. The argument to f is the full // object name including the prefix of the inspected directory. func (c *Container) Iter(ctx context.Context, dir string, f func(string) error) error { - // Ensure the object name actually ends with a dir suffix. Otherwise we'll just iterate the - // object itself as one prefix item. if dir != "" { - dir = strings.TrimSuffix(dir, DirDelim) + DirDelim + dir = strings.TrimSuffix(dir, string(DirDelim)) + string(DirDelim) } - - options := &objects.ListOpts{Full: false, Prefix: dir, Delimiter: DirDelim} - return objects.List(c.client, c.name, options).EachPage(func(page pagination.Page) (bool, error) { - objectNames, err := objects.ExtractNames(page) + return c.connection.ObjectsWalk(c.name, &swift.ObjectsOpts{Prefix: dir, Delimiter: DirDelim}, func(opts *swift.ObjectsOpts) (interface{}, error) { + objects, err := c.connection.ObjectNames(c.name, opts) if err != nil { - return false, err + return objects, errors.Wrap(err, "swift list object names") } - for _, objectName := range objectNames { - if err := f(objectName); err != nil { - return false, err + for _, object := range objects { + if err := f(object); err != nil { + return objects, errors.Wrap(err, "swift iteration over objects") } } - - return true, nil + return objects, nil }) } -// Get returns a reader for the given object name. -func (c *Container) Get(ctx context.Context, name string) (io.ReadCloser, error) { +func (c *Container) get(name string, headers swift.Headers, checkHash bool) (io.ReadCloser, error) { if name == "" { - return nil, errors.New("error, empty container name passed") + return nil, errors.New("Object name cannot be empty") } - response := objects.Download(c.client, c.name, name, nil) - return response.Body, response.Err + file, headers, err := c.connection.ObjectOpen(c.name, name, checkHash, headers) + if err != nil { + return nil, errors.Wrap(err, "swift open object") + } + return file, err +} + +// Get returns a reader for the given object name. +func (c *Container) Get(ctx context.Context, name string) (io.ReadCloser, error) { + return c.get(name, swift.Headers{}, true) } -// GetRange returns a new range reader for the given object name and range. func (c *Container) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error) { - options := objects.DownloadOpts{ - Newest: true, - Range: fmt.Sprintf("bytes=%d-%d", off, off+length-1), + // Set Range HTTP header, see the docs https://docs.openstack.org/api-ref/object-store/?expanded=show-container-details-and-list-objects-detail,get-object-content-and-metadata-detail#id76. + bytesRange := fmt.Sprintf("bytes=%d-", off) + if length != -1 { + bytesRange = fmt.Sprintf("%s%d", bytesRange, off+length-1) } - response := objects.Download(c.client, c.name, name, options) - return response.Body, response.Err + return c.get(name, swift.Headers{"Range": bytesRange}, false) } // Attributes returns information about the specified object. func (c *Container) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error) { - response := objects.Get(c.client, c.name, name, nil) - headers, err := response.Extract() + if name == "" { + return objstore.ObjectAttributes{}, errors.New("Object name cannot be empty") + } + info, _, err := c.connection.Object(c.name, name) if err != nil { - return objstore.ObjectAttributes{}, err + return objstore.ObjectAttributes{}, errors.Wrap(err, "swift get object attributes") } - return objstore.ObjectAttributes{ - Size: headers.ContentLength, - LastModified: headers.LastModified, + Size: info.Bytes, + LastModified: info.LastModified, }, nil } // Exists checks if the given object exists. func (c *Container) Exists(ctx context.Context, name string) (bool, error) { - err := objects.Get(c.client, c.name, name, nil).Err - if err == nil { - return true, nil - } - - if _, ok := err.(gophercloud.ErrDefault404); ok { - return false, nil + _, _, err := c.connection.Object(c.name, name) + if err != nil { + if c.IsObjNotFoundErr(err) { + err = nil + } + return false, errors.Wrap(err, "swift check if file exists") } - - return false, err + return true, nil } // IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations. func (c *Container) IsObjNotFoundErr(err error) bool { - _, ok := err.(gophercloud.ErrDefault404) - return ok + return errors.Is(err, swift.ObjectNotFound) } // Upload writes the contents of the reader as an object into the container. func (c *Container) Upload(ctx context.Context, name string, r io.Reader) error { - options := &objects.CreateOpts{Content: r} - res := objects.Create(c.client, c.name, name, options) - return res.Err -} - -// Delete removes the object with the given name. -func (c *Container) Delete(ctx context.Context, name string) error { - return objects.Delete(c.client, c.name, name, nil).Err -} - -func (*Container) Close() error { - // Nothing to close. - return nil -} - -func parseConfig(conf []byte) (*SwiftConfig, error) { - var sc SwiftConfig - err := yaml.UnmarshalStrict(conf, &sc) - return &sc, err -} - -func authOptsFromConfig(sc *SwiftConfig) (gophercloud.AuthOptions, error) { - authOpts := gophercloud.AuthOptions{ - IdentityEndpoint: sc.AuthUrl, - Username: sc.Username, - UserID: sc.UserId, - Password: sc.Password, - DomainID: sc.DomainId, - DomainName: sc.DomainName, - TenantID: sc.ProjectID, - TenantName: sc.ProjectName, - - // Allow Gophercloud to re-authenticate automatically. - AllowReauth: true, + size, err := objstore.TryToGetSize(r) + if err != nil { + level.Warn(c.logger).Log("msg", "could not guess file size, using large object to avoid issues if the file is larger than limit", "name", name, "err", err) + // Anything higher or equal to chunk size so the SLO is used. + size = c.chunkSize } - - // Support for cross-domain scoping (user in different domain than project). - // If a userDomainName or userDomainID is given, the user is scoped to this domain. - switch { - case sc.UserDomainName != "": - authOpts.DomainName = sc.UserDomainName - case sc.UserDomainID != "": - authOpts.DomainID = sc.UserDomainID + var file io.WriteCloser + if size >= c.chunkSize { + file, err = c.connection.StaticLargeObjectCreateFile(&swift.LargeObjectOpts{ + Container: c.name, + ObjectName: name, + ChunkSize: c.chunkSize, + SegmentContainer: c.segmentsContainer, + CheckHash: true, + }) + } else { + file, err = c.connection.ObjectCreate(c.name, name, true, "", "", swift.Headers{}) } - - // A token can be scoped to a domain or project. - // The project can be in another domain than the user, which is indicated by setting either projectDomainName or projectDomainID. - switch { - case sc.ProjectDomainName != "": - authOpts.Scope = &gophercloud.AuthScope{ - DomainName: sc.ProjectDomainName, + if err != nil { + return errors.Wrap(err, "swift failed to create file") + } + defer runutil.CloseWithLogOnErr(c.logger, file, "swift upload obj close") + switch f := file.(type) { + case io.ReaderFrom: + if _, err := f.ReadFrom(r); err != nil { + return errors.Wrap(err, "swift failed to write uploaded file") } - case sc.ProjectDomainID != "": - authOpts.Scope = &gophercloud.AuthScope{ - DomainID: sc.ProjectDomainID, + default: + buf, err := ioutil.ReadAll(r) + if err != nil { + return errors.Wrap(err, "swift failed to read uploaded file") } - } - if authOpts.Scope != nil { - switch { - case sc.ProjectName != "": - authOpts.Scope.ProjectName = sc.ProjectName - case sc.ProjectID != "": - authOpts.Scope.ProjectID = sc.ProjectID + if _, err = file.Write(buf); err != nil { + return errors.Wrap(err, "swift failed to write uploaded file") } } - return authOpts, nil -} - -func (c *Container) createContainer(name string) error { - return containers.Create(c.client, name, nil).Err -} - -func (c *Container) deleteContainer(name string) error { - return containers.Delete(c.client, name).Err + return nil } -func configFromEnv() SwiftConfig { - c := SwiftConfig{ - AuthUrl: os.Getenv("OS_AUTH_URL"), - Username: os.Getenv("OS_USERNAME"), - Password: os.Getenv("OS_PASSWORD"), - RegionName: os.Getenv("OS_REGION_NAME"), - ContainerName: os.Getenv("OS_CONTAINER_NAME"), - ProjectID: os.Getenv("OS_PROJECT_ID"), - ProjectName: os.Getenv("OS_PROJECT_NAME"), - UserDomainID: os.Getenv("OS_USER_DOMAIN_ID"), - UserDomainName: os.Getenv("OS_USER_DOMAIN_NAME"), - ProjectDomainID: os.Getenv("OS_PROJET_DOMAIN_ID"), - ProjectDomainName: os.Getenv("OS_PROJECT_DOMAIN_NAME"), - } - - return c +// Delete removes the object with the given name. +func (c *Container) Delete(ctx context.Context, name string) error { + return errors.Wrap(c.connection.LargeObjectDelete(c.name, name), "swift delete object") } -// validateForTests checks to see the config options for tests are set. -func validateForTests(conf SwiftConfig) error { - if conf.AuthUrl == "" || - conf.Username == "" || - conf.Password == "" || - (conf.ProjectName == "" && conf.ProjectID == "") || - conf.RegionName == "" { - return errors.New("insufficient swift test configuration information") - } +func (*Container) Close() error { + // Nothing to close. return nil } // NewTestContainer creates test objStore client that before returning creates temporary container. // In a close function it empties and deletes the container. func NewTestContainer(t testing.TB) (objstore.Bucket, func(), error) { - config := configFromEnv() - if err := validateForTests(config); err != nil { - return nil, nil, err - } - containerConfig, err := yaml.Marshal(config) + config, err := configFromEnv() if err != nil { - return nil, nil, err + return nil, nil, errors.Wrap(err, "swift loading config from ENV") } - - c, err := NewContainer(log.NewNopLogger(), containerConfig) - if err != nil { - return nil, nil, err - } - if config.ContainerName != "" { if os.Getenv("THANOS_ALLOW_EXISTING_BUCKET_USE") == "" { return nil, nil, errors.New("OS_CONTAINER_NAME is defined. Normally this tests will create temporary container " + @@ -295,30 +345,34 @@ func NewTestContainer(t testing.TB) (objstore.Bucket, func(), error) { "needs to be manually cleared. This means that it is only useful to run one test in a time. This is due " + "to safety (accidentally pointing prod container for test) as well as swift not being fully strong consistent.") } - + c, err := NewContainerFromConfig(log.NewNopLogger(), config, false) + if err != nil { + return nil, nil, errors.Wrap(err, "swift initializing new container") + } if err := c.Iter(context.Background(), "", func(f string) error { - return errors.Errorf("container %s is not empty", config.ContainerName) + return errors.Errorf("container %s is not empty", c.Name()) }); err != nil { - return nil, nil, errors.Wrapf(err, "swift check container %s", config.ContainerName) + return nil, nil, errors.Wrapf(err, "swift check container %s", c.Name()) } - - t.Log("WARNING. Reusing", config.ContainerName, "container for Swift tests. Manual cleanup afterwards is required") + t.Log("WARNING. Reusing", c.Name(), "container for Swift tests. Manual cleanup afterwards is required") return c, func() {}, nil } - - tmpContainerName := objstore.CreateTemporaryTestBucketName(t) - - if err := c.createContainer(tmpContainerName); err != nil { - return nil, nil, err + config.ContainerName = objstore.CreateTemporaryTestBucketName(t) + config.SegmentContainerName = config.ContainerName + c, err := NewContainerFromConfig(log.NewNopLogger(), config, true) + if err != nil { + return nil, nil, errors.Wrap(err, "swift initializing new container") } - - c.name = tmpContainerName - t.Log("created temporary container for swift tests with name", tmpContainerName) + t.Log("created temporary container for swift tests with name", c.Name()) return c, func() { objstore.EmptyBucket(t, context.Background(), c) - if err := c.deleteContainer(tmpContainerName); err != nil { - t.Logf("deleting container %s failed: %s", tmpContainerName, err) + time.Sleep(time.Second) + if err := c.connection.ContainerDelete(c.name); err != nil { + t.Logf("deleting container %s failed: %s", c.Name(), err) + } + if err := c.connection.ContainerDelete(c.segmentsContainer); err != nil { + t.Logf("deleting segments container %s failed: %s", c.segmentsContainer, err) } }, nil } diff --git a/pkg/objstore/swift/swift_test.go b/pkg/objstore/swift/swift_test.go index 0a789cc8df6..c4aac41860d 100644 --- a/pkg/objstore/swift/swift_test.go +++ b/pkg/objstore/swift/swift_test.go @@ -34,22 +34,3 @@ tenant_name: something`) // Must result in unmarshal error as there's no `tenant_name` in SwiftConfig. testutil.NotOk(t, err) } - -func TestAuthOptsFromConfig(t *testing.T) { - input := &SwiftConfig{ - AuthUrl: "http://identity.something.com/v3", - Username: "thanos", - UserDomainName: "userDomain", - ProjectName: "thanosProject", - ProjectDomainName: "projectDomain", - } - - authOpts, err := authOptsFromConfig(input) - testutil.Ok(t, err) - - testutil.Equals(t, "http://identity.something.com/v3", authOpts.IdentityEndpoint) - testutil.Equals(t, "thanos", authOpts.Username) - testutil.Equals(t, "userDomain", authOpts.DomainName) - testutil.Equals(t, "projectDomain", authOpts.Scope.DomainName) - testutil.Equals(t, "thanosProject", authOpts.Scope.ProjectName) -} diff --git a/scripts/cfggen/main.go b/scripts/cfggen/main.go index 1f764eaad14..aab609d173b 100644 --- a/scripts/cfggen/main.go +++ b/scripts/cfggen/main.go @@ -42,7 +42,7 @@ var ( client.AZURE: azure.Config{}, client.GCS: gcs.Config{}, client.S3: s3.DefaultConfig, - client.SWIFT: swift.SwiftConfig{}, + client.SWIFT: swift.DefaultConfig, client.COS: cos.Config{}, client.ALIYUNOSS: oss.Config{}, client.FILESYSTEM: filesystem.Config{},