feat(storage): support for soft delete policies and restore

go.work.sum

@@ -22,6 +22,8 @@ github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vl
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
+go.opentelemetry.io/contrib/detectors/gcp v1.22.0 h1:V/Wk7MofWKuX2zIA+xIx2M9/NDTDNCoIfPWZ0vm/pGQ=
+go.opentelemetry.io/contrib/detectors/gcp v1.22.0/go.mod h1:MiK4JblrPbDE6sDVZgP6WrcF+ARzSN1cIlvYCusAwyc=
 go.opentelemetry.io/otel/bridge/opencensus v0.40.0 h1:pqDiayRhBgoqy1vwnscik+TizcImJ58l053NScJyZso=
 go.opentelemetry.io/otel/bridge/opencensus v0.40.0/go.mod h1:1NvVHb6tLTe5A9qCYz+eErW0t8iPn4ZfR6tDKcqlGTM=
 go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=

storage/bucket.go

@@ -49,6 +49,7 @@ type BucketHandle struct {
 	userProject           string // project for Requester Pays buckets
 	retry                 *retryConfig
 	enableObjectRetention *bool
+	softDeleted           bool
 }
 
 // Bucket returns a BucketHandle, which provides operations on the named bucket.
@@ -140,6 +141,7 @@ func (b *BucketHandle) Object(name string) *ObjectHandle {
 		gen:         -1,
 		userProject: b.userProject,
 		retry:       retry,
+		softDeleted: b.softDeleted,
 	}
 }
 
@@ -479,6 +481,13 @@ type BucketAttrs struct {
 	// cannot be modified once the bucket is created.
 	// ObjectRetention cannot be configured or reported through the gRPC API.
 	ObjectRetentionMode string
+
+	// SoftDeletePolicy contains the bucket's soft delete policy, which defines
+	// the period of time that soft-deleted objects will be retained, and cannot
+	// be permanently deleted. By default, new buckets will be created with a
+	// 7 day retention duration. In order to fully disable soft delete, you need
+	// to set a policy with a RetentionDuration of 0.
+	SoftDeletePolicy *SoftDeletePolicy
 }
 
 // BucketPolicyOnly is an alias for UniformBucketLevelAccess.
@@ -766,6 +775,19 @@ type Autoclass struct {
 	TerminalStorageClassUpdateTime time.Time
 }
 
+// SoftDeletePolicy contains the bucket's soft delete policy, which defines the
+// period of time that soft-deleted objects will be retained, and cannot be
+// permanently deleted.
+type SoftDeletePolicy struct {
+	// EffectiveTime indicates the time from which the policy, or one with a
+	// greater retention, was effective. This field is read-only.
+	EffectiveTime time.Time
+
+	// RetentionDuration is the amount of time that soft-deleted objects in the
+	// bucket will be retained and cannot be permanently deleted.
+	RetentionDuration time.Duration
+}
+
 func newBucket(b *raw.Bucket) (*BucketAttrs, error) {
 	if b == nil {
 		return nil, nil
@@ -803,6 +825,7 @@ func newBucket(b *raw.Bucket) (*BucketAttrs, error) {
 		RPO:                      toRPO(b),
 		CustomPlacementConfig:    customPlacementFromRaw(b.CustomPlacementConfig),
 		Autoclass:                toAutoclassFromRaw(b.Autoclass),
+		SoftDeletePolicy:         toSoftDeletePolicyFromRaw(b.SoftDeletePolicy),
 	}, nil
 }
 
@@ -836,6 +859,7 @@ func newBucketFromProto(b *storagepb.Bucket) *BucketAttrs {
 		CustomPlacementConfig:    customPlacementFromProto(b.GetCustomPlacementConfig()),
 		ProjectNumber:            parseProjectNumber(b.GetProject()), // this can return 0 the project resource name is ID based
 		Autoclass:                toAutoclassFromProto(b.GetAutoclass()),
+		SoftDeletePolicy:         toSoftDeletePolicyFromProto(b.SoftDeletePolicy),
 	}
 }
 
@@ -891,6 +915,7 @@ func (b *BucketAttrs) toRawBucket() *raw.Bucket {
 		Rpo:                   b.RPO.String(),
 		CustomPlacementConfig: b.CustomPlacementConfig.toRawCustomPlacement(),
 		Autoclass:             b.Autoclass.toRawAutoclass(),
+		SoftDeletePolicy:      b.SoftDeletePolicy.toRawSoftDeletePolicy(),
 	}
 }
 
@@ -951,6 +976,7 @@ func (b *BucketAttrs) toProtoBucket() *storagepb.Bucket {
 		Rpo:                   b.RPO.String(),
 		CustomPlacementConfig: b.CustomPlacementConfig.toProtoCustomPlacement(),
 		Autoclass:             b.Autoclass.toProtoAutoclass(),
+		SoftDeletePolicy:      b.SoftDeletePolicy.toProtoSoftDeletePolicy(),
 	}
 }
 
@@ -1032,6 +1058,7 @@ func (ua *BucketAttrsToUpdate) toProtoBucket() *storagepb.Bucket {
 		IamConfig:             bktIAM,
 		Rpo:                   ua.RPO.String(),
 		Autoclass:             ua.Autoclass.toProtoAutoclass(),
+		SoftDeletePolicy:      ua.SoftDeletePolicy.toProtoSoftDeletePolicy(),
 		Labels:                ua.setLabels,
 	}
 }
@@ -1152,6 +1179,9 @@ type BucketAttrsToUpdate struct {
 	// See https://cloud.google.com/storage/docs/using-autoclass for more information.
 	Autoclass *Autoclass
 
+	// If set, updates the soft delete policy of the bucket.
+	SoftDeletePolicy *SoftDeletePolicy
+
 	// acl is the list of access control rules on the bucket.
 	// It is unexported and only used internally by the gRPC client.
 	// Library users should use ACLHandle methods directly.
@@ -1273,6 +1303,14 @@ func (ua *BucketAttrsToUpdate) toRawBucket() *raw.Bucket {
 		}
 		rb.ForceSendFields = append(rb.ForceSendFields, "Autoclass")
 	}
+	if ua.SoftDeletePolicy != nil {
+		if ua.SoftDeletePolicy.RetentionDuration == 0 {
+			rb.NullFields = append(rb.NullFields, "SoftDeletePolicy")
+			rb.SoftDeletePolicy = nil
+		} else {
+			rb.SoftDeletePolicy = ua.SoftDeletePolicy.toRawSoftDeletePolicy()
+		}
+	}
 	if ua.PredefinedACL != "" {
 		// Clear ACL or the call will fail.
 		rb.Acl = nil
@@ -2053,14 +2091,69 @@ func toAutoclassFromProto(a *storagepb.Bucket_Autoclass) *Autoclass {
 	}
 }
 
+func (p *SoftDeletePolicy) toRawSoftDeletePolicy() *raw.BucketSoftDeletePolicy {
+	if p == nil {
+		return nil
+	}
+	// Excluding read only field EffectiveTime.
+	return &raw.BucketSoftDeletePolicy{
+		RetentionDurationSeconds: int64(p.RetentionDuration.Seconds()),
+	}
+}
+
+func (p *SoftDeletePolicy) toProtoSoftDeletePolicy() *storagepb.Bucket_SoftDeletePolicy {
+	if p == nil {
+		return nil
+	}
+	// Excluding read only field EffectiveTime.
+	return &storagepb.Bucket_SoftDeletePolicy{
+		RetentionDuration: durationpb.New(p.RetentionDuration),
+	}
+}
+
+func toSoftDeletePolicyFromRaw(p *raw.BucketSoftDeletePolicy) *SoftDeletePolicy {
+	if p == nil {
+		return nil
+	}
+
+	policy := &SoftDeletePolicy{
+		RetentionDuration: time.Duration(p.RetentionDurationSeconds) * time.Second,
+	}
+
+	// Return EffectiveTime only if parsed to a valid value.
+	if t, err := time.Parse(time.RFC3339, p.EffectiveTime); err == nil {
+		policy.EffectiveTime = t
+	}
+
+	return policy
+}
+
+func toSoftDeletePolicyFromProto(p *storagepb.Bucket_SoftDeletePolicy) *SoftDeletePolicy {
+	if p == nil {
+		return nil
+	}
+	return &SoftDeletePolicy{
+		EffectiveTime:     p.GetEffectiveTime().AsTime(),
+		RetentionDuration: p.GetRetentionDuration().AsDuration(),
+	}
+}
+
 // Objects returns an iterator over the objects in the bucket that match the
 // Query q. If q is nil, no filtering is done. Objects will be iterated over
 // lexicographically by name.
 //
 // Note: The returned iterator is not safe for concurrent operations without explicit synchronization.
 func (b *BucketHandle) Objects(ctx context.Context, q *Query) *ObjectIterator {
 	o := makeStorageOpts(true, b.retry, b.userProject)
-	return b.c.tc.ListObjects(ctx, b.name, q, o...)
+	return b.c.tc.ListObjects(ctx, b.name, q, b.softDeleted, o...)
+}
+
+// SoftDeleted returns a new BucketHandle with the option to include
+// soft-deleted items in list results.
+func (b *BucketHandle) SoftDeleted(include bool) *BucketHandle {
+	b2 := *b
+	b2.softDeleted = include
+	return &b2
 }
 
 // Retryer returns a bucket handle that is configured with custom retry

storage/bucket_test.go

@@ -62,10 +62,11 @@ func TestBucketAttrsToRawBucket(t *testing.T) {
 				ResponseHeaders: []string{"FOO"},
 			},
 		},
-		Encryption: &BucketEncryption{DefaultKMSKeyName: "key"},
-		Logging:    &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
-		Website:    &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
-		Autoclass:  &Autoclass{Enabled: true, TerminalStorageClass: "NEARLINE"},
+		Encryption:       &BucketEncryption{DefaultKMSKeyName: "key"},
+		Logging:          &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
+		Website:          &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
+		Autoclass:        &Autoclass{Enabled: true, TerminalStorageClass: "NEARLINE"},
+		SoftDeletePolicy: &SoftDeletePolicy{RetentionDuration: time.Hour},
 		Lifecycle: Lifecycle{
 			Rules: []LifecycleRule{{
 				Action: LifecycleAction{
@@ -166,10 +167,11 @@ func TestBucketAttrsToRawBucket(t *testing.T) {
 				ResponseHeader: []string{"FOO"},
 			},
 		},
-		Encryption: &raw.BucketEncryption{DefaultKmsKeyName: "key"},
-		Logging:    &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
-		Website:    &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
-		Autoclass:  &raw.BucketAutoclass{Enabled: true, TerminalStorageClass: "NEARLINE"},
+		Encryption:       &raw.BucketEncryption{DefaultKmsKeyName: "key"},
+		Logging:          &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
+		Website:          &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
+		Autoclass:        &raw.BucketAutoclass{Enabled: true, TerminalStorageClass: "NEARLINE"},
+		SoftDeletePolicy: &raw.BucketSoftDeletePolicy{RetentionDurationSeconds: 60 * 60},
 		Lifecycle: &raw.BucketLifecycle{
 			Rule: []*raw.BucketLifecycleRule{{
 				Action: &raw.BucketLifecycleRuleAction{
@@ -395,10 +397,11 @@ func TestBucketAttrsToUpdateToRawBucket(t *testing.T) {
 				},
 			},
 		},
-		Logging:      &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
-		Website:      &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
-		StorageClass: "NEARLINE",
-		Autoclass:    &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"},
+		Logging:          &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
+		Website:          &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
+		StorageClass:     "NEARLINE",
+		Autoclass:        &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"},
+		SoftDeletePolicy: &SoftDeletePolicy{RetentionDuration: time.Hour},
 	}
 	au.SetLabel("a", "foo")
 	au.DeleteLabel("b")
@@ -439,11 +442,12 @@ func TestBucketAttrsToUpdateToRawBucket(t *testing.T) {
 				},
 			},
 		},
-		Logging:         &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
-		Website:         &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
-		StorageClass:    "NEARLINE",
-		Autoclass:       &raw.BucketAutoclass{Enabled: true, TerminalStorageClass: "ARCHIVE", ForceSendFields: []string{"Enabled"}},
-		ForceSendFields: []string{"DefaultEventBasedHold", "Lifecycle", "Autoclass"},
+		Logging:          &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
+		Website:          &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
+		StorageClass:     "NEARLINE",
+		Autoclass:        &raw.BucketAutoclass{Enabled: true, TerminalStorageClass: "ARCHIVE", ForceSendFields: []string{"Enabled"}},
+		SoftDeletePolicy: &raw.BucketSoftDeletePolicy{RetentionDurationSeconds: 3600},
+		ForceSendFields:  []string{"DefaultEventBasedHold", "Lifecycle", "Autoclass"},
 	}
 	if msg := testutil.Diff(got, want); msg != "" {
 		t.Error(msg)
@@ -463,14 +467,15 @@ func TestBucketAttrsToUpdateToRawBucket(t *testing.T) {
 
 	// Test nulls.
 	au3 := &BucketAttrsToUpdate{
-		RetentionPolicy: &RetentionPolicy{},
-		Encryption:      &BucketEncryption{},
-		Logging:         &BucketLogging{},
-		Website:         &BucketWebsite{},
+		RetentionPolicy:  &RetentionPolicy{},
+		Encryption:       &BucketEncryption{},
+		Logging:          &BucketLogging{},
+		Website:          &BucketWebsite{},
+		SoftDeletePolicy: &SoftDeletePolicy{},
 	}
 	got = au3.toRawBucket()
 	want = &raw.Bucket{
-		NullFields: []string{"RetentionPolicy", "Encryption", "Logging", "Website"},
+		NullFields: []string{"RetentionPolicy", "Encryption", "Logging", "Website", "SoftDeletePolicy"},
 	}
 	if msg := testutil.Diff(got, want); msg != "" {
 		t.Error(msg)
@@ -656,6 +661,10 @@ func TestNewBucket(t *testing.T) {
 			TerminalStorageClass:           "NEARLINE",
 			TerminalStorageClassUpdateTime: "2017-10-23T04:05:06Z",
 		},
+		SoftDeletePolicy: &raw.BucketSoftDeletePolicy{
+			EffectiveTime:            "2017-10-23T04:05:06Z",
+			RetentionDurationSeconds: 3600,
+		},
 	}
 	want := &BucketAttrs{
 		Name:                  "name",
@@ -713,6 +722,10 @@ func TestNewBucket(t *testing.T) {
 			TerminalStorageClass:           "NEARLINE",
 			TerminalStorageClassUpdateTime: time.Date(2017, 10, 23, 4, 5, 6, 0, time.UTC),
 		},
+		SoftDeletePolicy: &SoftDeletePolicy{
+			EffectiveTime:     time.Date(2017, 10, 23, 4, 5, 6, 0, time.UTC),
+			RetentionDuration: time.Hour,
+		},
 	}
 	got, err := newBucket(rb)
 	if err != nil {
@@ -768,6 +781,10 @@ func TestNewBucketFromProto(t *testing.T) {
 			TerminalStorageClass:           &autoclassTSC,
 			TerminalStorageClassUpdateTime: toProtoTimestamp(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)),
 		},
+		SoftDeletePolicy: &storagepb.Bucket_SoftDeletePolicy{
+			RetentionDuration: durationpb.New(3 * time.Hour),
+			EffectiveTime:     toProtoTimestamp(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)),
+		},
 		Lifecycle: &storagepb.Bucket_Lifecycle{
 			Rule: []*storagepb.Bucket_Lifecycle_Rule{
 				{
@@ -809,6 +826,10 @@ func TestNewBucketFromProto(t *testing.T) {
 		Logging:    &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
 		Website:    &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
 		Autoclass:  &Autoclass{Enabled: true, ToggleTime: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC), TerminalStorageClass: "NEARLINE", TerminalStorageClassUpdateTime: time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)},
+		SoftDeletePolicy: &SoftDeletePolicy{
+			EffectiveTime:     time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC),
+			RetentionDuration: time.Hour * 3,
+		},
 		Lifecycle: Lifecycle{
 			Rules: []LifecycleRule{{
 				Action: LifecycleAction{
@@ -853,10 +874,11 @@ func TestBucketAttrsToProtoBucket(t *testing.T) {
 				ResponseHeaders: []string{"FOO"},
 			},
 		},
-		Encryption: &BucketEncryption{DefaultKMSKeyName: "key"},
-		Logging:    &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
-		Website:    &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
-		Autoclass:  &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"},
+		Encryption:       &BucketEncryption{DefaultKMSKeyName: "key"},
+		Logging:          &BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
+		Website:          &BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
+		Autoclass:        &Autoclass{Enabled: true, TerminalStorageClass: "ARCHIVE"},
+		SoftDeletePolicy: &SoftDeletePolicy{RetentionDuration: time.Hour * 2},
 		Lifecycle: Lifecycle{
 			Rules: []LifecycleRule{{
 				Action: LifecycleAction{
@@ -903,10 +925,11 @@ func TestBucketAttrsToProtoBucket(t *testing.T) {
 				ResponseHeader: []string{"FOO"},
 			},
 		},
-		Encryption: &storagepb.Bucket_Encryption{DefaultKmsKey: "key"},
-		Logging:    &storagepb.Bucket_Logging{LogBucket: "projects/_/buckets/lb", LogObjectPrefix: "p"},
-		Website:    &storagepb.Bucket_Website{MainPageSuffix: "mps", NotFoundPage: "404"},
-		Autoclass:  &storagepb.Bucket_Autoclass{Enabled: true, TerminalStorageClass: &autoclassTSC},
+		Encryption:       &storagepb.Bucket_Encryption{DefaultKmsKey: "key"},
+		Logging:          &storagepb.Bucket_Logging{LogBucket: "projects/_/buckets/lb", LogObjectPrefix: "p"},
+		Website:          &storagepb.Bucket_Website{MainPageSuffix: "mps", NotFoundPage: "404"},
+		Autoclass:        &storagepb.Bucket_Autoclass{Enabled: true, TerminalStorageClass: &autoclassTSC},
+		SoftDeletePolicy: &storagepb.Bucket_SoftDeletePolicy{RetentionDuration: durationpb.New(2 * time.Hour)},
 		Lifecycle: &storagepb.Bucket_Lifecycle{
 			Rule: []*storagepb.Bucket_Lifecycle_Rule{
 				{