Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add ability to configure and utilize soft-delete and restore #2425

Merged
merged 1 commit into from
Mar 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions src/bucket.ts
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,7 @@ export interface GetFilesOptions {
maxApiCalls?: number;
maxResults?: number;
pageToken?: string;
softDeleted?: boolean;
startOffset?: string;
userProject?: string;
versions?: boolean;
Expand Down Expand Up @@ -341,6 +342,10 @@ export interface BucketMetadata extends BaseMetadata {
retentionPeriod?: string | number;
} | null;
rpo?: string;
softDeletePolicy?: {
retentionDurationSeconds?: string | number;
readonly effectiveTime?: string;
};
storageClass?: string;
timeCreated?: string;
updated?: string;
Expand Down Expand Up @@ -2624,6 +2629,9 @@ class Bucket extends ServiceObject<Bucket, BucketMetadata> {
* or 1 page of results will be returned per call.
* @property {string} [pageToken] A previously-returned page token
* representing part of the larger set of results to view.
* @property {boolean} [softDeleted] If true, only soft-deleted object versions will be
* listed as distinct results in order of generation number. Note `soft_deleted` and
* `versions` cannot be set to true simultaneously.
* @property {string} [startOffset] Filter results to objects whose names are
* lexicographically equal to or after startOffset. If endOffset is also set,
* the objects listed have names between startOffset (inclusive) and endOffset (exclusive).
Expand Down Expand Up @@ -2662,6 +2670,9 @@ class Bucket extends ServiceObject<Bucket, BucketMetadata> {
* or 1 page of results will be returned per call.
* @param {string} [query.pageToken] A previously-returned page token
* representing part of the larger set of results to view.
* @param {boolean} [query.softDeleted] If true, only soft-deleted object versions will be
* listed as distinct results in order of generation number. Note `soft_deleted` and
* `versions` cannot be set to true simultaneously.
* @param {string} [query.startOffset] Filter results to objects whose names are
* lexicographically equal to or after startOffset. If endOffset is also set,
* the objects listed have names between startOffset (inclusive) and endOffset (exclusive).
Expand Down
69 changes: 69 additions & 0 deletions src/file.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ import {
BaseMetadata,
DeleteCallback,
DeleteOptions,
GetResponse,
InstanceResponseCallback,
RequestResponse,
SetMetadataOptions,
} from './nodejs-common/service-object.js';
Expand Down Expand Up @@ -172,6 +174,8 @@ export interface GetFileMetadataCallback {

export interface GetFileOptions extends GetConfig {
userProject?: string;
generation?: number;
softDeleted?: boolean;
}

export type GetFileResponse = [File, unknown];
Expand Down Expand Up @@ -418,6 +422,11 @@ export interface SetStorageClassCallback {
(err?: Error | null, apiResponse?: unknown): void;
}

export interface RestoreOptions extends PreconditionOptions {
generation: number;
projection?: 'full' | 'noAcl';
}

export interface FileMetadata extends BaseMetadata {
acl?: AclMetadata[] | null;
bucket?: string;
Expand All @@ -436,6 +445,7 @@ export interface FileMetadata extends BaseMetadata {
eventBasedHold?: boolean | null;
readonly eventBasedHoldReleaseTime?: string;
generation?: string | number;
hardDeleteTime?: string;
kmsKeyName?: string;
md5Hash?: string;
mediaLink?: string;
Expand All @@ -454,6 +464,7 @@ export interface FileMetadata extends BaseMetadata {
} | null;
retentionExpirationTime?: string;
size?: string | number;
softDeleteTime?: string;
storageClass?: string;
temporaryHold?: boolean | null;
timeCreated?: string;
Expand Down Expand Up @@ -803,6 +814,9 @@ class File extends ServiceObject<File, FileMetadata> {
* @param {options} [options] Configuration options.
* @param {string} [options.userProject] The ID of the project which will be
* billed for the request.
* @param {number} [options.generation] The generation number to get
* @param {boolean} [options.softDeleted] If true, returns the soft-deleted object.
Object `generation` is required if `softDeleted` is set to True.
* @param {GetFileCallback} [callback] Callback function.
* @returns {Promise<GetFileResponse>}
*
Expand Down Expand Up @@ -2344,6 +2358,27 @@ class File extends ServiceObject<File, FileMetadata> {
return this;
}

get(options?: GetFileOptions): Promise<GetResponse<File>>;
get(callback: InstanceResponseCallback<File>): void;
get(options: GetFileOptions, callback: InstanceResponseCallback<File>): void;
get(
optionsOrCallback?: GetFileOptions | InstanceResponseCallback<File>,
cb?: InstanceResponseCallback<File>
): Promise<GetResponse<File>> | void {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const options: any =
typeof optionsOrCallback === 'object' ? optionsOrCallback : {};
cb =
typeof optionsOrCallback === 'function'
? (optionsOrCallback as InstanceResponseCallback<File>)
: cb;

super
.get(options)
.then(resp => cb!(null, ...resp))
.catch(cb!);
}

getExpirationDate(): Promise<GetExpirationDateResponse>;
getExpirationDate(callback: GetExpirationDateCallback): void;
/**
Expand Down Expand Up @@ -3597,6 +3632,39 @@ class File extends ServiceObject<File, FileMetadata> {
this.move(destinationFile, options, callback);
}

/**
* @typedef {object} RestoreOptions Options for File#restore(). See an
* {@link https://cloud.google.com/storage/docs/json_api/v1/objects#resource| Object resource}.
* @param {string} [userProject] The ID of the project which will be
* billed for the request.
* @param {number} [generation] If present, selects a specific revision of this object.
* @param {string} [projection] Specifies the set of properties to return. If used, must be 'full' or 'noAcl'.
* @param {string | number} [ifGenerationMatch] Request proceeds if the generation of the target resource
* matches the value used in the precondition.
* If the values don't match, the request fails with a 412 Precondition Failed response.
* @param {string | number} [ifGenerationNotMatch] Request proceeds if the generation of the target resource does
* not match the value used in the precondition. If the values match, the request fails with a 304 Not Modified response.
* @param {string | number} [ifMetagenerationMatch] Request proceeds if the meta-generation of the target resource
* matches the value used in the precondition.
* If the values don't match, the request fails with a 412 Precondition Failed response.
* @param {string | number} [ifMetagenerationNotMatch] Request proceeds if the meta-generation of the target resource does
* not match the value used in the precondition. If the values match, the request fails with a 304 Not Modified response.
*/
/**
* Restores a soft-deleted file
* @param {RestoreOptions} options Restore options.
* @returns {Promise<File>}
*/
async restore(options: RestoreOptions): Promise<File> {
const [file] = await this.request({
method: 'POST',
uri: '/restore',
qs: options,
});

return file as File;
}

request(reqOpts: DecorateRequestOptions): Promise<RequestResponse>;
request(
reqOpts: DecorateRequestOptions,
Expand Down Expand Up @@ -4244,6 +4312,7 @@ promisifyAll(File, {
'setEncryptionKey',
'shouldRetryBasedOnPreconditionAndIdempotencyStrat',
'getBufferFromReadable',
'restore',
],
});

Expand Down
78 changes: 78 additions & 0 deletions system-test/storage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -773,6 +773,10 @@ describe('storage', function () {

beforeEach(createBucket);

afterEach(async () => {
await bucket.delete();
});

it("sets bucket's RPO to ASYNC_TURBO", async () => {
await setTurboReplication(bucket, RPO_ASYNC_TURBO);
const [bucketMetadata] = await bucket.getMetadata();
Expand All @@ -786,6 +790,80 @@ describe('storage', function () {
});
});

describe('soft-delete', () => {
let bucket: Bucket;
const SOFT_DELETE_RETENTION_SECONDS = 7 * 24 * 60 * 60; //7 days in seconds;

beforeEach(async () => {
bucket = storage.bucket(generateName());
await bucket.create();
await bucket.setMetadata({
softDeletePolicy: {
retentionDurationSeconds: SOFT_DELETE_RETENTION_SECONDS,
},
});
});

afterEach(async () => {
await bucket.deleteFiles({force: true, versions: true});
await bucket.delete();
});

it('should set softDeletePolicy correctly', async () => {
const metadata = await bucket.getMetadata();
assert(metadata[0].softDeletePolicy);
assert(metadata[0].softDeletePolicy.effectiveTime);
assert.deepStrictEqual(
metadata[0].softDeletePolicy.retentionDurationSeconds,
SOFT_DELETE_RETENTION_SECONDS.toString()
);
});

it('should LIST soft-deleted files', async () => {
const f1 = bucket.file('file1');
const f2 = bucket.file('file2');
await f1.save('file1');
await f2.save('file2');
await f1.delete();
await f2.delete();
const [notSoftDeletedFiles] = await bucket.getFiles();
assert.strictEqual(notSoftDeletedFiles.length, 0);
const [softDeletedFiles] = await bucket.getFiles({softDeleted: true});
assert.strictEqual(softDeletedFiles.length, 2);
});

it('should GET a soft-deleted file', async () => {
const f1 = bucket.file('file3');
await f1.save('file3');
const [metadata] = await f1.getMetadata();
await f1.delete();
const [softDeletedFile] = await f1.get({
softDeleted: true,
generation: parseInt(metadata.generation?.toString() || '0'),
});
assert(softDeletedFile);
assert.strictEqual(
softDeletedFile.metadata.generation,
metadata.generation
);
});

it('should restore a soft-deleted file', async () => {
const f1 = bucket.file('file4');
await f1.save('file4');
const [metadata] = await f1.getMetadata();
await f1.delete();
let [files] = await bucket.getFiles();
assert.strictEqual(files.length, 0);
const restoredFile = await f1.restore({
generation: parseInt(metadata.generation?.toString() || '0'),
});
assert(restoredFile);
[files] = await bucket.getFiles();
assert.strictEqual(files.length, 1);
});
});

describe('dual-region', () => {
let bucket: Bucket;

Expand Down
19 changes: 19 additions & 0 deletions test/bucket.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1848,6 +1848,25 @@ describe('Bucket', () => {
});
});

it('should return soft-deleted Files if queried for softDeleted', done => {
const softDeletedTime = new Date('1/1/2024').toISOString();
bucket.request = (
reqOpts: DecorateRequestOptions,
callback: Function
) => {
callback(null, {
items: [{name: 'fake-file-name', generation: 1, softDeletedTime}],
});
};

bucket.getFiles({softDeleted: true}, (err: Error, files: FakeFile[]) => {
assert.ifError(err);
assert(files[0] instanceof FakeFile);
assert.strictEqual(files[0].metadata.softDeletedTime, softDeletedTime);
done();
});
});

it('should set kmsKeyName on file', done => {
const kmsKeyName = 'kms-key-name';

Expand Down
21 changes: 21 additions & 0 deletions test/file.ts
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ const fakePromisify = {
'setEncryptionKey',
'shouldRetryBasedOnPreconditionAndIdempotencyStrat',
'getBufferFromReadable',
'restore',
]);
},
};
Expand Down Expand Up @@ -4145,6 +4146,26 @@ describe('File', () => {
});
});

describe('restore', () => {
it('should pass options to underlying request call', async () => {
file.parent.request = function (
reqOpts: DecorateRequestOptions,
callback_: Function
) {
assert.strictEqual(this, file);
assert.deepStrictEqual(reqOpts, {
method: 'POST',
uri: '/restore',
qs: {generation: 123},
});
assert.strictEqual(callback_, undefined);
return [];
};

await file.restore({generation: 123});
});
});

describe('request', () => {
it('should call the parent request function', () => {
const options = {};
Expand Down
Loading