diff --git a/.changeset/six-melons-doubt.md b/.changeset/six-melons-doubt.md new file mode 100644 index 000000000..891d61ac1 --- /dev/null +++ b/.changeset/six-melons-doubt.md @@ -0,0 +1,55 @@ +--- +"@vercel/blob": minor +"vercel-storage-integration-test-suite": patch +--- + +feat(blob): Add multipart option to reliably upload medium and large files + +It turns out, uploading large files using Vercel Blob has been a struggle for users. +Before this change, file uploads were limited to around 200MB for technical reasons. +Before this change, even uploading a file of 100MB could fail for various reasons (network being one of them). + +To solve this for good, we're introducting a new option to `put` and `upload` calls: `multipart: true`. This new option will make sure your file is uploaded parts by parts to Vercel Blob, and when some parts are failing, we will retry them. This option is available for server and client uploads. + +Usage: +```ts +const blob = await put('file.png', file, { + access: 'public', + multipart: true // `false` by default +}); + +// and: +const blob = await upload('file.png', file, { + access: 'public', + handleUploadUrl: '/api/upload', + multipart: true +}); +``` + +If your `file` is a Node.js stream or a [ReadableStream](https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream) then we will gradually read and upload it without blowing out your server or browser memory. + +More examples: + +```ts +import { createReadStream } from 'node:fs'; + +const blob = await vercelBlob.put( + 'elon.mp4', + // this works 👍, it will gradually read the file from the system and upload it + createReadStream('/users/Elon/me.mp4'), + { access: 'public', multipart: true } +); +``` + +```ts +const response = await fetch( + 'https://example-files.online-convert.com/video/mp4/example_big.mp4', +); + +const blob = await vercelBlob.put( + 'example_big.mp4', + // this works too 👍, it will gradually read the file from internet and upload it + response.body, + { access: 'public', multipart: true }, +); +``` diff --git a/packages/blob/package.json b/packages/blob/package.json index a01e756ea..63fd8d6da 100644 --- a/packages/blob/package.json +++ b/packages/blob/package.json @@ -25,7 +25,8 @@ "module": "./dist/index.js", "browser": { "undici": "./dist/undici-browser.js", - "crypto": "./dist/crypto-browser.js" + "crypto": "./dist/crypto-browser.js", + "stream": "./dist/stream-browser.js" }, "typesVersions": { "*": { @@ -39,7 +40,7 @@ ], "scripts": { "build": "tsup && pnpm run copy-shims", - "copy-shims": "cp src/undici-browser.js dist/undici-browser.js && cp src/crypto-browser.js dist/crypto-browser.js", + "copy-shims": "cp src/*-browser.js dist/", "dev": "pnpm run copy-shims && tsup --watch --clean=false", "lint": "eslint --max-warnings=0 .", "prepublishOnly": "pnpm run build", @@ -59,11 +60,15 @@ } }, "dependencies": { + "async-retry": "1.3.3", + "bytes": "3.1.2", "undici": "5.28.2" }, "devDependencies": { "@edge-runtime/jest-environment": "2.3.7", "@edge-runtime/types": "2.2.7", + "@types/async-retry": "1.4.8", + "@types/bytes": "3.1.4", "@types/jest": "29.5.11", "@types/node": "20.10.4", "eslint": "8.55.0", diff --git a/packages/blob/src/client.browser.test.ts b/packages/blob/src/client.browser.test.ts index ee35da976..e6bd0caae 100644 --- a/packages/blob/src/client.browser.test.ts +++ b/packages/blob/src/client.browser.test.ts @@ -55,7 +55,7 @@ describe('upload()', () => { 1, 'http://localhost:3000/api/upload', { - body: '{"type":"blob.generate-client-token","payload":{"pathname":"foo.txt","callbackUrl":"http://localhost:3000/api/upload"}}', + body: '{"type":"blob.generate-client-token","payload":{"pathname":"foo.txt","callbackUrl":"http://localhost:3000/api/upload","clientPayload":null,"multipart":false}}', headers: { 'content-type': 'application/json' }, method: 'POST', }, diff --git a/packages/blob/src/client.node.test.ts b/packages/blob/src/client.node.test.ts index 292390a02..4744e26ff 100644 --- a/packages/blob/src/client.node.test.ts +++ b/packages/blob/src/client.node.test.ts @@ -87,6 +87,8 @@ describe('client uploads', () => { payload: { pathname: 'newfile.txt', callbackUrl: 'https://example.com', + multipart: false, + clientPayload: null, }, }, onBeforeGenerateToken: async (pathname) => { @@ -102,7 +104,7 @@ describe('client uploads', () => { }); expect(jsonResponse).toMatchInlineSnapshot(` { - "clientToken": "vercel_blob_client_12345fakeStoreId_ODBiNjcyZDgyZTNkOTYyNTcwMTQ4NTFhNzJlOTEzZmI0MzQ4NWEzNzE0NzhjNGE0ZGRlN2IxMzRmYjI0NTkxOS5leUowYjJ0bGJsQmhlV3h2WVdRaU9pSnVaWGRtYVd4bExuUjRkQ0lzSW5CaGRHaHVZVzFsSWpvaWJtVjNabWxzWlM1MGVIUWlMQ0p2YmxWd2JHOWhaRU52YlhCc1pYUmxaQ0k2ZXlKallXeHNZbUZqYTFWeWJDSTZJbWgwZEhCek9pOHZaWGhoYlhCc1pTNWpiMjBpTENKMGIydGxibEJoZVd4dllXUWlPaUp1WlhkbWFXeGxMblI0ZENKOUxDSjJZV3hwWkZWdWRHbHNJam94TmpjeU5UTXhNak13TURBd2ZRPT0=", + "clientToken": "vercel_blob_client_12345fakeStoreId_Y2JhNTlmNWM3MmZmMGZmM2I2YzVlYzgwNTU3MDgwMWE1YTA4ZGU2MjIyNTFkNjRiYTI1NjVjNmRjYmFkYmQ5Yy5leUowYjJ0bGJsQmhlV3h2WVdRaU9pSnVaWGRtYVd4bExuUjRkQ0lzSW5CaGRHaHVZVzFsSWpvaWJtVjNabWxzWlM1MGVIUWlMQ0p2YmxWd2JHOWhaRU52YlhCc1pYUmxaQ0k2ZXlKallXeHNZbUZqYTFWeWJDSTZJbWgwZEhCek9pOHZaWGhoYlhCc1pTNWpiMjBpTENKMGIydGxibEJoZVd4dllXUWlPaUp1WlhkbWFXeGxMblI0ZENKOUxDSjJZV3hwWkZWdWRHbHNJam94TmpjeU5UTTBPREF3TURBd2ZRPT0=", "type": "blob.generate-client-token", } `); @@ -117,7 +119,7 @@ describe('client uploads', () => { tokenPayload: 'newfile.txt', }, pathname: 'newfile.txt', - validUntil: 1672531230000, + validUntil: 1672534800000, }); }); @@ -176,6 +178,7 @@ describe('client uploads', () => { pathname: 'newfile.txt', callbackUrl: 'https://example.com', clientPayload: 'custom-metadata-from-client', + multipart: false, }, }, onBeforeGenerateToken: async () => { @@ -191,7 +194,7 @@ describe('client uploads', () => { }); expect(jsonResponse).toMatchInlineSnapshot(` { - "clientToken": "vercel_blob_client_12345fakeStoreId_YjgzZDU4YzFkZjM3MmNlN2JhMTk1MmVlYjE4YWMwOTczNGI3NjhlOTljMmE0ZTdiM2M0MTliOGJlNDg5YTFiZS5leUpoWkdSU1lXNWtiMjFUZFdabWFYZ2lPbVpoYkhObExDSndZWFJvYm1GdFpTSTZJbTVsZDJacGJHVXVkSGgwSWl3aWIyNVZjR3h2WVdSRGIyMXdiR1YwWldRaU9uc2lZMkZzYkdKaFkydFZjbXdpT2lKb2RIUndjem92TDJWNFlXMXdiR1V1WTI5dElpd2lkRzlyWlc1UVlYbHNiMkZrSWpvaVkzVnpkRzl0TFcxbGRHRmtZWFJoTFdaeWIyMHRZMnhwWlc1MEluMHNJblpoYkdsa1ZXNTBhV3dpT2pFMk56STFNekV5TXpBd01EQjk=", + "clientToken": "vercel_blob_client_12345fakeStoreId_NThhZGE3YTVkODBjNTcxMmIyMzJlMTAzMDM3MTgwYzI5NzVlMjUzYjhkYzU4MzFkZTZjMzk4ZmEwNmY2ODI5Ny5leUpoWkdSU1lXNWtiMjFUZFdabWFYZ2lPbVpoYkhObExDSndZWFJvYm1GdFpTSTZJbTVsZDJacGJHVXVkSGgwSWl3aWIyNVZjR3h2WVdSRGIyMXdiR1YwWldRaU9uc2lZMkZzYkdKaFkydFZjbXdpT2lKb2RIUndjem92TDJWNFlXMXdiR1V1WTI5dElpd2lkRzlyWlc1UVlYbHNiMkZrSWpvaVkzVnpkRzl0TFcxbGRHRmtZWFJoTFdaeWIyMHRZMnhwWlc1MEluMHNJblpoYkdsa1ZXNTBhV3dpT2pFMk56STFNelE0TURBd01EQjk=", "type": "blob.generate-client-token", } `); @@ -207,7 +210,7 @@ describe('client uploads', () => { "tokenPayload": "custom-metadata-from-client", }, "pathname": "newfile.txt", - "validUntil": 1672531230000, + "validUntil": 1672534800000, } `); }); @@ -228,6 +231,7 @@ describe('client uploads', () => { pathname: 'newfile.txt', callbackUrl: 'https://example.com', clientPayload: 'custom-metadata-from-client-we-expect', + multipart: false, }, }, onBeforeGenerateToken: async (pathname, clientPayload) => { diff --git a/packages/blob/src/client.ts b/packages/blob/src/client.ts index f9d4f84f4..1b59a063e 100644 --- a/packages/blob/src/client.ts +++ b/packages/blob/src/client.ts @@ -62,6 +62,10 @@ export interface UploadOptions { * Additional data which will be sent to your `handleUpload` route. */ clientPayload?: string; + /** + * Whether to use multipart upload. Use this when uploading large files. It will split the file into multiple parts, upload them in parallel and retry failed parts. + */ + multipart?: boolean; } /** @@ -103,7 +107,8 @@ export const upload = createPutMethod({ const clientToken = await retrieveClientToken({ handleUploadUrl: options.handleUploadUrl, pathname, - clientPayload: options.clientPayload, + clientPayload: options.clientPayload ?? null, + multipart: options.multipart ?? false, }); return clientToken; }, @@ -211,13 +216,18 @@ const EventTypes = { interface GenerateClientTokenEvent { type: (typeof EventTypes)['generateClientToken']; - payload: { pathname: string; callbackUrl: string; clientPayload?: string }; + payload: { + pathname: string; + callbackUrl: string; + multipart: boolean; + clientPayload: string | null; + }; } interface UploadCompletedEvent { type: (typeof EventTypes)['uploadCompleted']; payload: { blob: PutBlobResult; - tokenPayload?: string; + tokenPayload?: string | null; }; } @@ -229,7 +239,8 @@ export interface HandleUploadOptions { body: HandleUploadBody; onBeforeGenerateToken: ( pathname: string, - clientPayload?: string, + clientPayload: string | null, + multipart: boolean, ) => Promise< Pick< GenerateClientTokenOptions, @@ -238,7 +249,7 @@ export interface HandleUploadOptions { | 'validUntil' | 'addRandomSuffix' | 'cacheControlMaxAge' - > & { tokenPayload?: string } + > & { tokenPayload?: string | null } >; onUploadCompleted: (body: UploadCompletedEvent['payload']) => Promise; token?: string; @@ -260,10 +271,21 @@ export async function handleUpload({ const type = body.type; switch (type) { case 'blob.generate-client-token': { - const { pathname, callbackUrl, clientPayload } = body.payload; - const payload = await onBeforeGenerateToken(pathname, clientPayload); + const { pathname, callbackUrl, clientPayload, multipart } = body.payload; + const payload = await onBeforeGenerateToken( + pathname, + clientPayload, + multipart, + ); const tokenPayload = payload.tokenPayload ?? clientPayload; + // one hour + const oneHourInSeconds = 60 * 60; + const now = new Date(); + const validUntil = + payload.validUntil ?? + now.setSeconds(now.getSeconds() + oneHourInSeconds); + return { type, clientToken: await generateClientTokenFromReadWriteToken({ @@ -274,6 +296,7 @@ export async function handleUpload({ callbackUrl, tokenPayload, }, + validUntil, }), }; } @@ -309,7 +332,8 @@ export async function handleUpload({ async function retrieveClientToken(options: { pathname: string; handleUploadUrl: string; - clientPayload?: string; + clientPayload: string | null; + multipart: boolean; }): Promise { const { handleUploadUrl, pathname } = options; const url = isAbsoluteUrl(handleUploadUrl) @@ -322,6 +346,7 @@ async function retrieveClientToken(options: { pathname, callbackUrl: url, clientPayload: options.clientPayload, + multipart: options.multipart, }, }; @@ -400,7 +425,7 @@ export interface GenerateClientTokenOptions extends BlobCommandOptions { pathname: string; onUploadCompleted?: { callbackUrl: string; - tokenPayload?: string; + tokenPayload?: string | null; }; maximumSizeInBytes?: number; allowedContentTypes?: string[]; diff --git a/packages/blob/src/debug.ts b/packages/blob/src/debug.ts new file mode 100644 index 000000000..0c6ec4993 --- /dev/null +++ b/packages/blob/src/debug.ts @@ -0,0 +1,21 @@ +let debugIsActive = false; + +// wrapping this code in a try/catch in case some env doesn't support process.env (vite by default) +try { + if ( + process.env.DEBUG?.includes('blob') || + process.env.NEXT_PUBLIC_DEBUG?.includes('blob') + ) { + debugIsActive = true; + } +} catch (error) { + // noop +} + +// Set process.env.DEBUG = 'blob' to enable debug logging +export function debug(message: string, ...args: unknown[]): void { + if (debugIsActive) { + // eslint-disable-next-line no-console -- Ok for debugging + console.debug(`vercel-blob: ${message}`, ...args); + } +} diff --git a/packages/blob/src/helpers.ts b/packages/blob/src/helpers.ts index 7c5ab891c..5bb870b52 100644 --- a/packages/blob/src/helpers.ts +++ b/packages/blob/src/helpers.ts @@ -32,6 +32,11 @@ export interface CreateBlobCommandOptions extends BlobCommandOptions { * @defaultvalue 365 * 24 * 60 * 60 (1 Year) */ cacheControlMaxAge?: number; + /** + * Whether to use multipart upload. Use this when uploading large files. It will split the file into multiple parts, upload them in parallel and retry failed parts. + * @defaultvalue false + */ + multipart?: boolean; } export function getTokenFromOptionsOrEnv(options?: BlobCommandOptions): string { @@ -56,25 +61,25 @@ export class BlobError extends Error { export class BlobAccessError extends BlobError { constructor() { - super('Access denied, please provide a valid token for this resource'); + super('Access denied, please provide a valid token for this resource.'); } } export class BlobStoreNotFoundError extends BlobError { constructor() { - super('This store does not exist'); + super('This store does not exist.'); } } export class BlobStoreSuspendedError extends BlobError { constructor() { - super('This store has been suspended'); + super('This store has been suspended.'); } } export class BlobUnknownError extends BlobError { constructor() { - super('Unknown error, please visit https://vercel.com/help'); + super('Unknown error, please visit https://vercel.com/help.'); } } @@ -86,7 +91,7 @@ export class BlobNotFoundError extends BlobError { export class BlobServiceNotAvailable extends BlobError { constructor() { - super('The blob service is currently not available. Please try again'); + super('The blob service is currently not available. Please try again.'); } } diff --git a/packages/blob/src/index.node.test.ts b/packages/blob/src/index.node.test.ts index 5451805d9..02de36a95 100644 --- a/packages/blob/src/index.node.test.ts +++ b/packages/blob/src/index.node.test.ts @@ -84,7 +84,7 @@ describe('blob client', () => { await expect(head(`${BLOB_STORE_BASE_URL}/foo-id.txt`)).rejects.toThrow( new Error( - 'Vercel Blob: Access denied, please provide a valid token for this resource', + 'Vercel Blob: Access denied, please provide a valid token for this resource.', ), ); }); @@ -99,7 +99,7 @@ describe('blob client', () => { await expect(head(`${BLOB_STORE_BASE_URL}/foo-id.txt`)).rejects.toThrow( new Error( - 'Vercel Blob: Unknown error, please visit https://vercel.com/help', + 'Vercel Blob: Unknown error, please visit https://vercel.com/help.', ), ); }); @@ -123,7 +123,7 @@ describe('blob client', () => { .reply(403, { error: { code: 'store_suspended' } }); await expect(head(`${BLOB_STORE_BASE_URL}/foo-id.txt`)).rejects.toThrow( - new Error('Vercel Blob: This store has been suspended'), + new Error('Vercel Blob: This store has been suspended.'), ); }); @@ -136,7 +136,7 @@ describe('blob client', () => { .reply(403, { error: { code: 'store_not_found' } }); await expect(head(`${BLOB_STORE_BASE_URL}/foo-id.txt`)).rejects.toThrow( - new Error('Vercel Blob: This store does not exist'), + new Error('Vercel Blob: This store does not exist.'), ); }); @@ -225,7 +225,7 @@ describe('blob client', () => { await expect(del(`${BLOB_STORE_BASE_URL}/foo-id.txt`)).rejects.toThrow( new Error( - 'Vercel Blob: Access denied, please provide a valid token for this resource', + 'Vercel Blob: Access denied, please provide a valid token for this resource.', ), ); }); @@ -240,7 +240,7 @@ describe('blob client', () => { await expect(del(`${BLOB_STORE_BASE_URL}/foo-id.txt`)).rejects.toThrow( new Error( - 'Vercel Blob: Unknown error, please visit https://vercel.com/help', + 'Vercel Blob: Unknown error, please visit https://vercel.com/help.', ), ); }); @@ -310,7 +310,7 @@ describe('blob client', () => { await expect(list()).rejects.toThrow( new Error( - 'Vercel Blob: Access denied, please provide a valid token for this resource', + 'Vercel Blob: Access denied, please provide a valid token for this resource.', ), ); }); @@ -324,7 +324,7 @@ describe('blob client', () => { .reply(500, 'Invalid token'); await expect(list()).rejects.toThrow( new Error( - 'Vercel Blob: Unknown error, please visit https://vercel.com/help', + 'Vercel Blob: Unknown error, please visit https://vercel.com/help.', ), ); }); @@ -446,7 +446,7 @@ describe('blob client', () => { }), ).rejects.toThrow( new Error( - 'Vercel Blob: Access denied, please provide a valid token for this resource', + 'Vercel Blob: Access denied, please provide a valid token for this resource.', ), ); }); @@ -465,7 +465,7 @@ describe('blob client', () => { }), ).rejects.toThrow( new Error( - 'Vercel Blob: Unknown error, please visit https://vercel.com/help', + 'Vercel Blob: Unknown error, please visit https://vercel.com/help.', ), ); }); diff --git a/packages/blob/src/put-multipart.ts b/packages/blob/src/put-multipart.ts new file mode 100644 index 000000000..26b48628b --- /dev/null +++ b/packages/blob/src/put-multipart.ts @@ -0,0 +1,458 @@ +// eslint-disable-next-line unicorn/prefer-node-protocol -- node:stream does not resolve correctly in browser and edge +import { Readable } from 'stream'; +import type { BodyInit } from 'undici'; +import { fetch } from 'undici'; +import retry from 'async-retry'; +import bytes from 'bytes'; +import type { PutBlobApiResponse, PutBlobResult, PutBody } from './put'; +import { + BlobServiceNotAvailable, + getApiUrl, + validateBlobApiResponse, +} from './helpers'; +import { debug } from './debug'; + +// Most browsers will cap requests at 6 concurrent uploads per domain (Vercel Blob API domain) +// In other environments, we can afford to be more aggressive +const maxConcurrentUploads = typeof window !== 'undefined' ? 6 : 8; + +// 5MB is the minimum part size accepted by Vercel Blob, but we set our default part size to 8mb like the aws cli +const partSizeInBytes = 8 * 1024 * 1024; + +const maxBytesInMemory = maxConcurrentUploads * partSizeInBytes * 2; + +interface CreateMultiPartUploadApiResponse { + uploadId: string; + key: string; +} + +interface UploadPartApiResponse { + etag: string; +} + +export async function multipartPut( + pathname: string, + body: PutBody, + headers: Record, +): Promise { + debug('mpu: init', 'pathname:', pathname, 'headers:', headers); + + const stream = toReadableStream(body); + + // Step 1: Start multipart upload + const createMultipartUploadResponse = await createMultiPartUpload( + pathname, + headers, + ); + + // Step 2: Upload parts one by one + const parts = await uploadParts( + createMultipartUploadResponse.uploadId, + createMultipartUploadResponse.key, + pathname, + stream, + headers, + ); + + // Step 3: Complete multipart upload + const blob = await completeMultiPartUpload( + createMultipartUploadResponse.uploadId, + createMultipartUploadResponse.key, + pathname, + parts, + headers, + ); + + return blob; +} + +async function completeMultiPartUpload( + uploadId: string, + key: string, + pathname: string, + parts: CompletedPart[], + headers: Record, +): Promise { + const apiUrl = new URL(getApiUrl(`/mpu/${pathname}`)); + try { + const apiResponse = await fetch(apiUrl, { + method: 'POST', + headers: { + ...headers, + 'content-type': 'application/json', + 'x-mpu-action': 'complete', + 'x-mpu-upload-id': uploadId, + // key can be any utf8 character so we need to encode it as HTTP headers can only be us-ascii + // https://www.rfc-editor.org/rfc/rfc7230#section-3.2.4 + 'x-mpu-key': encodeURI(key), + }, + body: JSON.stringify(parts), + }); + + await validateBlobApiResponse(apiResponse); + + return (await apiResponse.json()) as PutBlobApiResponse; + } catch (error: unknown) { + if ( + error instanceof TypeError && + (error.message === 'Failed to fetch' || error.message === 'fetch failed') + ) { + throw new BlobServiceNotAvailable(); + } else { + throw error; + } + } +} + +async function createMultiPartUpload( + pathname: string, + headers: Record, +): Promise { + debug('mpu: create', 'pathname:', pathname); + + const apiUrl = new URL(getApiUrl(`/mpu/${pathname}`)); + try { + const apiResponse = await fetch(apiUrl, { + method: 'POST', + headers: { + ...headers, + 'x-mpu-action': 'create', + }, + }); + + await validateBlobApiResponse(apiResponse); + + const json = await apiResponse.json(); + + debug('mpu: create', json); + + return json as CreateMultiPartUploadApiResponse; + } catch (error: unknown) { + if ( + error instanceof TypeError && + (error.message === 'Failed to fetch' || error.message === 'fetch failed') + ) { + throw new BlobServiceNotAvailable(); + } else { + throw error; + } + } +} + +interface UploadPart { + partNumber: number; + blob: Blob; +} + +interface CompletedPart { + partNumber: number; + etag: string; +} + +// Can we rewrite this function without new Promise? +function uploadParts( + uploadId: string, + key: string, + pathname: string, + stream: ReadableStream, + headers: Record, +): Promise { + debug('mpu: upload init', 'key:', key); + const internalAbortController = new AbortController(); + + return new Promise((resolve, reject) => { + const partsToUpload: UploadPart[] = []; + const completedParts: CompletedPart[] = []; + const reader = stream.getReader(); + let activeUploads = 0; + let reading = false; + let currentPartNumber = 1; + // this next variable is used to escape the read loop when an error occurs + let rejected = false; + let currentBytesInMemory = 0; + let doneReading = false; + let bytesSent = 0; + + // This must be outside the read loop, in case we reach the maxBytesInMemory and + // we exit the loop but some bytes are still to be sent on the next read invocation. + let arrayBuffers: ArrayBuffer[] = []; + let currentPartBytesRead = 0; + + read().catch(cancel); + + async function read(): Promise { + debug( + 'mpu: upload read start', + 'activeUploads:', + activeUploads, + 'currentBytesInMemory:', + `${bytes(currentBytesInMemory)}/${bytes(maxBytesInMemory)}`, + 'bytesSent:', + bytes(bytesSent), + ); + + reading = true; + + while (currentBytesInMemory < maxBytesInMemory && !rejected) { + try { + // eslint-disable-next-line no-await-in-loop -- A for loop is fine here. + const { value, done } = await reader.read(); + + if (done) { + doneReading = true; + debug('mpu: upload read consumed the whole stream'); + // done is sent when the stream is fully consumed. That's why we're not using the value here. + if (arrayBuffers.length > 0) { + partsToUpload.push({ + partNumber: currentPartNumber++, + blob: new Blob(arrayBuffers, { + type: 'application/octet-stream', + }), + }); + + sendParts(); + } + reading = false; + return; + } + + currentBytesInMemory += value.byteLength; + + // This code ensures that each part will be exactly of `partSizeInBytes` size + // Otherwise R2 will refuse it. AWS S3 is fine with parts of different sizes. + let valueOffset = 0; + while (valueOffset < value.byteLength) { + const remainingPartSize = partSizeInBytes - currentPartBytesRead; + const endOffset = Math.min( + valueOffset + remainingPartSize, + value.byteLength, + ); + + const chunk = value.slice(valueOffset, endOffset); + + arrayBuffers.push(chunk); + currentPartBytesRead += chunk.byteLength; + valueOffset = endOffset; + + if (currentPartBytesRead === partSizeInBytes) { + partsToUpload.push({ + partNumber: currentPartNumber++, + blob: new Blob(arrayBuffers, { + type: 'application/octet-stream', + }), + }); + + arrayBuffers = []; + currentPartBytesRead = 0; + sendParts(); + } + } + } catch (error) { + cancel(error); + } + } + + debug( + 'mpu: upload read end', + 'activeUploads:', + activeUploads, + 'currentBytesInMemory:', + `${bytes(currentBytesInMemory)}/${bytes(maxBytesInMemory)}`, + 'bytesSent:', + bytes(bytesSent), + ); + + reading = false; + } + + async function sendPart(part: UploadPart): Promise { + activeUploads++; + + debug( + 'mpu: upload send part start', + 'partNumber:', + part.partNumber, + 'size:', + part.blob.size, + 'activeUploads:', + activeUploads, + 'currentBytesInMemory:', + `${bytes(currentBytesInMemory)}/${bytes(maxBytesInMemory)}`, + 'bytesSent:', + bytes(bytesSent), + ); + + const apiUrl = new URL(getApiUrl(`/mpu/${pathname}`)); + + try { + const apiResponse = await retry( + async () => { + const res = await fetch(apiUrl, { + signal: internalAbortController.signal, + method: 'POST', + headers: { + ...headers, + 'x-mpu-action': 'upload', + 'x-mpu-key': encodeURI(key), + 'x-mpu-upload-id': uploadId, + 'x-mpu-part-number': part.partNumber.toString(), + }, + // weird things between undici types and native fetch types + body: part.blob as BodyInit, + }); + + if (res.status >= 500) { + // this will be retried + throw new BlobServiceNotAvailable(); + } + + return res; + }, + { + onRetry: (error) => { + // eslint-disable-next-line no-console -- Ok for debugging + console.log('retrying', error.message); + }, + }, + ); + + try { + await validateBlobApiResponse(apiResponse); + } catch (error) { + cancel(error); + return; + } + + debug( + 'mpu: upload send part end', + 'partNumber:', + part.partNumber, + 'activeUploads', + activeUploads, + 'currentBytesInMemory:', + `${bytes(currentBytesInMemory)}/${bytes(maxBytesInMemory)}`, + 'bytesSent:', + bytes(bytesSent), + ); + + if (rejected) { + return; + } + + const completedPart = + (await apiResponse.json()) as UploadPartApiResponse; + completedParts.push({ + partNumber: part.partNumber, + etag: completedPart.etag, + }); + + currentBytesInMemory -= part.blob.size; + activeUploads--; + bytesSent += part.blob.size; + + if (partsToUpload.length > 0) { + sendParts(); + } + + if (doneReading) { + if (activeUploads === 0) { + reader.releaseLock(); + resolve(completedParts); + } + return; + } + + if (!reading) { + read().catch(cancel); + } + } catch (error) { + cancel(error); + } + } + + function sendParts(): void { + if (rejected) { + return; + } + + debug( + 'send parts', + 'activeUploads', + activeUploads, + 'partsToUpload', + partsToUpload.length, + ); + while (activeUploads < maxConcurrentUploads && partsToUpload.length > 0) { + const partToSend = partsToUpload.shift(); + if (partToSend) { + void sendPart(partToSend); + } + } + } + + function cancel(error: unknown): void { + // a previous call already rejected the whole call, ignore + if (rejected) { + return; + } + rejected = true; + internalAbortController.abort(); + reader.releaseLock(); + if ( + error instanceof TypeError && + (error.message === 'Failed to fetch' || + error.message === 'fetch failed') + ) { + reject(new BlobServiceNotAvailable()); + } else { + reject(error); + } + } + }); +} + +function toReadableStream(value: PutBody): ReadableStream { + // Already a ReadableStream, nothing to do + if (value instanceof ReadableStream) { + return value as ReadableStream; + } + + // In the case of a Blob or File (which inherits from Blob), we could use .slice() to create pointers + // to the original data instead of loading data in memory gradually. + // Here's an explanation on this subject: https://stackoverflow.com/a/24834417 + if (value instanceof Blob) { + return value.stream(); + } + + if (isNodeJsReadableStream(value)) { + return Readable.toWeb(value) as ReadableStream; + } + + const streamValue = + value instanceof ArrayBuffer ? value : stringToUint8Array(value); + + // from https://github.com/sindresorhus/to-readable-stream/blob/main/index.js + return new ReadableStream({ + start(controller) { + controller.enqueue(streamValue); + controller.close(); + }, + }); +} + +// From https://github.com/sindresorhus/is-stream/ +function isNodeJsReadableStream(value: PutBody): value is Readable { + return ( + typeof value === 'object' && + typeof (value as Readable).pipe === 'function' && + (value as Readable).readable && + typeof (value as Readable)._read === 'function' && + // @ts-expect-error _readableState does exists on Readable + typeof value._readableState === 'object' + ); +} + +function stringToUint8Array(s: string): Uint8Array { + const enc = new TextEncoder(); + return enc.encode(s); +} diff --git a/packages/blob/src/put.ts b/packages/blob/src/put.ts index 19fcebe30..c153f54e0 100644 --- a/packages/blob/src/put.ts +++ b/packages/blob/src/put.ts @@ -1,4 +1,5 @@ -import type { Readable } from 'node:stream'; +// eslint-disable-next-line unicorn/prefer-node-protocol -- node:stream does not resolve correctly in browser and edge +import type { Readable } from 'stream'; import type { BodyInit } from 'undici'; import { fetch } from 'undici'; import type { ClientPutCommandOptions } from './client'; @@ -10,6 +11,7 @@ import { BlobError, validateBlobApiResponse, } from './helpers'; +import { multipartPut } from './put-multipart'; // eslint-disable-next-line @typescript-eslint/no-empty-interface -- expose option interface for each API method for better extensibility in the future export interface PutCommandOptions extends CreateBlobCommandOptions {} @@ -29,6 +31,14 @@ export interface PutBlobResult { export type PutBlobApiResponse = PutBlobResult; +export type PutBody = + | string + | Readable // Node.js streams + | Blob + | ArrayBuffer + | ReadableStream // Streams API (= Web streams in Node.js) + | File; + type PartialBy = Omit & Partial>; export function createPutMethod< @@ -44,16 +54,7 @@ export function createPutMethod< }) { return async function put( pathname: TPath, - bodyOrOptions: TPath extends `${string}/` - ? T - : - | string - | Readable - | Blob - | ArrayBuffer - | FormData - | ReadableStream - | File, + bodyOrOptions: TPath extends `${string}/` ? T : PutBody, optionsInput?: T, ): Promise { if (!pathname) { @@ -73,7 +74,7 @@ export function createPutMethod< } // avoid using the options as body - const body = isFolderCreation ? undefined : (bodyOrOptions as BodyInit); + const body = isFolderCreation ? undefined : bodyOrOptions; // when no body is required options are the second argument const options = isFolderCreation ? (bodyOrOptions as T) : optionsInput; @@ -119,9 +120,13 @@ export function createPutMethod< options.cacheControlMaxAge.toString(); } + if (options.multipart === true && body) { + return multipartPut(pathname, body, headers); + } + const blobApiResponse = await fetch(getApiUrl(`/${pathname}`), { method: 'PUT', - body, + body: body as BodyInit, headers, // required in order to stream some body types to Cloudflare // currently only supported in Node.js, we may have to feature detect this diff --git a/packages/blob/src/stream-browser.js b/packages/blob/src/stream-browser.js new file mode 100644 index 000000000..7d3e8398b --- /dev/null +++ b/packages/blob/src/stream-browser.js @@ -0,0 +1,11 @@ +// This file is here because Edge Functions have no support for Node.js streams by default +// It's unlikely someone would try to read/use a Node.js stream in an Edge function but we still put +// a message in case this happens + +export const Readable = { + toWeb() { + throw new Error( + 'Vercel Blob: Sorry, we cannot get a Readable stream in this environment. If you see this message please open an issue here: https://github.com/vercel/storage/ with details on your environment.', + ); + }, +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ea23acb1c..450448513 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -47,6 +47,12 @@ importers: packages/blob: dependencies: + async-retry: + specifier: 1.3.3 + version: 1.3.3 + bytes: + specifier: 3.1.2 + version: 3.1.2 undici: specifier: 5.28.2 version: 5.28.2 @@ -57,6 +63,12 @@ importers: '@edge-runtime/types': specifier: 2.2.7 version: 2.2.7 + '@types/async-retry': + specifier: 1.4.8 + version: 1.4.8 + '@types/bytes': + specifier: 3.1.4 + version: 3.1.4 '@types/jest': specifier: 29.5.11 version: 29.5.11 @@ -303,6 +315,9 @@ importers: test/next: dependencies: + '@tailwindcss/forms': + specifier: 0.5.7 + version: 0.5.7(tailwindcss@3.3.6) '@types/node': specifier: 20.10.4 version: 20.10.4 @@ -1991,6 +2006,15 @@ packages: defer-to-connect: 2.0.1 dev: false + /@tailwindcss/forms@0.5.7(tailwindcss@3.3.6): + resolution: {integrity: sha512-QE7X69iQI+ZXwldE+rzasvbJiyV/ju1FGHH0Qn2W3FKbuYtqp8LKcy6iSw79fVUT5/Vvf+0XgLCeYVG+UV6hOw==} + peerDependencies: + tailwindcss: '>=3.0.0 || >= 3.0.0-alpha.1' + dependencies: + mini-svg-data-uri: 1.4.4 + tailwindcss: 3.3.6(ts-node@10.9.1) + dev: false + /@tootallnate/once@2.0.0: resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==} engines: {node: '>= 10'} @@ -2008,6 +2032,12 @@ packages: /@tsconfig/node16@1.0.3: resolution: {integrity: sha512-yOlFc+7UtL/89t2ZhjPvvB/DeAr3r+Dq58IgzsFkOAvVC6NMJXmCGjbptdXdR9qsX7pKcTL+s87FtYREi2dEEQ==} + /@types/async-retry@1.4.8: + resolution: {integrity: sha512-Qup/B5PWLe86yI5I3av6ePGaeQrIHNKCwbsQotD6aHQ6YkHsMUxVZkZsmx/Ry3VZQ6uysHwTjQ7666+k6UjVJA==} + dependencies: + '@types/retry': 0.12.5 + dev: true + /@types/babel__core@7.20.0: resolution: {integrity: sha512-+n8dL/9GWblDO0iU6eZAwEIJVr5DWigtle+Q6HLOrh/pdbXOhOtqzq8VPPE2zvNJzSKY4vH/z3iT3tn0A3ypiQ==} dependencies: @@ -2033,6 +2063,10 @@ packages: dependencies: '@babel/types': 7.23.4 + /@types/bytes@3.1.4: + resolution: {integrity: sha512-A0uYgOj3zNc4hNjHc5lYUfJQ/HVyBXiUMKdXd7ysclaE6k9oJdavQzODHuwjpUu2/boCP8afjQYi8z/GtvNCWA==} + dev: true + /@types/graceful-fs@4.1.6: resolution: {integrity: sha512-Sig0SNORX9fdW+bQuTEovKj3uHcUL6LQKbCrrqb1X7J6/ReAbhCXRAhc+SMejhLELFj2QcyuxmUooZ4bt5ReSw==} dependencies: @@ -2121,6 +2155,10 @@ packages: csstype: 3.1.3 dev: false + /@types/retry@0.12.5: + resolution: {integrity: sha512-3xSjTp3v03X/lSQLkczaN9UIEwJMoMCA1+Nb5HfbJEQWogdeQIyVtTvxPXDQjZ5zws8rFQfVfRdz03ARihPJgw==} + dev: true + /@types/scheduler@0.16.8: resolution: {integrity: sha512-WZLiwShhwLRmeV6zH+GkbOFT6Z6VklCItrDioxUnv+u4Ll+8vKeFySoFyK/0ctcRpOmwAicELfmys1sDc/Rw+A==} dev: false @@ -2657,6 +2695,12 @@ packages: /ast-types-flow@0.0.8: resolution: {integrity: sha512-OH/2E5Fg20h2aPrbe+QL8JZQFko0YZaF+j4mnQ7BGhfavO7OpSLa8a0y9sBwomHdSbkhTS8TQNayBfnW5DwbvQ==} + /async-retry@1.3.3: + resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==} + dependencies: + retry: 0.13.1 + dev: false + /asynciterator.prototype@1.0.0: resolution: {integrity: sha512-wwHYEIS0Q80f5mosx3L/dfG5t5rjEa9Ft51GTaNt862EnpyGHpgz2RkZvLPp1oF5TnAiTohkEKVEu8pQPJI7Vg==} dependencies: @@ -2878,6 +2922,11 @@ packages: streamsearch: 1.1.0 dev: false + /bytes@3.1.2: + resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} + engines: {node: '>= 0.8'} + dev: false + /cac@6.7.14: resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} engines: {node: '>=8'} @@ -5726,6 +5775,11 @@ packages: resolution: {integrity: sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==} engines: {node: '>=4'} + /mini-svg-data-uri@1.4.4: + resolution: {integrity: sha512-r9deDe9p5FJUPZAk3A59wGH7Ii9YrjjWw0jmw/liSbHl2CHiyXj6FcDXDu2K3TjVAXqiJdaw3xxwlZZr9E6nHg==} + hasBin: true + dev: false + /minimatch@3.1.2: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} dependencies: @@ -6588,6 +6642,11 @@ packages: signal-exit: 3.0.7 dev: true + /retry@0.13.1: + resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} + engines: {node: '>= 4'} + dev: false + /reusify@1.0.4: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} diff --git a/test/next/package.json b/test/next/package.json index d15a592a6..27ee5115d 100644 --- a/test/next/package.json +++ b/test/next/package.json @@ -11,6 +11,7 @@ "type-check": "tsc --noEmit" }, "dependencies": { + "@tailwindcss/forms": "0.5.7", "@types/node": "20.10.4", "@types/react": "18.2.42", "@types/react-dom": "18.2.17", diff --git a/test/next/public/big-video.mp4 b/test/next/public/big-video.mp4 new file mode 100644 index 000000000..69913670e Binary files /dev/null and b/test/next/public/big-video.mp4 differ diff --git a/test/next/src/app/vercel/blob/app/body/edge/page.tsx b/test/next/src/app/vercel/blob/app/body/edge/page.tsx index a5349f2e4..edb246cd3 100644 --- a/test/next/src/app/vercel/blob/app/body/edge/page.tsx +++ b/test/next/src/app/vercel/blob/app/body/edge/page.tsx @@ -5,7 +5,9 @@ import { FormBodyUpload } from '../../../form-body-upload'; export default function AppBodyEdge(): JSX.Element { return ( <> -

App Router direct body upload example via an Edge Function

+

+ App Router direct body upload example via an Edge Function +

This Next.js App Router{' '} example uses a{' '} diff --git a/test/next/src/app/vercel/blob/app/body/serverless/page.tsx b/test/next/src/app/vercel/blob/app/body/serverless/page.tsx index e0bab4434..88863d455 100644 --- a/test/next/src/app/vercel/blob/app/body/serverless/page.tsx +++ b/test/next/src/app/vercel/blob/app/body/serverless/page.tsx @@ -5,7 +5,9 @@ import { FormBodyUpload } from '../../../form-body-upload'; export default function AppFormDataServerless(): JSX.Element { return ( <> -

App Router direct body upload example via a Serverless Function

+

+ App Router direct body upload example via a Serverless Function +

This Next.js App Router{' '} example uses a{' '} diff --git a/test/next/src/app/vercel/blob/app/client-multipart/page.tsx b/test/next/src/app/vercel/blob/app/client-multipart/page.tsx new file mode 100644 index 000000000..7c837ef43 --- /dev/null +++ b/test/next/src/app/vercel/blob/app/client-multipart/page.tsx @@ -0,0 +1,58 @@ +'use client'; + +import { type PutBlobResult } from '@vercel/blob'; +import { upload } from '@vercel/blob/client'; +import { useRef, useState } from 'react'; + +export default function AppClientUpload(): JSX.Element { + const inputFileRef = useRef(null); + const [blob, setBlob] = useState(null); + return ( + <> +

App Router Client Upload (multipart)

+ +
=> { + event.preventDefault(); + + const file = inputFileRef.current?.files?.[0]; + if (!file) { + return; + } + + try { + const blobResult = await upload(file.name, file, { + access: 'public', + handleUploadUrl: `/vercel/blob/api/app/handle-blob-upload/edge`, + multipart: true, + }); + + setBlob(blobResult); + } catch (error: unknown) { + // eslint-disable-next-line no-console -- Fine for tests + console.log('error', error); + } + }} + > + + +
+ {blob ? ( +
+ Blob url: {blob.url} + {blob.url.endsWith('.mp4') ? ( + // eslint-disable-next-line jsx-a11y/media-has-caption -- no caption for tests, this is fine + + ) : null} +
+ ) : null} + + ); +} diff --git a/test/next/src/app/vercel/blob/app/client/page.tsx b/test/next/src/app/vercel/blob/app/client/page.tsx index 606ab09d1..4ca3ee0ad 100644 --- a/test/next/src/app/vercel/blob/app/client/page.tsx +++ b/test/next/src/app/vercel/blob/app/client/page.tsx @@ -9,7 +9,7 @@ export default function AppClientUpload(): JSX.Element { const [blob, setBlob] = useState(null); return ( <> -

App Router Client Upload

+

App Router Client Upload

=> { @@ -29,7 +29,12 @@ export default function AppClientUpload(): JSX.Element { }} > - +
{blob ? (
diff --git a/test/next/src/app/vercel/blob/app/formdata/edge/page.tsx b/test/next/src/app/vercel/blob/app/formdata/edge/page.tsx index 8a0f2c446..7849f5af8 100644 --- a/test/next/src/app/vercel/blob/app/formdata/edge/page.tsx +++ b/test/next/src/app/vercel/blob/app/formdata/edge/page.tsx @@ -5,7 +5,9 @@ import { FormDataUpload } from '../../../form-data-upload'; export default function AppFormDataEdge(): JSX.Element { return ( <> -

App Router Form Data upload example via an Edge Function

+

+ App Router Form Data upload example via an Edge Function +

This Next.js App Router{' '} example uses a{' '} diff --git a/test/next/src/app/vercel/blob/app/formdata/serverless/page.tsx b/test/next/src/app/vercel/blob/app/formdata/serverless/page.tsx index dc1a78f21..3951ecb50 100644 --- a/test/next/src/app/vercel/blob/app/formdata/serverless/page.tsx +++ b/test/next/src/app/vercel/blob/app/formdata/serverless/page.tsx @@ -5,7 +5,9 @@ import { FormDataUpload } from '../../../form-data-upload'; export default function AppFormDataServerless(): JSX.Element { return ( <> -

App Router Form Data upload example via a Serverless Function

+

+ App Router Form Data upload example via a Serverless Function +

This Next.js App Router{' '} example uses a{' '} diff --git a/test/next/src/app/vercel/blob/app/list/page.tsx b/test/next/src/app/vercel/blob/app/list/page.tsx index 334713a61..993327ab0 100644 --- a/test/next/src/app/vercel/blob/app/list/page.tsx +++ b/test/next/src/app/vercel/blob/app/list/page.tsx @@ -82,7 +82,7 @@ export default function AppList(): JSX.Element { return (

-

App Router List blob items

+

App Router List blob items

(''); const [blob, setBlob] = useState(null); @@ -17,16 +17,19 @@ export default function AppBodyClient({ const blobResult = await upload(filename, `Hello from ${filename}`, { access: 'public', handleUploadUrl: callback, + multipart: multipart === '1', }); setBlob(blobResult); setContent(await fetch(blobResult.url).then((r) => r.text())); }; void doUpload(); - }, [filename, callback]); + }, [filename, callback, multipart]); if (!blob || !content) return
Loading...
; return ( <> -

App Router direct string upload example via a Client {callback}

+

+ App Router direct string upload example via a Client {callback} +

{blob.pathname}

{content}

diff --git a/test/next/src/app/vercel/blob/app/test/edge/page.tsx b/test/next/src/app/vercel/blob/app/test/edge/page.tsx index 47deb842d..1662e731c 100644 --- a/test/next/src/app/vercel/blob/app/test/edge/page.tsx +++ b/test/next/src/app/vercel/blob/app/test/edge/page.tsx @@ -14,7 +14,9 @@ export default async function AppBodyEdge({ const content = await fetch(blob.url).then((r) => r.text()); return ( <> -

App Router direct string upload example via an Edge Function

+

+ App Router direct string upload example via an Edge Function +

{blob.pathname}

{content}

diff --git a/test/next/src/app/vercel/blob/app/test/serverless/page.tsx b/test/next/src/app/vercel/blob/app/test/serverless/page.tsx index cb911d868..dd2624ed2 100644 --- a/test/next/src/app/vercel/blob/app/test/serverless/page.tsx +++ b/test/next/src/app/vercel/blob/app/test/serverless/page.tsx @@ -12,7 +12,9 @@ export default async function AppBodyServerless({ const content = await fetch(blob.url).then((r) => r.text()); return ( <> -

App Router direct string upload example via a Serverless Function

+

+ App Router direct string upload example via a Serverless Function +

{blob.pathname}

{content}

diff --git a/test/next/src/app/vercel/blob/form-body-upload.tsx b/test/next/src/app/vercel/blob/form-body-upload.tsx index 09e2d519c..a633c03d6 100644 --- a/test/next/src/app/vercel/blob/form-body-upload.tsx +++ b/test/next/src/app/vercel/blob/form-body-upload.tsx @@ -29,7 +29,12 @@ export function FormBodyUpload({ action }: { action: string }): JSX.Element { }} > - + {blob ? (
diff --git a/test/next/src/app/vercel/blob/form-data-upload.tsx b/test/next/src/app/vercel/blob/form-data-upload.tsx index d2c12790d..5f33e6f22 100644 --- a/test/next/src/app/vercel/blob/form-data-upload.tsx +++ b/test/next/src/app/vercel/blob/form-data-upload.tsx @@ -25,7 +25,12 @@ export function FormDataUpload({ action }: { action: string }): JSX.Element { }} > - + {blob ? (
diff --git a/test/next/src/app/vercel/blob/handle-blob-upload.ts b/test/next/src/app/vercel/blob/handle-blob-upload.ts index 45311d725..98bbcd90d 100644 --- a/test/next/src/app/vercel/blob/handle-blob-upload.ts +++ b/test/next/src/app/vercel/blob/handle-blob-upload.ts @@ -39,13 +39,6 @@ export async function handleUploadHandler( } return { - maximumSizeInBytes: 10_000_000, - allowedContentTypes: [ - 'image/jpeg', - 'image/png', - 'image/gif', - 'text/plain', - ], tokenPayload: JSON.stringify({ userId: user?.id, }), diff --git a/test/next/src/app/vercel/blob/handle-body.ts b/test/next/src/app/vercel/blob/handle-body.ts index 73b647729..f82b1ff01 100644 --- a/test/next/src/app/vercel/blob/handle-body.ts +++ b/test/next/src/app/vercel/blob/handle-body.ts @@ -5,6 +5,7 @@ import { validateUploadToken } from './validate-upload-token'; export async function handleBody(request: Request): Promise { const { searchParams } = new URL(request.url); const pathname = searchParams.get('filename'); + const multipart = searchParams.get('multipart') === '1'; if (!request.body || pathname === null) { return NextResponse.json( @@ -27,6 +28,7 @@ export async function handleBody(request: Request): Promise { // Note: this will stream the file to Vercel's Blob Store const blob = await vercelBlob.put(pathname, request.body, { access: 'public', + multipart, }); return NextResponse.json(blob); diff --git a/test/next/src/app/vercel/blob/layout.tsx b/test/next/src/app/vercel/blob/layout.tsx index d3b336545..a8b72102f 100644 --- a/test/next/src/app/vercel/blob/layout.tsx +++ b/test/next/src/app/vercel/blob/layout.tsx @@ -6,8 +6,8 @@ export default function Layout({ children: React.ReactNode; }): JSX.Element { return ( -
-
+
+ {children} diff --git a/test/next/src/app/vercel/blob/page.tsx b/test/next/src/app/vercel/blob/page.tsx index 5854c8cd8..071672241 100644 --- a/test/next/src/app/vercel/blob/page.tsx +++ b/test/next/src/app/vercel/blob/page.tsx @@ -1,7 +1,7 @@ export default function Home(): JSX.Element { return (
-

+

Vercel Blob Next.js Examples ( code on GitHub @@ -42,6 +42,10 @@ export default function Home(): JSX.Element {
  • Client Upload → /app/client
    • +
  • + Client Upload (multipart) →{' '} + /app/client-multipart +
  • List blob items → /app/list
    • diff --git a/test/next/src/app/vercel/blob/script.mts b/test/next/src/app/vercel/blob/script.mts index c71c98917..c13e2b477 100644 --- a/test/next/src/app/vercel/blob/script.mts +++ b/test/next/src/app/vercel/blob/script.mts @@ -32,6 +32,8 @@ async function run(): Promise { weirdCharactersExample(), copyTextFile(), listFolders(), + multipartNodeJsFileStream(), + fetchExampleMultipart(), createFolder(), ]); @@ -63,7 +65,7 @@ async function run(): Promise { async function textFileExample(): Promise { const start = Date.now(); - const blob = await vercelBlob.put('folder/test.txt', 'Hello, world!', { + const blob = await vercelBlob.put('folderé/test.txt', 'Hello, world!', { access: 'public', }); console.log('Text file example:', blob.url, `(${Date.now() - start}ms)`); @@ -293,6 +295,51 @@ async function listFolders() { return blob.url; } +async function multipartNodeJsFileStream() { + const pathname = 'big-video.mp4'; + const fullPath = `public/${pathname}`; + const stream = createReadStream(fullPath); + stream.once('error', (error) => { + console.log(error); + throw error; + }); + + const start = Date.now(); + + // testing with an accent + const blob = await vercelBlob.put(`éllo/${pathname}`, stream, { + access: 'public', + multipart: true, + }); + + console.log( + 'Node.js multipart file stream example:', + blob.url, + `(${Date.now() - start}ms)`, + ); + + return blob.url; +} + +async function fetchExampleMultipart(): Promise { + const start = Date.now(); + + const response = await fetch( + 'https://example-files.online-convert.com/video/mp4/example_big.mp4', + ); + + const blob = await vercelBlob.put( + 'example_big.mp4', + response.body as ReadableStream, + { + access: 'public', + multipart: true, + }, + ); + + console.log('fetch example:', blob.url, `(${Date.now() - start}ms)`); + return blob.url; +} async function createFolder() { const start = Date.now(); diff --git a/test/next/src/pages/api/vercel/blob/pages/serverless.ts b/test/next/src/pages/api/vercel/blob/pages/serverless.ts index ac819fb4e..3d727c4f9 100644 --- a/test/next/src/pages/api/vercel/blob/pages/serverless.ts +++ b/test/next/src/pages/api/vercel/blob/pages/serverless.ts @@ -11,6 +11,7 @@ export default async function handleBody( response: NextApiResponse, ): Promise { const pathname = request.query.filename as string; + const multipart = request.query.multipart === '1'; if (!request.body || !pathname) { response.status(400).json({ message: 'No file to upload.' }); @@ -25,6 +26,7 @@ export default async function handleBody( // Note: this will stream the file to Vercel's Blob Store const blob = await vercelBlob.put(pathname, request.body as string, { access: 'public', + multipart, }); response.json(blob); diff --git a/test/next/src/pages/vercel/pages/blob/image.tsx b/test/next/src/pages/vercel/pages/blob/image.tsx index 2e18103af..a59b4c369 100644 --- a/test/next/src/pages/vercel/pages/blob/image.tsx +++ b/test/next/src/pages/vercel/pages/blob/image.tsx @@ -22,7 +22,7 @@ export const getServerSideProps: GetServerSideProps = async (req) => { export default function Blob(props: vercelBlob.PutBlobResult): JSX.Element { return (
    -

    Render an upload image on the browser

    +

    Render an upload image on the browser

    {/* eslint-disable-next-line @next/next/no-img-element -- we want an image element here, fine */} test diff --git a/test/next/src/pages/vercel/pages/blob/index.tsx b/test/next/src/pages/vercel/pages/blob/index.tsx index 76a5f26b1..c5408e503 100644 --- a/test/next/src/pages/vercel/pages/blob/index.tsx +++ b/test/next/src/pages/vercel/pages/blob/index.tsx @@ -21,7 +21,7 @@ export default function Blob( ): JSX.Element { return (
    -

    blob

    +

    blob

    {props.pathname}

    {props.content}

    diff --git a/test/next/tailwind.config.js b/test/next/tailwind.config.js index 66e198bfc..d0e42fd26 100644 --- a/test/next/tailwind.config.js +++ b/test/next/tailwind.config.js @@ -14,5 +14,5 @@ module.exports = { }, }, }, - plugins: [], + plugins: [require('@tailwindcss/forms')], }; diff --git a/test/next/test/@vercel/blob/index.test.ts b/test/next/test/@vercel/blob/index.test.ts index 9f3e22d22..f028d2313 100644 --- a/test/next/test/@vercel/blob/index.test.ts +++ b/test/next/test/@vercel/blob/index.test.ts @@ -96,6 +96,62 @@ test.describe('@vercel/blob', () => { }); }); }); + + test.describe('multipart upload', () => { + test('multipart client upload', async ({ browser }) => { + const callback = '/vercel/blob/api/app/handle-blob-upload/serverless'; + const browserContext = await browser.newContext(); + await browserContext.addCookies([ + { + name: 'clientUpload', + value: process.env.BLOB_UPLOAD_SECRET ?? '', + path: '/', + domain: ( + process.env.PLAYWRIGHT_TEST_BASE_URL ?? 'localhost' + ).replace('https://', ''), + }, + ]); + const page = await browserContext.newPage(); + await page.goto( + `vercel/blob/app/test/client?filename=${prefix}/test-app-client.txt&callback=${callback}&multipart=1`, + ); + + const textContent = await page.locator('#blob-path').textContent(); + expect(textContent).toBe(`${prefix}/test-app-client.txt`); + expect(await page.locator('#blob-content').textContent()).toBe( + `Hello from ${prefix}/test-app-client.txt`, + ); + }); + + test.describe('multipart server upload (app router)', () => { + [ + 'vercel/blob/api/app/body/edge', + 'vercel/blob/api/app/body/serverless', + 'api/vercel/blob/pages/edge', + 'api/vercel/blob/pages/serverless', + ].forEach((path) => { + test(path, async ({ request }) => { + const data = (await request + .post(`${path}?filename=${prefix}/test.txt&multipart=1`, { + data: `Hello world ${path} ${prefix}`, + headers: { + cookie: `clientUpload=${ + process.env.BLOB_UPLOAD_SECRET ?? '' + }`, + }, + }) + .then((r) => r.json())) as PutBlobResult; + expect(data.contentDisposition).toBe( + 'attachment; filename="test.txt"', + ); + expect(data.contentType).toBe('text/plain'); + expect(data.pathname).toBe(`${prefix}/test.txt`); + const content = await request.get(data.url).then((r) => r.text()); + expect(content).toBe(`Hello world ${path} ${prefix}`); + }); + }); + }); + }); }); test.afterAll(async ({ request }) => {