From 331e46cdf486894f2a88b75bf7a27088b8461f21 Mon Sep 17 00:00:00 2001 From: Espen Hovlandsdal Date: Wed, 10 Apr 2024 13:33:40 -0700 Subject: [PATCH] chore: move `@sanity/export` out of monorepo (#6314) --- packages/@repo/test-exports/.depcheckrc.json | 1 - packages/@repo/test-exports/package.json | 1 - packages/@sanity/export/.eslintrc.cjs | 16 - packages/@sanity/export/.gitignore | 16 - packages/@sanity/export/LICENSE | 21 - packages/@sanity/export/README.md | 59 --- packages/@sanity/export/jest.config.cjs | 8 - packages/@sanity/export/package.json | 54 --- packages/@sanity/export/src/AssetHandler.js | 382 ------------------ packages/@sanity/export/src/debug.js | 1 - packages/@sanity/export/src/export.js | 215 ---------- .../@sanity/export/src/filterDocumentTypes.js | 14 - packages/@sanity/export/src/filterDrafts.js | 12 - .../export/src/filterSystemDocuments.js | 14 - .../@sanity/export/src/getDocumentsStream.js | 15 - packages/@sanity/export/src/logFirstChunk.js | 15 - .../@sanity/export/src/rejectOnApiError.js | 16 - packages/@sanity/export/src/requestStream.js | 56 --- .../@sanity/export/src/stringifyStream.js | 4 - packages/@sanity/export/src/tryParseJson.js | 13 - packages/@sanity/export/src/util/rimraf.js | 4 - .../@sanity/export/src/validateOptions.js | 56 --- packages/@sanity/export/test/.eslintrc | 3 - .../@sanity/export/test/AssetHandler.test.js | 138 ------- .../__snapshots__/AssetHandler.test.js.snap | 95 ----- packages/@sanity/export/test/helpers/index.js | 48 --- pnpm-lock.yaml | 88 ++-- 27 files changed, 40 insertions(+), 1325 deletions(-) delete mode 100644 packages/@sanity/export/.eslintrc.cjs delete mode 100644 packages/@sanity/export/.gitignore delete mode 100644 packages/@sanity/export/LICENSE delete mode 100644 packages/@sanity/export/README.md delete mode 100644 packages/@sanity/export/jest.config.cjs delete mode 100644 packages/@sanity/export/package.json delete mode 100644 packages/@sanity/export/src/AssetHandler.js delete mode 100644 packages/@sanity/export/src/debug.js delete mode 100644 packages/@sanity/export/src/export.js delete mode 100644 packages/@sanity/export/src/filterDocumentTypes.js delete mode 100644 packages/@sanity/export/src/filterDrafts.js delete mode 100644 packages/@sanity/export/src/filterSystemDocuments.js delete mode 100644 packages/@sanity/export/src/getDocumentsStream.js delete mode 100644 packages/@sanity/export/src/logFirstChunk.js delete mode 100644 packages/@sanity/export/src/rejectOnApiError.js delete mode 100644 packages/@sanity/export/src/requestStream.js delete mode 100644 packages/@sanity/export/src/stringifyStream.js delete mode 100644 packages/@sanity/export/src/tryParseJson.js delete mode 100644 packages/@sanity/export/src/util/rimraf.js delete mode 100644 packages/@sanity/export/src/validateOptions.js delete mode 100644 packages/@sanity/export/test/.eslintrc delete mode 100644 packages/@sanity/export/test/AssetHandler.test.js delete mode 100644 packages/@sanity/export/test/__snapshots__/AssetHandler.test.js.snap delete mode 100644 packages/@sanity/export/test/helpers/index.js diff --git a/packages/@repo/test-exports/.depcheckrc.json b/packages/@repo/test-exports/.depcheckrc.json index c3192c893f3..ad32a1d6ba7 100644 --- a/packages/@repo/test-exports/.depcheckrc.json +++ b/packages/@repo/test-exports/.depcheckrc.json @@ -5,7 +5,6 @@ "@sanity/cli", "@sanity/codegen", "@sanity/diff", - "@sanity/export", "@sanity/import", "@sanity/import-cli", "@sanity/migrate", diff --git a/packages/@repo/test-exports/package.json b/packages/@repo/test-exports/package.json index 0b46862019f..45961b79059 100644 --- a/packages/@repo/test-exports/package.json +++ b/packages/@repo/test-exports/package.json @@ -14,7 +14,6 @@ "@sanity/cli": "workspace:*", "@sanity/codegen": "workspace:*", "@sanity/diff": "workspace:*", - "@sanity/export": "workspace:*", "@sanity/import": "workspace:*", "@sanity/import-cli": "workspace:*", "@sanity/migrate": "workspace:*", diff --git a/packages/@sanity/export/.eslintrc.cjs b/packages/@sanity/export/.eslintrc.cjs deleted file mode 100644 index 109de0ef37c..00000000000 --- a/packages/@sanity/export/.eslintrc.cjs +++ /dev/null @@ -1,16 +0,0 @@ -'use strict' - -const path = require('path') - -const ROOT_PATH = path.resolve(__dirname, '../../..') - -module.exports = { - env: { - browser: false, - node: true, - }, - rules: { - '@typescript-eslint/no-var-requires': 'off', - 'import/no-extraneous-dependencies': ['error', {packageDir: [ROOT_PATH, __dirname]}], - }, -} diff --git a/packages/@sanity/export/.gitignore b/packages/@sanity/export/.gitignore deleted file mode 100644 index 5df57fd32a6..00000000000 --- a/packages/@sanity/export/.gitignore +++ /dev/null @@ -1,16 +0,0 @@ -# Logs -/logs -*.log - -# Coverage directory used by tools like istanbul -/coverage - -# Dependency directories -/node_modules - -# Compiled code -/lib - -# Dev-fixtures -moviedb.ndjson -dist diff --git a/packages/@sanity/export/LICENSE b/packages/@sanity/export/LICENSE deleted file mode 100644 index c5f080fd508..00000000000 --- a/packages/@sanity/export/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2016 - 2024 Sanity.io - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/packages/@sanity/export/README.md b/packages/@sanity/export/README.md deleted file mode 100644 index 26d7062bbdc..00000000000 --- a/packages/@sanity/export/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# @sanity/export - -Exports documents and assets from a Sanity dataset - -## Installing - -``` -npm install --save @sanity/export -``` - -## Usage - -```js -const exportDataset = require('@sanity/export') - -exportDataset({ - // Instance of @sanity/client configured to correct project ID and dataset - client: someInstantiatedSanityClientInstance, - - // Name of dataset to export - dataset: 'myDataset', - - // Path to write tar.gz-archive file to, or `-` for stdout - outputPath: '/home/your-user/myDataset.tar.gz', - - // Whether or not to export assets. Note that this operation is currently slightly lossy; - // metadata stored on the asset document itself (original filename, for instance) might be lost - // Default: `true` - assets: false, - - // Exports documents only, without downloading or rewriting asset references - // Default: `false` - raw: true, - - // Whether or not to export drafts - // Default: `true` - drafts: true, - - // Export only given document types (`_type`) - // Optional, default: all types - types: ['products', 'shops'], - - // Run 12 concurrent asset downloads - assetConcurrency: 12, -}) -``` - -## Future improvements - -- Restore original filenames, keep track of duplicates, increase counter (`filename ().ext`) -- Skip archiving on raw/no-asset mode? - -## CLI-tool - -This functionality is built in to the `@sanity/cli` package as `sanity dataset export` - -## License - -MIT-licensed. See LICENSE. diff --git a/packages/@sanity/export/jest.config.cjs b/packages/@sanity/export/jest.config.cjs deleted file mode 100644 index 51ecfb62217..00000000000 --- a/packages/@sanity/export/jest.config.cjs +++ /dev/null @@ -1,8 +0,0 @@ -'use strict' - -const {createJestConfig} = require('../../../test/config.cjs') - -module.exports = createJestConfig({ - displayName: require('./package.json').name, - testEnvironment: 'node', -}) diff --git a/packages/@sanity/export/package.json b/packages/@sanity/export/package.json deleted file mode 100644 index abfa597e905..00000000000 --- a/packages/@sanity/export/package.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "name": "@sanity/export", - "version": "3.37.2", - "description": "Export Sanity documents and assets", - "keywords": [ - "sanity", - "cms", - "headless", - "realtime", - "content", - "export", - "ndjson" - ], - "homepage": "https://www.sanity.io/", - "bugs": { - "url": "https://github.com/sanity-io/sanity/issues" - }, - "repository": { - "type": "git", - "url": "git+https://github.com/sanity-io/sanity.git", - "directory": "packages/@sanity/export" - }, - "license": "MIT", - "author": "Sanity.io ", - "main": "./src/export.js", - "files": [ - "src" - ], - "scripts": { - "lint": "eslint .", - "test": "jest" - }, - "dependencies": { - "@sanity/util": "3.37.2", - "archiver": "^7.0.0", - "debug": "^4.3.4", - "get-it": "^8.4.18", - "lodash": "^4.17.21", - "mississippi": "^4.0.0", - "p-queue": "^2.3.0", - "rimraf": "^3.0.2", - "split2": "^4.2.0" - }, - "devDependencies": { - "@jest/globals": "^29.7.0", - "string-to-stream": "^1.1.0" - }, - "engines": { - "node": ">=18" - }, - "publishConfig": { - "access": "public" - } -} diff --git a/packages/@sanity/export/src/AssetHandler.js b/packages/@sanity/export/src/AssetHandler.js deleted file mode 100644 index 80df5bcf999..00000000000 --- a/packages/@sanity/export/src/AssetHandler.js +++ /dev/null @@ -1,382 +0,0 @@ -const crypto = require('crypto') -const {mkdirSync, createWriteStream} = require('fs') -const path = require('path') -const {parse: parseUrl, format: formatUrl} = require('url') -const {omit, noop} = require('lodash') -const miss = require('mississippi') -const PQueue = require('p-queue') -const pkg = require('../package.json') -const debug = require('./debug') -const requestStream = require('./requestStream') -const rimraf = require('./util/rimraf') - -const EXCLUDE_PROPS = ['_id', '_type', 'assetId', 'extension', 'mimeType', 'path', 'url'] -const ACTION_REMOVE = 'remove' -const ACTION_REWRITE = 'rewrite' -const ASSET_DOWNLOAD_CONCURRENCY = 8 - -const retryHelper = (times, fn, onError) => { - let attempt = 0 - const caller = (...args) => { - return fn(...args).catch((err) => { - if (onError) { - onError(err, attempt) - } - if (attempt < times) { - attempt++ - return caller(...args) - } - - throw err - }) - } - return caller -} - -class AssetHandler { - constructor(options) { - const concurrency = options.concurrency || ASSET_DOWNLOAD_CONCURRENCY - debug('Using asset download concurrency of %d', concurrency) - - this.client = options.client - this.tmpDir = options.tmpDir - this.assetDirsCreated = false - - this.downloading = [] - this.assetsSeen = new Map() - this.assetMap = {} - this.filesWritten = 0 - this.queueSize = 0 - this.queue = options.queue || new PQueue({concurrency}) - - this.rejectedError = null - this.reject = (err) => { - this.rejectedError = err - } - } - - clear() { - this.assetsSeen.clear() - this.queue.clear() - this.queueSize = 0 - } - - finish() { - return new Promise((resolve, reject) => { - if (this.rejectedError) { - reject(this.rejectedError) - return - } - - this.reject = reject - this.queue.onIdle().then(() => resolve(this.assetMap)) - }) - } - - // Called when we want to download all assets to local filesystem and rewrite documents to hold - // placeholder asset references (_sanityAsset: 'image@file:///local/path') - rewriteAssets = miss.through.obj(async (doc, enc, callback) => { - if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) { - const type = doc._type === 'sanity.imageAsset' ? 'image' : 'file' - const filePath = `${type}s/${generateFilename(doc._id)}` - this.assetsSeen.set(doc._id, type) - this.queueAssetDownload(doc, filePath, type) - callback() - return - } - - callback(null, this.findAndModify(doc, ACTION_REWRITE)) - }) - - // Called in the case where we don't _want_ assets, so basically just remove all asset documents - // as well as references to assets (*.asset._ref ^= (image|file)-) - stripAssets = miss.through.obj(async (doc, enc, callback) => { - if (['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type)) { - callback() - return - } - - callback(null, this.findAndModify(doc, ACTION_REMOVE)) - }) - - // Called when we are using raw export mode along with `assets: false`, where we simply - // want to skip asset documents but retain asset references (useful for data mangling) - skipAssets = miss.through.obj((doc, enc, callback) => { - const isAsset = ['sanity.imageAsset', 'sanity.fileAsset'].includes(doc._type) - if (isAsset) { - callback() - return - } - - callback(null, doc) - }) - - noop = miss.through.obj((doc, enc, callback) => callback(null, doc)) - - queueAssetDownload(assetDoc, dstPath, type) { - if (!assetDoc.url) { - debug('Asset document "%s" does not have a URL property, skipping', assetDoc._id) - return - } - - debug('Adding download task for %s (destination: %s)', assetDoc._id, dstPath) - this.queueSize++ - this.downloading.push(assetDoc.url) - - const doDownload = retryHelper( - 10, // try 10 times - () => this.downloadAsset(assetDoc, dstPath), - (err, attempt) => { - debug( - `Error downloading asset %s (destination: %s), attempt %d`, - assetDoc._id, - dstPath, - attempt, - err, - ) - }, - ) - this.queue.add(() => - doDownload().catch((err) => { - debug('Error downloading asset', err) - this.queue.clear() - this.reject(err) - }), - ) - } - - maybeCreateAssetDirs() { - if (this.assetDirsCreated) { - return - } - - /* eslint-disable no-sync */ - mkdirSync(path.join(this.tmpDir, 'files'), {recursive: true}) - mkdirSync(path.join(this.tmpDir, 'images'), {recursive: true}) - /* eslint-enable no-sync */ - this.assetDirsCreated = true - } - - getAssetRequestOptions(assetDoc) { - const token = this.client.config().token - const headers = {'User-Agent': `${pkg.name}@${pkg.version}`} - const isImage = assetDoc._type === 'sanity.imageAsset' - - const url = parseUrl(assetDoc.url, true) - if (isImage && ['cdn.sanity.io', 'cdn.sanity.work'].includes(url.hostname) && token) { - headers.Authorization = `Bearer ${token}` - url.query = {...(url.query || {}), dlRaw: 'true'} - } - - return {url: formatUrl(url), headers} - } - - // eslint-disable-next-line max-statements - async downloadAsset(assetDoc, dstPath) { - const {url} = assetDoc - const options = this.getAssetRequestOptions(assetDoc) - - let stream - try { - stream = await requestStream(options) - } catch (err) { - throw new Error('Failed create asset stream', {cause: err}) - } - - if (stream.statusCode !== 200) { - this.queue.clear() - try { - const err = await tryGetErrorFromStream(stream) - let errMsg = `Referenced asset URL "${url}" returned HTTP ${stream.statusCode}` - if (err) { - errMsg = `${errMsg}:\n\n${err}` - } - - throw new Error(errMsg) - } catch (err) { - throw new Error('Failed to parse error response from asset stream', {cause: err}) - } - } - - this.maybeCreateAssetDirs() - - debug('Asset stream ready, writing to filesystem at %s', dstPath) - const tmpPath = path.join(this.tmpDir, dstPath) - let sha1 = '' - let md5 = '' - let size = 0 - try { - const res = await writeHashedStream(tmpPath, stream) - sha1 = res.sha1 - md5 = res.md5 - size = res.size - } catch (err) { - throw new Error('Failed to write asset stream to filesystem', {cause: err}) - } - - // Verify it against our downloaded stream to make sure we have the same copy - const contentLength = stream.headers['content-length'] - const remoteSha1 = stream.headers['x-sanity-sha1'] - const remoteMd5 = stream.headers['x-sanity-md5'] - const hasHash = Boolean(remoteSha1 || remoteMd5) - const method = sha1 ? 'sha1' : 'md5' - - // Asset validity is primarily determined by the sha1 hash. However, the sha1 hash is computed - // before certain processes (i.e. svg sanitization) which can result in a different hash. - // When the sha1 hashes don't match, fallback to using the md5 hash. - const sha1Differs = remoteSha1 && sha1 !== remoteSha1 - const md5Differs = remoteMd5 && md5 !== remoteMd5 - const differs = sha1Differs && md5Differs - - if (differs) { - const details = [ - hasHash && - (method === 'md5' - ? `md5 should be ${remoteMd5}, got ${md5}` - : `sha1 should be ${remoteSha1}, got ${sha1}`), - - contentLength && - parseInt(contentLength, 10) !== size && - `Asset should be ${contentLength} bytes, got ${size}`, - - `Did not succeed after ${attemptNum} attempts.`, - ] - - const detailsString = `Details:\n - ${details.filter(Boolean).join('\n - ')}` - - await rimraf(tmpPath) - - throw new Error(`Failed to download asset at ${assetDoc.url}. ${detailsString}`) - } - - const isImage = assetDoc._type === 'sanity.imageAsset' - const type = isImage ? 'image' : 'file' - const id = `${type}-${sha1}` - - const metaProps = omit(assetDoc, EXCLUDE_PROPS) - if (Object.keys(metaProps).length > 0) { - this.assetMap[id] = metaProps - } - - this.downloading.splice( - this.downloading.findIndex((datUrl) => datUrl === url), - 1, - ) - - this.filesWritten++ - return true - } - - findAndModify = (item, action) => { - if (Array.isArray(item)) { - const children = item.map((child) => this.findAndModify(child, action)) - return children.filter(Boolean) - } - - if (!item || typeof item !== 'object') { - return item - } - - const isAsset = isAssetField(item) - if (isAsset && action === ACTION_REMOVE) { - return undefined - } - - if (isAsset && action === ACTION_REWRITE) { - const {asset, ...other} = item - const assetId = asset._ref - const assetType = getAssetType(item) - const filePath = `${assetType}s/${generateFilename(assetId)}` - return { - _sanityAsset: `${assetType}@file://./${filePath}`, - ...this.findAndModify(other, action), - } - } - - const newItem = {} - const keys = Object.keys(item) - for (let i = 0; i < keys.length; i++) { - const key = keys[i] - const value = item[key] - - newItem[key] = this.findAndModify(value, action) - - if (typeof newItem[key] === 'undefined') { - delete newItem[key] - } - } - - return newItem - } -} - -function isAssetField(item) { - return item.asset && item.asset._ref && isSanityAsset(item.asset._ref) -} - -function getAssetType(item) { - if (!item.asset || typeof item.asset._ref !== 'string') { - return null - } - - const [, type] = item.asset._ref.match(/^(image|file)-/) || [] - return type || null -} - -function isSanityAsset(assetId) { - return ( - /^image-[a-f0-9]{40}-\d+x\d+-[a-z]+$/.test(assetId) || - /^file-[a-f0-9]{40}-[a-z0-9]+$/.test(assetId) - ) -} - -function generateFilename(assetId) { - const [, , asset, ext] = assetId.match(/^(image|file)-(.*?)(-[a-z]+)?$/) || [] - const extension = (ext || 'bin').replace(/^-/, '') - return asset ? `${asset}.${extension}` : `${assetId}.bin` -} - -function writeHashedStream(filePath, stream) { - let size = 0 - const md5 = crypto.createHash('md5') - const sha1 = crypto.createHash('sha1') - - const hasher = miss.through((chunk, enc, cb) => { - size += chunk.length - md5.update(chunk) - sha1.update(chunk) - cb(null, chunk) - }) - - return new Promise((resolve, reject) => - miss.pipe(stream, hasher, createWriteStream(filePath), (err) => { - if (err) { - reject(err) - return - } - - resolve({ - size, - sha1: sha1.digest('hex'), - md5: md5.digest('hex'), - }) - }), - ) -} - -function tryGetErrorFromStream(stream) { - return new Promise((resolve, reject) => { - miss.pipe(stream, miss.concat(parse), (err) => (err ? reject(err) : noop)) - - function parse(body) { - try { - const parsed = JSON.parse(body.toString('utf8')) - resolve(parsed.message || parsed.error || null) - } catch (err) { - resolve(body.toString('utf8').slice(0, 16000)) - } - } - }) -} - -module.exports = AssetHandler diff --git a/packages/@sanity/export/src/debug.js b/packages/@sanity/export/src/debug.js deleted file mode 100644 index 3e9bce51fec..00000000000 --- a/packages/@sanity/export/src/debug.js +++ /dev/null @@ -1 +0,0 @@ -module.exports = require('debug')('sanity:export') diff --git a/packages/@sanity/export/src/export.js b/packages/@sanity/export/src/export.js deleted file mode 100644 index 52c602a8d50..00000000000 --- a/packages/@sanity/export/src/export.js +++ /dev/null @@ -1,215 +0,0 @@ -const fs = require('fs') -const os = require('os') -const path = require('path') -const zlib = require('zlib') -const archiver = require('archiver') -const miss = require('mississippi') -const split = require('split2') -const AssetHandler = require('./AssetHandler') -const debug = require('./debug') -const filterDocumentTypes = require('./filterDocumentTypes') -const filterDrafts = require('./filterDrafts') -const filterSystemDocuments = require('./filterSystemDocuments') -const getDocumentsStream = require('./getDocumentsStream') -const logFirstChunk = require('./logFirstChunk') -const rejectOnApiError = require('./rejectOnApiError') -const stringifyStream = require('./stringifyStream') -const tryParseJson = require('./tryParseJson') -const rimraf = require('./util/rimraf') -const validateOptions = require('./validateOptions') - -const noop = () => null - -function exportDataset(opts) { - const options = validateOptions(opts) - const onProgress = options.onProgress || noop - const archive = archiver('tar', { - gzip: true, - gzipOptions: {level: options.compress ? zlib.Z_DEFAULT_COMPRESSION : zlib.Z_NO_COMPRESSION}, - }) - - const slugDate = new Date() - .toISOString() - .replace(/[^a-z0-9]/gi, '-') - .toLowerCase() - - const prefix = `${opts.dataset}-export-${slugDate}` - const tmpDir = path.join(os.tmpdir(), prefix) - const cleanup = () => - rimraf(tmpDir).catch((err) => { - debug(`Error while cleaning up temporary files: ${err.message}`) - }) - - const assetHandler = new AssetHandler({ - client: options.client, - tmpDir, - prefix, - concurrency: options.assetConcurrency, - }) - - debug('Outputting assets (temporarily) to %s', tmpDir) - debug('Outputting to %s', options.outputPath === '-' ? 'stdout' : options.outputPath) - - let outputStream - if (isWritableStream(options.outputPath)) { - outputStream = options.outputPath - } else { - outputStream = - options.outputPath === '-' ? process.stdout : fs.createWriteStream(options.outputPath) - } - - let assetStreamHandler = assetHandler.noop - if (!options.raw) { - assetStreamHandler = options.assets ? assetHandler.rewriteAssets : assetHandler.stripAssets - } - - return new Promise(async (resolve, reject) => { - miss.finished(archive, async (archiveErr) => { - if (archiveErr) { - debug('Archiving errored! %s', archiveErr.stack) - await cleanup() - reject(archiveErr) - return - } - - debug('Archive finished!') - }) - - debug('Getting dataset export stream') - onProgress({step: 'Exporting documents...'}) - - let documentCount = 0 - let lastReported = Date.now() - const reportDocumentCount = (chunk, enc, cb) => { - ++documentCount - - const now = Date.now() - if (now - lastReported > 50) { - onProgress({ - step: 'Exporting documents...', - current: documentCount, - total: '?', - update: true, - }) - - lastReported = now - } - - cb(null, chunk) - } - - const inputStream = await getDocumentsStream(options.client, options.dataset) - debug('Got HTTP %d', inputStream.statusCode) - debug('Response headers: %o', inputStream.headers) - - const jsonStream = miss.pipeline( - inputStream, - logFirstChunk(), - split(tryParseJson), - rejectOnApiError(), - filterSystemDocuments(), - assetStreamHandler, - filterDocumentTypes(options.types), - options.drafts ? miss.through.obj() : filterDrafts(), - stringifyStream(), - miss.through(reportDocumentCount), - ) - - miss.finished(jsonStream, async (err) => { - if (err) { - return - } - - onProgress({ - step: 'Exporting documents...', - current: documentCount, - total: documentCount, - update: true, - }) - - if (!options.raw && options.assets) { - onProgress({step: 'Downloading assets...'}) - } - - let prevCompleted = 0 - const progressInterval = setInterval(() => { - const completed = - assetHandler.queueSize - assetHandler.queue.size - assetHandler.queue.pending - - if (prevCompleted === completed) { - return - } - - prevCompleted = completed - onProgress({ - step: 'Downloading assets...', - current: completed, - total: assetHandler.queueSize, - update: true, - }) - }, 500) - - debug('Waiting for asset handler to complete downloads') - try { - const assetMap = await assetHandler.finish() - - // Make sure we mark the progress as done (eg 100/100 instead of 99/100) - onProgress({ - step: 'Downloading assets...', - current: assetHandler.queueSize, - total: assetHandler.queueSize, - update: true, - }) - - archive.append(JSON.stringify(assetMap), {name: 'assets.json', prefix}) - clearInterval(progressInterval) - } catch (assetErr) { - clearInterval(progressInterval) - await cleanup() - reject(assetErr) - return - } - - // Add all downloaded assets to archive - archive.directory(path.join(tmpDir, 'files'), `${prefix}/files`, {store: true}) - archive.directory(path.join(tmpDir, 'images'), `${prefix}/images`, {store: true}) - - debug('Finalizing archive, flushing streams') - onProgress({step: 'Adding assets to archive...'}) - archive.finalize() - }) - - archive.on('warning', (err) => { - debug('Archive warning: %s', err.message) - }) - - archive.append(jsonStream, {name: 'data.ndjson', prefix}) - miss.pipe(archive, outputStream, onComplete) - - async function onComplete(err) { - onProgress({step: 'Clearing temporary files...'}) - await cleanup() - - if (!err) { - resolve() - return - } - - debug('Error during streaming: %s', err.stack) - assetHandler.clear() - reject(err) - } - }) -} - -function isWritableStream(val) { - return ( - val !== null && - typeof val === 'object' && - typeof val.pipe === 'function' && - typeof val._write === 'function' && - typeof val._writableState === 'object' - ) -} - -module.exports = exportDataset diff --git a/packages/@sanity/export/src/filterDocumentTypes.js b/packages/@sanity/export/src/filterDocumentTypes.js deleted file mode 100644 index 5602a0fc7d2..00000000000 --- a/packages/@sanity/export/src/filterDocumentTypes.js +++ /dev/null @@ -1,14 +0,0 @@ -const miss = require('mississippi') - -module.exports = (allowedTypes) => - allowedTypes - ? miss.through.obj((doc, enc, callback) => { - const type = doc && doc._type - if (allowedTypes.includes(type)) { - callback(null, doc) - return - } - - callback() - }) - : miss.through.obj() diff --git a/packages/@sanity/export/src/filterDrafts.js b/packages/@sanity/export/src/filterDrafts.js deleted file mode 100644 index ba9c686a2b2..00000000000 --- a/packages/@sanity/export/src/filterDrafts.js +++ /dev/null @@ -1,12 +0,0 @@ -const miss = require('mississippi') - -const isDraft = (doc) => doc && doc._id && doc._id.indexOf('drafts.') === 0 - -module.exports = () => - miss.through.obj((doc, enc, callback) => { - if (isDraft(doc)) { - return callback() - } - - return callback(null, doc) - }) diff --git a/packages/@sanity/export/src/filterSystemDocuments.js b/packages/@sanity/export/src/filterSystemDocuments.js deleted file mode 100644 index c731aa62fa7..00000000000 --- a/packages/@sanity/export/src/filterSystemDocuments.js +++ /dev/null @@ -1,14 +0,0 @@ -const miss = require('mississippi') -const debug = require('./debug') - -const isSystemDocument = (doc) => doc && doc._id && doc._id.indexOf('_.') === 0 - -module.exports = () => - miss.through.obj((doc, enc, callback) => { - if (isSystemDocument(doc)) { - debug('%s is a system document, skipping', doc && doc._id) - return callback() - } - - return callback(null, doc) - }) diff --git a/packages/@sanity/export/src/getDocumentsStream.js b/packages/@sanity/export/src/getDocumentsStream.js deleted file mode 100644 index 875dc983bb2..00000000000 --- a/packages/@sanity/export/src/getDocumentsStream.js +++ /dev/null @@ -1,15 +0,0 @@ -const pkg = require('../package.json') -const requestStream = require('./requestStream') - -module.exports = (client, dataset) => { - // Sanity client doesn't handle streams natively since we want to support node/browser - // with same API. We're just using it here to get hold of URLs and tokens. - const url = client.getUrl(`/data/export/${dataset}`) - const token = client.config().token - const headers = { - 'User-Agent': `${pkg.name}@${pkg.version}`, - ...(token ? {Authorization: `Bearer ${token}`} : {}), - } - - return requestStream({url, headers}) -} diff --git a/packages/@sanity/export/src/logFirstChunk.js b/packages/@sanity/export/src/logFirstChunk.js deleted file mode 100644 index a58a173f0bb..00000000000 --- a/packages/@sanity/export/src/logFirstChunk.js +++ /dev/null @@ -1,15 +0,0 @@ -const miss = require('mississippi') -const debug = require('./debug') - -module.exports = () => { - let firstChunk = true - return miss.through((chunk, enc, callback) => { - if (firstChunk) { - const string = chunk.toString('utf8').split('\n')[0] - debug('First chunk received: %s', string.slice(0, 300)) - firstChunk = false - } - - callback(null, chunk) - }) -} diff --git a/packages/@sanity/export/src/rejectOnApiError.js b/packages/@sanity/export/src/rejectOnApiError.js deleted file mode 100644 index da11dbb882c..00000000000 --- a/packages/@sanity/export/src/rejectOnApiError.js +++ /dev/null @@ -1,16 +0,0 @@ -const miss = require('mississippi') - -module.exports = () => - miss.through.obj((doc, enc, callback) => { - if (doc.error && doc.statusCode) { - callback(new Error([doc.statusCode, doc.error].join(': '))) - return - } - - if (!doc._id && doc.error) { - callback(new Error(doc.error.description || doc.error.message || JSON.stringify(doc))) - return - } - - callback(null, doc) - }) diff --git a/packages/@sanity/export/src/requestStream.js b/packages/@sanity/export/src/requestStream.js deleted file mode 100644 index 830eac35aed..00000000000 --- a/packages/@sanity/export/src/requestStream.js +++ /dev/null @@ -1,56 +0,0 @@ -const {getIt} = require('get-it') -// eslint-disable-next-line import/extensions -const {keepAlive, promise} = require('get-it/middleware') -const debug = require('./debug') - -const request = getIt([keepAlive(), promise({onlyBody: true})]) -const socketsWithTimeout = new WeakSet() - -const CONNECTION_TIMEOUT = 15 * 1000 // 15 seconds -const READ_TIMEOUT = 3 * 60 * 1000 // 3 minutes -const MAX_RETRIES = 5 - -function delay(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)) -} - -/* eslint-disable no-await-in-loop, max-depth */ -module.exports = async (options) => { - let error - for (let i = 0; i < MAX_RETRIES; i++) { - try { - const response = await request({ - ...options, - stream: true, - maxRedirects: 0, - timeout: {connect: CONNECTION_TIMEOUT, socket: READ_TIMEOUT}, - }) - - if ( - response.connection && - typeof response.connection.setTimeout === 'function' && - !socketsWithTimeout.has(response.connection) - ) { - socketsWithTimeout.add(response.connection) - response.connection.setTimeout(READ_TIMEOUT, () => { - response.destroy( - new Error(`Read timeout: No data received on socket for ${READ_TIMEOUT} ms`), - ) - }) - } - - return response - } catch (err) { - error = err - - if (err.response && err.response.statusCode && err.response.statusCode < 500) { - break - } - - debug('Error, retrying after 1500ms: %s', err.message) - await delay(1500) - } - } - - throw error -} diff --git a/packages/@sanity/export/src/stringifyStream.js b/packages/@sanity/export/src/stringifyStream.js deleted file mode 100644 index 8940916e7e8..00000000000 --- a/packages/@sanity/export/src/stringifyStream.js +++ /dev/null @@ -1,4 +0,0 @@ -const miss = require('mississippi') - -module.exports = () => - miss.through.obj((doc, enc, callback) => callback(null, `${JSON.stringify(doc)}\n`)) diff --git a/packages/@sanity/export/src/tryParseJson.js b/packages/@sanity/export/src/tryParseJson.js deleted file mode 100644 index b05b8ffd6c0..00000000000 --- a/packages/@sanity/export/src/tryParseJson.js +++ /dev/null @@ -1,13 +0,0 @@ -const {createSafeJsonParser} = require('@sanity/util/createSafeJsonParser') - -/** - * Safe JSON parser that is able to handle lines interrupted by an error object. - * - * This may occur when streaming NDJSON from the Export HTTP API. - * - * @internal - * @see {@link https://github.com/sanity-io/sanity/pull/1787 | Initial pull request} - */ -module.exports = createSafeJsonParser({ - errorLabel: 'Error streaming dataset', -}) diff --git a/packages/@sanity/export/src/util/rimraf.js b/packages/@sanity/export/src/util/rimraf.js deleted file mode 100644 index 44932166746..00000000000 --- a/packages/@sanity/export/src/util/rimraf.js +++ /dev/null @@ -1,4 +0,0 @@ -const {promisify} = require('util') -const rimrafCb = require('rimraf') - -module.exports = promisify(rimrafCb) diff --git a/packages/@sanity/export/src/validateOptions.js b/packages/@sanity/export/src/validateOptions.js deleted file mode 100644 index af07b98c055..00000000000 --- a/packages/@sanity/export/src/validateOptions.js +++ /dev/null @@ -1,56 +0,0 @@ -const defaults = require('lodash/defaults') - -const clientMethods = ['getUrl', 'config'] -const booleanFlags = ['assets', 'raw', 'compress', 'drafts'] -const exportDefaults = { - compress: true, - drafts: true, - assets: true, - raw: false, -} - -function validateOptions(opts) { - const options = defaults({}, opts, exportDefaults) - - if (typeof options.dataset !== 'string' || options.dataset.length < 1) { - throw new Error(`options.dataset must be a valid dataset name`) - } - - if (options.onProgress && typeof options.onProgress !== 'function') { - throw new Error(`options.onProgress must be a function`) - } - - if (!options.client) { - throw new Error('`options.client` must be set to an instance of @sanity/client') - } - - const missing = clientMethods.find((key) => typeof options.client[key] !== 'function') - if (missing) { - throw new Error( - `\`options.client\` is not a valid @sanity/client instance - no "${missing}" method found`, - ) - } - - const clientConfig = options.client.config() - if (!clientConfig.token) { - throw new Error('Client is not instantiated with a `token`') - } - - booleanFlags.forEach((flag) => { - if (typeof options[flag] !== 'boolean') { - throw new Error(`Flag ${flag} must be a boolean (true/false)`) - } - }) - - if (!options.outputPath) { - throw new Error('outputPath must be specified (- for stdout)') - } - - if (options.assetConcurrency && (options.assetConcurrency < 1 || options.assetConcurrency > 24)) { - throw new Error('`assetConcurrency` must be between 1 and 24') - } - - return options -} - -module.exports = validateOptions diff --git a/packages/@sanity/export/test/.eslintrc b/packages/@sanity/export/test/.eslintrc deleted file mode 100644 index 3143cd17ecb..00000000000 --- a/packages/@sanity/export/test/.eslintrc +++ /dev/null @@ -1,3 +0,0 @@ -{ - "env": {"jest": true} -} diff --git a/packages/@sanity/export/test/AssetHandler.test.js b/packages/@sanity/export/test/AssetHandler.test.js deleted file mode 100644 index 0e205af1ecf..00000000000 --- a/packages/@sanity/export/test/AssetHandler.test.js +++ /dev/null @@ -1,138 +0,0 @@ -import {afterAll, describe, expect, test} from '@jest/globals' - -const os = require('os') -const path = require('path') -const miss = require('mississippi') -const split = require('split2') -const rimraf = require('../src/util/rimraf') -const {getAssetHandler, arrayToStream} = require('./helpers') - -const docById = (docs, id) => docs.find((doc) => doc._id === id) - -describe('asset handler', () => { - afterAll(async () => { - await rimraf(path.join(os.tmpdir(), 'asset-handler-tests')) - }) - - test('can rewrite documents / queue downloads', (done) => { - // prettier-ignore - const docs = [ - {_id: 'doc1', _type: 'bike', name: 'Scooter', image: {_type: 'image', caption: 'Scooter bike', asset: {_ref: 'image-6dcdb82c282dbe0a09ff7a6b58b639732f2fb8de-3360x840-png'}, nested: {_type: 'image', asset: {_ref: 'image-6dcdb82c282dbe0a09ff7a6b58b639732f2fb8de-3360x840-png'}}}}, - {_id: 'doc2', _type: 'bike', name: 'Dupe', image: {_type: 'image', asset: {_ref: 'image-6dcdb82c282dbe0a09ff7a6b58b639732f2fb8de-3360x840-png'}}}, - {_id: 'doc3', _type: 'bike', name: 'Tandem', image: {_type: 'image', asset: {_ref: 'image-31befc6dfa0315d6c535b8a57e34f86c18eb5f20-310x282-jpg'}}}, - {_id: 'datMuxAsset', _type: 'mux.videoAsset', assetId: 'someAssetId'}, - {_id: 'refsDatMuxAsset', _type: 'bike', video: {_type: 'mux.video', asset: {_ref: 'datMuxAsset'}}}, - {_id: 'mzFgq1cvHSEeGscxBsRFoqKG', _type: 'sanity.imageAsset', url: 'https://cdn.sanity.io/images/__fixtures__/__test__/mzFgq1cvHSEeGscxBsRFoqKG-2048x1364.jpg'}, - {_id: 'image-54e9595477915230501dfec656c8e86235bb470a-dads3360x840-png', _type: 'sanity.imageAsset', url: 'https://cdn.sanity.io/images/__fixtures__/__test__/6dcdb82c282dbe0a09ff7a6b58b639732f2fb8de-3360x840.png'}, - {_id: 'image-31befc6dfa0315d6c535b8a57e34f86c18eb5f20-310x282-jpg', _type: 'sanity.imageAsset', url: 'https://cdn.sanity.io/images/__fixtures__/__test__/31befc6dfa0315d6c535b8a57e34f86c18eb5f20-310x282.jpg'}, - {_id: 'plain', _type: 'bike', name: 'Broom'} - ] - - const assetHandler = getAssetHandler() - arrayToStream(docs) - .pipe(split(JSON.parse)) - .pipe(assetHandler.rewriteAssets) - .pipe(miss.concat(onComplete)) - - async function onComplete(newDocs) { - expect(newDocs).toHaveLength(6) - expect(docById(newDocs, 'doc1')).toMatchSnapshot('Rewritten asset for doc1') - expect(docById(newDocs, 'doc2')).toMatchSnapshot('Rewritten asset for doc2') - expect(docById(newDocs, 'doc3')).toMatchSnapshot('Rewritten asset for doc3') - expect(docById(newDocs, 'plain')).toMatchSnapshot('Nothing rewritten in assetless doc') - expect(docById(newDocs, 'refsDatMuxAsset')).toMatchSnapshot( - 'Nothing rewritten if pattern doesnt match Sanity asset', - ) - - await assetHandler.finish() - expect(assetHandler.filesWritten).toEqual(3) - done() - } - }) - - test('can remove asset documents', (done) => { - // prettier-ignore - const docs = [ - {_id: 'doc1', _type: 'bike', name: 'Scooter', image: {_type: 'image', caption: 'Scooter bike', asset: {_ref: 'image-6dcdb82c282dbe0a09ff7a6b58b639732f2fb8de-3360x840-png'}}}, - {_id: 'doc2', _type: 'bike', name: 'Dupe', image: {_type: 'image', asset: {_ref: 'image-6dcdb82c282dbe0a09ff7a6b58b639732f2fb8de-3360x840-png'}}}, - {_id: 'doc3', _type: 'bike', name: 'Tandem', image: {_type: 'image', asset: {_ref: 'image-31befc6dfa0315d6c535b8a57e34f86c18eb5f20-310x282-jpg'}}}, - {_id: 'mzFgq1cvHSEeGscxBsRFoqKG', _type: 'sanity.imageAsset', url: 'https://cdn.sanity.io/images/__fixtures__/__test__/mzFgq1cvHSEeGscxBsRFoqKG-2048x1364.jpg'}, - {_id: 'image-6dcdb82c282dbe0a09ff7a6b58b639732f2fb8de-3360x840-png', _type: 'sanity.imageAsset', url: 'https://cdn.sanity.io/images/__fixtures__/__test__/6dcdb82c282dbe0a09ff7a6b58b639732f2fb8de-3360x840.png'}, - {_id: 'image-31befc6dfa0315d6c535b8a57e34f86c18eb5f20-310x282-jpg', _type: 'sanity.imageAsset', url: 'https://cdn.sanity.io/images/__fixtures__/__test__/31befc6dfa0315d6c535b8a57e34f86c18eb5f20-310x282.jpg'}, - {_id: 'plain', _type: 'bike', name: 'Broom'} - ] - - const assetHandler = getAssetHandler() - arrayToStream(docs) - .pipe(split(JSON.parse)) - .pipe(assetHandler.stripAssets) - .pipe(miss.concat(onComplete)) - - async function onComplete(newDocs) { - expect(newDocs).toHaveLength(4) - expect(docById(newDocs, 'doc1')).toMatchSnapshot('doc1 with no image asset') - expect(docById(newDocs, 'doc2')).toMatchSnapshot('doc2 with no image asset') - expect(docById(newDocs, 'doc3')).toMatchSnapshot('doc3 with no image asset') - expect(docById(newDocs, 'plain')).toMatchSnapshot('Nothing removed in assetless doc') - - await assetHandler.finish() - expect(assetHandler.filesWritten).toEqual(0) - done() - } - }) - - test('downloads assets that are not referenced by documents', (done) => { - // prettier-ignore - const docs = [ - {_id: 'image-6dcdb82c282dbe0a09ff7a6b58b639732f2fb8de-3360x840-png', _type: 'sanity.imageAsset', url: 'https://cdn.sanity.io/images/__fixtures__/__test__/6dcdb82c282dbe0a09ff7a6b58b639732f2fb8de-3360x840.png'}, - {_id: 'mzFgq1cvHSEeGscxBsRFoqKG', _type: 'sanity.imageAsset', url: 'https://cdn.sanity.io/images/__fixtures__/__test__/mzFgq1cvHSEeGscxBsRFoqKG-2048x1364.jpg'}, - {_id: 'plain', _type: 'bike', name: 'Broom'} - ] - - const assetHandler = getAssetHandler() - arrayToStream(docs) - .pipe(split(JSON.parse)) - .pipe(assetHandler.rewriteAssets) - .pipe(miss.concat(onComplete)) - - async function onComplete(newDocs) { - expect(newDocs).toHaveLength(1) - expect(docById(newDocs, 'plain')).toMatchObject(docs[2]) - - await assetHandler.finish() - expect(assetHandler.filesWritten).toEqual(2) - done() - } - }) - - test('can skip asset documents', (done) => { - // prettier-ignore - const docs = [ - {_id: 'doc1', _type: 'bike', name: 'Scooter', image: {_type: 'image', caption: 'Scooter bike', asset: {_ref: 'image-6dcdb82c282dbe0a09ff7a6b58b639732f2fb8de-3360x840-png'}}}, - {_id: 'doc2', _type: 'bike', name: 'Dupe', image: {_type: 'image', asset: {_ref: 'image-6dcdb82c282dbe0a09ff7a6b58b639732f2fb8de-3360x840-png'}}}, - {_id: 'doc3', _type: 'bike', name: 'Tandem', image: {_type: 'image', asset: {_ref: 'image-31befc6dfa0315d6c535b8a57e34f86c18eb5f20-310x282-jpg'}}}, - {_id: 'mzFgq1cvHSEeGscxBsRFoqKG', _type: 'sanity.imageAsset', url: 'https://cdn.sanity.io/images/__fixtures__/__test__/mzFgq1cvHSEeGscxBsRFoqKG-2048x1364.jpg'}, - {_id: 'image-6dcdb82c282dbe0a09ff7a6b58b639732f2fb8de-3360x840-png', _type: 'sanity.imageAsset', url: 'https://cdn.sanity.io/images/__fixtures__/__test__/6dcdb82c282dbe0a09ff7a6b58b639732f2fb8de-3360x840.png'}, - {_id: 'image-31befc6dfa0315d6c535b8a57e34f86c18eb5f20-310x282-jpg', _type: 'sanity.imageAsset', url: 'https://cdn.sanity.io/images/__fixtures__/__test__/31befc6dfa0315d6c535b8a57e34f86c18eb5f20-310x282.jpg'}, - {_id: 'plain', _type: 'bike', name: 'Broom'} - ] - - const assetHandler = getAssetHandler() - arrayToStream(docs) - .pipe(split(JSON.parse)) - .pipe(assetHandler.skipAssets) - .pipe(miss.concat(onComplete)) - - async function onComplete(newDocs) { - expect(newDocs).toHaveLength(4) - expect(docById(newDocs, 'doc1')).toMatchObject(docs[0]) - expect(docById(newDocs, 'doc2')).toMatchObject(docs[1]) - expect(docById(newDocs, 'doc3')).toMatchObject(docs[2]) - expect(docById(newDocs, 'plain')).toMatchObject(docs[6]) - - await assetHandler.finish() - expect(assetHandler.filesWritten).toEqual(0) - done() - } - }) -}) diff --git a/packages/@sanity/export/test/__snapshots__/AssetHandler.test.js.snap b/packages/@sanity/export/test/__snapshots__/AssetHandler.test.js.snap deleted file mode 100644 index f6f62769014..00000000000 --- a/packages/@sanity/export/test/__snapshots__/AssetHandler.test.js.snap +++ /dev/null @@ -1,95 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`asset handler can remove asset documents: Nothing removed in assetless doc 1`] = ` -Object { - "_id": "plain", - "_type": "bike", - "name": "Broom", -} -`; - -exports[`asset handler can remove asset documents: doc1 with no image asset 1`] = ` -Object { - "_id": "doc1", - "_type": "bike", - "name": "Scooter", -} -`; - -exports[`asset handler can remove asset documents: doc2 with no image asset 1`] = ` -Object { - "_id": "doc2", - "_type": "bike", - "name": "Dupe", -} -`; - -exports[`asset handler can remove asset documents: doc3 with no image asset 1`] = ` -Object { - "_id": "doc3", - "_type": "bike", - "name": "Tandem", -} -`; - -exports[`asset handler can rewrite documents / queue downloads: Nothing rewritten if pattern doesnt match Sanity asset 1`] = ` -Object { - "_id": "refsDatMuxAsset", - "_type": "bike", - "video": Object { - "_type": "mux.video", - "asset": Object { - "_ref": "datMuxAsset", - }, - }, -} -`; - -exports[`asset handler can rewrite documents / queue downloads: Nothing rewritten in assetless doc 1`] = ` -Object { - "_id": "plain", - "_type": "bike", - "name": "Broom", -} -`; - -exports[`asset handler can rewrite documents / queue downloads: Rewritten asset for doc1 1`] = ` -Object { - "_id": "doc1", - "_type": "bike", - "image": Object { - "_sanityAsset": "image@file://./images/6dcdb82c282dbe0a09ff7a6b58b639732f2fb8de-3360x840.png", - "_type": "image", - "caption": "Scooter bike", - "nested": Object { - "_sanityAsset": "image@file://./images/6dcdb82c282dbe0a09ff7a6b58b639732f2fb8de-3360x840.png", - "_type": "image", - }, - }, - "name": "Scooter", -} -`; - -exports[`asset handler can rewrite documents / queue downloads: Rewritten asset for doc2 1`] = ` -Object { - "_id": "doc2", - "_type": "bike", - "image": Object { - "_sanityAsset": "image@file://./images/6dcdb82c282dbe0a09ff7a6b58b639732f2fb8de-3360x840.png", - "_type": "image", - }, - "name": "Dupe", -} -`; - -exports[`asset handler can rewrite documents / queue downloads: Rewritten asset for doc3 1`] = ` -Object { - "_id": "doc3", - "_type": "bike", - "image": Object { - "_sanityAsset": "image@file://./images/31befc6dfa0315d6c535b8a57e34f86c18eb5f20-310x282.jpg", - "_type": "image", - }, - "name": "Tandem", -} -`; diff --git a/packages/@sanity/export/test/helpers/index.js b/packages/@sanity/export/test/helpers/index.js deleted file mode 100644 index 151b4a05c6e..00000000000 --- a/packages/@sanity/export/test/helpers/index.js +++ /dev/null @@ -1,48 +0,0 @@ -import {jest} from '@jest/globals' - -const os = require('os') -const path = require('path') -const stringToStream = require('string-to-stream') -const AssetHandler = require('../../src/AssetHandler') - -const getMockClient = () => ({ - config: () => ({projectId: '__fixtures__', dataset: '__test__'}), - fetch: (query, params) => - query.endsWith('._type') ? `sanity.imageAsset` : `http://localhost:32323/${params.id}.jpg`, -}) - -const getMockArchive = () => ({append: jest.fn(), abort: jest.fn()}) - -const getMockQueue = () => { - const ops = [] - return { - add: (task) => ops.push(task), - __size: () => ops.length, - __run: () => ops.forEach((fn) => fn()), - } -} - -const arrayToStream = (docs) => stringToStream(docs.map((doc) => JSON.stringify(doc)).join('\n')) - -const ndjsonToArray = (ndjson) => - ndjson - .toString('utf8') - .split('\n') - .filter(Boolean) - .map((line) => JSON.parse(line)) - -const getAssetHandler = () => - new AssetHandler({ - prefix: 'test', - client: getMockClient(), - tmpDir: path.join(os.tmpdir(), 'asset-handler-tests', `${Date.now()}`), - }) - -module.exports = { - getAssetHandler, - getMockClient, - getMockArchive, - getMockQueue, - arrayToStream, - ndjsonToArray, -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7810c65a0cc..2851489e238 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -620,9 +620,6 @@ importers: '@sanity/diff': specifier: workspace:* version: link:../../@sanity/diff - '@sanity/export': - specifier: workspace:* - version: link:../../@sanity/export '@sanity/import': specifier: workspace:* version: link:../../@sanity/import @@ -1006,43 +1003,6 @@ importers: specifier: ^3.0.2 version: 3.0.2 - packages/@sanity/export: - dependencies: - '@sanity/util': - specifier: 3.37.2 - version: link:../util - archiver: - specifier: ^7.0.0 - version: 7.0.1 - debug: - specifier: ^4.3.4 - version: 4.3.4(supports-color@9.4.0) - get-it: - specifier: ^8.4.18 - version: 8.4.18 - lodash: - specifier: ^4.17.21 - version: 4.17.21 - mississippi: - specifier: ^4.0.0 - version: 4.0.0 - p-queue: - specifier: ^2.3.0 - version: 2.4.2 - rimraf: - specifier: ^3.0.2 - version: 3.0.2 - split2: - specifier: ^4.2.0 - version: 4.2.0 - devDependencies: - '@jest/globals': - specifier: ^29.7.0 - version: 29.7.0 - string-to-stream: - specifier: ^1.1.0 - version: 1.1.1 - packages/@sanity/import: dependencies: '@sanity/asset-utils': @@ -1604,7 +1564,7 @@ importers: version: 5.0.1 '@sanity/export': specifier: 3.37.2 - version: link:../@sanity/export + version: 3.37.2 '@sanity/icons': specifier: ^2.11.0 version: 2.11.8(react@18.2.0) @@ -6085,6 +6045,23 @@ packages: event-source-polyfill: 1.0.31 eventsource: 2.0.2 + /@sanity/export@3.37.2: + resolution: {integrity: sha512-bHMPrNyPx4f64VQYGq+KRRlp/gCxkdF3NNQei8+LQNJ+Ge23znIiItIdHNHZVftG6ilIqfMrzGXIpKEP5ytkxw==} + engines: {node: '>=18'} + dependencies: + '@sanity/util': 3.37.2 + archiver: 7.0.1 + debug: 4.3.4(supports-color@9.4.0) + get-it: 8.4.18 + lodash: 4.17.21 + mississippi: 4.0.0 + p-queue: 2.4.2 + rimraf: 3.0.2 + split2: 4.2.0 + transitivePeerDependencies: + - supports-color + dev: false + /@sanity/generate-help-url@3.0.0: resolution: {integrity: sha512-wtMYcV5GIDIhVyF/jjmdwq1GdlK07dRL40XMns73VbrFI7FteRltxv48bhYVZPcLkRXb0SHjpDS/icj9/yzbVA==} @@ -6401,6 +6378,15 @@ packages: - supports-color - terser + /@sanity/types@3.37.2: + resolution: {integrity: sha512-1EfKkNlJ86wIDtc7oFHb79JI8lKDOxKDYrkmwhvuHgJY83GpSABc1kFdbwAtWZfrWVWyqVXUv/KlNwA3b99y/g==} + dependencies: + '@sanity/client': 6.15.11 + '@types/react': 18.2.75 + transitivePeerDependencies: + - supports-color + dev: false + /@sanity/ui-workshop@1.2.11(@sanity/icons@2.11.8)(@sanity/ui@2.1.2)(@types/node@18.19.29)(react-dom@18.2.0)(react@18.2.0)(styled-components@6.1.8): resolution: {integrity: sha512-vzj7upIF7wq2W1HEA0D5VSkR8axaH4Rt07yNTAaas7CLgjSE9r2d+Gnkrq4FIbIuN1GYhhCD+D3/s60GaZrpQw==} hasBin: true @@ -6460,6 +6446,19 @@ packages: react-refractor: 2.1.7(react@18.2.0) styled-components: 6.1.8(react-dom@18.2.0)(react@18.2.0) + /@sanity/util@3.37.2: + resolution: {integrity: sha512-hq0eLjyV2iaOm9ivtPw12YTQ4QsE3jnV/Ui0zhclEhu8Go5JiaEhFt2+WM2lLGRH6qcSA414QbsCNCcyhJL6rA==} + engines: {node: '>=18'} + dependencies: + '@sanity/client': 6.15.11 + '@sanity/types': 3.37.2 + get-random-values-esm: 1.0.2 + moment: 2.30.1 + rxjs: 7.8.1 + transitivePeerDependencies: + - supports-color + dev: false + /@sanity/uuid@3.0.2: resolution: {integrity: sha512-vzdhqOrX7JGbMyK40KuIwwyXHm7GMLOGuYgn3xlC09e4ZVNofUO5mgezQqnRv0JAMthIRhofqs9f6ufUjMKOvw==} dependencies: @@ -17388,13 +17387,6 @@ packages: strip-ansi: 6.0.1 dev: true - /string-to-stream@1.1.1: - resolution: {integrity: sha512-QySF2+3Rwq0SdO3s7BAp4x+c3qsClpPQ6abAmb0DGViiSBAkT5kL6JT2iyzEVP+T1SmzHrQD1TwlP9QAHCc+Sw==} - dependencies: - inherits: 2.0.4 - readable-stream: 2.3.8 - dev: true - /string-width@2.1.1: resolution: {integrity: sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==} engines: {node: '>=4'}