fix(@sanity): issue where hidden unicode characters were bloating doc…

…ument in PTE (#6424)

* fix(portable-text-editor): issue shown in tests re stega. use duplicate code

* test(playwright-ct): add test

* chore(sanity): remove prettier linting

* test(sanity): fix missing snapshot

* test(sanity): update test after realising the test would pass always if comparing object number

* chore: test unicode removal

* chore: test unicode removal

packages/@sanity/portable-text-editor/package.json

@@ -62,6 +62,7 @@
     "@sanity/schema": "3.38.0",
     "@sanity/types": "3.38.0",
     "@sanity/util": "3.38.0",
+    "@vercel/stega": "0.1.0",
     "debug": "^3.2.7",
     "is-hotkey-esm": "^1.0.0",
     "lodash": "^4.17.21",

packages/@sanity/portable-text-editor/src/editor/plugins/createWithInsertData.ts

@@ -12,6 +12,7 @@ import {
 import {debugWithName} from '../../utils/debug'
 import {validateValue} from '../../utils/validateValue'
 import {fromSlateValue, isEqualToEmptyEditor, toSlateValue} from '../../utils/values'
+import {vercelStegaCleanAll} from '../../utils/vercelStegaCleanAll'
 
 const debug = debugWithName('plugin:withInsertData')
 
@@ -167,8 +168,9 @@ export function createWithInsertData(
         return false
       }
       change$.next({type: 'loading', isLoading: true}) // This could potentially take some time
-      const html = data.getData('text/html')
-      const text = data.getData('text/plain')
+      // vercelStegaCleanAll will make sure that no invisible unicode characters will bog down PTE
+      const html = vercelStegaCleanAll(data.getData('text/html'))
+      const text = vercelStegaCleanAll(data.getData('text/plain'))
       if (html || text) {
         debug('Inserting data', data)
         let portableText: PortableTextBlock[]

packages/@sanity/portable-text-editor/src/utils/vercelStegaCleanAll.ts

@@ -0,0 +1,24 @@
+import {vercelStegaSplit} from '@vercel/stega'
+
+/**
+ * This is a duplicate code from `@sanity/client/stega`
+ * Unfortunately, as it stands, the e2e process is pulling in the node version of `@sanity/client` and so we don't have access to the utility as it stands
+ * @todo remove once this utility is available in `@vercel/stega`
+ *
+ * Can take a `result` JSON from a `const {result} = client.fetch(query, params, {filterResponse: false})`
+ * and remove all stega-encoded data from it.
+ * @alpha
+ * @hidden
+ */
+export function vercelStegaCleanAll<Result = unknown>(result: Result): Result {
+  try {
+    return JSON.parse(
+      JSON.stringify(result, (key, value) => {
+        if (typeof value !== 'string') return value
+        return vercelStegaSplit(value).cleaned
+      }),
+    )
+  } catch {
+    return result
+  }
+}

packages/@sanity/portable-text-editor/tsdoc.json

@@ -0,0 +1,33 @@
+{
+  "$schema": "https://developer.microsoft.com/json-schemas/tsdoc/v0/tsdoc.schema.json",
+  "tagDefinitions": [
+    {
+      "tagName": "@hidden",
+      "syntaxKind": "block",
+      "allowMultiple": true
+    },
+    {
+      "tagName": "@todo",
+      "syntaxKind": "block",
+      "allowMultiple": true
+    }
+  ],
+  "supportForTags": {
+    "@hidden": true,
+    "@beta": true,
+    "@internal": true,
+    "@public": true,
+    "@experimental": true,
+    "@see": true,
+    "@link": true,
+    "@example": true,
+    "@deprecated": true,
+    "@alpha": true,
+    "@param": true,
+    "@returns": true,
+    "@remarks": true,
+    "@throws": true,
+    "@defaultValue": true,
+    "@todo": true
+  }
+}

packages/sanity/playwright-ct/tests/formBuilder/inputs/PortableText/copyPaste/CopyPaste.spec.tsx

@@ -4,7 +4,13 @@ import {type Path, type SanityDocument} from '@sanity/types'
 
 import {testHelpers} from '../../../../utils/testHelpers'
 import CopyPasteStory from './CopyPasteStory'
-import {GDOCS_INPUT, NORMALIZED_INPUT_SNAPSHOT, REMOVED_INPUT_SNAPSHOT} from './input'
+import {
+  CLEANED_UNICODE_INPUT_SNAPSHOT,
+  GDOCS_INPUT,
+  NORMALIZED_INPUT_SNAPSHOT,
+  REMOVED_INPUT_SNAPSHOT,
+  UNICODE_TEXT,
+} from './input'
 
 export type UpdateFn = () => {focusPath: Path; document: SanityDocument}
 
@@ -68,4 +74,31 @@ test.describe('Portable Text Input', () => {
       await expect(documentState?.bodyNormalized?.length || 0).toEqual(snapshotLength)
     })
   })
+
+  test.describe('Should be able to paste text that has hidden unicode characters without bloating the PTE', () => {
+    test(`Removed unicode characters`, async ({mount, page}) => {
+      const {getFocusedPortableTextEditor, insertPortableTextCopyPaste, waitForDocumentState} =
+        testHelpers({page})
+
+      await mount(<CopyPasteStory document={document} />)
+
+      const $pte = await getFocusedPortableTextEditor('field-body')
+
+      await insertPortableTextCopyPaste(UNICODE_TEXT, $pte)
+
+      const documentState = await waitForDocumentState((documentStateValue) => {
+        return (documentStateValue?.body?.length || 0) > 0
+      })
+
+      // strigify is needed in these cases in order to get the correct length for the content within the children
+      // prettier-ignore
+      const bodyLength = await JSON.stringify(documentState?.body).length || 0
+      // prettier-ignore
+      const snapshotLength = JSON.stringify(CLEANED_UNICODE_INPUT_SNAPSHOT).length
+
+      // Ideally we would compare the snapshot with the document, but the keys will be different each time
+      // We therefore compare the length of the body to the snapshot length here instead.
+      await expect(bodyLength).toEqual(snapshotLength)
+    })
+  })
 })