Support Y.js

addons/dexie-cloud/src/helpers/dbOnClosed.ts

@@ -4,16 +4,16 @@ import Dexie from "dexie";
  */
 export function dbOnClosed(db: Dexie, handler: () => void) {
   db.on.close.subscribe(handler);
-  // @ts-ignore
+  /*// @ts-ignore
   const origClose = db._close;
   // @ts-ignore
   db._close = function () {
     origClose.call(this);
     handler();
-  };
+  };*/
   return () => {
     db.on.close.unsubscribe(handler);
     // @ts-ignore
-    db._close = origClose;
+    //db._close = origClose;
   };
 }

addons/dexie-cloud/src/sync/DEXIE_CLOUD_SYNCER_ID.ts

@@ -0,0 +1,2 @@
+
+export const DEXIE_CLOUD_SYNCER_ID = 'dexie-cloud-syncer';

addons/dexie-cloud/src/sync/applyYMessages.ts

@@ -0,0 +1,44 @@
+import { InsertType, YSyncer, YUpdateRow } from 'dexie';
+import { DexieCloudDB } from '../db/DexieCloudDB';
+import { YServerMessage } from 'dexie-cloud-common/src/YMessage';
+import { DEXIE_CLOUD_SYNCER_ID } from './DEXIE_CLOUD_SYNCER_ID';
+
+export async function applyYServerMessages(
+  yMessages: YServerMessage[],
+  db: DexieCloudDB
+): Promise<void> {
+  for (const m of yMessages) {
+    switch (m.type) {
+      case 'u-s': {
+        await db.table(m.utbl).add({
+          k: m.k,
+          u: m.u,
+        } satisfies InsertType<YUpdateRow, 'i'>);
+        break;
+      }
+      case 'u-ack': {
+        await db.transaction('rw', m.utbl, async (tx) => {
+          let syncer = (await tx.table(m.utbl).get(DEXIE_CLOUD_SYNCER_ID)) as
+            | YSyncer
+            | undefined;
+          await tx.table(m.utbl).put(DEXIE_CLOUD_SYNCER_ID, {
+            ...(syncer || { i: DEXIE_CLOUD_SYNCER_ID }),
+            unsentFrom: Math.max(syncer?.unsentFrom || 0, m.i + 1),
+          } as YSyncer);
+        });
+        break;
+      }
+      case 'u-reject': {
+        // Acces control or constraint rejected the update.
+        // We delete it. It's not going to be sent again.
+        // What's missing is a way to notify consumers, such as Tiptap editor, that the update was rejected.
+        // This is only an issue when the document is open. We could find the open document and
+        // in a perfect world, we should send a reverse update to the open document to undo the change.
+        // See my question in https://discuss.yjs.dev/t/generate-an-inverse-update/2765
+        console.debug(`Y update rejected. Deleting it.`);
+        await db.table(m.utbl).delete(m.i);
+        break;
+      }
+    }
+  }
+}

addons/dexie-cloud/src/sync/listYClientMessages.ts

@@ -0,0 +1,38 @@
+import { DexieYProvider, Table, YSyncer, YUpdateRow } from 'dexie';
+import { getTableFromMutationTable } from '../helpers/getTableFromMutationTable';
+import { DexieCloudDB } from '../db/DexieCloudDB';
+import { DBOperation, DBOperationsSet } from 'dexie-cloud-common';
+import { flatten } from '../helpers/flatten';
+import { YClientMessage } from 'dexie-cloud-common/src/YMessage';
+import { DEXIE_CLOUD_SYNCER_ID } from './DEXIE_CLOUD_SYNCER_ID';
+
+export async function listYClientMessages(
+  db: DexieCloudDB
+): Promise<YClientMessage[]> {
+  const result: YClientMessage[] = [];
+  for (const table of db.tables) {
+    for (const yProp of table.schema.yProps || []) {
+      const yTable = db.table(yProp.updTable);
+      const syncer = (await yTable.get(DEXIE_CLOUD_SYNCER_ID)) as YSyncer | undefined;
+      const unsentFrom = syncer?.unsentFrom || 0;
+      const updates = await yTable
+        .where('i')
+        .aboveOrEqual(unsentFrom)
+        .toArray();
+      result.push(
+        ...updates
+          .filter((update) => update.f & 0x01) // Don't send back updates that we got from server or other clients.
+          .map(({ i, k, u }: YUpdateRow) => {
+            return {
+              type: 'u-c',
+              utbl: yProp.updTable,
+              i,
+              k,
+              u,
+            } satisfies YClientMessage;
+          })
+      );
+    }
+  }
+  return result;
+}

addons/dexie-cloud/src/sync/sync.ts

@@ -26,6 +26,8 @@ import { updateBaseRevs } from './updateBaseRevs';
 import { getLatestRevisionsPerTable } from './getLatestRevisionsPerTable';
 import { applyServerChanges } from './applyServerChanges';
 import { checkSyncRateLimitDelay } from './ratelimit';
+import { listYClientMessages } from './listYClientMessages';
+import { applyYServerMessages } from './applyYMessages';
 
 export const CURRENT_SYNC_WORKER = 'currentSyncWorker';
 
@@ -147,13 +149,14 @@ async function _sync(
   //
   // List changes to sync
   //
-  const [clientChangeSet, syncState, baseRevs] = await db.transaction(
+  const [clientChangeSet, syncState, baseRevs, yMessages] = await db.transaction(
     'r',
     db.tables,
     async () => {
       const syncState = await db.getPersistedSyncState();
       const baseRevs = await db.$baseRevs.toArray();
       let clientChanges = await listClientChanges(mutationTables, db);
+      const yMessages = await listYClientMessages(db);
       throwIfCancelled(cancelToken);
       if (doSyncify) {
         const alreadySyncedRealms = [
@@ -168,15 +171,15 @@ async function _sync(
         );
         throwIfCancelled(cancelToken);
         clientChanges = clientChanges.concat(syncificationInserts);
-        return [clientChanges, syncState, baseRevs];
+        return [clientChanges, syncState, baseRevs, yMessages];
       }
-      return [clientChanges, syncState, baseRevs];
+      return [clientChanges, syncState, baseRevs, yMessages];
     }
   );
 
   const pushSyncIsNeeded = clientChangeSet.some((set) =>
     set.muts.some((mut) => mut.keys.length > 0)
-  );
+  ) || yMessages.length > 0;
   if (justCheckIfNeeded) {
     console.debug('Sync is needed:', pushSyncIsNeeded);
     return pushSyncIsNeeded;
@@ -199,6 +202,7 @@ async function _sync(
   throwIfCancelled(cancelToken);
   const res = await syncWithServer(
     clientChangeSet,
+    yMessages,
     syncState,
     baseRevs,
     db,
@@ -328,6 +332,11 @@ async function _sync(
     //
     await applyServerChanges(filteredChanges, db);
 
+    //
+    // apply yMessages
+    //
+    await applyYServerMessages(res.yMessages, db);
+
     //
     // Update syncState
     //

addons/dexie-cloud/src/sync/syncWithServer.ts

@@ -14,10 +14,12 @@ import {
 import { encodeIdsForServer } from './encodeIdsForServer';
 import { UserLogin } from '../db/entities/UserLogin';
 import { updateSyncRateLimitDelays } from './ratelimit';
+import { YClientMessage } from 'dexie-cloud-common/src/YMessage';
 //import {BisonWebStreamReader} from "dreambase-library/dist/typeson-simplified/BisonWebStreamReader";
 
 export async function syncWithServer(
   changes: DBOperationsSet,
+  y: YClientMessage[],
   syncState: PersistedSyncState | undefined,
   baseRevs: BaseRevisionMapEntry[],
   db: DexieCloudDB,
@@ -63,6 +65,7 @@ export async function syncWithServer(
       : undefined,
     baseRevs,
     changes: encodeIdsForServer(db.dx.core.schema, currentUser, changes),
+    y
   };
   console.debug('Sync request', syncRequest);
   db.syncStateChangedEvent.next({

import-wrapper-prod.mjs

@@ -9,7 +9,9 @@ if (_Dexie.semVer !== Dexie.semVer) {
 }
 const {
     liveQuery, mergeRanges, rangesOverlap, RangeSet, cmp, Entity,
-    PropModSymbol, PropModification, replacePrefix, add, remove } = Dexie;
+    PropModSymbol, PropModification, replacePrefix, add, remove,
+    DexieYProvider } = Dexie;
 export { liveQuery, mergeRanges, rangesOverlap, RangeSet, cmp, Dexie, Entity,
-    PropModSymbol, PropModification, replacePrefix, add, remove };
+    PropModSymbol, PropModification, replacePrefix, add, remove,
+    DexieYProvider};
 export default Dexie;

import-wrapper.mjs

@@ -8,7 +8,10 @@ if (_Dexie.semVer !== Dexie.semVer) {
     throw new Error(`Two different versions of Dexie loaded in the same app: ${_Dexie.semVer} and ${Dexie.semVer}`);
 }
 const { liveQuery, mergeRanges, rangesOverlap, RangeSet, cmp, Entity,
-    PropModSymbol, PropModification, replacePrefix, add, remove } = Dexie;
+    PropModSymbol, PropModification, replacePrefix, add, remove,
+    DexieYProvider } = Dexie;
 export { liveQuery, mergeRanges, rangesOverlap, RangeSet, cmp, Dexie, Entity,
-    PropModSymbol, PropModification, replacePrefix, add, remove };
+    PropModSymbol, PropModification, replacePrefix, add, remove,
+    DexieYProvider};
+
 export default Dexie;

libs/dexie-cloud-common/src/SyncRequest.ts

@@ -1,6 +1,7 @@
 import { BaseRevisionMapEntry } from './BaseRevisionMapEntry.js';
 import { DBOperationsSet } from './DBOperationsSet.js';
 import { DexieCloudSchema } from './DexieCloudSchema.js';
+import { YClientMessage } from './YMessage.js';
 
 export interface SyncRequest {
   v?: number;
@@ -14,5 +15,6 @@ export interface SyncRequest {
   };
   baseRevs: BaseRevisionMapEntry[];
   changes: DBOperationsSet;
+  y?: YClientMessage[];
   //invites: {[inviteId: string]: "accept" | "reject"}
 }

libs/dexie-cloud-common/src/SyncResponse.ts

@@ -1,5 +1,6 @@
 import { DBOperationsSet } from './DBOperationsSet.js';
 import { DexieCloudSchema } from './DexieCloudSchema.js';
+import { YServerMessage } from './YMessage.js';
 
 export interface SyncResponse {
   serverRevision: string | bigint; // string "[1,\"2823\"]" in protocol version 2. bigint in version 1.
@@ -9,5 +10,6 @@ export interface SyncResponse {
   schema: DexieCloudSchema;
   changes: DBOperationsSet<string>;
   rejections: { name: string; message: string; txid: string }[];
+  yMessages: YServerMessage[];
   //invites: DBInvite[];
 }

libs/dexie-cloud-common/src/YMessage.ts

@@ -0,0 +1,41 @@
+
+export type YMessage = YClientMessage | YServerMessage;
+export type YClientMessage = YUpdateFromClientRequest | YAwarenessUpdate;
+export type YServerMessage = YUpdateFromClientAck | YUpdateFromClientReject | YUpdateFromServerMessage | YAwarenessUpdate;
+
+export interface YUpdateFromClientRequest {
+  type: 'u-c';
+  utbl: string;
+  k: any;
+  u: Uint8Array;
+  i: number;
+}
+
+export interface YUpdateFromClientAck {
+  type: 'u-ack';
+  utbl: string;
+  i: number;
+}
+
+export interface YUpdateFromClientReject {
+  type: 'u-reject';
+  utbl: string;
+  i: number;
+}
+
+
+export interface YUpdateFromServerMessage {
+  type: 'u-s';
+  utbl: string;
+  k: any;
+  u: Uint8Array;
+  realmSetHash: Uint8Array;
+  newRev: string;
+}
+
+export interface YAwarenessUpdate {
+  type: 'awareness';
+  utbl: string;
+  k: any;
+  u: Uint8Array;
+}

package.json

@@ -77,7 +77,7 @@
       "# Build dist/dexie.js, dist/dexie.mjs and dist/dexie.d.ts",
       "cd src",
       "tsc [--watch 'Watching for file changes']",
-      "tsc --target es2020 --outdir ../tools/tmp/modern/src/",
+      "tsc --target es2021 --outdir ../tools/tmp/modern/src/",
       "rollup -c ../tools/build-configs/rollup.config.js",
       "rollup -c ../tools/build-configs/rollup.umd.config.js",
       "rollup -c ../tools/build-configs/rollup.modern.config.js",
@@ -118,7 +118,7 @@
   "devDependencies": {
     "@lambdatest/node-tunnel": "^4.0.7",
     "cross-env": "^7.0.3",
-    "dts-bundle-generator": "^5.9.0",
+    "dts-bundle-generator": "^9.3.1",
     "just-build": "^0.9.24",
     "karma": "^6.1.1",
     "karma-chrome-launcher": "^3.1.0",
@@ -140,6 +140,8 @@
     "terser": "^5.3.1",
     "tslib": "^2.1.0",
     "typescript": "^5.3.3",
-    "uglify-js": "^3.9.2"
+    "uglify-js": "^3.9.2",
+    "y-protocols": "^1.0.6",
+    "yjs": "^13.6.16"
   }
 }