feat(store-sync): add syncToStash util (#3192)

Co-authored-by: Kevin Ingersoll <kingersoll@gmail.com>

.changeset/lucky-cows-fail.md

@@ -0,0 +1,23 @@
+---
+"@latticexyz/store-sync": patch
+---
+
+Added a `syncToStash` util to hydrate a `stash` client store from MUD contract state. This is currently exported from `@latticexyz/store-sync/internal` while Stash package is unstable/experimental.
+
+```ts
+import { createClient, http } from "viem";
+import { anvil } from "viem/chains";
+import { createStash } from "@latticexyz/stash/internal";
+import { syncToStash } from "@latticexyz/store-sync/internal";
+import config from "../mud.config";
+
+const client = createClient({
+  chain: anvil,
+  transport: http(),
+});
+
+const address = "0x...";
+
+const stash = createStash(config);
+const sync = await syncToStash({ stash, client, address });
+```

packages/stash/package.json

@@ -11,8 +11,7 @@
   "type": "module",
   "exports": {
     ".": "./dist/index.js",
-    "./internal": "./dist/internal.js",
-    "./recs": "./dist/recs.js"
+    "./internal": "./dist/internal.js"
   },
   "typesVersions": {
     "*": {

packages/stash/src/createStash.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it, vi } from "vitest";
 import { attest } from "@arktype/attest";
-import { CreateStoreResult, createStash } from "./createStash";
+import { CreateStashResult, createStash } from "./createStash";
 import { defineStore, defineTable } from "@latticexyz/store/config/v2";
 import { Hex } from "viem";
 
@@ -60,7 +60,7 @@ describe("createStash", () => {
       value: { field1: "hello" },
     });
 
-    attest<CreateStoreResult<typeof config>>(stash);
+    attest<CreateStashResult<typeof config>>(stash);
     attest<{
       config: {
         namespace1: {

packages/stash/src/createStash.ts

@@ -5,12 +5,12 @@ import { Table } from "@latticexyz/store/config/v2";
 
 export type Config = StoreConfig;
 
-export type CreateStoreResult<config extends Config = Config> = Stash<config> & DefaultActions<config>;
+export type CreateStashResult<config extends Config = Config> = Stash<config> & DefaultActions<config>;
 
 /**
  * Initializes a Stash based on the provided store config.
  */
-export function createStash<config extends Config>(storeConfig?: config): CreateStoreResult<config> {
+export function createStash<config extends Config>(storeConfig?: config): CreateStashResult<config> {
   const tableSubscribers: TableSubscribers = {};
   const storeSubscribers: StoreSubscribers = new Set();
 

packages/store-sync/package.json

@@ -73,6 +73,7 @@
     "@latticexyz/protocol-parser": "workspace:*",
     "@latticexyz/recs": "workspace:*",
     "@latticexyz/schema-type": "workspace:*",
+    "@latticexyz/stash": "workspace:*",
     "@latticexyz/store": "workspace:*",
     "@latticexyz/world": "workspace:*",
     "@trpc/client": "10.34.0",

packages/store-sync/src/common.ts

@@ -3,6 +3,7 @@ import { StoreEventsAbiItem, StoreEventsAbi } from "@latticexyz/store";
 import { Observable } from "rxjs";
 import { UnionPick } from "@latticexyz/common/type-utils";
 import {
+  ValueArgs,
   getKeySchema,
   getSchemaPrimitives,
   getSchemaTypes,
@@ -140,3 +141,9 @@ export const schemasTable = {
   keySchema: getSchemaTypes(getKeySchema(mudTables.Tables)),
   valueSchema: getSchemaTypes(getValueSchema(mudTables.Tables)),
 };
+
+export const emptyValueArgs = {
+  staticData: "0x",
+  encodedLengths: "0x",
+  dynamicData: "0x",
+} as const satisfies ValueArgs;

packages/store-sync/src/exports/internal.ts

@@ -1 +1,2 @@
 export * from "../sql";
+export * from "../stash";

packages/store-sync/src/stash/createStorageAdapter.test.ts

@@ -0,0 +1,92 @@
+import { beforeAll, describe, expect, it } from "vitest";
+import { storeEventsAbi } from "@latticexyz/store";
+import { createStorageAdapter } from "./createStorageAdapter";
+import { config, deployMockGame } from "../../test/mockGame";
+import { fetchAndStoreLogs } from "../fetchAndStoreLogs";
+import { testClient } from "../../test/common";
+import { getBlockNumber } from "viem/actions";
+import { createStash } from "@latticexyz/stash/internal";
+
+describe("createStorageAdapter", async () => {
+  beforeAll(async () => {
+    await deployMockGame();
+  });
+
+  it("sets component values from logs", async () => {
+    const stash = createStash(config);
+    const storageAdapter = createStorageAdapter({ stash });
+
+    console.log("fetching blocks");
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    for await (const block of fetchAndStoreLogs({
+      storageAdapter,
+      publicClient: testClient,
+      events: storeEventsAbi,
+      fromBlock: 0n,
+      toBlock: await getBlockNumber(testClient),
+    })) {
+      //
+    }
+
+    expect(stash.get().records).toMatchInlineSnapshot(`
+      {
+        "": {
+          "Health": {
+            "0x078cf0753dd50f7C56F20B3Ae02719EA199BE2eb": {
+              "health": 0n,
+              "player": "0x078cf0753dd50f7C56F20B3Ae02719EA199BE2eb",
+            },
+            "0x1D96F2f6BeF1202E4Ce1Ff6Dad0c2CB002861d3e": {
+              "health": 5n,
+              "player": "0x1D96F2f6BeF1202E4Ce1Ff6Dad0c2CB002861d3e",
+            },
+            "0x328809Bc894f92807417D2dAD6b7C998c1aFdac6": {
+              "health": 5n,
+              "player": "0x328809Bc894f92807417D2dAD6b7C998c1aFdac6",
+            },
+          },
+          "Inventory": {},
+          "Position": {
+            "0x078cf0753dd50f7C56F20B3Ae02719EA199BE2eb": {
+              "player": "0x078cf0753dd50f7C56F20B3Ae02719EA199BE2eb",
+              "x": 3,
+              "y": 5,
+            },
+            "0x1D96F2f6BeF1202E4Ce1Ff6Dad0c2CB002861d3e": {
+              "player": "0x1D96F2f6BeF1202E4Ce1Ff6Dad0c2CB002861d3e",
+              "x": 1,
+              "y": -1,
+            },
+            "0x328809Bc894f92807417D2dAD6b7C998c1aFdac6": {
+              "player": "0x328809Bc894f92807417D2dAD6b7C998c1aFdac6",
+              "x": 3,
+              "y": 5,
+            },
+            "0xdBa86119a787422C593ceF119E40887f396024E2": {
+              "player": "0xdBa86119a787422C593ceF119E40887f396024E2",
+              "x": 100,
+              "y": 100,
+            },
+          },
+          "Score": {},
+          "Terrain": {
+            "3|5": {
+              "terrainType": 2,
+              "x": 3,
+              "y": 5,
+            },
+          },
+          "Winner": {},
+        },
+      }
+    `);
+
+    expect(stash.getRecord({ table: config.tables.Terrain, key: { x: 3, y: 5 } })).toMatchInlineSnapshot(`
+      {
+        "terrainType": 2,
+        "x": 3,
+        "y": 5,
+      }
+    `);
+  });
+});

packages/store-sync/src/stash/createStorageAdapter.ts

@@ -0,0 +1,75 @@
+import { Stash, deleteRecord, getRecord, setRecord } from "@latticexyz/stash/internal";
+import {
+  decodeKey,
+  decodeValueArgs,
+  encodeValueArgs,
+  getKeySchema,
+  getSchemaTypes,
+  getValueSchema,
+} from "@latticexyz/protocol-parser/internal";
+import { spliceHex } from "@latticexyz/common";
+import { size } from "viem";
+import { Table } from "@latticexyz/config";
+import { StorageAdapter, StorageAdapterBlock, emptyValueArgs } from "../common";
+
+export type CreateStorageAdapter = {
+  stash: Stash;
+};
+
+export function createStorageAdapter({ stash }: CreateStorageAdapter): StorageAdapter {
+  const tablesById = Object.fromEntries(
+    Object.values(stash.get().config)
+      .flatMap((namespace) => Object.values(namespace) as readonly Table[])
+      .map((table) => [table.tableId, table]),
+  );
+
+  return async function storageAdapter({ logs }: StorageAdapterBlock): Promise<void> {
+    for (const log of logs) {
+      const table = tablesById[log.args.tableId];
+      if (!table) continue;
+
+      const valueSchema = getSchemaTypes(getValueSchema(table));
+      const keySchema = getSchemaTypes(getKeySchema(table));
+      const key = decodeKey(keySchema, log.args.keyTuple);
+
+      if (log.eventName === "Store_SetRecord") {
+        const value = decodeValueArgs(valueSchema, log.args);
+        setRecord({ stash, table, key, value });
+      } else if (log.eventName === "Store_SpliceStaticData") {
+        const previousValue = getRecord({ stash, table, key });
+
+        const {
+          staticData: previousStaticData,
+          encodedLengths,
+          dynamicData,
+        } = previousValue ? encodeValueArgs(valueSchema, previousValue) : emptyValueArgs;
+
+        const staticData = spliceHex(previousStaticData, log.args.start, size(log.args.data), log.args.data);
+        const value = decodeValueArgs(valueSchema, {
+          staticData,
+          encodedLengths,
+          dynamicData,
+        });
+
+        setRecord({ stash, table, key, value });
+      } else if (log.eventName === "Store_SpliceDynamicData") {
+        const previousValue = getRecord({ stash, table, key });
+
+        const { staticData, dynamicData: previousDynamicData } = previousValue
+          ? encodeValueArgs(valueSchema, previousValue)
+          : emptyValueArgs;
+
+        const dynamicData = spliceHex(previousDynamicData, log.args.start, log.args.deleteCount, log.args.data);
+        const value = decodeValueArgs(valueSchema, {
+          staticData,
+          encodedLengths: log.args.encodedLengths,
+          dynamicData,
+        });
+
+        setRecord({ stash, table, key, value });
+      } else if (log.eventName === "Store_DeleteRecord") {
+        deleteRecord({ stash, table, key });
+      }
+    }
+  };
+}

packages/store-sync/src/stash/index.ts

@@ -0,0 +1,2 @@
+export * from "./createStorageAdapter";
+export * from "./syncToStash";

packages/store-sync/src/stash/syncToStash.ts

@@ -0,0 +1,76 @@
+import { getRecord, setRecord, registerTable, Stash } from "@latticexyz/stash/internal";
+import { Address, Client, publicActions } from "viem";
+import { createStorageAdapter } from "./createStorageAdapter";
+import { defineTable } from "@latticexyz/store/config/v2";
+import { SyncStep } from "../SyncStep";
+import { SyncResult } from "../common";
+import { createStoreSync } from "../createStoreSync";
+import { getSchemaPrimitives, getValueSchema } from "@latticexyz/protocol-parser/internal";
+
+export const SyncProgress = defineTable({
+  namespaceLabel: "syncToStash",
+  label: "SyncProgress",
+  schema: {
+    step: "string",
+    percentage: "uint32",
+    latestBlockNumber: "uint256",
+    lastBlockNumberProcessed: "uint256",
+    message: "string",
+  },
+  key: [],
+});
+
+export const initialProgress = {
+  step: SyncStep.INITIALIZE,
+  percentage: 0,
+  latestBlockNumber: 0n,
+  lastBlockNumberProcessed: 0n,
+  message: "Connecting",
+} satisfies getSchemaPrimitives<getValueSchema<typeof SyncProgress>>;
+
+export type SyncToStashOptions = {
+  stash: Stash;
+  client: Client;
+  address: Address;
+  startSync?: boolean;
+};
+
+export type SyncToStashResult = Omit<SyncResult, "waitForTransaction"> & {
+  waitForStateChange: SyncResult["waitForTransaction"];
+  stopSync: () => void;
+};
+
+export async function syncToStash({
+  stash,
+  client,
+  address,
+  startSync = true,
+}: SyncToStashOptions): Promise<SyncToStashResult> {
+  registerTable({ stash, table: SyncProgress });
+
+  const storageAdapter = createStorageAdapter({ stash });
+
+  const { waitForTransaction: waitForStateChange, ...sync } = await createStoreSync({
+    storageAdapter,
+    publicClient: client.extend(publicActions) as never,
+    address,
+    onProgress: (nextValue) => {
+      const currentValue = getRecord({ stash, table: SyncProgress, key: {} });
+      // update sync progress until we're caught up and live
+      if (currentValue?.step !== SyncStep.LIVE) {
+        setRecord({ stash, table: SyncProgress, key: {}, value: nextValue });
+      }
+    },
+  });
+
+  const sub = startSync ? sync.storedBlockLogs$.subscribe() : null;
+  function stopSync(): void {
+    sub?.unsubscribe();
+  }
+
+  return {
+    ...sync,
+    waitForStateChange,
+    stopSync,
+  };
+}

tsconfig.paths.json

@@ -25,6 +25,8 @@
       "@latticexyz/recs/deprecated": ["./packages/recs/src/deprecated/index.ts"],
       "@latticexyz/schema-type": ["./packages/schema-type/src/typescript/index.ts"],
       "@latticexyz/schema-type/*": ["./packages/schema-type/src/typescript/exports/*.ts"],
+      "@latticexyz/stash": ["./packages/stash/ts/exports/index.ts"],
+      "@latticexyz/stash/internal": ["./packages/stash/ts/exports/internal.ts"],
       "@latticexyz/store": ["./packages/store/ts/exports/index.ts"],
       "@latticexyz/store/mud.config": ["./packages/store/mud.config.ts"],
       "@latticexyz/store/codegen": ["./packages/store/ts/codegen/index.ts"],