Serialize tree structure in file map cache (#1006)

Summary: Pull Request resolved: #1006 Serialise `metro-file-map`'s `TreeFS` by cloning the tree, instead of converting to and from a flat `Map`. - Cache `fileSystemData` is now fully cross-platform (it contains no path separators). - Serialised data is now a closer match with the internal representation. The new structure is as fast to write, faster to read, and smaller. Changelog: ``` - **[Performance]**: Improved startup speed via a new file map cache format. ``` Reviewed By: motiz88 Differential Revision: D46598820 fbshipit-source-id: c6ab0cf33e28e7dcb2e7c3896676b2104b8808b5
facebook · Jun 24, 2023 · 166477e · 166477e
1 parent 9b9711c
commit 166477e
Show file tree

Hide file tree

Showing 4 changed files with 110 additions and 73 deletions.
diff --git a/packages/metro-file-map/src/__tests__/index-test.js b/packages/metro-file-map/src/__tests__/index-test.js
@@ -10,6 +10,7 @@
 
 import crypto from 'crypto';
 import * as path from 'path';
+import {serialize} from 'v8';
 
 jest.useRealTimers();
 
@@ -153,7 +154,13 @@ jest.mock('fs', () => ({
 }));
 
 const object = data => Object.assign(Object.create(null), data);
-const createMap = obj => new Map(Object.keys(obj).map(key => [key, obj[key]]));
+const createMap = obj => new Map(Object.entries(obj));
+const assertFileSystemEqual = (fileSystem: FileSystem, fileData: FileData) => {
+  expect(fileSystem.getDifference(fileData)).toEqual({
+    changedFiles: new Map(),
+    removedFiles: new Set(),
+  });
+};
 
 // Jest toEqual does not match Map instances from different contexts
 // This normalizes them for the uses cases in this test
@@ -406,11 +413,12 @@ describe('HasteMap', () => {
       mocksPattern: '__mocks__',
     });
 
-    await hasteMap.build();
+    const {fileSystem} = await hasteMap.build();
 
     expect(cacheContent.clocks).toEqual(mockClocks);
 
-    expect(cacheContent.files).toEqual(
+    assertFileSystemEqual(
+      fileSystem,
       createMap({
         [path.join('fruits', 'Banana.js')]: [
           'Banana',
@@ -581,7 +589,7 @@ describe('HasteMap', () => {
 
         await hasteMap.build();
 
-        expect(cacheContent.files).toEqual(
+        expect(
           createMap({
             [path.join('fruits', 'Banana.js')]: [
               'Banana',
@@ -693,11 +701,14 @@ describe('HasteMap', () => {
       roots: [...defaultConfig.roots, path.join('/', 'project', 'video')],
     });
 
-    await hasteMap.build();
+    const {fileSystem} = await hasteMap.build();
     const data = cacheContent;
 
     expect(data.map.get('IRequireAVideo')).toBeDefined();
-    expect(data.files.get(path.join('video', 'video.mp4'))).toBeDefined();
+    expect(fileSystem.linkStats(path.join('video', 'video.mp4'))).toEqual({
+      fileType: 'f',
+      modifiedTime: 32,
+    });
     expect(fs.readFileSync.mock.calls.map(call => call[0])).not.toContain(
       path.join('video', 'video.mp4'),
     );
@@ -716,15 +727,15 @@ describe('HasteMap', () => {
       retainAllFiles: true,
     });
 
-    await hasteMap.build();
+    const {fileSystem} = await hasteMap.build();
 
     // Expect the node module to be part of files but make sure it wasn't
     // read.
     expect(
-      cacheContent.files.get(
+      fileSystem.linkStats(
         path.join('fruits', 'node_modules', 'fbjs', 'fbjs.js'),
       ),
-    ).toEqual(['', 32, 42, 0, [], null, 0]);
+    ).toEqual({fileType: 'f', modifiedTime: 32});
 
     expect(cacheContent.map.get('fbjs')).not.toBeDefined();
 
@@ -835,9 +846,10 @@ describe('HasteMap', () => {
       const Blackberry = require("Blackberry");
     `;
 
-    await new HasteMap(defaultConfig).build();
+    const {fileSystem} = await new HasteMap(defaultConfig).build();
 
-    expect(cacheContent.files).toEqual(
+    assertFileSystemEqual(
+      fileSystem,
       createMap({
         [path.join('fruits', 'Strawberry.android.js')]: [
           'Strawberry',
@@ -915,13 +927,22 @@ describe('HasteMap', () => {
     // Expect no fs reads, because there have been no changes
     expect(fs.readFileSync.mock.calls.length).toBe(0);
     expect(deepNormalize(data.clocks)).toEqual(mockClocks);
-    expect(deepNormalize(data.files)).toEqual(initialData.files);
+    expect(serialize(data.fileSystem)).toEqual(
+      serialize(initialData.fileSystem),
+    );
     expect(deepNormalize(data.map)).toEqual(initialData.map);
   });
 
   it('only does minimal file system access when files change', async () => {
     // Run with a cold cache initially
-    await new HasteMap(defaultConfig).build();
+    const {fileSystem: initialFileSystem} = await new HasteMap(
+      defaultConfig,
+    ).build();
+
+    expect(
+      initialFileSystem.getDependencies(path.join('fruits', 'Banana.js')),
+    ).toEqual(['Strawberry']);
+
     const initialData = cacheContent;
     fs.readFileSync.mockClear();
     expect(mockCacheManager.read).toHaveBeenCalledTimes(1);
@@ -939,7 +960,7 @@ describe('HasteMap', () => {
       vegetables: 'c:fake-clock:2',
     });
 
-    await new HasteMap(defaultConfig).build();
+    const {fileSystem} = await new HasteMap(defaultConfig).build();
     const data = cacheContent;
 
     expect(mockCacheManager.read).toHaveBeenCalledTimes(2);
@@ -950,18 +971,9 @@ describe('HasteMap', () => {
 
     expect(deepNormalize(data.clocks)).toEqual(mockClocks);
 
-    const files = new Map(initialData.files);
-    files.set(path.join('fruits', 'Banana.js'), [
-      'Banana',
-      32,
-      42,
-      1,
-      'Kiwi',
-      null,
-      0,
-    ]);
-
-    expect(deepNormalize(data.files)).toEqual(files);
+    expect(
+      fileSystem.getDependencies(path.join('fruits', 'Banana.js')),
+    ).toEqual(['Kiwi']);
 
     const map = new Map(initialData.map);
     expect(deepNormalize(data.map)).toEqual(map);
@@ -984,16 +996,13 @@ describe('HasteMap', () => {
       vegetables: 'c:fake-clock:2',
     });
 
-    await new HasteMap(defaultConfig).build();
-    const data = cacheContent;
+    const {fileSystem} = await new HasteMap(defaultConfig).build();
 
-    const files = new Map(initialData.files);
-    files.delete(path.join('fruits', 'Banana.js'));
-    expect(deepNormalize(data.files)).toEqual(files);
+    expect(fileSystem.exists(path.join('fruits', 'Banana.js'))).toEqual(false);
 
     const map = new Map(initialData.map);
     map.delete('Banana');
-    expect(deepNormalize(data.map)).toEqual(map);
+    expect(deepNormalize(cacheContent.map)).toEqual(map);
   });
 
   it('correctly handles platform-specific file additions', async () => {
@@ -1258,11 +1267,11 @@ describe('HasteMap', () => {
       };
     });
 
-    await new HasteMap(defaultConfig).build();
-    expect(cacheContent.files.size).toBe(5);
+    const {fileSystem} = await new HasteMap(defaultConfig).build();
+    expect(fileSystem.getDifference(new Map()).removedFiles.size).toBe(5);
 
     // Ensure this file is not part of the file list.
-    expect(cacheContent.files.get(invalidFilePath)).toBe(undefined);
+    expect(fileSystem.exists(invalidFilePath)).toBe(false);
   });
 
   it('distributes work across workers', async () => {
@@ -1362,11 +1371,13 @@ describe('HasteMap', () => {
       });
     });
 
-    await new HasteMap(defaultConfig).build();
+    const {fileSystem} = await new HasteMap(defaultConfig).build();
+
     expect(watchman).toBeCalled();
     expect(node).toBeCalled();
 
-    expect(cacheContent.files).toEqual(
+    assertFileSystemEqual(
+      fileSystem,
       createMap({
         [path.join('fruits', 'Banana.js')]: [
           'Banana',
@@ -1399,12 +1410,13 @@ describe('HasteMap', () => {
       });
     });
 
-    await new HasteMap(defaultConfig).build();
+    const {fileSystem} = await new HasteMap(defaultConfig).build();
 
     expect(watchman).toBeCalled();
     expect(node).toBeCalled();
 
-    expect(cacheContent.files).toEqual(
+    assertFileSystemEqual(
+      fileSystem,
       createMap({
         [path.join('fruits', 'Banana.js')]: [
           'Banana',

diff --git a/packages/metro-file-map/src/flow-types.js b/packages/metro-file-map/src/flow-types.js
@@ -50,7 +50,7 @@ export type CacheData = $ReadOnly<{
   map: RawModuleMap['map'],
   mocks: RawModuleMap['mocks'],
   duplicates: RawModuleMap['duplicates'],
-  files: FileData,
+  fileSystemData: mixed,
 }>;
 
 export type CacheDelta = $ReadOnly<{
@@ -177,7 +177,7 @@ export interface FileSystem {
   };
   getModuleName(file: Path): ?string;
   getRealPath(file: Path): ?string;
-  getSerializableSnapshot(): FileData;
+  getSerializableSnapshot(): CacheData['fileSystemData'];
   getSha1(file: Path): ?string;
 
   /**

diff --git a/packages/metro-file-map/src/index.js b/packages/metro-file-map/src/index.js
@@ -143,7 +143,7 @@ export type {
 // This should be bumped whenever a code change to `metro-file-map` itself
 // would cause a change to the cache data structure and/or content (for a given
 // filesystem state and build parameters).
-const CACHE_BREAKER = '4';
+const CACHE_BREAKER = '5';
 
 const CHANGE_INTERVAL = 30;
 const NODE_MODULES = path.sep + 'node_modules' + path.sep;
@@ -341,30 +341,29 @@ export default class HasteMap extends EventEmitter {
         }
         if (!initialData) {
           debug('Not using a cache');
-          initialData = {
-            files: new Map(),
-            map: new Map(),
-            duplicates: new Map(),
-            clocks: new Map(),
-            mocks: new Map(),
-          };
         } else {
-          debug(
-            'Cache loaded (%d file(s), %d clock(s))',
-            initialData.files.size,
-            initialData.clocks.size,
-          );
+          debug('Cache loaded (%d clock(s))', initialData.clocks.size);
         }
 
         const rootDir = this._options.rootDir;
-        const fileData = initialData.files;
         this._startupPerfLogger?.point('constructFileSystem_start');
-        const fileSystem = new TreeFS({
-          files: fileData,
-          rootDir,
-        });
+        const fileSystem =
+          initialData != null
+            ? TreeFS.fromDeserializedSnapshot({
+                rootDir,
+                // Typed `mixed` because we've read this from an external
+                // source. It'd be too expensive to validate at runtime, so
+                // trust our cache manager that this is correct.
+                // $FlowIgnore
+                fileSystemData: initialData.fileSystemData,
+              })
+            : new TreeFS({rootDir});
         this._startupPerfLogger?.point('constructFileSystem_end');
-        const {map, mocks, duplicates} = initialData;
+        const {map, mocks, duplicates} = initialData ?? {
+          map: new Map(),
+          mocks: new Map(),
+          duplicates: new Map(),
+        };
         const rawModuleMap: RawModuleMap = {
           duplicates,
           map,
@@ -374,7 +373,7 @@ export default class HasteMap extends EventEmitter {
 
         const fileDelta = await this._buildFileDelta({
           fileSystem,
-          clocks: initialData.clocks,
+          clocks: initialData?.clocks ?? new Map(),
         });
 
         await this._applyFileDelta(fileSystem, rawModuleMap, fileDelta);
@@ -386,7 +385,11 @@ export default class HasteMap extends EventEmitter {
           fileDelta.changedFiles,
           fileDelta.removedFiles,
         );
-        debug('Finished mapping %d files.', fileData.size);
+        debug(
+          'Finished mapping files (%d changes, %d removed).',
+          fileDelta.changedFiles.size,
+          fileDelta.removedFiles.size,
+        );
 
         await this._watch(fileSystem, rawModuleMap);
         return {
@@ -802,7 +805,7 @@ export default class HasteMap extends EventEmitter {
     const {map, duplicates, mocks} = deepCloneRawModuleMap(moduleMap);
     await this._cacheManager.write(
       {
-        files: fileSystem.getSerializableSnapshot(),
+        fileSystemData: fileSystem.getSerializableSnapshot(),
         map,
         clocks: new Map(clocks),
         duplicates,

diff --git a/packages/metro-file-map/src/lib/TreeFS.js b/packages/metro-file-map/src/lib/TreeFS.js
@@ -9,6 +9,7 @@
  */
 
 import type {
+  CacheData,
   FileData,
   FileMetaData,
   FileStats,
@@ -35,20 +36,29 @@ type MixedNode = FileNode | DirectoryNode;
 export default class TreeFS implements MutableFileSystem {
   +#cachedNormalSymlinkTarkets: WeakMap<FileNode, Path> = new WeakMap();
   +#rootDir: Path;
-  +#rootNode: DirectoryNode = new Map();
+  #rootNode: DirectoryNode = new Map();
 
-  constructor({rootDir, files}: {rootDir: Path, files: FileData}) {
+  constructor({rootDir, files}: {rootDir: Path, files?: FileData}) {
     this.#rootDir = rootDir;
-    this.bulkAddOrModify(files);
+    if (files != null) {
+      this.bulkAddOrModify(files);
+    }
   }
 
-  getSerializableSnapshot(): FileData {
-    return new Map(
-      Array.from(
-        this._metadataIterator(this.#rootNode, {includeSymlinks: true}),
-        ({normalPath, metadata}) => [normalPath, [...metadata]],
-      ),
-    );
+  getSerializableSnapshot(): CacheData['fileSystemData'] {
+    return this._cloneTree(this.#rootNode);
+  }
+
+  static fromDeserializedSnapshot({
+    rootDir,
+    fileSystemData,
+  }: {
+    rootDir: string,
+    fileSystemData: DirectoryNode,
+  }): TreeFS {
+    const tfs = new TreeFS({rootDir});
+    tfs.#rootNode = fileSystemData;
+    return tfs;
   }
 
   getModuleName(mixedPath: Path): ?string {
@@ -548,4 +558,16 @@ export default class TreeFS implements MutableFileSystem {
     }
     return result.node;
   }
+
+  _cloneTree(root: DirectoryNode): DirectoryNode {
+    const clone: DirectoryNode = new Map();
+    for (const [name, node] of root) {
+      if (node instanceof Map) {
+        clone.set(name, this._cloneTree(node));
+      } else {
+        clone.set(name, [...node]);
+      }
+    }
+    return clone;
+  }
 }