deduplicate references lists in tsbuildinfo

src/compiler/builder.ts

@@ -695,12 +695,87 @@ namespace ts {
  export interface ProgramBuildInfo {
  fileInfos: MapLike<BuilderState.FileInfo>;
  options: CompilerOptions;
- referencedMap?: MapLike<string[]>;
- exportedModulesMap?: MapLike<string[]>;
+ referencedMap?: MapLike<string[] | number>;
+ referencedMapLists?: string[][];
+ exportedModulesMap?: MapLike<string[] | number>;
+ exportedModulesMapLists?: string[][];
  semanticDiagnosticsPerFile?: ProgramBuildInfoDiagnostic[];
  affectedFilesPendingEmit?: ProgramBuilderInfoFilePendingEmit[];
  }
 
+ /**
+ * Runs deduplication on a map of references lists to reduce storage cost
+ */
+ function deduplicateReferencedMap(inputMap: ReadonlyESMap<Path, BuilderState.ReferencedSet>, relativeToBuildInfo: (path: Path) => string): { map: MapLike<string[] | number>, lists?: string[][] } {
+ const map: MapLike<string[] | number> = {};
+ const lists: string[][] = [];
+ const sharedMap = new Map<BuilderState.ReferencedSet, number>();
+ const notSharedMap = new Map<BuilderState.ReferencedSet, string>();
+ const seenSetsBySize = new Map<number, BuilderState.ReferencedSet[]>();
+ for (const key of arrayFrom(inputMap.keys()).sort(compareStringsCaseSensitive)) {
+ const set = inputMap.get(key)!;
+ let sharedSet;
+ let notSharedSet;
+ // already shared
+ if (sharedMap.has(set)) {
+ sharedSet = set;
+ }
+ // already seen, but not shared yet
+ else if (notSharedMap.has(set)) {
+ notSharedSet = set;
+ }
+ // compare with all seen sets to find duplicates without same identity
+ else {
+ const potentialDuplicates = seenSetsBySize.get(set.size);
+ if (potentialDuplicates !== undefined) {
+ outer: for (const otherSet of potentialDuplicates) {
+ const it = set.keys();
+ let itResult;
+ while (!(itResult = it.next()).done) {
+ if (!otherSet.has(itResult.value)) continue outer;
+ }
+ if (sharedMap.has(otherSet)) {
+ sharedSet = otherSet;
+ }
+ else {
+ notSharedSet = otherSet;
+ }
+ break;
+ }
+ }
+ }
+
+ if (sharedSet !== undefined) {
+ // use the existing id
+ map[relativeToBuildInfo(key)] = sharedMap.get(sharedSet)!;
+ }
+ else if(notSharedSet !== undefined) {
+ // create a new shared list and assign both sets to it
+ // also update the existing key with the new shared index
+ const idx = lists.length;
+ const existingKey = notSharedMap.get(notSharedSet)!;
+ sharedMap.set(set, idx);
+ sharedMap.set(notSharedSet, idx);
+ lists.push(map[existingKey] as string[]);
+ notSharedMap.delete(notSharedSet);
+ map[existingKey] = idx;
+ map[relativeToBuildInfo(key)] = idx;
+ }
+ else {
+ const relativeKey = relativeToBuildInfo(key);
+ notSharedMap.set(set, relativeKey);
+ map[relativeKey] = arrayFrom(set.keys(), relativeToBuildInfo).sort(compareStringsCaseSensitive);
+ if (seenSetsBySize.has(set.size)) {
+ seenSetsBySize.get(set.size)!.push(set);
+ }
+ else {
+ seenSetsBySize.set(set.size, [set]);
+ }
+ }
+ }
+ return { map, lists: lists.length > 0 ? lists : undefined };
+ }
+
  /**
  * Gets the program information to be emitted in buildInfo so that we can use it to create new program
  */
@@ -719,23 +794,15 @@ namespace ts {
  options: convertToReusableCompilerOptions(state.compilerOptions, relativeToBuildInfoEnsuringAbsolutePath)
  };
  if (state.referencedMap) {
- const referencedMap: MapLike<string[]> = {};
- for (const key of arrayFrom(state.referencedMap.keys()).sort(compareStringsCaseSensitive)) {
- referencedMap[relativeToBuildInfo(key)] = arrayFrom(state.referencedMap.get(key)!.keys(), relativeToBuildInfo).sort(compareStringsCaseSensitive);
- }
- result.referencedMap = referencedMap;
+ const { map, lists } = deduplicateReferencedMap(state.referencedMap, relativeToBuildInfo);
+ result.referencedMap = map;
+ result.referencedMapLists = lists;
  }
 
  if (state.exportedModulesMap) {
- const exportedModulesMap: MapLike<string[]> = {};
- for (const key of arrayFrom(state.exportedModulesMap.keys()).sort(compareStringsCaseSensitive)) {
- const newValue = state.currentAffectedFilesExportedModulesMap && state.currentAffectedFilesExportedModulesMap.get(key);
- // Not in temporary cache, use existing value
- if (newValue === undefined) exportedModulesMap[relativeToBuildInfo(key)] = arrayFrom(state.exportedModulesMap.get(key)!.keys(), relativeToBuildInfo).sort(compareStringsCaseSensitive);
- // Value in cache and has updated value map, use that
- else if (newValue) exportedModulesMap[relativeToBuildInfo(key)] = arrayFrom(newValue.keys(), relativeToBuildInfo).sort(compareStringsCaseSensitive);
- }
- result.exportedModulesMap = exportedModulesMap;
+ const { map, lists } = deduplicateReferencedMap(state.exportedModulesMap, relativeToBuildInfo);
+ result.exportedModulesMap = map;
+ result.exportedModulesMapLists = lists;
  }
 
  if (state.semanticDiagnosticsPerFile) {
@@ -1167,14 +1234,29 @@ namespace ts {
  }
  }
 
- function getMapOfReferencedSet(mapLike: MapLike<readonly string[]> | undefined, toPath: (path: string) => Path): ReadonlyESMap<Path, BuilderState.ReferencedSet> | undefined {
+ function getMapOfReferencedSet(mapLike: MapLike<readonly string[] | number> | undefined, lists: (readonly string[])[] | undefined, toPath: (path: string) => Path): ReadonlyESMap<Path, BuilderState.ReferencedSet> | undefined {
  if (!mapLike) return undefined;
  const map = new Map<Path, BuilderState.ReferencedSet>();
+ const cache = new Map<number, BuilderState.ReferencedSet>();
  // Copies keys/values from template. Note that for..in will not throw if
  // template is undefined, and instead will just exit the loop.
  for (const key in mapLike) {
  if (hasProperty(mapLike, key)) {
- map.set(toPath(key), new Set(mapLike[key].map(toPath)));
+ const value = mapLike[key];
+ if (typeof value === "number") {
+ if (cache.has(value)) {
+ map.set(toPath(key), cache.get(value)!);
+ }
+ else {
+ if(!lists) return undefined;
+ const list = new Set(lists[value].map(toPath));
+ cache.set(value, list);
+ map.set(toPath(key), list);
+ }
+ }
+ else {
+ map.set(toPath(key), new Set(value.map(toPath)));
+ }
  }
  }
  return map;
@@ -1194,8 +1276,8 @@ namespace ts {
  const state: ReusableBuilderProgramState = {
  fileInfos,
  compilerOptions: convertToOptionsWithAbsolutePaths(program.options, toAbsolutePath),
- referencedMap: getMapOfReferencedSet(program.referencedMap, toPath),
- exportedModulesMap: getMapOfReferencedSet(program.exportedModulesMap, toPath),
+ referencedMap: getMapOfReferencedSet(program.referencedMap, program.referencedMapLists, toPath),
+ exportedModulesMap: getMapOfReferencedSet(program.exportedModulesMap, program.exportedModulesMapLists, toPath),
  semanticDiagnosticsPerFile: program.semanticDiagnosticsPerFile && arrayToMap(program.semanticDiagnosticsPerFile, value => toPath(isString(value) ? value : value[0]), value => isString(value) ? emptyArray : value[1]),
  hasReusableDiagnostic: true,
  affectedFilesPendingEmit: map(program.affectedFilesPendingEmit, value => toPath(value[0])),

tests/baselines/reference/tsbuild/demo/initial-build/in-master-branch-with-everything-setup-correctly-and-reports-no-error.js

@@ -398,29 +398,31 @@ exports.lastElementOf = lastElementOf;
  "configFilePath": "../../zoo/tsconfig.json"
  },
  "referencedMap": {
- "../animals/dog.d.ts": [
- "../animals/index.d.ts"
- ],
+ "../animals/dog.d.ts": 0,
  "../animals/index.d.ts": [
  "../animals/animal.d.ts",
  "../animals/dog.d.ts"
  ],
- "../../zoo/zoo.ts": [
+ "../../zoo/zoo.ts": 0
+ },
+ "referencedMapLists": [
+ [
  "../animals/index.d.ts"
  ]
- },
+ ],
  "exportedModulesMap": {
- "../animals/dog.d.ts": [
- "../animals/index.d.ts"
- ],
+ "../animals/dog.d.ts": 0,
  "../animals/index.d.ts": [
  "../animals/animal.d.ts",
  "../animals/dog.d.ts"
  ],
- "../../zoo/zoo.ts": [
+ "../../zoo/zoo.ts": 0
+ },
+ "exportedModulesMapLists": [
+ [
  "../animals/index.d.ts"
  ]
- },
+ ],
  "semanticDiagnosticsPerFile": [
  "../../../lib/lib.d.ts",
  "../animals/animal.d.ts",

tests/baselines/reference/tsbuild/sample1/incremental-declaration-changes/rebuilds-when-tsconfig-changes.js

@@ -85,13 +85,14 @@ exitCode:: ExitStatus.Success
  ]
  },
  "exportedModulesMap": {
- "../logic/index.d.ts": [
- "../core/anothermodule.d.ts"
- ],
- "./index.ts": [
+ "../logic/index.d.ts": 0,
+ "./index.ts": 0
+ },
+ "exportedModulesMapLists": [
+ [
  "../core/anothermodule.d.ts"
  ]
- },
+ ],
  "semanticDiagnosticsPerFile": [
  "../../lib/lib.d.ts",
  "../core/anothermodule.d.ts",

tests/baselines/reference/tsbuild/sample1/initial-build/always-builds-under-with-force-option.js

@@ -320,13 +320,14 @@ exports.m = mod;
  ]
  },
  "exportedModulesMap": {
- "../logic/index.d.ts": [
- "../core/anothermodule.d.ts"
- ],
- "./index.ts": [
+ "../logic/index.d.ts": 0,
+ "./index.ts": 0
+ },
+ "exportedModulesMapLists": [
+ [
  "../core/anothermodule.d.ts"
  ]
- },
+ ],
  "semanticDiagnosticsPerFile": [
  "../../lib/lib.d.ts",
  "../core/anothermodule.d.ts",

tests/baselines/reference/tsbuild/sample1/initial-build/builds-correctly-when-declarationDir-is-specified.js

@@ -307,13 +307,14 @@ exports.m = mod;
  ]
  },
  "exportedModulesMap": {
- "../logic/out/decls/index.d.ts": [
- "../core/anothermodule.d.ts"
- ],
- "./index.ts": [
+ "../logic/out/decls/index.d.ts": 0,
+ "./index.ts": 0
+ },
+ "exportedModulesMapLists": [
+ [
  "../core/anothermodule.d.ts"
  ]
- },
+ ],
  "semanticDiagnosticsPerFile": [
  "../../lib/lib.d.ts",
  "../core/anothermodule.d.ts",

tests/baselines/reference/tsbuild/sample1/initial-build/builds-correctly-when-outDir-is-specified.js

@@ -307,13 +307,14 @@ exports.m = mod;
  ]
  },
  "exportedModulesMap": {
- "../logic/outdir/index.d.ts": [
- "../core/anothermodule.d.ts"
- ],
- "./index.ts": [
+ "../logic/outdir/index.d.ts": 0,
+ "./index.ts": 0
+ },
+ "exportedModulesMapLists": [
+ [
  "../core/anothermodule.d.ts"
  ]
- },
+ ],
  "semanticDiagnosticsPerFile": [
  "../../lib/lib.d.ts",
  "../core/anothermodule.d.ts",

tests/baselines/reference/tsbuild/sample1/initial-build/can-detect-when-and-what-to-rebuild.js

@@ -236,13 +236,14 @@ Input::
  ]
  },
  "exportedModulesMap": {
- "../logic/index.d.ts": [
- "../core/anothermodule.d.ts"
- ],
- "./index.ts": [
+ "../logic/index.d.ts": 0,
+ "./index.ts": 0
+ },
+ "exportedModulesMapLists": [
+ [
  "../core/anothermodule.d.ts"
  ]
- },
+ ],
  "semanticDiagnosticsPerFile": [
  "../../lib/lib.d.ts",
  "../core/anothermodule.d.ts",

tests/baselines/reference/tsbuild/sample1/initial-build/explainFiles.js

@@ -372,13 +372,14 @@ exports.m = mod;
  ]
  },
  "exportedModulesMap": {
- "../logic/index.d.ts": [
- "../core/anothermodule.d.ts"
- ],
- "./index.ts": [
+ "../logic/index.d.ts": 0,
+ "./index.ts": 0
+ },
+ "exportedModulesMapLists": [
+ [
  "../core/anothermodule.d.ts"
  ]
- },
+ ],
  "semanticDiagnosticsPerFile": [
  "../../lib/lib.d.ts",
  "../core/anothermodule.d.ts",
@@ -641,13 +642,14 @@ exports.someClass = someClass;
  ]
  },
  "exportedModulesMap": {
- "../logic/index.d.ts": [
- "../core/anothermodule.d.ts"
- ],
- "./index.ts": [
+ "../logic/index.d.ts": 0,
+ "./index.ts": 0
+ },
+ "exportedModulesMapLists": [
+ [
  "../core/anothermodule.d.ts"
  ]
- },
+ ],
  "semanticDiagnosticsPerFile": [
  "../../lib/lib.d.ts",
  "../core/anothermodule.d.ts",

tests/baselines/reference/tsbuild/sample1/initial-build/listEmittedFiles.js

@@ -337,13 +337,14 @@ exports.m = mod;
  ]
  },
  "exportedModulesMap": {
- "../logic/index.d.ts": [
- "../core/anothermodule.d.ts"
- ],
- "./index.ts": [
+ "../logic/index.d.ts": 0,
+ "./index.ts": 0
+ },
+ "exportedModulesMapLists": [
+ [
  "../core/anothermodule.d.ts"
  ]
- },
+ ],
  "semanticDiagnosticsPerFile": [
  "../../lib/lib.d.ts",
  "../core/anothermodule.d.ts",
@@ -566,13 +567,14 @@ exports.someClass = someClass;
  ]
  },
  "exportedModulesMap": {
- "../logic/index.d.ts": [
- "../core/anothermodule.d.ts"
- ],
- "./index.ts": [
+ "../logic/index.d.ts": 0,
+ "./index.ts": 0
+ },
+ "exportedModulesMapLists": [
+ [
  "../core/anothermodule.d.ts"
  ]
- },
+ ],
  "semanticDiagnosticsPerFile": [
  "../../lib/lib.d.ts",
  "../core/anothermodule.d.ts",