deduplicate references lists in tsbuildinfo

src/compiler/builder.ts

@@ -695,12 +695,95 @@ namespace ts {
  export interface ProgramBuildInfo {
  fileInfos: MapLike<BuilderState.FileInfo>;
  options: CompilerOptions;
- referencedMap?: MapLike<string[]>;
- exportedModulesMap?: MapLike<string[]>;
+ referencedMap?: MapLike<number>;
+ exportedModulesMap?: MapLike<number>;
+ mapLists: number[][];
+ mapFiles: string[];
  semanticDiagnosticsPerFile?: ProgramBuildInfoDiagnostic[];
  affectedFilesPendingEmit?: ProgramBuilderInfoFilePendingEmit[];
  }
 
+ /**
+ * Runs deduplication on a map of lists of files to reduce storage cost
+ */
+ function deduplicateFileMaps(inputMaps: (ReadonlyESMap<Path, BuilderState.ReferencedSet> | undefined)[], relativeToBuildInfo: (path: Path) => string): { maps: (MapLike<number> | undefined)[], lists: number[][], files: string[] } {
+
+ // Discover all the files and Sets to bring them in a deterministic order
+ const allAbsoluteFiles = new Set<Path>();
+ const allSets = new Set<BuilderState.ReferencedSet>();
+ for(const inputMap of inputMaps) {
+ if(!inputMap) continue;
+ for(const key of arrayFrom(inputMap.keys())) {
+ allAbsoluteFiles.add(key);
+ const set = inputMap.get(key)!;
+ if(!allSets.has(set)) {
+ allSets.add(set);
+ const it = set.keys();
+ let itResult;
+ while (!(itResult = it.next()).done) {
+ allAbsoluteFiles.add(itResult.value);
+ }
+ }
+ }
+ }
+
+ // Get files list and map indicies
+ const fileToIndexMap = new Map<string, number>();
+ const filesWithInfo = arrayFrom(allAbsoluteFiles.keys(), file => ({ file, relativeFile: relativeToBuildInfo(file) })).sort((a, b) => compareStringsCaseSensitive(a.relativeFile, b.relativeFile));
+ for(let i = 0; i < filesWithInfo.length; i++) fileToIndexMap.set(filesWithInfo[i].file, i);
+ const files = filesWithInfo.map(info => info.relativeFile);
+
+ const setToArray = new Map<BuilderState.ReferencedSet, number[]>();
+ const setKeyToArray = new Map<string, number[]>();
+ const it = allSets.keys();
+ let itResult;
+ while (!(itResult = it.next()).done) {
+ const set = itResult.value;
+ const array = arrayFrom(set.keys(), file => fileToIndexMap.get(file)!).sort((a, b) => a - b);
+ const key = array.join();
+ let listArray = setKeyToArray.get(key);
+ if(listArray === undefined) {
+ listArray = array;
+ setKeyToArray.set(key, listArray);
+ }
+ setToArray.set(set, listArray);
+ }
+
+ // Create lists in a deterministic order
+ const lists = arrayFrom(setKeyToArray.values()).sort((a, b) => {
+ const d = a.length - b.length;
+ if(d !== 0) return d;
+ let i = 0;
+ while(true) {
+ const d = a[i] - b[i];
+ if(d !== 0) return d;
+ i++;
+ }
+ });
+ const arrayToIndexMap = new Map<number[], number>();
+ for(let i = 0; i < lists.length; i++) arrayToIndexMap.set(lists[i], i);
+
+ // Replace input maps with numbers
+ const maps = inputMaps.map(inputMap => {
+ if(!inputMap) return undefined;
+
+ const map: MapLike<number> = {};
+ const it = inputMap.keys();
+ let itResult;
+ while (!(itResult = it.next()).done) {
+ const key = itResult.value;
+ const set = inputMap.get(key)!;
+
+ // In an object, keys that look like numbers will always be in numeric order
+ // no need to sort them
+ map[fileToIndexMap.get(key)!] = arrayToIndexMap.get(setToArray.get(set)!)!;
+ }
+ return map;
+ });
+
+ return { maps, lists, files };
+ }
+
  /**
  * Gets the program information to be emitted in buildInfo so that we can use it to create new program
  */
@@ -714,29 +797,15 @@ namespace ts {
  fileInfos[relativeToBuildInfo(key)] = signature === undefined ? value : { version: value.version, signature, affectsGlobalScope: value.affectsGlobalScope };
  });
 
+ const { maps: [referencedMap, exportedModulesMap], lists, files } = deduplicateFileMaps([state.referencedMap, state.exportedModulesMap], relativeToBuildInfo);
  const result: ProgramBuildInfo = {
  fileInfos,
- options: convertToReusableCompilerOptions(state.compilerOptions, relativeToBuildInfoEnsuringAbsolutePath)
+ options: convertToReusableCompilerOptions(state.compilerOptions, relativeToBuildInfoEnsuringAbsolutePath),
+ referencedMap,
+ exportedModulesMap,
+ mapLists: lists,
+ mapFiles: files,
  };
- if (state.referencedMap) {
- const referencedMap: MapLike<string[]> = {};
- for (const key of arrayFrom(state.referencedMap.keys()).sort(compareStringsCaseSensitive)) {
- referencedMap[relativeToBuildInfo(key)] = arrayFrom(state.referencedMap.get(key)!.keys(), relativeToBuildInfo).sort(compareStringsCaseSensitive);
- }
- result.referencedMap = referencedMap;
- }
-
- if (state.exportedModulesMap) {
- const exportedModulesMap: MapLike<string[]> = {};
- for (const key of arrayFrom(state.exportedModulesMap.keys()).sort(compareStringsCaseSensitive)) {
- const newValue = state.currentAffectedFilesExportedModulesMap && state.currentAffectedFilesExportedModulesMap.get(key);
- // Not in temporary cache, use existing value
- if (newValue === undefined) exportedModulesMap[relativeToBuildInfo(key)] = arrayFrom(state.exportedModulesMap.get(key)!.keys(), relativeToBuildInfo).sort(compareStringsCaseSensitive);
- // Value in cache and has updated value map, use that
- else if (newValue) exportedModulesMap[relativeToBuildInfo(key)] = arrayFrom(newValue.keys(), relativeToBuildInfo).sort(compareStringsCaseSensitive);
- }
- result.exportedModulesMap = exportedModulesMap;
- }
 
  if (state.semanticDiagnosticsPerFile) {
  const semanticDiagnosticsPerFile: ProgramBuildInfoDiagnostic[] = [];
@@ -1167,17 +1236,19 @@ namespace ts {
  }
  }
 
- function getMapOfReferencedSet(mapLike: MapLike<readonly string[]> | undefined, toPath: (path: string) => Path): ReadonlyESMap<Path, BuilderState.ReferencedSet> | undefined {
- if (!mapLike) return undefined;
- const map = new Map<Path, BuilderState.ReferencedSet>();
- // Copies keys/values from template. Note that for..in will not throw if
- // template is undefined, and instead will just exit the loop.
- for (const key in mapLike) {
- if (hasProperty(mapLike, key)) {
- map.set(toPath(key), new Set(mapLike[key].map(toPath)));
+ function expandFileMaps(inputMaps: (MapLike<number> | undefined)[], lists: number[][], files: string[], toPath: (path: string) => Path): (ReadonlyESMap<Path, BuilderState.ReferencedSet> | undefined)[] {
+ const paths = files.map(toPath);
+ const sets = lists.map(list => new Set(list.map(i => paths[i])));
+ return inputMaps.map(inputMap => {
+ if(!inputMap) return undefined;
+ const map = new Map<Path, BuilderState.ReferencedSet>();
+ for (const key in inputMap) {
+ if (hasProperty(inputMap, key)) {
+ map.set(paths[+key], sets[+inputMap[key]]);
+ }
  }
- }
- return map;
+  return map;
+ });
  }
 
  export function createBuildProgramUsingProgramBuildInfo(program: ProgramBuildInfo, buildInfoPath: string, host: ReadBuildProgramHost): EmitAndSemanticDiagnosticsBuilderProgram {
@@ -1191,11 +1262,12 @@ namespace ts {
  }
  }
 
+ const [referencedMap, exportedModulesMap] = expandFileMaps([program.referencedMap, program.exportedModulesMap], program.mapLists, program.mapFiles, toPath);
  const state: ReusableBuilderProgramState = {
  fileInfos,
  compilerOptions: convertToOptionsWithAbsolutePaths(program.options, toAbsolutePath),
- referencedMap: getMapOfReferencedSet(program.referencedMap, toPath),
- exportedModulesMap: getMapOfReferencedSet(program.exportedModulesMap, toPath),
+ referencedMap,
+ exportedModulesMap,
  semanticDiagnosticsPerFile: program.semanticDiagnosticsPerFile && arrayToMap(program.semanticDiagnosticsPerFile, value => toPath(isString(value) ? value : value[0]), value => isString(value) ? emptyArray : value[1]),
  hasReusableDiagnostic: true,
  affectedFilesPendingEmit: map(program.affectedFilesPendingEmit, value => toPath(value[0])),

tests/baselines/reference/tsbuild/configFileErrors/incremental-declaration-changes/builds-after-fixing-config-file-errors.js

@@ -60,6 +60,8 @@ exports.bar = bar;
  },
  "referencedMap": {},
  "exportedModulesMap": {},
+ "mapLists": [],
+ "mapFiles": [],
  "semanticDiagnosticsPerFile": [
  "../lib/lib.d.ts",
  "./a.ts",

tests/baselines/reference/tsbuild/configFileExtends/initial-build/when-building-project-uses-reference-and-both-extend-config-with-include.js

@@ -100,6 +100,8 @@ exports.a = 1;
  },
  "referencedMap": {},
  "exportedModulesMap": {},
+ "mapLists": [],
+ "mapFiles": [],
  "semanticDiagnosticsPerFile": [
  "../../../lib/lib.d.ts",
  "../../shared/index.ts",
@@ -149,6 +151,8 @@ exports.b = 1;
  },
  "referencedMap": {},
  "exportedModulesMap": {},
+ "mapLists": [],
+ "mapFiles": [],
  "semanticDiagnosticsPerFile": [
  "../../../lib/lib.d.ts",
  "../../shared/typings-base/globals.d.ts",

tests/baselines/reference/tsbuild/configFileExtends/initial-build/when-building-solution-with-projects-extends-config-with-include.js

@@ -101,6 +101,8 @@ exports.a = 1;
  },
  "referencedMap": {},
  "exportedModulesMap": {},
+ "mapLists": [],
+ "mapFiles": [],
  "semanticDiagnosticsPerFile": [
  "../../../lib/lib.d.ts",
  "../../shared/index.ts",
@@ -150,6 +152,8 @@ exports.b = 1;
  },
  "referencedMap": {},
  "exportedModulesMap": {},
+ "mapLists": [],
+ "mapFiles": [],
  "semanticDiagnosticsPerFile": [
  "../../../lib/lib.d.ts",
  "../../shared/typings-base/globals.d.ts",

tests/baselines/reference/tsbuild/containerOnlyReferenced/initial-build/verify-that-subsequent-builds-after-initial-build-doesnt-build-anything.js

@@ -132,6 +132,8 @@ exports.x = 10;
  },
  "referencedMap": {},
  "exportedModulesMap": {},
+ "mapLists": [],
+ "mapFiles": [],
  "semanticDiagnosticsPerFile": [
  "../../../lib/lib.d.ts",
  "./index.ts"
@@ -172,6 +174,8 @@ exports.x = 10;
  },
  "referencedMap": {},
  "exportedModulesMap": {},
+ "mapLists": [],
+ "mapFiles": [],
  "semanticDiagnosticsPerFile": [
  "../../../lib/lib.d.ts",
  "./index.ts"
@@ -212,6 +216,8 @@ exports.x = 10;
  },
  "referencedMap": {},
  "exportedModulesMap": {},
+ "mapLists": [],
+ "mapFiles": [],
  "semanticDiagnosticsPerFile": [
  "../../lib/lib.d.ts",
  "./index.ts"

tests/baselines/reference/tsbuild/declarationEmit/initial-build/when-declaration-file-is-referenced-through-triple-slash-but-uses-no-references.js

@@ -148,24 +148,31 @@ exports.getVar = getVar;
  "configFilePath": "../tsconfig.json"
  },
  "referencedMap": {
- "../src/common/nominal.ts": [
- "../src/common/types.d.ts"
- ],
- "../src/subproject/index.ts": [
- "../src/common/nominal.ts"
- ],
- "../src/subproject2/index.ts": [
- "../src/subproject/index.ts"
- ]
+ "0": 1,
+ "2": 0,
+ "3": 2
  },
  "exportedModulesMap": {
- "../src/subproject/index.ts": [
- "../src/common/nominal.ts"
+ "2": 0,
+ "3": 2
+ },
+ "mapLists": [
+ [
+ 0
+ ],
+ [
+ 1
  ],
- "../src/subproject2/index.ts": [
- "../src/subproject/index.ts"
+ [
+ 2
  ]
- },
+ ],
+ "mapFiles": [
+ "../src/common/nominal.ts",
+ "../src/common/types.d.ts",
+ "../src/subproject/index.ts",
+ "../src/subproject2/index.ts"
+ ],
  "semanticDiagnosticsPerFile": [
  "../../../lib/lib.d.ts",
  "../src/common/nominal.ts",