Skip to content

Commit

Permalink
fix(prerender): cache writing hashed assets
Browse files Browse the repository at this point in the history
  • Loading branch information
@adamdbradley
adamdbradley committed Oct 27, 2020
1 parent 044aa96 commit 96c44f8
Show file tree
Hide file tree
Showing 3 changed files with 128 additions and 45 deletions.
118 changes: 86 additions & 32 deletions src/compiler/prerender/prerender-optimize.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { optimizeCss } from '../optimize/optimize-css';
import { optimizeJs } from '../optimize/optimize-js';
import { join } from 'path';
import { minifyCss } from '../optimize/minify-css';
import { PrerenderContext } from './prerender-worker-ctx';

export const inlineExternalStyleSheets = async (sys: d.CompilerSystem, appDir: string, doc: Document) => {
const documentLinks = Array.from(doc.querySelectorAll('link[rel=stylesheet]')) as HTMLLinkElement[];
Expand Down Expand Up @@ -95,7 +96,13 @@ export const minifyScriptElements = async (doc: Document, addMinifiedAttr: boole
);
};

export const minifyStyleElements = async (sys: d.CompilerSystem, appDir: string, doc: Document, currentUrl: URL, addMinifiedAttr: boolean) => {
export const minifyStyleElements = async (
sys: d.CompilerSystem,
appDir: string,
doc: Document,
currentUrl: URL,
addMinifiedAttr: boolean,
) => {
const styleElms = Array.from(doc.querySelectorAll('style')).filter(styleElm => {
if (styleElm.hasAttribute(dataMinifiedAttr)) {
return false;
Expand All @@ -115,7 +122,7 @@ export const minifyStyleElements = async (sys: d.CompilerSystem, appDir: string,
const hash = await getAssetFileHash(sys, appDir, assetUrl);
assetUrl.searchParams.append('v', hash);
return assetUrl.pathname + assetUrl.search;
}
},
});
if (optimizeResults.diagnostics.length === 0) {
styleElm.innerHTML = optimizeResults.output;
Expand All @@ -128,7 +135,11 @@ export const minifyStyleElements = async (sys: d.CompilerSystem, appDir: string,
);
};

export const excludeStaticComponents = (doc: Document, hydrateOpts: d.PrerenderHydrateOptions, hydrateResults: d.HydrateResults) => {
export const excludeStaticComponents = (
doc: Document,
hydrateOpts: d.PrerenderHydrateOptions,
hydrateResults: d.HydrateResults,
) => {
const staticComponents = hydrateOpts.staticComponents.filter(tag => {
return hydrateResults.components.some(cmp => cmp.tag === tag);
});
Expand Down Expand Up @@ -158,7 +169,12 @@ s&&((s['data-opts']=s['data-opts']||{}).exclude=__EXCLUDE__);
.replace(/\n/g, '')
.trim();

export const addModulePreloads = (doc: Document, hydrateOpts: d.PrerenderHydrateOptions, hydrateResults: d.HydrateResults, componentGraph: Map<string, string[]>) => {
export const addModulePreloads = (
doc: Document,
hydrateOpts: d.PrerenderHydrateOptions,
hydrateResults: d.HydrateResults,
componentGraph: Map<string, string[]>,
) => {
if (!componentGraph) {
return false;
}
Expand All @@ -167,7 +183,9 @@ export const addModulePreloads = (doc: Document, hydrateOpts: d.PrerenderHydrate

const cmpTags = hydrateResults.components.filter(cmp => !staticComponents.includes(cmp.tag));

const modulePreloads = unique(flatOne(cmpTags.map(cmp => getScopeId(cmp.tag, cmp.mode)).map(scopeId => componentGraph.get(scopeId) || [])));
const modulePreloads = unique(
flatOne(cmpTags.map(cmp => getScopeId(cmp.tag, cmp.mode)).map(scopeId => componentGraph.get(scopeId) || [])),
);

injectModulePreloads(doc, modulePreloads);
return true;
Expand All @@ -194,7 +212,15 @@ export const hasStencilScript = (doc: Document) => {
return !!doc.querySelector('script[data-stencil]');
};

export const hashAssets = async (sys: d.CompilerSystem, diagnostics: d.Diagnostic[], hydrateOpts: d.PrerenderHydrateOptions, appDir: string, doc: Document, currentUrl: URL) => {
export const hashAssets = async (
sys: d.CompilerSystem,
prerenderCtx: PrerenderContext,
diagnostics: d.Diagnostic[],
hydrateOpts: d.PrerenderHydrateOptions,
appDir: string,
doc: Document,
currentUrl: URL,
) => {
// do one at a time to prevent too many opened files and memory usage issues
// hash id is cached in each worker, so shouldn't have to do this for every page

Expand All @@ -208,18 +234,23 @@ export const hashAssets = async (sys: d.CompilerSystem, diagnostics: d.Diagnosti
if (currentUrl.host === stylesheetUrl.host) {
try {
const filePath = join(appDir, stylesheetUrl.pathname);
if (prerenderCtx.hashedFile.has(filePath)) {
continue;
}
prerenderCtx.hashedFile.add(filePath);

let css = await sys.readFile(filePath);
if (isString(css)) {
if (isString(css) && css.length > 0) {
css = await minifyCss({
css,
async resolveUrl(urlProp) {
const assetUrl = new URL(urlProp, stylesheetUrl);
const hash = await getAssetFileHash(sys, appDir, assetUrl);
assetUrl.searchParams.append('v', hash);
return assetUrl.pathname + assetUrl.search;
}
},
});
await sys.writeFile(filePath, css);
sys.writeFileSync(filePath, css);
}
} catch (e) {
catchError(diagnostics, e);
Expand All @@ -238,24 +269,36 @@ export const hashAssets = async (sys: d.CompilerSystem, diagnostics: d.Diagnosti
await hashAsset(sys, hydrateOpts, appDir, doc, currentUrl, 'script', ['src']);
await hashAsset(sys, hydrateOpts, appDir, doc, currentUrl, 'img', ['src', 'srcset']);
await hashAsset(sys, hydrateOpts, appDir, doc, currentUrl, 'picture > source', ['srcset']);

const pageStates = Array.from(doc.querySelectorAll('script[data-stencil-static="page.state"][type="application/json"]')) as HTMLScriptElement[];

const pageStates = Array.from(
doc.querySelectorAll('script[data-stencil-static="page.state"][type="application/json"]'),
) as HTMLScriptElement[];
if (pageStates.length > 0) {
await Promise.all(pageStates.map(async pageStateScript => {
const pageState = JSON.parse(pageStateScript.textContent);
if (pageState && Array.isArray(pageState.ast)) {
for (const node of pageState.ast) {
if (Array.isArray(node)) {
await hashPageStateAstAssets(sys, hydrateOpts, appDir, currentUrl, pageStateScript, node);
await Promise.all(
pageStates.map(async pageStateScript => {
const pageState = JSON.parse(pageStateScript.textContent);
if (pageState && Array.isArray(pageState.ast)) {
for (const node of pageState.ast) {
if (Array.isArray(node)) {
await hashPageStateAstAssets(sys, hydrateOpts, appDir, currentUrl, pageStateScript, node);
}
}
pageStateScript.textContent = JSON.stringify(pageState);
}
pageStateScript.textContent = JSON.stringify(pageState);
}
}));
}),
);
}
}
};

const hashAsset = async (sys: d.CompilerSystem, hydrateOpts: d.PrerenderHydrateOptions, appDir: string, doc: Document, currentUrl: URL, selector: string, srcAttrs: string[]) => {
const hashAsset = async (
sys: d.CompilerSystem,
hydrateOpts: d.PrerenderHydrateOptions,
appDir: string,
doc: Document,
currentUrl: URL,
selector: string,
srcAttrs: string[],
) => {
const elms = Array.from(doc.querySelectorAll(selector));

// do one at a time to prevent too many opened files and memory usage issues
Expand All @@ -281,7 +324,14 @@ const hashAsset = async (sys: d.CompilerSystem, hydrateOpts: d.PrerenderHydrateO
}
};

const hashPageStateAstAssets = async (sys: d.CompilerSystem, hydrateOpts: d.PrerenderHydrateOptions, appDir: string, currentUrl: URL, pageStateScript: HTMLScriptElement, node: any[]) => {
const hashPageStateAstAssets = async (
sys: d.CompilerSystem,
hydrateOpts: d.PrerenderHydrateOptions,
appDir: string,
currentUrl: URL,
pageStateScript: HTMLScriptElement,
node: any[],
) => {
const tagName = node[0];
const attrs = node[1];

Expand Down Expand Up @@ -318,21 +368,25 @@ const hashPageStateAstAssets = async (sys: d.CompilerSystem, hydrateOpts: d.Prer
};

export const getAttrUrls = (attrName: string, attrValue: string) => {
const srcValues: { src: string, descriptor?: string }[] = [];
const srcValues: { src: string; descriptor?: string }[] = [];
if (isString(attrValue)) {
if (attrName.toLowerCase() === 'srcset') {
attrValue.split(',').map(a => a.trim()).filter(a => a.length > 0).forEach(src => {
const spaceSplt = src.split(' ');
if (spaceSplt[0].length > 0) {
srcValues.push({ src: spaceSplt[0], descriptor: spaceSplt[1] });
}
});
attrValue
.split(',')
.map(a => a.trim())
.filter(a => a.length > 0)
.forEach(src => {
const spaceSplt = src.split(' ');
if (spaceSplt[0].length > 0) {
srcValues.push({ src: spaceSplt[0], descriptor: spaceSplt[1] });
}
});
} else {
srcValues.push({ src: attrValue });
}
}
return srcValues;
}
};

export const setAttrUrls = (url: URL, descriptor: string) => {
let src = url.pathname + url.search;
Expand All @@ -352,6 +406,6 @@ const getAssetFileHash = async (sys: d.CompilerSystem, appDir: string, assetUrl:
hashedAssets.set(assetUrl.pathname, p);
}
return p;
}
};

const dataMinifiedAttr = 'data-m';
31 changes: 31 additions & 0 deletions src/compiler/prerender/prerender-worker-ctx.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
import type * as d from '../../declarations';

export interface PrerenderContext {
buildId: string;
componentGraph: Map<string, string[]>;
prerenderConfig: d.PrerenderConfig;
ensuredDirs: Set<string>;
templateHtml: string;
hashedFile: Set<string>;
}

const prerenderCtx: PrerenderContext = {
buildId: null,
componentGraph: null,
prerenderConfig: null,
ensuredDirs: null,
templateHtml: null,
hashedFile: null,
};

export const getPrerenderCtx = (prerenderRequest: d.PrerenderUrlRequest) => {
if (prerenderRequest.buildId !== prerenderCtx.buildId) {
prerenderCtx.buildId = prerenderRequest.buildId;
prerenderCtx.componentGraph = null;
prerenderCtx.prerenderConfig = null;
prerenderCtx.ensuredDirs = new Set();
prerenderCtx.templateHtml = null;
prerenderCtx.hashedFile = new Set();
}
return prerenderCtx;
};
24 changes: 11 additions & 13 deletions src/compiler/prerender/prerender-worker.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,12 @@ import {
} from './prerender-optimize';
import { catchError, isPromise, isRootPath, normalizePath, isFunction } from '@utils';
import { crawlAnchorsForNextUrls } from './crawl-urls';
import { getPrerenderCtx, PrerenderContext } from './prerender-worker-ctx';
import { getHydrateOptions } from './prerender-hydrate-options';
import { getPrerenderConfig } from './prerender-config';
import { requireFunc } from '../sys/environment';
import { dirname, join } from 'path';

const prerenderCtx = {
componentGraph: null as Map<string, string[]>,
prerenderConfig: null as d.PrerenderConfig,
ensuredDirs: new Set<string>(),
templateHtml: null as string,
};

export const prerenderWorker = async (sys: d.CompilerSystem, prerenderRequest: d.PrerenderUrlRequest) => {
// worker thread!
const results: d.PrerenderUrlResults = {
Expand All @@ -31,9 +25,11 @@ export const prerenderWorker = async (sys: d.CompilerSystem, prerenderRequest: d
};

try {
const prerenderCtx = getPrerenderCtx(prerenderRequest);

const url = new URL(prerenderRequest.url, prerenderRequest.devServerHostUrl);
const baseUrl = new URL(prerenderRequest.baseUrl);
const componentGraph = getComponentGraph(sys, prerenderRequest.componentGraphPath);
const componentGraph = getComponentGraph(sys, prerenderCtx, prerenderRequest.componentGraphPath);

// webpack work-around/hack
const hydrateApp = requireFunc(prerenderRequest.hydrateAppFilePath);
Expand Down Expand Up @@ -128,12 +124,14 @@ export const prerenderWorker = async (sys: d.CompilerSystem, prerenderRequest: d

if (hydrateOpts.hashAssets && !prerenderRequest.isDebug) {
try {
docPromises.push(hashAssets(sys, results.diagnostics, hydrateOpts, prerenderRequest.appDir, doc, url));
docPromises.push(
hashAssets(sys, prerenderCtx, results.diagnostics, hydrateOpts, prerenderRequest.appDir, doc, url),
);
} catch (e) {
catchError(results.diagnostics, e);
}
}

if (docPromises.length > 0) {
await Promise.all(docPromises);
}
Expand Down Expand Up @@ -168,7 +166,7 @@ export const prerenderWorker = async (sys: d.CompilerSystem, prerenderRequest: d

const html = hydrateApp.serializeDocumentToString(doc, hydrateOpts);

prerenderEnsureDir(sys, results.filePath);
prerenderEnsureDir(sys, prerenderCtx, results.filePath);

const writePromise = sys.writeFile(results.filePath, html)