From ab55086b99157fa7570263c39d7959f1dbecdf01 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Fri, 18 Feb 2022 12:46:40 +0100 Subject: [PATCH 1/9] start materializing page mode sooner, wait for it to finish before emitting ENGINES_READY --- packages/gatsby/src/commands/build.ts | 9 ++++++--- packages/gatsby/src/utils/page-mode.ts | 2 ++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/gatsby/src/commands/build.ts b/packages/gatsby/src/commands/build.ts index 2ed486536da02..78b5d8b0f8ccc 100644 --- a/packages/gatsby/src/commands/build.ts +++ b/packages/gatsby/src/commands/build.ts @@ -286,8 +286,12 @@ module.exports = async function build( ) } + // Start saving page.mode in the main process (while queries run in workers in parallel) + let waitMaterializePageMode = Promise.resolve() + let waitForWorkerPoolRestart = Promise.resolve() if (process.env.GATSBY_EXPERIMENTAL_PARALLEL_QUERY_RUNNING) { + waitMaterializePageMode = materializePageMode() await runQueriesInWorkersQueue(workerPool, queryIds, { parentSpan: buildSpan, }) @@ -310,6 +314,7 @@ module.exports = async function build( parentSpan: buildSpan, store, }) + waitMaterializePageMode = materializePageMode() } // create scope so we don't leak state object @@ -322,6 +327,7 @@ module.exports = async function build( } if (process.send && shouldGenerateEngines()) { + await waitMaterializePageMode process.send({ type: `LOG_ACTION`, action: { @@ -401,9 +407,6 @@ module.exports = async function build( await waitForWorkerPoolRestart - // Start saving page.mode in the main process (while HTML is generated in workers in parallel) - const waitMaterializePageMode = materializePageMode() - const { toRegenerate, toDelete } = await buildHTMLPagesAndDeleteStaleArtifacts({ program, diff --git a/packages/gatsby/src/utils/page-mode.ts b/packages/gatsby/src/utils/page-mode.ts index 6574e5984b58c..08be1ddbc2a58 100644 --- a/packages/gatsby/src/utils/page-mode.ts +++ b/packages/gatsby/src/utils/page-mode.ts @@ -8,6 +8,7 @@ import { import { reportOnce } from "./report-once" import { ROUTES_DIRECTORY } from "../constants" import { Runner } from "../bootstrap/create-graphql-runner" +import { getDataStore } from "../datastore" type IPageConfigFn = (arg: { params: Record }) => { defer: boolean @@ -111,6 +112,7 @@ export async function materializePageMode(): Promise { await new Promise(resolve => setImmediate(resolve)) } } + await getDataStore().ready() } export async function preparePageTemplateConfigs( From 4b1a3f4ff14dce05a31383f74947517efcb1d301 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Fri, 18 Feb 2022 13:22:49 +0100 Subject: [PATCH 2/9] checksum --- .../src/schema/graphql-engine/bundle-webpack.ts | 4 ++++ .../gatsby/src/schema/graphql-engine/entry.ts | 11 +++++++++++ packages/gatsby/src/utils/lmdb-pages-checksum.ts | 15 +++++++++++++++ 3 files changed, 30 insertions(+) create mode 100644 packages/gatsby/src/utils/lmdb-pages-checksum.ts diff --git a/packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts b/packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts index b7248a3bbec30..0b0cbace7f45d 100644 --- a/packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts +++ b/packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts @@ -7,6 +7,7 @@ import { printQueryEnginePlugins } from "./print-plugins" import mod from "module" import { WebpackLoggingPlugin } from "../../utils/webpack/plugins/webpack-logging" import reporter from "gatsby-cli/lib/reporter" +import { generatePagesChecksum } from "../../utils/lmdb-pages-checksum" type Reporter = typeof reporter @@ -131,6 +132,9 @@ export async function createGraphqlEngineBundle( "process.env.GATSBY_SKIP_WRITING_SCHEMA_TO_FILE": `true`, SCHEMA_SNAPSHOT: JSON.stringify(schemaSnapshotString), "process.env.GATSBY_LOGGER": JSON.stringify(`yurnalist`), + "process.env.GATSBY_PAGES_CHECKSUM": JSON.stringify( + generatePagesChecksum() + ), }), process.env.GATSBY_WEBPACK_LOGGING?.includes(`query-engine`) && new WebpackLoggingPlugin(rootDir, reporter, isVerbose), diff --git a/packages/gatsby/src/schema/graphql-engine/entry.ts b/packages/gatsby/src/schema/graphql-engine/entry.ts index 41d9b237b5291..df462739d2bf2 100644 --- a/packages/gatsby/src/schema/graphql-engine/entry.ts +++ b/packages/gatsby/src/schema/graphql-engine/entry.ts @@ -24,6 +24,7 @@ import { // @ts-ignore } from ".cache/query-engine-plugins" import { initTracer } from "../../utils/tracer" +import { generatePagesChecksum } from "../../utils/lmdb-pages-checksum" type MaybePhantomActivity = | ReturnType @@ -107,6 +108,16 @@ export class GraphQLEngine { // The way internal runner works can change, so we should not make it a public API. // Here we just want to expose way to await it being ready await this.getRunner() + + if (process.env.GATSBY_PAGES_CHECKSUM) { + const engineChecksum = generatePagesChecksum() + // @ts-ignore PAGES_CHECKSUM is being "inlined" by bundler + if (engineChecksum !== process.env.GATSBY_PAGES_CHECKSUM) { + reporter.error( + `Pages checksum doesn't match: \n - calculated: "${engineChecksum}"\n - expected: "${process.env.GATSBY_PAGES_CHECKSUM}"` + ) + } + } } public async runQuery( diff --git a/packages/gatsby/src/utils/lmdb-pages-checksum.ts b/packages/gatsby/src/utils/lmdb-pages-checksum.ts new file mode 100644 index 0000000000000..ceb0d9c8d4768 --- /dev/null +++ b/packages/gatsby/src/utils/lmdb-pages-checksum.ts @@ -0,0 +1,15 @@ +import crypto from "crypto" +import { getDataStore } from "../datastore" + +export function generatePagesChecksum(): string { + const hash = crypto.createHash(`md5`) + + try { + for (const page of getDataStore().iterateNodesByType(`SitePage`)) { + hash.update(page?.path as string) + } + } catch (e) { + console.log(e) + } + return hash.digest(`hex`) +} From 9e4bb08dbeb5a7a17e9014d73ae626a8600236f7 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Fri, 18 Feb 2022 13:25:35 +0100 Subject: [PATCH 3/9] toggle page mode materialization with env var --- packages/gatsby/src/commands/build.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/gatsby/src/commands/build.ts b/packages/gatsby/src/commands/build.ts index 78b5d8b0f8ccc..5049b9a8f624e 100644 --- a/packages/gatsby/src/commands/build.ts +++ b/packages/gatsby/src/commands/build.ts @@ -291,7 +291,9 @@ module.exports = async function build( let waitForWorkerPoolRestart = Promise.resolve() if (process.env.GATSBY_EXPERIMENTAL_PARALLEL_QUERY_RUNNING) { - waitMaterializePageMode = materializePageMode() + if (process.env.GATSBY_MATERIALIZE_EARLY) { + waitMaterializePageMode = materializePageMode() + } await runQueriesInWorkersQueue(workerPool, queryIds, { parentSpan: buildSpan, }) @@ -314,7 +316,9 @@ module.exports = async function build( parentSpan: buildSpan, store, }) - waitMaterializePageMode = materializePageMode() + if (process.env.GATSBY_MATERIALIZE_EARLY) { + waitMaterializePageMode = materializePageMode() + } } // create scope so we don't leak state object @@ -407,6 +411,10 @@ module.exports = async function build( await waitForWorkerPoolRestart + if (!process.env.GATSBY_MATERIALIZE_EARLY) { + waitMaterializePageMode = materializePageMode() + } + const { toRegenerate, toDelete } = await buildHTMLPagesAndDeleteStaleArtifacts({ program, From 7fbbab6a69c9c108638aac5efefc7da589d97173 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Fri, 18 Feb 2022 15:31:04 +0100 Subject: [PATCH 4/9] move checksum comparison --- .../gatsby/src/schema/graphql-engine/entry.ts | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/gatsby/src/schema/graphql-engine/entry.ts b/packages/gatsby/src/schema/graphql-engine/entry.ts index df462739d2bf2..99a913648c6ff 100644 --- a/packages/gatsby/src/schema/graphql-engine/entry.ts +++ b/packages/gatsby/src/schema/graphql-engine/entry.ts @@ -90,6 +90,16 @@ export class GraphQLEngine { // Note: skipping inference metadata because we rely on schema snapshot await build({ fullMetadataBuild: false, parentSpan: wrapActivity.span }) + if (process.env.GATSBY_PAGES_CHECKSUM) { + const engineChecksum = generatePagesChecksum() + // @ts-ignore PAGES_CHECKSUM is being "inlined" by bundler + if (engineChecksum !== process.env.GATSBY_PAGES_CHECKSUM) { + reporter.error( + `Pages checksum doesn't match: \n - calculated: "${engineChecksum}"\n - expected: "${process.env.GATSBY_PAGES_CHECKSUM}"` + ) + } + } + return new GraphQLRunner(store) } finally { wrapActivity.end() @@ -108,16 +118,6 @@ export class GraphQLEngine { // The way internal runner works can change, so we should not make it a public API. // Here we just want to expose way to await it being ready await this.getRunner() - - if (process.env.GATSBY_PAGES_CHECKSUM) { - const engineChecksum = generatePagesChecksum() - // @ts-ignore PAGES_CHECKSUM is being "inlined" by bundler - if (engineChecksum !== process.env.GATSBY_PAGES_CHECKSUM) { - reporter.error( - `Pages checksum doesn't match: \n - calculated: "${engineChecksum}"\n - expected: "${process.env.GATSBY_PAGES_CHECKSUM}"` - ) - } - } } public async runQuery( From 6dcd1757fa1a1a336d86c7d5a32132805ab07243 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Sat, 19 Feb 2022 12:18:17 +0100 Subject: [PATCH 5/9] drop unneeded @ts-ignore --- packages/gatsby/src/schema/graphql-engine/entry.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/gatsby/src/schema/graphql-engine/entry.ts b/packages/gatsby/src/schema/graphql-engine/entry.ts index 99a913648c6ff..45e2e5371cd69 100644 --- a/packages/gatsby/src/schema/graphql-engine/entry.ts +++ b/packages/gatsby/src/schema/graphql-engine/entry.ts @@ -92,7 +92,6 @@ export class GraphQLEngine { if (process.env.GATSBY_PAGES_CHECKSUM) { const engineChecksum = generatePagesChecksum() - // @ts-ignore PAGES_CHECKSUM is being "inlined" by bundler if (engineChecksum !== process.env.GATSBY_PAGES_CHECKSUM) { reporter.error( `Pages checksum doesn't match: \n - calculated: "${engineChecksum}"\n - expected: "${process.env.GATSBY_PAGES_CHECKSUM}"` From 0da6f9206dbcf74ed825ca8ab7fc90d1b9f029fa Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Sat, 19 Feb 2022 12:18:58 +0100 Subject: [PATCH 6/9] drop GATSBY_MATERIALIZE_EARLY --- packages/gatsby/src/commands/build.ts | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/packages/gatsby/src/commands/build.ts b/packages/gatsby/src/commands/build.ts index 5049b9a8f624e..7b74b7a6386f8 100644 --- a/packages/gatsby/src/commands/build.ts +++ b/packages/gatsby/src/commands/build.ts @@ -291,9 +291,8 @@ module.exports = async function build( let waitForWorkerPoolRestart = Promise.resolve() if (process.env.GATSBY_EXPERIMENTAL_PARALLEL_QUERY_RUNNING) { - if (process.env.GATSBY_MATERIALIZE_EARLY) { - waitMaterializePageMode = materializePageMode() - } + waitMaterializePageMode = materializePageMode() + await runQueriesInWorkersQueue(workerPool, queryIds, { parentSpan: buildSpan, }) @@ -316,9 +315,8 @@ module.exports = async function build( parentSpan: buildSpan, store, }) - if (process.env.GATSBY_MATERIALIZE_EARLY) { - waitMaterializePageMode = materializePageMode() - } + + waitMaterializePageMode = materializePageMode() } // create scope so we don't leak state object @@ -411,10 +409,6 @@ module.exports = async function build( await waitForWorkerPoolRestart - if (!process.env.GATSBY_MATERIALIZE_EARLY) { - waitMaterializePageMode = materializePageMode() - } - const { toRegenerate, toDelete } = await buildHTMLPagesAndDeleteStaleArtifacts({ program, From d86fcaa7482560fdf20a130071c4a457ccd3bc14 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Mon, 21 Feb 2022 11:34:43 +0100 Subject: [PATCH 7/9] drop checksum --- .../src/schema/graphql-engine/bundle-webpack.ts | 4 ---- .../gatsby/src/schema/graphql-engine/entry.ts | 10 ---------- packages/gatsby/src/utils/lmdb-pages-checksum.ts | 15 --------------- 3 files changed, 29 deletions(-) delete mode 100644 packages/gatsby/src/utils/lmdb-pages-checksum.ts diff --git a/packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts b/packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts index 0b0cbace7f45d..b7248a3bbec30 100644 --- a/packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts +++ b/packages/gatsby/src/schema/graphql-engine/bundle-webpack.ts @@ -7,7 +7,6 @@ import { printQueryEnginePlugins } from "./print-plugins" import mod from "module" import { WebpackLoggingPlugin } from "../../utils/webpack/plugins/webpack-logging" import reporter from "gatsby-cli/lib/reporter" -import { generatePagesChecksum } from "../../utils/lmdb-pages-checksum" type Reporter = typeof reporter @@ -132,9 +131,6 @@ export async function createGraphqlEngineBundle( "process.env.GATSBY_SKIP_WRITING_SCHEMA_TO_FILE": `true`, SCHEMA_SNAPSHOT: JSON.stringify(schemaSnapshotString), "process.env.GATSBY_LOGGER": JSON.stringify(`yurnalist`), - "process.env.GATSBY_PAGES_CHECKSUM": JSON.stringify( - generatePagesChecksum() - ), }), process.env.GATSBY_WEBPACK_LOGGING?.includes(`query-engine`) && new WebpackLoggingPlugin(rootDir, reporter, isVerbose), diff --git a/packages/gatsby/src/schema/graphql-engine/entry.ts b/packages/gatsby/src/schema/graphql-engine/entry.ts index 45e2e5371cd69..41d9b237b5291 100644 --- a/packages/gatsby/src/schema/graphql-engine/entry.ts +++ b/packages/gatsby/src/schema/graphql-engine/entry.ts @@ -24,7 +24,6 @@ import { // @ts-ignore } from ".cache/query-engine-plugins" import { initTracer } from "../../utils/tracer" -import { generatePagesChecksum } from "../../utils/lmdb-pages-checksum" type MaybePhantomActivity = | ReturnType @@ -90,15 +89,6 @@ export class GraphQLEngine { // Note: skipping inference metadata because we rely on schema snapshot await build({ fullMetadataBuild: false, parentSpan: wrapActivity.span }) - if (process.env.GATSBY_PAGES_CHECKSUM) { - const engineChecksum = generatePagesChecksum() - if (engineChecksum !== process.env.GATSBY_PAGES_CHECKSUM) { - reporter.error( - `Pages checksum doesn't match: \n - calculated: "${engineChecksum}"\n - expected: "${process.env.GATSBY_PAGES_CHECKSUM}"` - ) - } - } - return new GraphQLRunner(store) } finally { wrapActivity.end() diff --git a/packages/gatsby/src/utils/lmdb-pages-checksum.ts b/packages/gatsby/src/utils/lmdb-pages-checksum.ts deleted file mode 100644 index ceb0d9c8d4768..0000000000000 --- a/packages/gatsby/src/utils/lmdb-pages-checksum.ts +++ /dev/null @@ -1,15 +0,0 @@ -import crypto from "crypto" -import { getDataStore } from "../datastore" - -export function generatePagesChecksum(): string { - const hash = crypto.createHash(`md5`) - - try { - for (const page of getDataStore().iterateNodesByType(`SitePage`)) { - hash.update(page?.path as string) - } - } catch (e) { - console.log(e) - } - return hash.digest(`hex`) -} From a1c0bf64c07e00bffd602c1311a2f2fb5d849b37 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Mon, 21 Feb 2022 12:50:45 +0100 Subject: [PATCH 8/9] non-pqr match pqr - materialize first, queries later --- packages/gatsby/src/commands/build.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/gatsby/src/commands/build.ts b/packages/gatsby/src/commands/build.ts index 7b74b7a6386f8..70797987f5d3c 100644 --- a/packages/gatsby/src/commands/build.ts +++ b/packages/gatsby/src/commands/build.ts @@ -302,6 +302,8 @@ module.exports = async function build( waitForWorkerPoolRestart = workerPool.restart() await mergeWorkerState(workerPool, buildSpan) } else { + waitMaterializePageMode = materializePageMode() + await runStaticQueries({ queryIds, parentSpan: buildSpan, @@ -315,8 +317,6 @@ module.exports = async function build( parentSpan: buildSpan, store, }) - - waitMaterializePageMode = materializePageMode() } // create scope so we don't leak state object From 2d2f2f9243f95d73059aab0b4815c836b368e695 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Mon, 21 Feb 2022 12:52:27 +0100 Subject: [PATCH 9/9] single start of materialization --- packages/gatsby/src/commands/build.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/gatsby/src/commands/build.ts b/packages/gatsby/src/commands/build.ts index 70797987f5d3c..f2e2aec0942c7 100644 --- a/packages/gatsby/src/commands/build.ts +++ b/packages/gatsby/src/commands/build.ts @@ -287,12 +287,10 @@ module.exports = async function build( } // Start saving page.mode in the main process (while queries run in workers in parallel) - let waitMaterializePageMode = Promise.resolve() + const waitMaterializePageMode = materializePageMode() let waitForWorkerPoolRestart = Promise.resolve() if (process.env.GATSBY_EXPERIMENTAL_PARALLEL_QUERY_RUNNING) { - waitMaterializePageMode = materializePageMode() - await runQueriesInWorkersQueue(workerPool, queryIds, { parentSpan: buildSpan, }) @@ -302,8 +300,6 @@ module.exports = async function build( waitForWorkerPoolRestart = workerPool.restart() await mergeWorkerState(workerPool, buildSpan) } else { - waitMaterializePageMode = materializePageMode() - await runStaticQueries({ queryIds, parentSpan: buildSpan,