Merge remote-tracking branch 'origin/master' into webpack/virtual-mod…

…ules

CODEOWNERS

@@ -5,6 +5,9 @@
 /docs/                                          @gatsbyjs/learning
 **/README.md                                    @gatsbyjs/learning
 
+# Benchmarks
+/benchmarks/                                    @duffn @pvdz @smthomas
+
 # All blog posts must be reviewed by the content team.
 /docs/blog/                                     @gatsbyjs/content
 

docs/sites.yml

@@ -10931,3 +10931,12 @@
   built_by: Nagarjun Palavalli
   built_by_url: https://twitter.com/palavalli
   featured: false
+- title: Bold.org
+  url: https://bold.org/
+  main_url: https://bold.org/
+  description: >
+    Fighting Student Debt. Create or apply to exclusive scholarships, fellowships, and grants, in minutes.
+  categories:
+    - Education
+  built_by: Bold.org
+  featured: false

packages/gatsby-core-utils/src/__tests__/create-content-digest.ts

@@ -23,6 +23,32 @@ describe(`Create content digest`, () => {
     )
   })
 
+  // @fixes https://github.com/gatsbyjs/gatsby/issues/21840
+  it(`returns the content digest when input is a Buffer`, () => {
+    const input = Buffer.from(`1234`)
+    const input2 = Buffer.from(`5678`)
+
+    const contentDigest = createContentDigest(input)
+    const contentDigest2 = createContentDigest(input2)
+
+    expect(typeof contentDigest).toEqual(`string`)
+    expect(contentDigest).toMatchInlineSnapshot(
+      `"81dc9bdb52d04dc20036dbd8313ed055"`
+    )
+    expect(contentDigest2).toMatchInlineSnapshot(
+      `"674f3c2c1a8a6f90461e8a66fb5550ba"`
+    )
+    expect(contentDigest).not.toEqual(contentDigest2)
+  })
+
+  // @fixes https://github.com/gatsbyjs/gatsby/issues/21840
+  it(`returns a deterministic hash from a Buffer`, () => {
+    const input = Buffer.from(`1234`)
+    const input2 = Buffer.from(`1234`)
+
+    expect(createContentDigest(input)).toEqual(createContentDigest(input2))
+  })
+
   it(`returns a deterministic hash from an object`, () => {
     const input = {
       id: `12345`,

packages/gatsby-core-utils/src/create-content-digest.ts

@@ -26,7 +26,7 @@ const hashPrimitive = (input: BinaryLike | string): string =>
 export const createContentDigest = (
   input: BinaryLike | string | any
 ): string => {
-  if (typeof input === `object`) {
+  if (typeof input === `object` && !Buffer.isBuffer(input)) {
     return hasher.hash(input)
   }
 

packages/gatsby-plugin-offline/src/gatsby-node.js

@@ -143,6 +143,11 @@ exports.onPostBuild = (
         urlPattern: /^https?:.*\page-data\/.*\/page-data\.json/,
         handler: `StaleWhileRevalidate`,
       },
+      {
+        // app-data.json is not content hashed
+        urlPattern: /^https?:.*\/page-data\/app-data\.json/,
+        handler: `StaleWhileRevalidate`,
+      },
       {
         // Add runtime caching of various other page resources
         urlPattern: /^https?:.*\.(png|jpg|jpeg|webp|svg|gif|tiff|js|woff|woff2|json|css)$/,
@@ -162,7 +167,9 @@ exports.onPostBuild = (
 
   const idbKeyvalFile = `idb-keyval-iife.min.js`
   const idbKeyvalSource = require.resolve(`idb-keyval/dist/${idbKeyvalFile}`)
-  const idbKeyvalDest = `public/${idbKeyvalFile}`
+  const idbKeyvalPackageJson = require(`idb-keyval/package.json`)
+  const idbKeyValVersioned = `idb-keyval-${idbKeyvalPackageJson.version}-iife.min.js`
+  const idbKeyvalDest = `public/${idbKeyValVersioned}`
   fs.createReadStream(idbKeyvalSource).pipe(fs.createWriteStream(idbKeyvalDest))
 
   const swDest = `public/sw.js`
@@ -183,6 +190,7 @@ exports.onPostBuild = (
 
       const swAppend = fs
         .readFileSync(`${__dirname}/sw-append.js`, `utf8`)
+        .replace(/%idbKeyValVersioned%/g, idbKeyValVersioned)
         .replace(/%pathPrefix%/g, pathPrefix)
         .replace(/%appFile%/g, appFile)
 

packages/gatsby-plugin-offline/src/sw-append.js

@@ -1,6 +1,5 @@
 /* global importScripts, workbox, idbKeyval */
-
-importScripts(`idb-keyval-iife.min.js`)
+importScripts(`%idbKeyValVersioned%`)
 
 const { NavigationRoute } = workbox.routing
 

packages/gatsby/src/bootstrap/create-graphql-runner.ts

@@ -28,7 +28,9 @@ export const createGraphQLRunner = (
   }
 ): Runner => {
   // TODO: Move tracking of changed state inside GraphQLRunner itself. https://github.com/gatsbyjs/gatsby/issues/20941
-  let runner = new GraphQLRunner(store, { graphqlTracing })
+  let runner: GraphQLRunner | undefined = new GraphQLRunner(store, {
+    graphqlTracing,
+  })
 
   const eventTypes: string[] = [
     `DELETE_CACHE`,
@@ -43,12 +45,17 @@ export const createGraphQLRunner = (
 
   eventTypes.forEach(type => {
     emitter.on(type, () => {
-      runner = new GraphQLRunner(store)
+      runner = undefined
     })
   })
 
-  return (query, context): ReturnType<Runner> =>
-    runner
+  return (query, context): ReturnType<Runner> => {
+    if (!runner) {
+      runner = new GraphQLRunner(store, {
+        graphqlTracing,
+      })
+    }
+    return runner
       .query(query, context, {
         queryName: `gatsby-node query`,
         parentSpan,
@@ -92,4 +99,5 @@ export const createGraphQLRunner = (
 
         return result
       })
+  }
 }

packages/gatsby/src/bootstrap/requires-writer.ts

@@ -9,6 +9,7 @@ import { joinPath } from "gatsby-core-utils"
 import { store, emitter } from "../redux/"
 import { IGatsbyState, IGatsbyPage } from "../redux/types"
 import { writeModule } from "../utils/gatsby-webpack-virtual-modules"
+import { markWebpackStatusAsPending } from "../utils/webpack-status"
 
 interface IGatsbyPageComponent {
   component: string
@@ -254,6 +255,7 @@ const debouncedWriteAll = _.debounce(
     const didRequiresChange = await writeAll(store.getState())
     if (didRequiresChange) {
       reporter.pendingActivity({ id: `webpack-develop` })
+      markWebpackStatusAsPending()
     }
     activity.end()
   },

packages/gatsby/src/commands/build.ts

@@ -14,6 +14,7 @@ import { initTracer, stopTracer } from "../utils/tracer"
 import db from "../db"
 import { store, readState } from "../redux"
 import * as appDataUtil from "../utils/app-data"
+import { flush as flushPendingPageDataWrites } from "../utils/page-data"
 import * as WorkerPool from "../utils/worker/pool"
 import { structureWebpackErrors } from "../utils/webpack-error-utils"
 import {
@@ -30,6 +31,10 @@ import {
   runStaticQueries,
   runPageQueries,
 } from "../services"
+import {
+  markWebpackStatusAsPending,
+  markWebpackStatusAsDone,
+} from "../utils/webpack-status"
 
 let cachedPageData
 let cachedWebpackCompilationHash
@@ -57,6 +62,8 @@ module.exports = async function build(program: IBuildArgs): Promise<void> {
     )
   }
 
+  markWebpackStatusAsPending()
+
   const publicDir = path.join(program.directory, `public`)
   initTracer(program.openTracingConfigFile)
   const buildActivity = report.phantomActivity(`build`)
@@ -144,6 +151,9 @@ module.exports = async function build(program: IBuildArgs): Promise<void> {
     store,
   })
 
+  await flushPendingPageDataWrites()
+  markWebpackStatusAsDone()
+
   if (process.env.GATSBY_EXPERIMENTAL_PAGE_BUILD_ON_DATA_CHANGES) {
     const { pages } = store.getState()
     if (cachedPageData) {

packages/gatsby/src/commands/develop-process.ts

@@ -62,6 +62,12 @@ import {
   showFeedbackRequest,
 } from "../utils/feedback"
 
+import { enqueueFlush } from "../utils/page-data"
+import {
+  markWebpackStatusAsPending,
+  markWebpackStatusAsDone,
+} from "../utils/webpack-status"
+
 import { Stage, IProgram } from "./types"
 import {
   calculateDirtyQueries,
@@ -397,6 +403,7 @@ module.exports = async (program: IProgram): Promise<void> => {
     )
   }
   initTracer(program.openTracingConfigFile)
+  markWebpackStatusAsPending()
   report.pendingActivity({ id: `webpack-develop` })
   telemetry.trackCli(`DEVELOP_START`)
   telemetry.startBackgroundUpdate()
@@ -610,9 +617,9 @@ module.exports = async (program: IProgram): Promise<void> => {
     })
   }
 
-  // compiler.hooks.invalid.tap(`log compiling`, function(...args) {
-  //   console.log(`set invalid`, args, this)
-  // })
+  compiler.hooks.invalid.tap(`log compiling`, function () {
+    markWebpackStatusAsPending()
+  })
 
   compiler.hooks.watchRun.tapAsync(`log compiling`, function (_, done) {
     if (webpackActivity) {
@@ -677,6 +684,9 @@ module.exports = async (program: IProgram): Promise<void> => {
       webpackActivity = null
     }
 
+    enqueueFlush()
+    markWebpackStatusAsDone()
+
     done()
   })
 }

packages/gatsby/src/query/index.js

@@ -8,6 +8,7 @@ const { boundActionCreators } = require(`../redux/actions`)
 const report = require(`gatsby-cli/lib/reporter`)
 const queryQueue = require(`./queue`)
 const { GraphQLRunner } = require(`./graphql-runner`)
+const pageDataUtil = require(`../utils/page-data`)
 
 const seenIdsWithoutDataDependencies = new Set()
 let queuedDirtyActions = []
@@ -306,6 +307,7 @@ const startListeningToDevelopQueue = ({ graphqlTracing } = {}) => {
     const activity = createQueryRunningActivity(queryJobs.length)
 
     const onFinish = (...arg) => {
+      pageDataUtil.enqueueFlush()
       activity.done()
       return callback(...arg)
     }

packages/gatsby/src/query/query-runner.ts

@@ -3,21 +3,19 @@ import _ from "lodash"
 import fs from "fs-extra"
 import report from "gatsby-cli/lib/reporter"
 import crypto from "crypto"
+import { ExecutionResult } from "graphql"
 
 import path from "path"
 import { store } from "../redux"
 import { boundActionCreators } from "../redux/actions"
-import { writePageData } from "../utils/page-data"
 import { getCodeFrame } from "./graphql-errors"
 import errorParser from "./error-parser"
 
 import { GraphQLRunner } from "./graphql-runner"
-import { ExecutionResult } from "graphql"
+import { IExecutionResult, PageContext } from "./types"
 
 const resultHashes = new Map()
 
-type PageContext = any
-
 interface IQueryJob {
   id: string
   hash?: string
@@ -28,10 +26,6 @@ interface IQueryJob {
   pluginCreatorId: string
 }
 
-interface IExecutionResult extends ExecutionResult {
-  pageContext?: PageContext
-}
-
 // Run query
 export const queryRunner = async (
   graphqlRunner: GraphQLRunner,
@@ -157,17 +151,21 @@ export const queryRunner = async (
     resultHashes.set(queryJob.id, resultHash)
 
     if (queryJob.isPage) {
-      const publicDir = path.join(program.directory, `public`)
-      const { pages } = store.getState()
-      const page = pages.get(queryJob.id)
-
-      if (!page) {
-        throw new Error(
-          `A page was not found for the queryJob. This is likely an internal bug to Gatsby and you should create an issue to report it.`
-        )
-      }
-
-      await writePageData(publicDir, page, result)
+      // We need to save this temporarily in cache because
+      // this might be incomplete at the moment
+      const resultPath = path.join(
+        program.directory,
+        `.cache`,
+        `json`,
+        `${queryJob.id.replace(/\//g, `_`)}.json`
+      )
+      await fs.outputFile(resultPath, resultJSON)
+      store.dispatch({
+        type: `ADD_PENDING_PAGE_DATA_WRITE`,
+        payload: {
+          path: queryJob.id,
+        },
+      })
     } else {
       // The babel plugin is hard-coded to load static queries from
       // public/static/d/

packages/gatsby/src/query/queue.js

@@ -43,12 +43,7 @@ const createDevelopQueue = getRunner => {
   const handler = ({ job: queryJob, activity }, callback) => {
     queryRunner(getRunner(), queryJob, activity?.span).then(
       result => {
-        if (queryJob.isPage) {
-          websocketManager.emitPageData({
-            result,
-            id: queryJob.id,
-          })
-        } else {
+        if (!queryJob.isPage) {
           websocketManager.emitStaticQueryData({
             result,
             id: queryJob.id,

packages/gatsby/src/query/types.ts

@@ -1,3 +1,5 @@
+import { ExecutionResult } from "graphql"
+
 export interface IGraphQLRunnerStats {
   totalQueries: number
   uniqueOperations: Set<string>
@@ -25,3 +27,9 @@ export interface IGraphQLRunnerStatResults {
   uniqueFilterPaths: number
   uniqueSorts: number
 }
+
+export type PageContext = any
+
+export interface IExecutionResult extends ExecutionResult {
+  pageContext?: PageContext
+}

packages/gatsby/src/redux/__tests__/__snapshots__/index.js.snap

@@ -54,6 +54,10 @@ Object {
   },
   "pageData": Map {},
   "pageDataStats": Map {},
+  "pendingPageDataWrites": Object {
+    "pagePaths": Set {},
+    "templatePaths": Set {},
+  },
   "staticQueryComponents": Map {},
   "status": Object {
     "PLUGINS_HASH": "",

packages/gatsby/src/redux/actions/public.js

@@ -888,7 +888,7 @@ type CreateNodeInput = {
  * @param {string} $0.fieldName [deprecated] the name for the field
  * @param {string} $0.fieldValue [deprecated] the value for the field
  * @param {string} $0.name the name for the field
- * @param {string} $0.value the value for the field
+ * @param {any} $0.value the value for the field
  * @example
  * createNodeField({
  *   node,

packages/gatsby/src/redux/index.ts

@@ -85,6 +85,7 @@ export const saveState = (): void => {
     webpackCompilationHash: state.webpackCompilationHash,
     pageDataStats: state.pageDataStats,
     pageData: state.pageData,
+    pendingPageDataWrites: state.pendingPageDataWrites,
   })
 }
 

packages/gatsby/src/redux/reducers/index.ts

@@ -22,6 +22,7 @@ import { programReducer } from "./program"
 import { resolvedNodesCacheReducer } from "./resolved-nodes"
 import { nodesTouchedReducer } from "./nodes-touched"
 import { flattenedPluginsReducer } from "./flattened-plugins"
+import { pendingPageDataWritesReducer } from "./pending-page-data-writes"
 import { schemaCustomizationReducer } from "./schema-customization"
 import { inferenceMetadataReducer } from "./inference-metadata"
 
@@ -55,4 +56,5 @@ export {
   inferenceMetadataReducer as inferenceMetadata,
   pageDataStatsReducer as pageDataStats,
   pageDataReducer as pageData,
+  pendingPageDataWritesReducer as pendingPageDataWrites,
 }