Fix out-of-memory error when building hundreds of pages

gatsbyjs · Nov 23, 2017 · 8ab36c6 · 8ab36c6
1 parent 17848d7
commit 8ab36c6
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 4 deletions.
diff --git a/packages/gatsby/src/internal-plugins/query-runner/page-query-runner.js b/packages/gatsby/src/internal-plugins/query-runner/page-query-runner.js
@@ -7,6 +7,7 @@
 
 const _ = require(`lodash`)
 const Promise = require(`bluebird`)
+const os = require(`os`)
 
 const { store, emitter } = require(`../../redux`)
 const queryRunner = require(`./query-runner`)
@@ -90,8 +91,9 @@ const runQueriesForIds = ids => {
     return Promise.resolve()
   }
   const state = store.getState()
-  return Promise.all(
-    ids.map(id => {
+  return Promise.map(
+    ids,
+    id => {
       const pagesAndLayouts = [...state.pages, ...state.layouts]
       const plObj = pagesAndLayouts.find(
         pl => pl.path === id || `LAYOUT___${pl.id}` === id
@@ -100,7 +102,8 @@ const runQueriesForIds = ids => {
         return queryRunner(plObj, state.components[plObj.component])
       }
       return null
-    })
+    },
+    { concurrency: Math.min(os.cpus().length, 32) }
   )
 }
 

diff --git a/packages/gatsby/src/internal-plugins/query-runner/query-watcher.js b/packages/gatsby/src/internal-plugins/query-runner/query-watcher.js
@@ -10,6 +10,7 @@
 
 const _ = require(`lodash`)
 const chokidar = require(`chokidar`)
+const os = require(`os`)
 
 const { store } = require(`../../redux/`)
 const { boundActionCreators } = require(`../../redux/actions`)
@@ -57,7 +58,11 @@ const runQueriesForComponent = componentPath => {
     pages.map(p => p.path || p.id)
   )
   const component = store.getState().components[componentPath]
-  return Promise.all(pages.map(p => queryRunner(p, component)))
+  return Promise.map(
+    pages,
+    p => queryRunner(p, component),
+    { concurrency: Math.min(os.cpus().length, 32) }
+  )
 }
 
 const getPagesForComponent = componentPath => {