fix(gatsby): queue fs writes so event loop not overwhelmed on large sites

packages/gatsby/package.json

@@ -72,6 +72,7 @@
     "execa": "^4.0.3",
     "express": "^4.17.1",
     "express-graphql": "^0.9.0",
+    "fastq": "^1.10.0",
     "fastest-levenshtein": "^1.0.12",
     "file-loader": "^1.1.11",
     "find-cache-dir": "^3.3.1",

packages/gatsby/src/utils/page-data.ts

@@ -1,6 +1,7 @@
 import { walkStream as fsWalkStream, Entry } from "@nodelib/fs.walk"
 import fs from "fs-extra"
 import reporter from "gatsby-cli/lib/reporter"
+import fastq from "fastq"
 import path from "path"
 import { IGatsbyPage } from "../redux/types"
 import { websocketManager } from "./websocket-manager"
@@ -134,7 +135,7 @@ export async function flush(): Promise<void> {
 
   const pagesToWrite = pagePaths.values()
 
-  for (const pagePath of pagesToWrite) {
+  const flushQueue = fastq(async (pagePath, cb) => {
     const page = pages.get(pagePath)
 
     // It's a gloomy day in Bombay, let me tell you a short story...
@@ -162,7 +163,7 @@ export async function flush(): Promise<void> {
 
         if (hasFlag(query.dirty, FLAG_DIRTY_NEW_PAGE)) {
           // query results are not written yet
-          continue
+          return cb(null, true)
         }
       }
 
@@ -184,12 +185,25 @@ export async function flush(): Promise<void> {
         })
       }
     }
+
     store.dispatch({
       type: `CLEAR_PENDING_PAGE_DATA_WRITE`,
       payload: {
         page: pagePath,
       },
     })
+
+    return cb(null, true)
+  }, 25)
+
+  for (const pagePath of pagesToWrite) {
+    flushQueue.push(pagePath, () => {})
+  }
+
+  if (!flushQueue.idle()) {
+    await new Promise(resolve => {
+      flushQueue.drain = resolve
+    })
   }
 
   isFlushing = false

yarn.lock

@@ -10927,6 +10927,13 @@ fastparse@^1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/fastparse/-/fastparse-1.1.1.tgz#d1e2643b38a94d7583b479060e6c4affc94071f8"
 
+fastq@^1.10.0:
+  version "1.10.0"
+  resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.10.0.tgz#74dbefccade964932cdf500473ef302719c652bb"
+  integrity sha512-NL2Qc5L3iQEsyYzweq7qfgy5OtXCmGzGvhElGEd/SoFWEMOEczNh5s5ocaF01HDetxz+p8ecjNPA6cZxxIHmzA==
+  dependencies:
+    reusify "^1.0.4"
+
 fastq@^1.6.0:
   version "1.6.0"
   resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.6.0.tgz#4ec8a38f4ac25f21492673adb7eae9cfef47d1c2"
@@ -22248,7 +22255,7 @@ retry@^0.10.0:
   version "0.10.1"
   resolved "https://registry.yarnpkg.com/retry/-/retry-0.10.1.tgz#e76388d217992c252750241d3d3956fed98d8ff4"
 
-reusify@^1.0.0:
+reusify@^1.0.0, reusify@^1.0.4:
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76"
   integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==