feat(webpack): webpackCompilationHash loaded in app-data file

The webpackCompilationHash is loaded in a new app-data.json file to reduce the blast radius of src
changes on generated files

gatsbyjs#15872

packages/gatsby/cache-dir/loader.js

@@ -38,9 +38,6 @@ const loadPageDataJson = loadObj => {
     if (status === 200) {
       try {
         const jsonPayload = JSON.parse(responseText)
-        if (jsonPayload.webpackCompilationHash === undefined) {
-          throw new Error(`not a valid pageData response`)
-        }
 
         return Object.assign(loadObj, {
           status: `success`,
@@ -173,8 +170,12 @@ export class BaseLoader {
       return this.inFlightDb.get(pagePath)
     }
 
-    const inFlight = this.loadPageDataJson(pagePath)
-      .then(result => {
+    const inFlight = Promise.all([
+      this.loadAppData(),
+      this.loadPageDataJson(pagePath),
+    ])
+      .then(allData => {
+        const result = allData[1]
         if (result.notFound) {
           // if request was a 404, we should fallback to findMatchPath.
           let foundMatchPatch = findMatchPath(pagePath)
@@ -199,7 +200,7 @@ export class BaseLoader {
           )
         }
 
-        const pageData = result.payload
+        let pageData = result.payload
         const { componentChunkName } = pageData
         return this.loadComponent(componentChunkName).then(component => {
           const finalResult = { createdAt: new Date() }
@@ -211,6 +212,9 @@ export class BaseLoader {
             if (result.notFound === true) {
               finalResult.notFound = true
             }
+            pageData = Object.assign(pageData, {
+              webpackCompilationHash: allData[0].appHash,
+            })
             pageResources = toPageResources(pageData, component)
             finalResult.payload = pageResources
             emitter.emit(`onPostLoadPageResources`, {
@@ -320,6 +324,30 @@ export class BaseLoader {
     const page = this.pageDb.get(pagePath)
     return page && page.notFound === true
   }
+
+  loadAppData() {
+    return doFetch(`${__PATH_PREFIX__}/app-data.json`).then(req => {
+      const { status, responseText } = req
+
+      let appData
+
+      // Handle 200
+      if (status === 200) {
+        try {
+          const jsonPayload = JSON.parse(responseText)
+          if (jsonPayload.appHash === undefined) {
+            throw new Error(`not a valid app-data response`)
+          }
+
+          appData = jsonPayload
+        } catch (err) {
+          // continue regardless of error
+        }
+      }
+
+      return appData
+    })
+  }
 }
 
 const createComponentUrls = componentChunkName =>
@@ -389,6 +417,7 @@ export const publicLoader = {
   prefetch: rawPath => instance.prefetch(rawPath),
   isPageNotFound: rawPath => instance.isPageNotFound(rawPath),
   hovering: rawPath => instance.hovering(rawPath),
+  loadAppData: () => instance.loadAppData(),
 }
 
 export default publicLoader

packages/gatsby/cache-dir/production-app.js

@@ -26,7 +26,6 @@ loader.setApiRunner(apiRunner)
 window.asyncRequires = asyncRequires
 window.___emitter = emitter
 window.___loader = publicLoader
-window.___webpackCompilationHash = window.webpackCompilationHash
 
 navigationInit()
 
@@ -125,6 +124,9 @@ apiRunnerAsync(`onClientEntry`).then(() => {
         } not found. Not rendering React`
       )
     }
+
+    window.___webpackCompilationHash = page.page.webpackCompilationHash
+
     const Root = () => (
       <Location>
         {locationContext => <LocationHandler {...locationContext} />}

packages/gatsby/cache-dir/static-entry.js

@@ -322,6 +322,15 @@ export default (pagePath, callback) => {
       )
     })
 
+  headComponents.push(
+    <link
+      as="fetch"
+      rel="preload"
+      key="/app-data.json"
+      href="/app-data.json"
+      crossOrigin="anonymous"
+    />
+  )
   if (pageData) {
     headComponents.push(
       <link
@@ -365,10 +374,8 @@ export default (pagePath, callback) => {
       }
     })
 
-  const webpackCompilationHash = pageData.webpackCompilationHash
-
   // Add page metadata for the current page
-  const windowPageData = `/*<![CDATA[*/window.pagePath="${pagePath}";window.webpackCompilationHash="${webpackCompilationHash}";/*]]>*/`
+  const windowPageData = `/*<![CDATA[*/window.pagePath="${pagePath}";/*]]>*/`
 
   postBodyComponents.push(
     <script

packages/gatsby/src/commands/build.js

@@ -15,7 +15,7 @@ const signalExit = require(`signal-exit`)
 const telemetry = require(`gatsby-telemetry`)
 const { store, emitter } = require(`../redux`)
 const queryUtil = require(`../query`)
-const pageDataUtil = require(`../utils/page-data`)
+const appDataUtil = require(`../utils/app-data`)
 const WorkerPool = require(`../utils/worker/pool`)
 const handleWebpackError = require(`../utils/webpack-error-parser`)
 
@@ -91,7 +91,10 @@ module.exports = async function build(program: BuildArgs) {
   const workerPool = WorkerPool.create()
 
   const webpackCompilationHash = stats.hash
-  if (webpackCompilationHash !== store.getState().webpackCompilationHash) {
+  if (
+    webpackCompilationHash !== store.getState().webpackCompilationHash ||
+    !appDataUtil.exists(publicDir)
+  ) {
     store.dispatch({
       type: `SET_WEBPACK_COMPILATION_HASH`,
       payload: webpackCompilationHash,
@@ -102,19 +105,7 @@ module.exports = async function build(program: BuildArgs) {
     })
     activity.start()
 
-    // We need to update all page-data.json files with the new
-    // compilation hash. As a performance optimization however, we
-    // don't update the files for `pageQueryIds` (dirty queries),
-    // since they'll be written after query execution.
-    const cleanPagePaths = _.difference(
-      [...store.getState().pages.keys()],
-      pageQueryIds
-    )
-    await pageDataUtil.updateCompilationHashes(
-      { publicDir, workerPool },
-      cleanPagePaths,
-      webpackCompilationHash
-    )
+    await appDataUtil.write(publicDir, webpackCompilationHash)
 
     activity.end()
   }

packages/gatsby/src/query/query-runner.js

@@ -24,12 +24,7 @@ type QueryJob = {
 
 // Run query
 module.exports = async (queryJob: QueryJob) => {
-  const {
-    schema,
-    schemaCustomization,
-    program,
-    webpackCompilationHash,
-  } = store.getState()
+  const { schema, schemaCustomization, program } = store.getState()
 
   const graphql = (query, context) =>
     graphqlFunction(
@@ -101,12 +96,7 @@ ${formatErrorDetails(errorDetails)}`)
       const publicDir = path.join(program.directory, `public`)
       const { pages } = store.getState()
       const page = pages.get(queryJob.id)
-      await pageDataUtil.write(
-        { publicDir },
-        page,
-        result,
-        webpackCompilationHash
-      )
+      await pageDataUtil.write({ publicDir }, page, result)
     } else {
       // The babel plugin is hard-coded to load static queries from
       // public/static/d/

packages/gatsby/src/utils/app-data.js

@@ -0,0 +1,18 @@
+const fs = require(`fs-extra`)
+const path = require(`path`)
+
+const APP_DATA_JSON = `app-data.json`
+
+const write = (publicDir, hash) => {
+  fs.writeJson(path.join(publicDir, APP_DATA_JSON), {
+    appHash: hash,
+  })
+}
+
+const exists = publicDir =>
+  fs.pathExistsSync(path.join(publicDir, APP_DATA_JSON))
+
+module.exports = {
+  write,
+  exists,
+}

packages/gatsby/src/utils/page-data.js

@@ -1,7 +1,5 @@
 const fs = require(`fs-extra`)
 const path = require(`path`)
-const Promise = require(`bluebird`)
-const { chunk } = require(`lodash`)
 
 const getFilePath = ({ publicDir }, pagePath) => {
   const fixedPagePath = pagePath === `/` ? `index` : pagePath
@@ -13,35 +11,18 @@ const read = async ({ publicDir }, pagePath) => {
   return JSON.parse(rawPageData)
 }
 
-const write = async ({ publicDir }, page, result, webpackCompilationHash) => {
+const write = async ({ publicDir }, page, result) => {
   const filePath = getFilePath({ publicDir }, page.path)
   const body = {
     componentChunkName: page.componentChunkName,
     path: page.path,
     matchPath: page.matchPath,
-    webpackCompilationHash,
     result,
   }
   await fs.outputFile(filePath, JSON.stringify(body))
 }
 
-const updateCompilationHashes = (
-  { publicDir, workerPool },
-  pagePaths,
-  webpackCompilationHash
-) => {
-  const segments = chunk(pagePaths, 50)
-  return Promise.map(segments, segment =>
-    workerPool.updateCompilationHashes(
-      { publicDir },
-      segment,
-      webpackCompilationHash
-    )
-  )
-}
-
 module.exports = {
   read,
   write,
-  updateCompilationHashes,
 }

packages/gatsby/src/utils/worker/page-data.js

@@ -1,32 +1,10 @@
-const fs = require(`fs-extra`)
 const path = require(`path`)
-const Promise = require(`bluebird`)
 
 const getFilePath = ({ publicDir }, pagePath) => {
   const fixedPagePath = pagePath === `/` ? `index` : pagePath
   return path.join(publicDir, `page-data`, fixedPagePath, `page-data.json`)
 }
 
-const updateJsonFileField = async (filename, fieldname, value) => {
-  const object = JSON.parse(await fs.readFile(filename, `utf-8`))
-  object[fieldname] = value
-  await fs.outputFile(filename, JSON.stringify(object), `utf-8`)
-}
-
-const updateCompilationHashes = (
-  { publicDir },
-  pagePaths,
-  webpackCompilationHash
-) =>
-  Promise.map(pagePaths, pagePath =>
-    updateJsonFileField(
-      getFilePath({ publicDir }, pagePath),
-      `webpackCompilationHash`,
-      webpackCompilationHash
-    )
-  )
-
 module.exports = {
   getFilePath,
-  updateCompilationHashes,
 }