feat(gatsby-source-drupal): Persist backreference data individually i…

…n cache so cheaper to get/set (#35323) * Persist backreference data individually in cache so cheaper to get/set * small updates to README * Update packages/gatsby-source-drupal/src/utils.js Co-authored-by: Ward Peeters <ward@coding-tech.com> * Update packages/gatsby-source-drupal/src/utils.js Co-authored-by: Tyler Barnes <tylerdbarnes@gmail.com> * Update packages/gatsby-source-drupal/src/utils.js Co-authored-by: Ward Peeters <ward@coding-tech.com> * more fixes * add set * Process nodes in parallel Co-authored-by: Ward Peeters <ward@coding-tech.com> Co-authored-by: Tyler Barnes <tylerdbarnes@gmail.com>
gatsbyjs · Apr 28, 2022 · 1257a8d · 1257a8d
1 parent 63828fc
commit 1257a8d
Show file tree

Hide file tree

Showing 4 changed files with 134 additions and 146 deletions.
diff --git a/packages/gatsby-source-drupal/README.md b/packages/gatsby-source-drupal/README.md
@@ -2,14 +2,14 @@
 
 Source plugin for pulling data (including images) into Gatsby from Drupal sites.
 
-It pulls data from Drupal 8 sites with the
+It pulls data from Drupal 8/9 sites with the
 [Drupal JSONAPI module](https://www.drupal.org/project/jsonapi) installed.
 
 An example site built with the headless Drupal distro
 [ContentaCMS](https://twitter.com/contentacms) is at
 https://using-drupal.gatsbyjs.org/
 
-`apiBase` Option allows changing the API entry point depending on the version of
+The `apiBase` option allows changing the API entry point depending on the version of
 jsonapi used by your Drupal instance. The default value is `jsonapi`, which has
 been used since jsonapi version `8.x-1.0-alpha4`.
 

diff --git a/packages/gatsby-source-drupal/src/__tests__/index.js b/packages/gatsby-source-drupal/src/__tests__/index.js
@@ -34,6 +34,7 @@ function makeCache() {
   return {
     get: async id => store.get(id),
     set: async (key, value) => store.set(key, value),
+    del: async key => store.delete(key),
     store,
   }
 }

diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js
@@ -22,8 +22,6 @@ const {
   imageCDNState,
 } = require(`./normalize`)
 const {
-  initRefsLookups,
-  storeRefsLookups,
   handleReferences,
   handleWebhookUpdate,
   createNodeIfItDoesNotExist,
@@ -189,8 +187,6 @@ exports.sourceNodes = async (
     unstable_createNodeManifest,
   } = actions
 
-  await initRefsLookups({ cache, getNode })
-
   // Update the concurrency limit from the plugin options
   requestQueue.concurrency = concurrentAPIRequests
 
@@ -234,6 +230,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
           const deletedNode = await handleDeletedNode({
             actions,
             getNode,
+            cache,
             node: nodeToDelete,
             createNodeId,
             createContentDigest,
@@ -243,7 +240,6 @@ ${JSON.stringify(webhookBody, null, 4)}`
         }
 
         changesActivity.end()
-        await storeRefsLookups({ cache })
         return
       }
 
@@ -285,7 +281,6 @@ ${JSON.stringify(webhookBody, null, 4)}`
       return
     }
     changesActivity.end()
-    await storeRefsLookups({ cache })
     return
   }
 
@@ -404,6 +399,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
               handleDeletedNode({
                 actions,
                 getNode,
+                cache,
                 node: nodeSyncData,
                 createNodeId,
                 createContentDigest,
@@ -417,23 +413,25 @@ ${JSON.stringify(webhookBody, null, 4)}`
                 nodesToUpdate = [nodeSyncData.data]
               }
 
-              for (const nodeToUpdate of nodesToUpdate) {
-                await handleWebhookUpdate(
-                  {
-                    nodeToUpdate,
-                    actions,
-                    cache,
-                    createNodeId,
-                    createContentDigest,
-                    getCache,
-                    getNode,
-                    reporter,
-                    store,
-                    languageConfig,
-                  },
-                  pluginOptions
+              await Promise.all(
+                nodesToUpdate.map(nodeToUpdate =>
+                  handleWebhookUpdate(
+                    {
+                      nodeToUpdate,
+                      actions,
+                      cache,
+                      createNodeId,
+                      createContentDigest,
+                      getCache,
+                      getNode,
+                      reporter,
+                      store,
+                      languageConfig,
+                    },
+                    pluginOptions
+                  )
                 )
-              }
+              )
             }
           }
 
@@ -445,7 +443,6 @@ ${JSON.stringify(webhookBody, null, 4)}`
 
         drupalFetchIncrementalActivity.end()
         fastBuildsSpan.finish()
-        await storeRefsLookups({ cache })
         return
       }
 
@@ -456,7 +453,6 @@ ${JSON.stringify(webhookBody, null, 4)}`
       initialSourcing = false
 
       if (!requireFullRebuild) {
-        await storeRefsLookups({ cache })
         return
       }
     }
@@ -696,15 +692,17 @@ ${JSON.stringify(webhookBody, null, 4)}`
   createNodesSpan.setTag(`sourceNodes.createNodes.count`, nodes.size)
 
   // second pass - handle relationships and back references
-  nodes.forEach(node => {
-    handleReferences(node, {
-      getNode: nodes.get.bind(nodes),
-      mutateNode: true,
-      createNodeId,
-      entityReferenceRevisions,
-      fileNodesExtendedData,
-    })
-  })
+  await Promise.all(
+    Array.from(nodes.values()).map(node =>
+      handleReferences(node, {
+        getNode: nodes.get.bind(nodes),
+        mutateNode: true,
+        createNodeId,
+        cache,
+        entityReferenceRevisions,
+      })
+    )
+  )
 
   if (skipFileDownloads) {
     reporter.info(`Skipping remote file download from Drupal`)
@@ -753,7 +751,7 @@ ${JSON.stringify(webhookBody, null, 4)}`
   initialSourcing = false
 
   createNodesSpan.finish()
-  await storeRefsLookups({ cache, getNodes })
+  return
 }
 
 // This is maintained for legacy reasons and will eventually be removed.