[1.0] refactor node structure to hide internal fields (#960)

* Initial refactor done * Remove unused code * Some fixes * Bump site version number cause it's getting better and stuff * Fix source-drupal plugin * Add basic documentation for node interface and hacker news source plugin
gatsbyjs · May 15, 2017 · daaa22c · daaa22c
1 parent d7e228b
commit daaa22c
Show file tree

Hide file tree

Showing 30 changed files with 529 additions and 363 deletions.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -10,7 +10,7 @@ If you want your issue to be resolved quickly, please include in your issue:
   changes you've made there.
 
 ## Contributing
-You can install the latest `master` version of Gatsby by following these steps:
+You can install the latest version of Gatsby by following these steps:
 
 * Clone the repo, navigate to its directory.
 * Install dependencies using `npm install` in the root of the repo.
@@ -20,7 +20,7 @@ The usual contributing steps are:
 * Fork the [official repository](https://github.com/gatsbyjs/gatsby).
 * Clone your fork: git clone `git@github.com:<your-username>/gatsby.git`
 * Install lerna, and gatsby-dev-cli globally: `npm install -g lerna gatsby-dev-cli@canary`
-* Checkout to the 1.0 branch: `git checkout 1.0`
+* Checkout the 1.0 branch: `git checkout 1.0`
 * Install dependencies: `npm install && lerna bootstrap`
 * Make sure tests are passing for you: `npm test`
 * Create a topic branch: `git checkout -b topics/new-feature-name`

diff --git a/docs/docs/node-interface.md b/docs/docs/node-interface.md
@@ -2,4 +2,54 @@
 title: Node Interface
 ---
 
-Coming soon.
+The "node" is the center of Gatsby's data system. All data that's
+added to Gatsby is modeled using nodes.
+
+The basic node data structure is as follows:
+
+```flow
+id: String,
+children: Array[String],
+parent: String,
+internal: {
+  contentDigest: String,
+  mediaType: String,
+  type: String,
+  pluginName: String,
+  content: String,
+}
+...other node type specific fields
+```
+
+## Source plugins
+
+New nodes are added to Gatsby by "source" plugins. A common one that many
+Gatsby sites use is the [Filesystem source
+plugin](/docs/packages/gatsby-source-filesystem/) which turns files on disk
+into File nodes.
+
+Other source plugins pull data from external APIs such as the [Drupal](/docs/packages/gatsby-source-drupal/) and
+[Hacker News](/docs/packages/gatsby-source-hacker-news/)
+
+## Transformer plugins
+
+Transformer plugins transform source nodes into new types of nodes. It is very
+common when building Gatsby sites to install both source plugin(s) and transformer
+plugins.
+
+Nodes created by transformer plugins are set as "children" of their "parent"
+nodes.
+
+* The [Remark
+(Markdown library) transformer
+plugin](/docs/packages/gatsby-transformer-remark/) looks for new nodes that
+are created with a `mediaType` of `text/x-markdown` and then transforms these
+nodes into `MarkdownRemark` nodes with an `html` field.
+* The [YAML transformer plugin](/docs/packages/gatsby-transformer-yaml/) looks
+for new nodes with a media type of `text/yaml` (e.g. a `.yaml` file) and creates new
+YAML child node(s) by parsing the YAML source into JavaScript objects.
+
+## GraphQL
+
+Gatsby automatically infers the structure of your site's nodes and creates
+a GraphQL schema which you can then query from your site's components.
diff --git a/docs/docs/plugins.md b/docs/docs/plugins.md
@@ -34,6 +34,7 @@ mean converting this file into a JS component).
 * [gatsby-plugin-typescript](/docs/packages/gatsby-plugin-typescript/)
 * [gatsby-source-filesystem](/docs/packages/gatsby-source-filesystem/)
 * [gatsby-source-drupal](/docs/packages/gatsby-source-drupal/)
+* [gatsby-source-hacker-news](/docs/packages/gatsby-source-hacker-news/)
 * [gatsby-remark-autolink-headers](/docs/packages/gatsby-remark-autolink-headers/)
 * [gatsby-remark-copy-linked-files](/docs/packages/gatsby-remark-copy-linked-files/)
 * [gatsby-remark-prismjs](/docs/packages/gatsby-remark-prismjs/)

diff --git a/packages/gatsby-plugin-sharp/src/index.js b/packages/gatsby-plugin-sharp/src/index.js
@@ -173,7 +173,7 @@ function queueImageResizing({ file, args = {} }) {
     return true
   })
   const sortedArgs = _.sortBy(filteredArgs, arg => arg[0] === `width`)
-  const imgSrc = `/${file.contentDigest}-${qs.stringify(_.fromPairs(sortedArgs))}.${file.extension}`
+  const imgSrc = `/${file.internal.contentDigest}-${qs.stringify(_.fromPairs(sortedArgs))}.${file.extension}`
   const filePath = `${process.cwd()}/public${imgSrc}`
   // Create function to call when the image is finished.
   let outsideResolve

diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js
@@ -9,9 +9,11 @@ const makeTypeName = type => {
 const processEntities = ents => {
   return ents.map(ent => {
     const newEnt = {
-      ...ent.attributes,
       id: ent.id,
-      type: ent.type,
+      internal: {
+        type: ent.type,
+      },
+      ...ent.attributes,
       created: new Date(ent.attributes.created * 1000).toJSON(),
       changed: new Date(ent.attributes.changed * 1000).toJSON(),
     }
@@ -43,15 +45,8 @@ exports.sourceNodes = async (
   })
 
   // Touch existing Drupal nodes so Gatsby doesn't garbage collect them.
-  // console.log(
-  // "existing drupal nodes",
-  // _.values(store.getState().nodes)
-  // .filter(n => n.type.slice(0, 8) === `drupal__`)
-  // .map(n => n.id)
-  // )
-
   _.values(store.getState().nodes)
-    .filter(n => n.type.slice(0, 8) === `drupal__`)
+    .filter(n => n.internal.type.slice(0, 8) === `drupal__`)
     .forEach(n => touchNode(n.id))
 
   // Fetch articles.
@@ -94,12 +89,15 @@ exports.sourceNodes = async (
 
     const gatsbyNode = {
       ...node,
-      parent: `__SOURCE__`,
-      type: makeTypeName(node.type),
       children: [],
-      content: nodeStr,
+      parent: `__SOURCE__`,
+      internal: {
+        ...node.internal,
+        type: makeTypeName(node.internal.type),
+        content: nodeStr,
+        mediaType: `application/json`,
+      },
       author___NODE: result.data.data[i].relationships.uid.data.id,
-      mediaType: `application/json`,
     }
 
     // Get content digest of node.
@@ -108,7 +106,7 @@ exports.sourceNodes = async (
       .update(JSON.stringify(gatsbyNode))
       .digest(`hex`)
 
-    gatsbyNode.contentDigest = contentDigest
+    gatsbyNode.internal.contentDigest = contentDigest
 
     createNode(gatsbyNode)
   })
@@ -124,11 +122,13 @@ exports.sourceNodes = async (
 
         const gatsbyUser = {
           ...user,
-          parent: `__SOURCE__`,
-          type: makeTypeName(user.type),
           children: [],
-          content: userStr,
-          mediaType: `application/json`,
+          parent: `__SOURCE__`,
+          internal: {
+            type: makeTypeName(user.internal.type),
+            content: userStr,
+            mediaType: `application/json`,
+          },
         }
 
         if (gatsbyUser.uid === 1) {
@@ -138,7 +138,7 @@ exports.sourceNodes = async (
         axios
           .get(
             userResult.data.data[i].relationships.user_picture.links.related,
-            { timeout: 3000 }
+            { timeout: 20000 }
           )
           .catch(() => console.log(`fail fetch`, gatsbyUser))
           .then(pictureResult => {
@@ -150,7 +150,7 @@ exports.sourceNodes = async (
               .update(JSON.stringify(gatsbyUser))
               .digest(`hex`)
 
-            gatsbyUser.contentDigest = contentDigest
+            gatsbyUser.internal.contentDigest = contentDigest
 
             createNode(gatsbyUser)
 

diff --git a/packages/gatsby-source-filesystem/src/gatsby-node.js b/packages/gatsby-source-filesystem/src/gatsby-node.js
@@ -26,12 +26,14 @@ function readFile(file, pluginOptions, cb) {
           // people will use the id for that and ids shouldn't be treated as
           // useful information.
           id: createId(file),
-          contentDigest: contentDigest,
           children: [],
           parent: `___SOURCE___`,
-          mediaType: mime.lookup(slashedFile.ext),
-          type: `File`,
-          sourceName: pluginOptions.name,
+          internal: {
+            contentDigest: contentDigest,
+            mediaType: mime.lookup(slashedFile.ext),
+            type: `File`,
+          },
+          sourceInstanceName: pluginOptions.name,
           absolutePath: slashedFile.absolutePath,
           relativePath: slash(
             path.relative(pluginOptions.path, slashedFile.absolutePath)

diff --git a/packages/gatsby-source-hacker-news/README.md b/packages/gatsby-source-hacker-news/README.md
@@ -0,0 +1,47 @@
+# gatsby-source-hacker-news
+
+Source plugin for pulling data into Gatsby from the [Hacker News
+API](https://github.com/HackerNews/API)
+
+## Install
+
+`npm install --save gatsby-source-hacker-news`
+
+## How to use
+
+```javascript
+// In your gatsby-config.js
+plugins: [
+  `gatsby-source-hacker-news`,
+]
+```
+
+## How to query
+
+You can query nodes created from Hacker News like the following:
+
+```graphql
+query StoriesQuery {
+  allHnStory(sortBy: {fields: [order]}) {
+    edges {
+      node {
+        id
+        title
+        score
+        order
+        domain
+        url
+        by
+        descendants
+        timeISO(fromNow: true)
+        children {
+          id
+          text
+          timeISO(fromNow: true)
+          by
+        }
+      }
+    }
+  }
+}
+```
diff --git a/packages/gatsby-source-hacker-news/src/gatsby-node.js b/packages/gatsby-source-hacker-news/src/gatsby-node.js
@@ -103,13 +103,15 @@ fragment commentsFragment on HackerNewsItem {
 
     const storyNode = {
       ...kidLessStory,
-      domain,
-      order: i + 1,
+      children: kids.kids.map(k => k.id),
       parent: `__SOURCE__`,
-      type: `HNStory`,
-      children: [...kids.kids.map(k => k.id)],
       content: storyStr,
-      mediaType: `application/json`,
+      internal: {
+        type: `HNStory`,
+        mediaType: `application/json`,
+      },
+      domain,
+      order: i + 1,
     }
 
     // Just store the user id
@@ -121,7 +123,7 @@ fragment commentsFragment on HackerNewsItem {
       .update(JSON.stringify(storyNode))
       .digest(`hex`)
 
-    storyNode.contentDigest = contentDigest
+    storyNode.internal.contentDigest = contentDigest
 
     createNode(storyNode)
 
@@ -133,11 +135,13 @@ fragment commentsFragment on HackerNewsItem {
         }
         let commentNode = {
           ..._.omit(comment, `kids`),
-          order: i + 1,
-          type: `HNComment`,
+          children: comment.kids.map(k => k.id),
           parent,
-          children: [...comment.kids.map(k => k.id)],
-          mediaType: `application/json`,
+          internal: {
+            type: `HNComment`,
+            mediaType: `application/json`,
+          },
+          order: i + 1,
         }
 
         commentNode.by = commentNode.by.id
@@ -149,8 +153,8 @@ fragment commentsFragment on HackerNewsItem {
           .update(nodeStr)
           .digest(`hex`)
 
-        commentNode.contentDigest = contentDigest
-        commentNode.content = nodeStr
+        commentNode.internal.contentDigest = contentDigest
+        commentNode.internal.content = nodeStr
 
         createNode(commentNode)
 

diff --git a/packages/gatsby-transformer-json/src/gatsby-node.js b/packages/gatsby-transformer-json/src/gatsby-node.js
@@ -11,12 +11,12 @@ async function onNodeCreate({ node, boundActionCreators, loadNodeContent }) {
   // but since this transformer creates new nodes with the same media-type
   // as its parent node, we have to add this check that we didn't create
   // the node).
-  if (node.pluginName === `gatsby-transformer-json`) {
+  if (node.internal.pluginName === `gatsby-transformer-json`) {
     return
   }
 
   // We only care about JSON content.
-  if (node.mediaType !== `application/json`) {
+  if (node.internal.mediaType !== `application/json`) {
     return
   }
 
@@ -28,15 +28,17 @@ async function onNodeCreate({ node, boundActionCreators, loadNodeContent }) {
     return {
       ...obj,
       id: obj.id ? obj.id : `${node.id} [${i}] >>> JSON`,
-      contentDigest,
-      mediaType: `application/json`,
-      parent: node.id,
-      // TODO make choosing the "type" a lot smarter. This assumes
-      // the parent node is a file.
-      // PascalCase
-      type: _.upperFirst(_.camelCase(`${node.name} Json`)),
       children: [],
-      content: objStr,
+      parent: node.id,
+      internal: {
+        contentDigest,
+        mediaType: `application/json`,
+        // TODO make choosing the "type" a lot smarter. This assumes
+        // the parent node is a file.
+        // PascalCase
+        type: _.upperFirst(_.camelCase(`${node.name} Json`)),
+        content: objStr,
+      },
     }
   })