feat(gatsby-plugin-functions): Add the ability to run functions local…

…ly and on Gatsby Cloud (#30192)

* Add new plugin

* Add functions plugin by default

* Add stub

* Bump up package.json

* wip

* Compile all functions found in build

* Add plugin by default only if GATSBY_EXPERIMENT_FUNCTIONS is set

* Move to public for now

* Move to functions dir

* Run a dev server for functions

* wip

* Move to onPreBootstrap

* Add target node

* Bump up package.json

* Bump up package.json

* Add support for env vars

* Publish 0.1.0-4

* Bump up peerDependencies

* Update yarn lock

* Use a parameter in express middleware and ignore extension (#30222)

* Publish 0.1.0-6

* Set a default functions directory path (#30277)

* feat(functions): Add body parser

* Require files from the .cache directory so user can run es/commonjs (#30338)

* Require files from the .cache directory so user can run es/commonjs

* Panic on build if we get an error

* Add some logging via reporter

* logging to func

* Update packages/gatsby-plugin-functions/src/gatsby-node.ts

Co-authored-by: Ward Peeters <ward@coding-tech.com>

* Function name does not include extension

* Print a relative path to the functions directory

Co-authored-by: Sidhartha Chatterjee <me@sidharthachatterjee.com>
Co-authored-by: Ward Peeters <ward@coding-tech.com>

* Publish 0.1.0-7

* Prefer directory from program in state

* Log invocations as verbose

* Add timeout constant

* Add internal plugin

* Load functions internal plugin

* Load functions during initialize and stick em in redux

* Clean up comments

* Move initialization code back to plugin

* Add experiment for Gatsby Functions

* Revert yarn changes

* Remove plugin package

* Do not extract comments

* Update yarn.lock

* Update packages/gatsby/src/internal-plugins/functions/gatsby-node.ts

Co-authored-by: Lennart <lekoarts@gmail.com>

* chore(functions): Remove console.logs

* chore(functions): Remove unused urlResolve import

* get internal plugin functions working + some cleanups

* Fix regex per @jamo's suggestion + update yarn.lock

* remove accidentally committed file

* globby isn't a dependency of gatsby

* fix(functions): End request with status 500 on function error

* Hot reload functions

* Load env variables from .env.* files

* Resolve result of function so we can catch errors from async functions

* Log webpack warnings/errors when compiling to the terminal

* Log how long function took to execute

* reload watch when .env files change & use all process.env vars

* Allow for arbitrarily deep functions

* Detect new functions and incorporate

* Log when functions rebuild or we restart the watcher

* List functions on dev-404-page

* Only show new API instructions when trying to reach an API route + add example

* Fix linting

* put new code behind conditional

* Properly close webpack watcher + handle deleting functions

* Ignore non-js extensions (so make typescript work)

* Write out manifest file so can serve functions from 'gatsby serve'

* Address @lekoart's comments

* Add default type for SiteFunction & fix small bugs when site doesn't yet have functions

* Add initial suite of tests

* Update snapshots

* Fix more tests

* Add functions tests to circleci setup

* Only parse once

* Small tweaks

* add comment

* Fix typescript support

* Add tests for apis with special characters h/t @LekoArts

Co-authored-by: abhiaiyer91 <abhiaiyer91@gmail.com>
Co-authored-by: gatsbybot <mathews.kyle+gatsbybot@gmail.com>
Co-authored-by: Julien Poissonnier <julien@caffeine.lu>
Co-authored-by: Ward Peeters <ward@coding-tech.com>
Co-authored-by: Lennart <lekoarts@gmail.com>
Co-authored-by: Kyle Mathews <mathews.kyle@gmail.com>

.circleci/config.yml

@@ -285,6 +285,13 @@ jobs:
       - store_artifacts:
           path: integration-tests/images/__diff_output__
 
+  integration_tests_functions:
+    executor: node
+    steps:
+      - e2e-test:
+          test_path: integration-tests/functions
+          test_command: yarn test
+
   e2e_tests_path-prefix:
     <<: *e2e-executor
     environment:
@@ -610,6 +617,8 @@ workflows:
           <<: *e2e-test-workflow
       - integration_tests_images:
           <<: *e2e-test-workflow
+      - integration_tests_functions:
+          <<: *e2e-test-workflow
       - integration_tests_gatsby_cli:
           requires:
             - bootstrap

integration-tests/functions/.env.development

@@ -0,0 +1 @@
+pickle=word

integration-tests/functions/.gitignore

@@ -0,0 +1,3 @@
+node_modules/
+.cache/
+public

integration-tests/functions/README.md

@@ -0,0 +1,54 @@
+<p align="center">
+  <a href="https://www.gatsbyjs.com/?utm_source=starter&utm_medium=readme&utm_campaign=minimal-starter">
+    <img alt="Gatsby" src="https://www.gatsbyjs.com/Gatsby-Monogram.svg" width="60" />
+  </a>
+</p>
+<h1 align="center">
+  Gatsby minimal starter
+</h1>
+
+## 🚀 Quick start
+
+1.  **Create a Gatsby site.**
+
+    Use the Gatsby CLI to create a new site, specifying the minimal starter.
+
+    ```shell
+    # create a new Gatsby site using the minimal starter
+    npm init gatsby
+    ```
+
+2.  **Start developing.**
+
+    Navigate into your new site’s directory and start it up.
+
+    ```shell
+    cd my-gatsby-site/
+    npm run develop
+    ```
+
+3.  **Open the code and start customizing!**
+
+    Your site is now running at http://localhost:8000!
+
+    Edit `src/pages/index.js` to see your site update in real-time!
+
+4.  **Learn more**
+
+    - [Documentation](https://www.gatsbyjs.com/docs/?utm_source=starter&utm_medium=readme&utm_campaign=minimal-starter)
+
+    - [Tutorials](https://www.gatsbyjs.com/tutorial/?utm_source=starter&utm_medium=readme&utm_campaign=minimal-starter)
+
+    - [Guides](https://www.gatsbyjs.com/tutorial/?utm_source=starter&utm_medium=readme&utm_campaign=minimal-starter)
+
+    - [API Reference](https://www.gatsbyjs.com/docs/api-reference/?utm_source=starter&utm_medium=readme&utm_campaign=minimal-starter)
+
+    - [Plugin Library](https://www.gatsbyjs.com/plugins?utm_source=starter&utm_medium=readme&utm_campaign=minimal-starter)
+
+    - [Cheat Sheet](https://www.gatsbyjs.com/docs/cheat-sheet/?utm_source=starter&utm_medium=readme&utm_campaign=minimal-starter)
+
+## 🚀 Quick start (Gatsby Cloud)
+
+Deploy this starter with one click on [Gatsby Cloud](https://www.gatsbyjs.com/cloud/):
+
+[<img src="https://www.gatsbyjs.com/deploynow.svg" alt="Deploy to Gatsby Cloud">](https://www.gatsbyjs.com/dashboard/deploynow?url=https://github.com/gatsbyjs/my-gatsby-site)

integration-tests/functions/__tests__/__snapshots__/functions.js.snap

@@ -0,0 +1,77 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`functions can parse different ways of sending data form data 1`] = `
+Object {
+  "a": "form-data",
+}
+`;
+
+exports[`functions can parse different ways of sending data form parameters 1`] = `
+Object {
+  "a": "form parameters",
+}
+`;
+
+exports[`functions can parse different ways of sending data json body 1`] = `
+Object {
+  "a": "json",
+}
+`;
+
+exports[`functions can parse different ways of sending data query string 1`] = `
+Object {
+  "amIReal": "true",
+}
+`;
+
+exports[`response formats returns json correctly 1`] = `
+Object {
+  "amIJSON": true,
+}
+`;
+
+exports[`response formats returns json correctly 2`] = `
+Object {
+  "access-control-allow-origin": "*",
+  "connection": "close",
+  "content-length": "16",
+  "content-type": "application/json; charset=utf-8",
+  "etag": "W/\\"10-R6td1pV+B+Xz9CJkNeaEI2kP+QY\\"",
+  "vary": "Accept-Encoding",
+  "x-powered-by": "Express",
+}
+`;
+
+exports[`response formats returns text correctly 1`] = `"I am typescript"`;
+
+exports[`response formats returns text correctly 2`] = `
+Object {
+  "access-control-allow-origin": "*",
+  "connection": "close",
+  "content-length": "15",
+  "content-type": "text/html; charset=utf-8",
+  "etag": "W/\\"f-zwggT56l/fnWBm4dI0PkA4E3i4E\\"",
+  "vary": "Accept-Encoding",
+  "x-powered-by": "Express",
+}
+`;
+
+exports[`routing routes with special characters 1`] = `"I-Am-Capitalized.js"`;
+
+exports[`routing routes with special characters 2`] = `"some whitespace.js"`;
+
+exports[`routing routes with special characters 3`] = `"with-äöü-umlaut.js"`;
+
+exports[`routing routes with special characters 4`] = `"some-àè-french.js"`;
+
+exports[`routing routes with special characters 5`] = `"some-אודות.js"`;
+
+exports[`routing secondary-level API 1`] = `"I am at a secondary-level"`;
+
+exports[`routing secondary-level API 2`] = `"I am another sub-directory function"`;
+
+exports[`routing secondary-level API with index.js 1`] = `"I am an index.js in a sub-directory!"`;
+
+exports[`routing top-level API 1`] = `"I am at the top-level"`;
+
+exports[`typescript typescript functions work 1`] = `"I am typescript"`;

integration-tests/functions/__tests__/fixtures/function-a.js

@@ -0,0 +1,3 @@
+export default function Function(req, res) {
+  res.send(`hi`)
+}

integration-tests/functions/__tests__/fixtures/test.txt

@@ -0,0 +1 @@
+hi friends

integration-tests/functions/__tests__/functions.js

@@ -0,0 +1,221 @@
+const fetch = require(`node-fetch`)
+const execa = require(`execa`)
+const fs = require(`fs-extra`)
+const path = require(`path`)
+const FormData = require("form-data")
+
+describe(`routing`, () => {
+  test(`top-level API`, async () => {
+    const result = await fetch(
+      `http://localhost:8000/api/top-level`
+    ).then(res => res.text())
+
+    expect(result).toMatchSnapshot()
+  })
+  test(`secondary-level API`, async () => {
+    const result = await fetch(
+      `http://localhost:8000/api/a-directory/function`
+    ).then(res => res.text())
+
+    expect(result).toMatchSnapshot()
+  })
+  test(`secondary-level API with index.js`, async () => {
+    const result = await fetch(
+      `http://localhost:8000/api/a-directory`
+    ).then(res => res.text())
+
+    expect(result).toMatchSnapshot()
+  })
+  test(`secondary-level API`, async () => {
+    const result = await fetch(
+      `http://localhost:8000/api/dir/function`
+    ).then(res => res.text())
+
+    expect(result).toMatchSnapshot()
+  })
+  test(`routes with special characters`, async () => {
+    const routes = [
+      `http://localhost:8000/api/I-Am-Capitalized`,
+      `http://localhost:8000/api/some whitespace`,
+      `http://localhost:8000/api/with-äöü-umlaut`,
+      `http://localhost:8000/api/some-àè-french`,
+      encodeURI(`http://localhost:8000/api/some-אודות`),
+    ]
+
+    for (const route of routes) {
+      const result = await fetch(route).then(res => res.text())
+
+      expect(result).toMatchSnapshot()
+    }
+  })
+})
+
+describe(`environment variables`, () => {
+  test(`can use inside functions`, async () => {
+    const result = await fetch(
+      `http://localhost:8000/api/env-variables`
+    ).then(res => res.text())
+
+    expect(result).toEqual(`word`)
+  })
+})
+
+describe(`typescript`, () => {
+  test(`typescript functions work`, async () => {
+    const result = await fetch(
+      `http://localhost:8000/api/i-am-typescript`
+    ).then(res => res.text())
+
+    expect(result).toMatchSnapshot()
+  })
+})
+
+describe(`response formats`, () => {
+  test(`returns json correctly`, async () => {
+    const res = await fetch(`http://localhost:8000/api/i-am-json`)
+    const result = await res.json()
+
+    const { date, ...headers } = Object.fromEntries(res.headers)
+    expect(result).toMatchSnapshot()
+    expect(headers).toMatchSnapshot()
+  })
+  test(`returns text correctly`, async () => {
+    const res = await fetch(`http://localhost:8000/api/i-am-typescript`)
+    const result = await res.text()
+
+    const { date, ...headers } = Object.fromEntries(res.headers)
+    expect(result).toMatchSnapshot()
+    expect(headers).toMatchSnapshot()
+  })
+})
+
+describe(`functions can send custom statuses`, () => {
+  test(`can return 200 status`, async () => {
+    const res = await fetch(`http://localhost:8000/api/status`)
+
+    expect(res.status).toEqual(200)
+  })
+
+  test(`can return 404 status`, async () => {
+    const res = await fetch(`http://localhost:8000/api/status?code=404`)
+
+    expect(res.status).toEqual(404)
+  })
+
+  test(`can return 500 status`, async () => {
+    const res = await fetch(`http://localhost:8000/api/status?code=500`)
+
+    expect(res.status).toEqual(500)
+  })
+})
+
+describe(`functions can parse different ways of sending data`, () => {
+  test(`query string`, async () => {
+    const result = await fetch(
+      `http://localhost:8000/api/parser?amIReal=true`
+    ).then(res => res.json())
+
+    expect(result).toMatchSnapshot()
+  })
+
+  test(`form parameters`, async () => {
+    const { URLSearchParams } = require("url")
+    const params = new URLSearchParams()
+    params.append("a", `form parameters`)
+    const result = await fetch(`http://localhost:8000/api/parser`, {
+      method: `POST`,
+      body: params,
+    }).then(res => res.json())
+
+    expect(result).toMatchSnapshot()
+  })
+
+  test(`form data`, async () => {
+    const FormData = require("form-data")
+
+    const form = new FormData()
+    form.append("a", `form-data`)
+    const result = await fetch(`http://localhost:8000/api/parser`, {
+      method: `POST`,
+      body: form,
+    }).then(res => res.json())
+
+    expect(result).toMatchSnapshot()
+  })
+
+  test(`json body`, async () => {
+    const body = { a: `json` }
+    const result = await fetch(`http://localhost:8000/api/parser`, {
+      method: `POST`,
+      body: JSON.stringify(body),
+      headers: { "Content-Type": "application/json" },
+    }).then(res => res.json())
+
+    expect(result).toMatchSnapshot()
+  })
+
+  // TODO enable when functions support uploading files.
+  // test(`file in multipart/form`, async () => {
+  // const { readFileSync } = require("fs")
+
+  // const file = readFileSync(path.join(__dirname, "./fixtures/test.txt"))
+
+  // const form = new FormData()
+  // form.append("file", file)
+  // const result = await fetch(`http://localhost:8000/api/parser`, {
+  // method: `POST`,
+  // body: form,
+  // }).then(res => res.json())
+
+  // console.log({ result })
+
+  // expect(result).toMatchSnapshot()
+  // })
+
+  // test(`stream a file`, async () => {
+  // const { createReadStream } = require("fs")
+
+  // const stream = createReadStream(path.join(__dirname, "./fixtures/test.txt"))
+  // const res = await fetch(`http://localhost:8000/api/parser`, {
+  // method: `POST`,
+  // body: stream,
+  // })
+
+  // console.log(res)
+
+  // expect(result).toMatchSnapshot()
+  // })
+})
+
+// TODO figure out why this gets into endless loops
+// describe.only(`hot reloading`, () => {
+// const fixturesDir = path.join(__dirname, `fixtures`)
+// const apiDir = path.join(__dirname, `../src/api`)
+// beforeAll(() => {
+// try {
+// fs.unlinkSync(path.join(apiDir, `function-a.js`))
+// } catch (e) {
+// // Ignore as this should mostly error with file not found.
+// // We delete to be sure it's not there.
+// }
+// })
+// afterAll(() => {
+// fs.unlinkSync(path.join(apiDir, `function-a.js`))
+// })
+
+// test(`new function`, cb => {
+// fs.copySync(
+// path.join(fixturesDir, `function-a.js`),
+// path.join(apiDir, `function-a.js`)
+// )
+// setTimeout(async () => {
+// const result = await fetch(
+// `http://localhost:8000/api/function-a`
+// ).then(res => res.text())
+
+// console.log(result)
+// expect(result).toMatchSnapshot()
+// cb()
+// }, 400)
+// })
+// })

integration-tests/functions/gatsby-config.js

@@ -0,0 +1,9 @@
+module.exports = {
+  flags: {
+    FUNCTIONS: true,
+  },
+  siteMetadata: {
+    title: "functions",
+  },
+  plugins: [],
+}