diff --git a/packages/gatsby/src/utils/__tests__/__snapshots__/webpack-utils.ts.snap b/packages/gatsby/src/utils/__tests__/__snapshots__/webpack-utils.ts.snap
index bbf7618b26235..7dd6d27ff33dd 100644
--- a/packages/gatsby/src/utils/__tests__/__snapshots__/webpack-utils.ts.snap
+++ b/packages/gatsby/src/utils/__tests__/__snapshots__/webpack-utils.ts.snap
@@ -10,6 +10,7 @@ Object {
       "loader": StringContaining "babel-loader",
       "options": Object {
         "babelrc": false,
+        "cacheDirectory": "<TEMP_DIR>/test/.cache/webpack/babel",
         "cacheIdentifier": "develop---gatsby-dependencies@1.0.0",
         "compact": false,
         "configFile": false,
@@ -37,6 +38,7 @@ Object {
     Object {
       "loader": StringContaining "babel-loader",
       "options": Object {
+        "cacheDirectory": "<TEMP_DIR>/test/.cache/webpack/babel",
         "compact": false,
         "configFile": true,
         "stage": "develop",
diff --git a/packages/gatsby/src/utils/__tests__/webpack-utils.ts b/packages/gatsby/src/utils/__tests__/webpack-utils.ts
index 87b30b241439c..9c14f12474221 100644
--- a/packages/gatsby/src/utils/__tests__/webpack-utils.ts
+++ b/packages/gatsby/src/utils/__tests__/webpack-utils.ts
@@ -1,3 +1,4 @@
+import * as os from "os"
 import { createWebpackUtils } from "../webpack-utils"
 import { Stage, IProgram } from "../../commands/types"
 
@@ -27,7 +28,9 @@ jest.mock(`autoprefixer`, () =>
 let config
 
 beforeAll(() => {
-  config = createWebpackUtils(Stage.Develop, {} as IProgram)
+  config = createWebpackUtils(Stage.Develop, {
+    directory: `${os.tmpdir()}/test`,
+  } as IProgram)
 })
 
 describe(`webpack utils`, () => {
diff --git a/packages/gatsby/src/utils/__tests__/webpack.config.js b/packages/gatsby/src/utils/__tests__/webpack.config.js
index da8786dce116f..f878032af96aa 100644
--- a/packages/gatsby/src/utils/__tests__/webpack.config.js
+++ b/packages/gatsby/src/utils/__tests__/webpack.config.js
@@ -55,6 +55,7 @@ beforeEach(() => {
 const getConfig = (args = {}) =>
   webpackConfig(
     {
+      directory: process.cwd(),
       extensions: [`.js`],
     },
     process.cwd(),
diff --git a/packages/gatsby/src/utils/babel-loader.js b/packages/gatsby/src/utils/babel-loader.js
index 3f5ad3c05e18c..2d8b5f6b6e729 100644
--- a/packages/gatsby/src/utils/babel-loader.js
+++ b/packages/gatsby/src/utils/babel-loader.js
@@ -30,7 +30,6 @@ module.exports = babelLoader.custom(babel => {
           stage,
         },
         loader: {
-          cacheDirectory: true,
           sourceType: `unambiguous`,
           ...getCustomOptions(stage),
           ...options,
diff --git a/packages/gatsby/src/utils/webpack-utils.ts b/packages/gatsby/src/utils/webpack-utils.ts
index b84b42af1d125..6391aac42a8d0 100644
--- a/packages/gatsby/src/utils/webpack-utils.ts
+++ b/packages/gatsby/src/utils/webpack-utils.ts
@@ -1,3 +1,4 @@
+import * as path from "path"
 import { Loader, RuleSetRule, Plugin } from "webpack"
 import { GraphQLSchema } from "graphql"
 import postcss from "postcss"
@@ -267,6 +268,13 @@ export const createWebpackUtils = (
       return {
         options: {
           stage,
+          // TODO add proper cache keys
+          cacheDirectory: path.join(
+            program.directory,
+            `.cache`,
+            `webpack`,
+            `babel`
+          ),
           ...options,
         },
         loader: require.resolve(`./babel-loader`),
@@ -275,7 +283,16 @@ export const createWebpackUtils = (
 
     dependencies: options => {
       return {
-        options,
+        options: {
+          // TODO add proper cache keys
+          cacheDirectory: path.join(
+            program.directory,
+            `.cache`,
+            `webpack`,
+            `babel`
+          ),
+          ...options,
+        },
         loader: require.resolve(`babel-loader`),
       }
     },
@@ -541,7 +558,8 @@ export const createWebpackUtils = (
     ...options
   }: { terserOptions?: TerserPlugin.TerserPluginOptions } = {}): Plugin =>
     new TerserPlugin({
-      cache: true,
+      // TODO add proper cache keys
+      cache: path.join(program.directory, `.cache`, `webpack`, `terser`),
       // We can't use parallel in WSL because of https://github.com/gatsbyjs/gatsby/issues/6540
       // This issue was fixed in https://github.com/gatsbyjs/gatsby/pull/12636
       parallel: !isWsl,