Add experimental options for more parallelization in webpack builds

errors/parallel-build-without-worker.mdx

@@ -0,0 +1,20 @@
+---
+title: Parallel Build Without Build Worker
+---
+
+## Why This Error Occurred
+
+The `experimental.parallelServerAndEdgeCompiles` and `experimental.parallelServerBuildTraces`
+options require that the `experimental.webpackBuildWorker` option is set to true. These
+options use workers to improve the paralellization of the build which may improve performance,
+but the build may use more memory at the same time.
+
+## Possible Ways to Fix It
+
+Build workers are enabled by default unless you have a custom webpack config.
+You can force enable the option by setting `config.experimental.webpackBuildWorker: true` in the config,
+but some webpack configurations may not be compatible.
+
+## Useful Links
+
+Also see https://nextjs.org/docs/messages/webpack-build-worker-opt-out

packages/next/src/build/index.ts

@@ -1098,6 +1098,12 @@ export default async function build(
         config.experimental.webpackBuildWorker ||
         (config.experimental.webpackBuildWorker === undefined &&
           !config.webpack)
+      const runServerAndEdgeInParallel =
+        config.experimental.parallelServerAndEdgeCompiles
+      const collectServerBuildTracesInParallel =
+        config.experimental.parallelServerBuildTraces ||
+        (config.experimental.parallelServerBuildTraces === undefined &&
+          isCompileMode)
 
       nextBuildSpan.setAttribute(
         'has-custom-webpack-config',
@@ -1113,43 +1119,65 @@ export default async function build(
           'Custom webpack configuration is detected. When using a custom webpack configuration, the Webpack build worker is disabled by default. To force enable it, set the "experimental.webpackBuildWorker" option to "true". Read more: https://nextjs.org/docs/messages/webpack-build-worker-opt-out'
         )
       }
+      if (
+        !useBuildWorker &&
+        (runServerAndEdgeInParallel || collectServerBuildTracesInParallel)
+      ) {
+        throw new Error(
+          'The "parallelServerBuildTraces" and "parallelServerAndEdgeCompiles" options may only be used when build workers can be used. Read more: https://nextjs.org/docs/messages/parallel-build-without-worker'
+        )
+      }
 
       if (!isGenerateMode) {
-        if (isCompileMode && useBuildWorker) {
+        if (runServerAndEdgeInParallel || collectServerBuildTracesInParallel) {
           let durationInSeconds = 0
 
-          await webpackBuild(useBuildWorker, ['server']).then((res) => {
+          const serverBuildPromise = webpackBuild(useBuildWorker, [
+            'server',
+          ]).then((res) => {
             buildTraceContext = res.buildTraceContext
             durationInSeconds += res.duration
-            const buildTraceWorker = new Worker(
-              require.resolve('./collect-build-traces'),
-              {
-                numWorkers: 1,
-                exposedMethods: ['collectBuildTraces'],
-              }
-            ) as Worker & typeof import('./collect-build-traces')
 
-            buildTracesPromise = buildTraceWorker
-              .collectBuildTraces({
-                dir,
-                config,
-                distDir,
-                // Serialize Map as this is sent to the worker.
-                pageInfos: serializePageInfos(new Map()),
-                staticPages: [],
-                hasSsrAmpPages: false,
-                buildTraceContext,
-                outputFileTracingRoot,
-              })
-              .catch((err) => {
-                console.error(err)
-                process.exit(1)
-              })
+            if (collectServerBuildTracesInParallel) {
+              const buildTraceWorker = new Worker(
+                require.resolve('./collect-build-traces'),
+                {
+                  numWorkers: 1,
+                  exposedMethods: ['collectBuildTraces'],
+                }
+              ) as Worker & typeof import('./collect-build-traces')
+
+              buildTracesPromise = buildTraceWorker
+                .collectBuildTraces({
+                  dir,
+                  config,
+                  distDir,
+                  // Serialize Map as this is sent to the worker.
+                  pageInfos: serializePageInfos(new Map()),
+                  staticPages: [],
+                  hasSsrAmpPages: false,
+                  buildTraceContext,
+                  outputFileTracingRoot,
+                })
+                .catch((err) => {
+                  console.error(err)
+                  process.exit(1)
+                })
+            }
           })
+          if (!runServerAndEdgeInParallel) {
+            await serverBuildPromise
+          }
 
-          await webpackBuild(useBuildWorker, ['edge-server']).then((res) => {
+          const edgeBuildPromise = webpackBuild(useBuildWorker, [
+            'edge-server',
+          ]).then((res) => {
             durationInSeconds += res.duration
           })
+          if (runServerAndEdgeInParallel) {
+            await serverBuildPromise
+          }
+          await edgeBuildPromise
 
           await webpackBuild(useBuildWorker, ['client']).then((res) => {
             durationInSeconds += res.duration

packages/next/src/lib/turbopack-warning.ts

@@ -88,6 +88,8 @@ const supportedTurbopackNextConfigOptions = [
   'experimental.memoryBasedWorkersCount',
   'experimental.clientRouterFilterRedirects',
   'experimental.webpackBuildWorker',
+  'experimental.parallelServerAndEdgeCompiles',
+  'experimental.parallelServerBuildTraces',
   'experimental.appDocumentPreloading',
   'experimental.incrementalCacheHandlerPath',
   'experimental.amp',

packages/next/src/server/config-schema.ts

@@ -285,6 +285,8 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() =>
         outputFileTracingIncludes: z
           .record(z.string(), z.array(z.string()))
           .optional(),
+        parallelServerAndEdgeCompiles: z.boolean().optional(),
+        parallelServerBuildTraces: z.boolean().optional(),
         ppr: z.boolean().optional(),
         taint: z.boolean().optional(),
         proxyTimeout: z.number().gte(0).optional(),

packages/next/src/server/config-shared.ts

@@ -282,6 +282,35 @@ export interface ExperimentalConfig {
    */
   typedRoutes?: boolean
 
+  /**
+   * Runs the compilations for server and edge in parallel instead of in serial.
+   * This will make builds faster if there is enough server and edge functions
+   * in the application at the cost of more memory.
+   *
+   * NOTE: This option is only valid when the build process can use workers. See
+   * the documentation for `webpackBuildWorker` for more details.
+   */
+  parallelServerAndEdgeCompiles?: boolean
+
+  /**
+   * Runs the logic to collect build traces for the server routes in parallel
+   * with other work during the compilation. This will increase the speed of
+   * the build at the cost of more memory. This option may incur some additional
+   * work compared to if the option was disabled since the work is started
+   * before data from the client compilation is available to potentially reduce
+   * the amount of code that needs to be traced. Despite that, this may still
+   * result in faster builds for some applications.
+   *
+   * Valid values are:
+   * - `true`: Collect the server build traces in parallel.
+   * - `false`: Do not collect the server build traces in parallel.
+   * - `undefined`: Collect server build traces in parallel only in the `experimental-compile` mode.
+   *
+   * NOTE: This option is only valid when the build process can use workers. See
+   * the documentation for `webpackBuildWorker` for more details.
+   */
+  parallelServerBuildTraces?: boolean
+
   /**
    * Run the Webpack build in a separate process to optimize memory usage during build.
    * Valid values are:
@@ -801,6 +830,8 @@ export const defaultConfig: NextConfig = {
     typedRoutes: false,
     instrumentationHook: false,
     bundlePagesExternals: false,
+    parallelServerAndEdgeCompiles: false,
+    parallelServerBuildTraces: false,
     ppr:
       // TODO: remove once we've made PPR default
       // If we're testing, and the `__NEXT_EXPERIMENTAL_PPR` environment variable

test/e2e/app-dir/app/next.config.js

@@ -1,6 +1,8 @@
 module.exports = {
   experimental: {
     clientRouterFilterRedirects: true,
+    parallelServerAndEdgeCompiles: true,
+    parallelServerBuildTraces: true,
     webpackBuildWorker: true,
   },
   // output: 'standalone',

test/production/app-dir/typed-routes-with-webpack-worker/good-routes/next.config.js

@@ -2,6 +2,7 @@
 module.exports = {
   experimental: {
     typedRoutes: true,
+    parallelServerBuildTraces: true,
     webpackBuildWorker: true,
   },
 }