From 718851bf7b25d8f9dd09fa4352a9781eef113f1c Mon Sep 17 00:00:00 2001 From: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com> Date: Thu, 1 Aug 2024 13:52:50 +0200 Subject: [PATCH 1/6] Also set turbo config fields in next-mdx --- packages/next-mdx/index.js | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/packages/next-mdx/index.js b/packages/next-mdx/index.js index 5d650c881acde..45cef3c863e0b 100644 --- a/packages/next-mdx/index.js +++ b/packages/next-mdx/index.js @@ -23,6 +23,27 @@ module.exports = } return Object.assign({}, nextConfig, { + experimental: Object.assign({}, nextConfig?.experimental, { + turbo: Object.assign({}, nextConfig?.experimental?.turbo, { + rules: Object.assign({}, nextConfig?.experimental?.turbo?.rules, { + '*.mdx': { + loaders: [loader], + as: '*.tsx', + }, + }), + resolveAlias: Object.assign( + {}, + nextConfig?.experimental?.turbo?.resolveAlias, + { + 'next-mdx-import-source-file': [ + 'private-next-root-dir/src/mdx-components', + 'private-next-root-dir/mdx-components', + '@mdx-js/react', + ], + } + ), + }), + }), webpack(config, options) { config.resolve.alias['next-mdx-import-source-file'] = [ 'private-next-root-dir/src/mdx-components', From 45ead3aee156692dfa1e2cef21b2a6f47ebe601b Mon Sep 17 00:00:00 2001 From: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com> Date: Thu, 1 Aug 2024 15:18:14 +0200 Subject: [PATCH 2/6] Use @vercel/turbopack-next/mdx-import-source --- crates/next-core/src/next_import_map.rs | 20 +++++++++----------- packages/next-mdx/index.js | 7 ++----- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/crates/next-core/src/next_import_map.rs b/crates/next-core/src/next_import_map.rs index b775f11b987c6..f86e47c732c3e 100644 --- a/crates/next-core/src/next_import_map.rs +++ b/crates/next-core/src/next_import_map.rs @@ -806,17 +806,15 @@ async fn insert_next_shared_aliases( ) -> Result<()> { let package_root = next_js_fs().root(); - if next_config.mdx_rs().await?.is_some() { - insert_alias_to_alternatives( - import_map, - mdx_import_source_file(), - vec![ - request_to_import_mapping(project_path, "./mdx-components"), - request_to_import_mapping(project_path, "./src/mdx-components"), - request_to_import_mapping(project_path, "@mdx-js/react"), - ], - ); - } + insert_alias_to_alternatives( + import_map, + mdx_import_source_file(), + vec![ + request_to_import_mapping(project_path, "./mdx-components"), + request_to_import_mapping(project_path, "./src/mdx-components"), + request_to_import_mapping(project_path, "@mdx-js/react"), + ], + ); insert_package_alias( import_map, diff --git a/packages/next-mdx/index.js b/packages/next-mdx/index.js index 45cef3c863e0b..4d894b9af41d4 100644 --- a/packages/next-mdx/index.js +++ b/packages/next-mdx/index.js @@ -35,11 +35,8 @@ module.exports = {}, nextConfig?.experimental?.turbo?.resolveAlias, { - 'next-mdx-import-source-file': [ - 'private-next-root-dir/src/mdx-components', - 'private-next-root-dir/mdx-components', - '@mdx-js/react', - ], + 'next-mdx-import-source-file': + '@vercel/turbopack-next/mdx-import-source', } ), }), From 07a07ab15f43de03beda51bdb9f6e8e2103cf460 Mon Sep 17 00:00:00 2001 From: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com> Date: Thu, 1 Aug 2024 15:19:28 +0200 Subject: [PATCH 3/6] Enable @next/mdx tests on Turbopack :tada: --- test/e2e/app-dir/mdx/mdx.test.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test/e2e/app-dir/mdx/mdx.test.ts b/test/e2e/app-dir/mdx/mdx.test.ts index af957f28e4fb6..516969ad9f27f 100644 --- a/test/e2e/app-dir/mdx/mdx.test.ts +++ b/test/e2e/app-dir/mdx/mdx.test.ts @@ -1,10 +1,6 @@ import { nextTestSetup } from 'e2e-utils' -for (const type of [ - 'with-mdx-rs', - // only mdx-rs should work with turbopack - ...(process.env.TURBOPACK ? [] : ['without-mdx-rs']), -]) { +for (const type of ['with-mdx-rs', 'without-mdx-rs']) { describe(`mdx ${type}`, () => { const { next } = nextTestSetup({ files: __dirname, From 514fad5cff89f659762aef170f77c9e642cc6ce6 Mon Sep 17 00:00:00 2001 From: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com> Date: Thu, 1 Aug 2024 12:04:30 +0200 Subject: [PATCH 4/6] Support Buffer results from Webpack loaders --- .../crates/turbopack-node/js/src/transforms/webpack-loaders.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbopack/crates/turbopack-node/js/src/transforms/webpack-loaders.ts b/turbopack/crates/turbopack-node/js/src/transforms/webpack-loaders.ts index ea0fb75b5cfbe..091451561f368 100644 --- a/turbopack/crates/turbopack-node/js/src/transforms/webpack-loaders.ts +++ b/turbopack/crates/turbopack-node/js/src/transforms/webpack-loaders.ts @@ -468,7 +468,7 @@ const transform = ( if (!result.result) return reject(new Error("No result from loaders")); const [source, map] = result.result; resolve({ - source, + source: Buffer.isBuffer(source) ? source.toString("utf8") : source, map: typeof map === "string" ? map From 9388834b62ea9d265d0bc78295bbf44e774ef2d7 Mon Sep 17 00:00:00 2001 From: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com> Date: Fri, 2 Aug 2024 10:58:14 +0200 Subject: [PATCH 5/6] Support Buffer results from Webpack loaders --- Cargo.lock | 4 ++++ turbopack/crates/turbopack-node/Cargo.toml | 1 + .../turbopack-node/js/src/transforms/webpack-loaders.ts | 2 +- turbopack/crates/turbopack-node/src/transforms/webpack.rs | 7 +++++-- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ad33f314a129f..88c0ec010bd30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2008,6 +2008,9 @@ name = "either" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +dependencies = [ + "serde", +] [[package]] name = "encode_unicode" @@ -9076,6 +9079,7 @@ dependencies = [ "async-stream", "async-trait", "const_format", + "either", "futures", "futures-retry", "indexmap 1.9.3", diff --git a/turbopack/crates/turbopack-node/Cargo.toml b/turbopack/crates/turbopack-node/Cargo.toml index 220319306fe60..9d06c0d3fc288 100644 --- a/turbopack/crates/turbopack-node/Cargo.toml +++ b/turbopack/crates/turbopack-node/Cargo.toml @@ -21,6 +21,7 @@ anyhow = { workspace = true } async-stream = "0.3.4" async-trait = { workspace = true } const_format = "0.2.30" +either = { workspace = true, features = ["serde"] } futures = { workspace = true } futures-retry = { workspace = true } indexmap = { workspace = true, features = ["serde"] } diff --git a/turbopack/crates/turbopack-node/js/src/transforms/webpack-loaders.ts b/turbopack/crates/turbopack-node/js/src/transforms/webpack-loaders.ts index 091451561f368..049af6d7b4392 100644 --- a/turbopack/crates/turbopack-node/js/src/transforms/webpack-loaders.ts +++ b/turbopack/crates/turbopack-node/js/src/transforms/webpack-loaders.ts @@ -468,7 +468,7 @@ const transform = ( if (!result.result) return reject(new Error("No result from loaders")); const [source, map] = result.result; resolve({ - source: Buffer.isBuffer(source) ? source.toString("utf8") : source, + source: Buffer.isBuffer(source) ? [...source] : source, map: typeof map === "string" ? map diff --git a/turbopack/crates/turbopack-node/src/transforms/webpack.rs b/turbopack/crates/turbopack-node/src/transforms/webpack.rs index a218d3f21a699..c4bd8596c1ea9 100644 --- a/turbopack/crates/turbopack-node/src/transforms/webpack.rs +++ b/turbopack/crates/turbopack-node/src/transforms/webpack.rs @@ -2,6 +2,7 @@ use std::mem::take; use anyhow::{bail, Context, Result}; use async_trait::async_trait; +use either::Either; use serde::{Deserialize, Serialize}; use serde_json::{json, Value as JsonValue}; use turbo_tasks::{ @@ -56,7 +57,9 @@ use crate::{ #[serde(rename_all = "camelCase")] #[turbo_tasks::value(serialization = "custom")] struct WebpackLoadersProcessingResult { - source: RcStr, + #[serde(with = "either::serde_untagged")] + #[turbo_tasks(trace_ignore)] + source: Either>, map: Option, #[turbo_tasks(trace_ignore)] assets: Option>, @@ -250,7 +253,7 @@ impl WebpackLoadersProcessedAsset { } else { None }; - let file = File::from(processed.source); + let file = either::for_both!(processed.source, source => File::from(source)); let assets = emitted_assets_to_virtual_sources(processed.assets); let content = AssetContent::File(FileContent::Content(file).cell()).cell(); Ok(ProcessWebpackLoadersResult { From d57f15d0112ffbaa4887611672f96916edab196f Mon Sep 17 00:00:00 2001 From: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com> Date: Fri, 2 Aug 2024 12:21:37 +0200 Subject: [PATCH 6/6] Use base64 for binary results from Webpack loaders --- Cargo.lock | 1 + turbopack/crates/turbopack-node/Cargo.toml | 1 + .../js/src/transforms/webpack-loaders.ts | 2 +- .../turbopack-node/src/transforms/webpack.rs | 17 ++++++++++++++--- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88c0ec010bd30..8e148c7125081 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9091,6 +9091,7 @@ dependencies = [ "regex", "serde", "serde_json", + "serde_with", "tokio", "tracing", "turbo-tasks", diff --git a/turbopack/crates/turbopack-node/Cargo.toml b/turbopack/crates/turbopack-node/Cargo.toml index 9d06c0d3fc288..0b7ea08e374a3 100644 --- a/turbopack/crates/turbopack-node/Cargo.toml +++ b/turbopack/crates/turbopack-node/Cargo.toml @@ -34,6 +34,7 @@ regex = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } #serde_qs = { workspace = true } +serde_with = { workspace = true, features = ["base64"] } tokio = { workspace = true, features = ["full"] } tracing = { workspace = true } turbo-tasks = { workspace = true } diff --git a/turbopack/crates/turbopack-node/js/src/transforms/webpack-loaders.ts b/turbopack/crates/turbopack-node/js/src/transforms/webpack-loaders.ts index 049af6d7b4392..d598627cb099c 100644 --- a/turbopack/crates/turbopack-node/js/src/transforms/webpack-loaders.ts +++ b/turbopack/crates/turbopack-node/js/src/transforms/webpack-loaders.ts @@ -468,7 +468,7 @@ const transform = ( if (!result.result) return reject(new Error("No result from loaders")); const [source, map] = result.result; resolve({ - source: Buffer.isBuffer(source) ? [...source] : source, + source: Buffer.isBuffer(source) ? {binary: source.toString('base64')} : source, map: typeof map === "string" ? map diff --git a/turbopack/crates/turbopack-node/src/transforms/webpack.rs b/turbopack/crates/turbopack-node/src/transforms/webpack.rs index c4bd8596c1ea9..c486cdaec5800 100644 --- a/turbopack/crates/turbopack-node/src/transforms/webpack.rs +++ b/turbopack/crates/turbopack-node/src/transforms/webpack.rs @@ -5,6 +5,7 @@ use async_trait::async_trait; use either::Either; use serde::{Deserialize, Serialize}; use serde_json::{json, Value as JsonValue}; +use serde_with::serde_as; use turbo_tasks::{ trace::TraceRawVcs, Completion, RcStr, TaskInput, TryJoinIterExt, Value, ValueToString, Vc, }; @@ -53,13 +54,20 @@ use crate::{ AssetsForSourceMapping, }; +#[serde_as] +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +struct BytesBase64 { + #[serde_as(as = "serde_with::base64::Base64")] + binary: Vec, +} + #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] #[turbo_tasks::value(serialization = "custom")] struct WebpackLoadersProcessingResult { #[serde(with = "either::serde_untagged")] - #[turbo_tasks(trace_ignore)] - source: Either>, + #[turbo_tasks(debug_ignore, trace_ignore)] + source: Either, map: Option, #[turbo_tasks(trace_ignore)] assets: Option>, @@ -253,7 +261,10 @@ impl WebpackLoadersProcessedAsset { } else { None }; - let file = either::for_both!(processed.source, source => File::from(source)); + let file = match processed.source { + Either::Left(str) => File::from(str), + Either::Right(bytes) => File::from(bytes.binary), + }; let assets = emitted_assets_to_virtual_sources(processed.assets); let content = AssetContent::File(FileContent::Content(file).cell()).cell(); Ok(ProcessWebpackLoadersResult {