diff --git a/Cargo.lock b/Cargo.lock index c0fa63f1437019..8a5393e7cce11c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16,6 +16,15 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" +[[package]] +name = "ahash" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" +dependencies = [ + "const-random", +] + [[package]] name = "aho-corasick" version = "0.7.13" @@ -265,6 +274,26 @@ dependencies = [ "bitflags", ] +[[package]] +name = "const-random" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f1af9ac737b2dd2d577701e59fd09ba34822f6f2ebdb30a7647405d9e55e16a" +dependencies = [ + "const-random-macro", + "proc-macro-hack", +] + +[[package]] +name = "const-random-macro" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25e4c606eb459dd29f7c57b2e0879f2b6f14ee130918c2b78ccb58a9624e6c7a" +dependencies = [ + "getrandom", + "proc-macro-hack", +] + [[package]] name = "crc32fast" version = "1.2.0" @@ -330,6 +359,17 @@ dependencies = [ "syn 1.0.36", ] +[[package]] +name = "dashmap" +version = "3.11.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f260e2fc850179ef410018660006951c1b55b79e8087e87111a2c388994b9b5" +dependencies = [ + "ahash", + "cfg-if", + "num_cpus", +] + [[package]] name = "deno" version = "1.3.2" @@ -2263,6 +2303,7 @@ checksum = "c3e99fc5c5b87871fa19036fd8a622ecf6b2a29b30b4d632e48f6a1923e393ea" dependencies = [ "Inflector", "arrayvec", + "dashmap", "either", "fxhash", "indexmap", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 7c8414e55d76dd..e9770ff6ba64de 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -62,7 +62,7 @@ serde_json = { version = "1.0.57", features = [ "preserve_order" ] } sys-info = "0.7.0" sourcemap = "6.0.1" swc_common = { version = "=0.10.2", features = ["sourcemap"] } -swc_ecmascript = { version = "=0.6.3", features = ["codegen", "parser", "transforms", "visit"] } +swc_ecmascript = { version = "=0.6.3", features = ["codegen", "parser", "react", "transforms", "visit"] } tempfile = "3.1.0" termcolor = "1.1.0" tokio = { version = "0.2.22", features = ["full"] } diff --git a/cli/ast.rs b/cli/ast.rs new file mode 100644 index 00000000000000..c60492b4e8873e --- /dev/null +++ b/cli/ast.rs @@ -0,0 +1,449 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +use crate::msg::MediaType; + +use deno_core::ErrBox; +use std::error::Error; +use std::fmt; +use std::rc::Rc; +use std::sync::Arc; +use std::sync::RwLock; +use swc_common::chain; +use swc_common::comments::Comment; +use swc_common::comments::SingleThreadedComments; +use swc_common::errors::Diagnostic; +use swc_common::errors::DiagnosticBuilder; +use swc_common::errors::Emitter; +use swc_common::errors::Handler; +use swc_common::errors::HandlerFlags; +use swc_common::FileName; +use swc_common::Globals; +use swc_common::Loc; +pub use swc_common::SourceMap; +pub use swc_common::Span; +use swc_ecmascript::ast::Module; +use swc_ecmascript::ast::Program; +use swc_ecmascript::codegen::text_writer::JsWriter; +use swc_ecmascript::codegen::Config; +use swc_ecmascript::codegen::Node; +use swc_ecmascript::parser::lexer::Lexer; +use swc_ecmascript::parser::EsConfig; +use swc_ecmascript::parser::JscTarget; +use swc_ecmascript::parser::StringInput; +use swc_ecmascript::parser::Syntax; +use swc_ecmascript::parser::TsConfig; +use swc_ecmascript::transforms::fixer; +use swc_ecmascript::transforms::helpers; +use swc_ecmascript::transforms::pass::Optional; +use swc_ecmascript::transforms::proposals::decorators; +use swc_ecmascript::transforms::react; +use swc_ecmascript::transforms::typescript; +use swc_ecmascript::visit::FoldWith; + +static TARGET: JscTarget = JscTarget::Es2020; + +#[derive(Debug, Clone, Default, Eq, PartialEq)] +pub struct Location { + pub filename: String, + pub line: usize, + pub col: usize, +} + +impl Into for Loc { + fn into(self) -> Location { + use swc_common::FileName::*; + + let filename = match &self.file.name { + Real(path_buf) => path_buf.to_string_lossy().to_string(), + Custom(str_) => str_.to_string(), + _ => panic!("invalid filename"), + }; + + Location { + filename, + line: self.line, + col: self.col_display, + } + } +} + +/// A buffer for collecting diagnostic messages from the AST parser. +#[derive(Debug)] +pub struct DiagnosticBuffer(Vec); + +impl Error for DiagnosticBuffer {} + +impl fmt::Display for DiagnosticBuffer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s = self.0.join(","); + f.pad(&s) + } +} + +impl DiagnosticBuffer { + pub fn from_error_buffer(error_buffer: ErrorBuffer, get_loc: F) -> Self + where + F: Fn(Span) -> Loc, + { + let s = error_buffer.0.read().unwrap().clone(); + let diagnostics = s + .iter() + .map(|d| { + let mut msg = d.message(); + + if let Some(span) = d.span.primary_span() { + let loc = get_loc(span); + let file_name = match &loc.file.name { + FileName::Custom(n) => n, + _ => unreachable!(), + }; + msg = format!( + "{} at {}:{}:{}", + msg, file_name, loc.line, loc.col_display + ); + } + + msg + }) + .collect::>(); + + Self(diagnostics) + } +} + +#[derive(Debug, Clone)] +pub struct ErrorBuffer(Arc>>); + +impl ErrorBuffer { + pub fn new() -> Self { + Self(Arc::new(RwLock::new(Vec::new()))) + } +} + +impl Emitter for ErrorBuffer { + fn emit(&mut self, db: &DiagnosticBuilder) { + self.0.write().unwrap().push((**db).clone()); + } +} + +fn get_es_config(jsx: bool) -> EsConfig { + EsConfig { + class_private_methods: true, + class_private_props: true, + class_props: true, + dynamic_import: true, + export_default_from: true, + export_namespace_from: true, + import_meta: true, + jsx, + nullish_coalescing: true, + num_sep: true, + optional_chaining: true, + top_level_await: true, + ..EsConfig::default() + } +} + +fn get_ts_config(tsx: bool) -> TsConfig { + TsConfig { + decorators: true, + dynamic_import: true, + tsx, + ..TsConfig::default() + } +} + +pub fn get_syntax_for_media_type(media_type: MediaType) -> Syntax { + match media_type { + MediaType::JavaScript => Syntax::Es(get_es_config(false)), + MediaType::JSX => Syntax::Es(get_es_config(true)), + MediaType::TypeScript => Syntax::Typescript(get_ts_config(false)), + MediaType::TSX => Syntax::Typescript(get_ts_config(true)), + _ => Syntax::Es(get_es_config(false)), + } +} + +#[derive(Debug, Clone)] +pub struct EmitTranspileOptions { + /// When emitting a legacy decorator, also emit experimental decorator meta + /// data. Defaults to `false`. + pub emit_metadata: bool, + /// Should the source map be inlined in the emitted code file, or provided + /// as a separate file. Defaults to `true`. + pub inline_source_map: bool, + /// When transforming JSX, what value should be used for the JSX factory. + /// Defaults to `React.createElement`. + pub jsx_factory: String, + /// When transforming JSX, what value should be used for the JSX fragment + /// factory. Defaults to `React.Fragment`. + pub jsx_fragment_factory: String, + /// Should JSX be transformed or preserved. Defaults to `true`. + pub transform_jsx: bool, +} + +impl Default for EmitTranspileOptions { + fn default() -> Self { + EmitTranspileOptions { + emit_metadata: false, + inline_source_map: true, + jsx_factory: "React.createElement".into(), + jsx_fragment_factory: "React.Fragment".into(), + transform_jsx: true, + } + } +} + +/// A logical structure to hold the value of a parsed module for further +/// processing. +#[derive(Clone)] +pub struct ParsedModule { + comments: SingleThreadedComments, + /// Leading comments for the module, where references and pragmas might be + /// contained. + pub leading_comments: Vec, + pub module: Module, + /// The AST source for the module. + pub source_map: Rc, +} + +impl fmt::Debug for ParsedModule { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "ParsedModule {{ }}") + } +} + +impl ParsedModule { + /// Transpile a module to JavaScript supportable by Deno, erasing TypeScript + /// types and performing other transforms. + pub fn transpile( + self, + options: &EmitTranspileOptions, + ) -> Result<(String, Option), ErrBox> { + let program = Program::Module(self.module); + + let jsx_pass = react::react( + self.source_map.clone(), + react::Options { + pragma: options.jsx_factory.clone(), + pragma_frag: options.jsx_fragment_factory.clone(), + // this will use `Object.assign()` instead of the `_extends` helper + // when spreading props. + use_builtins: true, + ..Default::default() + }, + ); + let mut passes = chain!( + Optional::new(jsx_pass, options.transform_jsx), + decorators::decorators(decorators::Config { + legacy: true, + emit_metadata: options.emit_metadata, + }), + typescript::strip(), + fixer(Some(&self.comments)), + ); + + let program = swc_common::GLOBALS.set(&Globals::new(), || { + helpers::HELPERS.set(&helpers::Helpers::new(false), || { + program.fold_with(&mut passes) + }) + }); + + let mut mappings = Vec::new(); + let mut buf = Vec::new(); + { + let wr = Box::new(JsWriter::new( + self.source_map.clone(), + "\n", + &mut buf, + Some(&mut mappings), + )); + let cfg = Config { minify: false }; + let mut emitter = swc_ecmascript::codegen::Emitter { + cfg, + comments: Some(&self.comments), + cm: self.source_map.clone(), + wr, + }; + program.emit_with(&mut emitter)?; + } + let mut code = String::from_utf8(buf)?; + let mut map: Option = None; + { + let mut buf = Vec::new(); + self + .source_map + .build_source_map_from(&mut mappings, None) + .to_writer(&mut buf)?; + + if options.inline_source_map { + code.push_str("//# sourceMappingURL=data:application/json;base64,"); + let encoded_map = base64::encode(buf); + code.push_str(&encoded_map); + } else { + map = Some(String::from_utf8(buf)?); + } + } + + Ok((code, map)) + } +} + +/// For a given specifier, source, and media type, parse the source of the +/// module and return a representation which can be further processed. +/// +/// # Arguments +/// +/// - `specifier` - The module specifier for the module. +/// - `source` - The source code for the module. +/// - `media_type` - The media type for the module. +/// +pub fn parse( + specifier: &str, + source: &str, + media_type: MediaType, +) -> Result { + let source_map = SourceMap::default(); + let source_file = source_map.new_source_file( + FileName::Custom(specifier.to_string()), + source.to_string(), + ); + let error_buffer = ErrorBuffer::new(); + let syntax = get_syntax_for_media_type(media_type); + let input = StringInput::from(&*source_file); + let comments = SingleThreadedComments::default(); + + let handler = Handler::with_emitter_and_flags( + Box::new(error_buffer.clone()), + HandlerFlags { + can_emit_warnings: true, + dont_buffer_diagnostics: true, + ..HandlerFlags::default() + }, + ); + + let lexer = Lexer::new(syntax, TARGET, input, Some(&comments)); + let mut parser = swc_ecmascript::parser::Parser::new_from(lexer); + + let sm = &source_map; + let module = parser.parse_module().map_err(move |err| { + let mut diagnostic = err.into_diagnostic(&handler); + diagnostic.emit(); + + ErrBox::from(DiagnosticBuffer::from_error_buffer(error_buffer, |span| { + sm.lookup_char_pos(span.lo) + })) + })?; + let leading_comments = + comments.with_leading(module.span.lo, |comments| comments.to_vec()); + + Ok(ParsedModule { + leading_comments, + module, + source_map: Rc::new(source_map), + comments, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parsed_module_get_leading_comments() { + let specifier = "https://deno.land/x/mod.ts"; + let source = r#"// this is the first comment + // this is the second comment + import * as bar from "./test.ts";"#; + let parsed_module = parse(&specifier, source, MediaType::TypeScript) + .expect("could not parse module"); + let dependencies = parsed_module.get_dependencies(); + let leading_comments: Vec<&str> = dependencies[0] + .leading_comments + .iter() + .map(|c| c.text.as_str()) + .collect(); + assert_eq!( + leading_comments, + vec![" this is the first comment", " this is the second comment"] + ); + } + + #[test] + fn test_transpile() { + let specifier = "https://deno.land/x/mod.ts"; + let source = r#" + enum D { + A, + B, + C, + } + + export class A { + private b: string; + protected c: number = 1; + e: "foo"; + constructor (public d = D.A) { + const e = "foo" as const; + this.e = e; + } + } + "#; + let module = parse(&specifier, source, MediaType::TypeScript) + .expect("could not parse module"); + let (code, maybe_map) = module + .transpile(&EmitTranspileOptions::default()) + .expect("could not strip types"); + assert!(code.starts_with("var D;\n(function(D) {\n")); + assert!( + code.contains("\n//# sourceMappingURL=data:application/json;base64,") + ); + assert!(maybe_map.is_none()); + } + + #[test] + fn test_transpile_tsx() { + let specifier = "https://deno.land/x/mod.ts"; + let source = r#" + export class A { + render() { + return
+ } + } + "#; + let module = parse(&specifier, source, MediaType::TSX) + .expect("could not parse module"); + let (code, _) = module + .transpile(&EmitTranspileOptions::default()) + .expect("could not strip types"); + assert!(code.contains("React.createElement(\"div\", null")); + } + + #[test] + fn test_transpile_decorators() { + let specifier = "https://deno.land/x/mod.ts"; + let source = r#" + function enumerable(value: boolean) { + return function ( + _target: any, + _propertyKey: string, + descriptor: PropertyDescriptor, + ) { + descriptor.enumerable = value; + }; + } + + export class A { + @enumerable(false) + a() { + Test.value; + } + } + "#; + let module = parse(&specifier, source, MediaType::TypeScript) + .expect("could not parse module"); + let (code, _) = module + .transpile(&EmitTranspileOptions::default()) + .expect("could not strip types"); + assert!(code.contains("_applyDecoratedDescriptor(")); + } +} diff --git a/cli/swc_util.rs b/cli/swc_util.rs index 7c7d0183237e30..954082453bdcdc 100644 --- a/cli/swc_util.rs +++ b/cli/swc_util.rs @@ -29,6 +29,10 @@ use swc_ecmascript::parser::StringInput; use swc_ecmascript::parser::Syntax; use swc_ecmascript::parser::TsConfig; use swc_ecmascript::transforms::fixer; +use swc_ecmascript::transforms::helpers; +use swc_ecmascript::transforms::pass::Optional; +use swc_ecmascript::transforms::proposals::decorators; +use swc_ecmascript::transforms::react; use swc_ecmascript::transforms::typescript; use swc_ecmascript::visit::FoldWith; @@ -230,52 +234,6 @@ impl AstParser { }) } - pub fn strip_types( - &self, - file_name: &str, - media_type: MediaType, - source_code: &str, - ) -> Result { - let module = self.parse_module(file_name, media_type, source_code)?; - let program = Program::Module(module); - let mut compiler_pass = - chain!(typescript::strip(), fixer(Some(&self.comments))); - let program = program.fold_with(&mut compiler_pass); - - let mut src_map_buf = vec![]; - let mut buf = vec![]; - { - let writer = Box::new(JsWriter::new( - self.source_map.clone(), - "\n", - &mut buf, - Some(&mut src_map_buf), - )); - let config = swc_ecmascript::codegen::Config { minify: false }; - let mut emitter = swc_ecmascript::codegen::Emitter { - cfg: config, - comments: Some(&self.comments), - cm: self.source_map.clone(), - wr: writer, - }; - program.emit_with(&mut emitter)?; - } - let mut src = String::from_utf8(buf)?; - { - let mut buf = vec![]; - self - .source_map - .build_source_map_from(&mut src_map_buf, None) - .to_writer(&mut buf)?; - let map = String::from_utf8(buf)?; - - src.push_str("//# sourceMappingURL=data:application/json;base64,"); - let encoded_map = base64::encode(map.as_bytes()); - src.push_str(&encoded_map); - } - Ok(src) - } - pub fn get_span_location(&self, span: Span) -> swc_common::Loc { self.source_map.lookup_char_pos(span.lo()) } @@ -290,13 +248,122 @@ impl AstParser { } } -#[test] -fn test_strip_types() { +#[derive(Debug, Clone)] +pub struct EmitTranspileOptions { + /// When emitting a legacy decorator, also emit experimental decorator meta + /// data. Defaults to `false`. + pub emit_metadata: bool, + /// Should the source map be inlined in the emitted code file, or provided + /// as a separate file. Defaults to `true`. + pub inline_source_map: bool, + /// When transforming JSX, what value should be used for the JSX factory. + /// Defaults to `React.createElement`. + pub jsx_factory: String, + /// When transforming JSX, what value should be used for the JSX fragment + /// factory. Defaults to `React.Fragment`. + pub jsx_fragment_factory: String, + /// Should JSX be transformed or preserved. Defaults to `true`. + pub transform_jsx: bool, +} + +impl Default for EmitTranspileOptions { + fn default() -> Self { + EmitTranspileOptions { + emit_metadata: false, + inline_source_map: true, + jsx_factory: "React.createElement".into(), + jsx_fragment_factory: "React.Fragment".into(), + transform_jsx: true, + } + } +} + +pub fn transpile( + file_name: &str, + media_type: MediaType, + source_code: &str, + options: &EmitTranspileOptions, +) -> Result<(String, Option), ErrBox> { let ast_parser = AstParser::default(); - let result = ast_parser - .strip_types("test.ts", MediaType::TypeScript, "const a: number = 10;") - .unwrap(); - assert!(result.starts_with( + let module = ast_parser.parse_module(file_name, media_type, source_code)?; + let program = Program::Module(module); + + let jsx_pass = react::react( + ast_parser.source_map.clone(), + react::Options { + pragma: options.jsx_factory.clone(), + pragma_frag: options.jsx_fragment_factory.clone(), + // this will use `Object.assign()` instead of the `_extends` helper + // when spreading props. + use_builtins: true, + ..Default::default() + }, + ); + let mut passes = chain!( + Optional::new(jsx_pass, options.transform_jsx), + decorators::decorators(decorators::Config { + legacy: true, + emit_metadata: options.emit_metadata, + }), + typescript::strip(), + fixer(Some(&ast_parser.comments)), + ); + + let program = swc_common::GLOBALS.set(&Globals::new(), || { + helpers::HELPERS.set(&helpers::Helpers::new(false), || { + program.fold_with(&mut passes) + }) + }); + + let mut src_map_buf = vec![]; + let mut buf = vec![]; + { + let writer = Box::new(JsWriter::new( + ast_parser.source_map.clone(), + "\n", + &mut buf, + Some(&mut src_map_buf), + )); + let config = swc_ecmascript::codegen::Config { minify: false }; + let mut emitter = swc_ecmascript::codegen::Emitter { + cfg: config, + comments: Some(&ast_parser.comments), + cm: ast_parser.source_map.clone(), + wr: writer, + }; + program.emit_with(&mut emitter)?; + } + let mut src = String::from_utf8(buf)?; + let mut map: Option = None; + { + let mut buf = Vec::new(); + ast_parser + .source_map + .build_source_map_from(&mut src_map_buf, None) + .to_writer(&mut buf)?; + + if options.inline_source_map { + src.push_str("//# sourceMappingURL=data:application/json;base64,"); + let encoded_map = base64::encode(buf); + src.push_str(&encoded_map); + } else { + map = Some(String::from_utf8(buf)?); + } + } + Ok((src, map)) +} + +#[test] +fn test_transpile() { + let result = transpile( + "test.ts", + MediaType::TypeScript, + "const a: number = 10;", + &EmitTranspileOptions::default(), + ) + .unwrap(); + let (src, _maybe_source_map) = result; + assert!(src.starts_with( "const a = 10;\n//# sourceMappingURL=data:application/json;base64," )); } diff --git a/cli/tsc.rs b/cli/tsc.rs index cc902d196db4fb..79967369497ad5 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -18,6 +18,7 @@ use crate::permissions::Permissions; use crate::source_maps::SourceMapGetter; use crate::startup_data; use crate::state::State; +use crate::swc_util; use crate::swc_util::AstParser; use crate::swc_util::Location; use crate::swc_util::SwcDiagnosticBuffer; @@ -418,6 +419,16 @@ struct CompileResponse { stats: Option>, } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct TranspileTsOptions { + check_js: bool, + emit_decorator_metadata: bool, + jsx: String, + jsx_factory: String, + jsx_fragment_factory: String, +} + // TODO(bartlomieju): possible deduplicate once TS refactor is stabilized #[derive(Deserialize)] #[serde(rename_all = "camelCase")] @@ -795,12 +806,39 @@ impl TsCompiler { let mut emit_map = HashMap::new(); + let mut compiler_options = json!({ + "checkJs": false, + "emitDecoratorMetadata": false, + "jsx": "react", + "jsxFactory": "React.createElement", + "jsxFragmentFactory": "React.Fragment", + }); + + let compiler_config = self.config.clone(); + + tsc_config::json_merge(&mut compiler_options, &compiler_config.options); + + warn_ignored_options( + compiler_config.maybe_ignored_options, + compiler_config.path.as_ref().unwrap(), + ); + + let compiler_options: TranspileTsOptions = + serde_json::from_value(compiler_options)?; + + let transpile_options = swc_util::EmitTranspileOptions { + emit_metadata: compiler_options.emit_decorator_metadata, + inline_source_map: true, + jsx_factory: compiler_options.jsx_factory, + jsx_fragment_factory: compiler_options.jsx_fragment_factory, + transform_jsx: compiler_options.jsx == "react", + }; for source_file in source_files { - let parser = AstParser::default(); - let stripped_source = parser.strip_types( + let (stripped_source, _maybe_source_map) = swc_util::transpile( &source_file.file_name, MediaType::TypeScript, &source_file.source_code, + &transpile_options, )?; // TODO(bartlomieju): this is superfluous, just to make caching function happy