From 671fdacb23aab6129a7dd463c4e9c84d427b1ed6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Sat, 22 Aug 2020 21:18:55 +0200 Subject: [PATCH 1/2] backport ast::ParsedModule --- cli/ast.rs | 449 ++++++++++++++++++++++++++++++++++++++++++++++++++++ cli/lint.rs | 4 +- cli/main.rs | 3 +- cli/tsc.rs | 8 +- 4 files changed, 458 insertions(+), 6 deletions(-) create mode 100644 cli/ast.rs diff --git a/cli/ast.rs b/cli/ast.rs new file mode 100644 index 00000000000000..c60492b4e8873e --- /dev/null +++ b/cli/ast.rs @@ -0,0 +1,449 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +use crate::msg::MediaType; + +use deno_core::ErrBox; +use std::error::Error; +use std::fmt; +use std::rc::Rc; +use std::sync::Arc; +use std::sync::RwLock; +use swc_common::chain; +use swc_common::comments::Comment; +use swc_common::comments::SingleThreadedComments; +use swc_common::errors::Diagnostic; +use swc_common::errors::DiagnosticBuilder; +use swc_common::errors::Emitter; +use swc_common::errors::Handler; +use swc_common::errors::HandlerFlags; +use swc_common::FileName; +use swc_common::Globals; +use swc_common::Loc; +pub use swc_common::SourceMap; +pub use swc_common::Span; +use swc_ecmascript::ast::Module; +use swc_ecmascript::ast::Program; +use swc_ecmascript::codegen::text_writer::JsWriter; +use swc_ecmascript::codegen::Config; +use swc_ecmascript::codegen::Node; +use swc_ecmascript::parser::lexer::Lexer; +use swc_ecmascript::parser::EsConfig; +use swc_ecmascript::parser::JscTarget; +use swc_ecmascript::parser::StringInput; +use swc_ecmascript::parser::Syntax; +use swc_ecmascript::parser::TsConfig; +use swc_ecmascript::transforms::fixer; +use swc_ecmascript::transforms::helpers; +use swc_ecmascript::transforms::pass::Optional; +use swc_ecmascript::transforms::proposals::decorators; +use swc_ecmascript::transforms::react; +use swc_ecmascript::transforms::typescript; +use swc_ecmascript::visit::FoldWith; + +static TARGET: JscTarget = JscTarget::Es2020; + +#[derive(Debug, Clone, Default, Eq, PartialEq)] +pub struct Location { + pub filename: String, + pub line: usize, + pub col: usize, +} + +impl Into for Loc { + fn into(self) -> Location { + use swc_common::FileName::*; + + let filename = match &self.file.name { + Real(path_buf) => path_buf.to_string_lossy().to_string(), + Custom(str_) => str_.to_string(), + _ => panic!("invalid filename"), + }; + + Location { + filename, + line: self.line, + col: self.col_display, + } + } +} + +/// A buffer for collecting diagnostic messages from the AST parser. +#[derive(Debug)] +pub struct DiagnosticBuffer(Vec); + +impl Error for DiagnosticBuffer {} + +impl fmt::Display for DiagnosticBuffer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s = self.0.join(","); + f.pad(&s) + } +} + +impl DiagnosticBuffer { + pub fn from_error_buffer(error_buffer: ErrorBuffer, get_loc: F) -> Self + where + F: Fn(Span) -> Loc, + { + let s = error_buffer.0.read().unwrap().clone(); + let diagnostics = s + .iter() + .map(|d| { + let mut msg = d.message(); + + if let Some(span) = d.span.primary_span() { + let loc = get_loc(span); + let file_name = match &loc.file.name { + FileName::Custom(n) => n, + _ => unreachable!(), + }; + msg = format!( + "{} at {}:{}:{}", + msg, file_name, loc.line, loc.col_display + ); + } + + msg + }) + .collect::>(); + + Self(diagnostics) + } +} + +#[derive(Debug, Clone)] +pub struct ErrorBuffer(Arc>>); + +impl ErrorBuffer { + pub fn new() -> Self { + Self(Arc::new(RwLock::new(Vec::new()))) + } +} + +impl Emitter for ErrorBuffer { + fn emit(&mut self, db: &DiagnosticBuilder) { + self.0.write().unwrap().push((**db).clone()); + } +} + +fn get_es_config(jsx: bool) -> EsConfig { + EsConfig { + class_private_methods: true, + class_private_props: true, + class_props: true, + dynamic_import: true, + export_default_from: true, + export_namespace_from: true, + import_meta: true, + jsx, + nullish_coalescing: true, + num_sep: true, + optional_chaining: true, + top_level_await: true, + ..EsConfig::default() + } +} + +fn get_ts_config(tsx: bool) -> TsConfig { + TsConfig { + decorators: true, + dynamic_import: true, + tsx, + ..TsConfig::default() + } +} + +pub fn get_syntax_for_media_type(media_type: MediaType) -> Syntax { + match media_type { + MediaType::JavaScript => Syntax::Es(get_es_config(false)), + MediaType::JSX => Syntax::Es(get_es_config(true)), + MediaType::TypeScript => Syntax::Typescript(get_ts_config(false)), + MediaType::TSX => Syntax::Typescript(get_ts_config(true)), + _ => Syntax::Es(get_es_config(false)), + } +} + +#[derive(Debug, Clone)] +pub struct EmitTranspileOptions { + /// When emitting a legacy decorator, also emit experimental decorator meta + /// data. Defaults to `false`. + pub emit_metadata: bool, + /// Should the source map be inlined in the emitted code file, or provided + /// as a separate file. Defaults to `true`. + pub inline_source_map: bool, + /// When transforming JSX, what value should be used for the JSX factory. + /// Defaults to `React.createElement`. + pub jsx_factory: String, + /// When transforming JSX, what value should be used for the JSX fragment + /// factory. Defaults to `React.Fragment`. + pub jsx_fragment_factory: String, + /// Should JSX be transformed or preserved. Defaults to `true`. + pub transform_jsx: bool, +} + +impl Default for EmitTranspileOptions { + fn default() -> Self { + EmitTranspileOptions { + emit_metadata: false, + inline_source_map: true, + jsx_factory: "React.createElement".into(), + jsx_fragment_factory: "React.Fragment".into(), + transform_jsx: true, + } + } +} + +/// A logical structure to hold the value of a parsed module for further +/// processing. +#[derive(Clone)] +pub struct ParsedModule { + comments: SingleThreadedComments, + /// Leading comments for the module, where references and pragmas might be + /// contained. + pub leading_comments: Vec, + pub module: Module, + /// The AST source for the module. + pub source_map: Rc, +} + +impl fmt::Debug for ParsedModule { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "ParsedModule {{ }}") + } +} + +impl ParsedModule { + /// Transpile a module to JavaScript supportable by Deno, erasing TypeScript + /// types and performing other transforms. + pub fn transpile( + self, + options: &EmitTranspileOptions, + ) -> Result<(String, Option), ErrBox> { + let program = Program::Module(self.module); + + let jsx_pass = react::react( + self.source_map.clone(), + react::Options { + pragma: options.jsx_factory.clone(), + pragma_frag: options.jsx_fragment_factory.clone(), + // this will use `Object.assign()` instead of the `_extends` helper + // when spreading props. + use_builtins: true, + ..Default::default() + }, + ); + let mut passes = chain!( + Optional::new(jsx_pass, options.transform_jsx), + decorators::decorators(decorators::Config { + legacy: true, + emit_metadata: options.emit_metadata, + }), + typescript::strip(), + fixer(Some(&self.comments)), + ); + + let program = swc_common::GLOBALS.set(&Globals::new(), || { + helpers::HELPERS.set(&helpers::Helpers::new(false), || { + program.fold_with(&mut passes) + }) + }); + + let mut mappings = Vec::new(); + let mut buf = Vec::new(); + { + let wr = Box::new(JsWriter::new( + self.source_map.clone(), + "\n", + &mut buf, + Some(&mut mappings), + )); + let cfg = Config { minify: false }; + let mut emitter = swc_ecmascript::codegen::Emitter { + cfg, + comments: Some(&self.comments), + cm: self.source_map.clone(), + wr, + }; + program.emit_with(&mut emitter)?; + } + let mut code = String::from_utf8(buf)?; + let mut map: Option = None; + { + let mut buf = Vec::new(); + self + .source_map + .build_source_map_from(&mut mappings, None) + .to_writer(&mut buf)?; + + if options.inline_source_map { + code.push_str("//# sourceMappingURL=data:application/json;base64,"); + let encoded_map = base64::encode(buf); + code.push_str(&encoded_map); + } else { + map = Some(String::from_utf8(buf)?); + } + } + + Ok((code, map)) + } +} + +/// For a given specifier, source, and media type, parse the source of the +/// module and return a representation which can be further processed. +/// +/// # Arguments +/// +/// - `specifier` - The module specifier for the module. +/// - `source` - The source code for the module. +/// - `media_type` - The media type for the module. +/// +pub fn parse( + specifier: &str, + source: &str, + media_type: MediaType, +) -> Result { + let source_map = SourceMap::default(); + let source_file = source_map.new_source_file( + FileName::Custom(specifier.to_string()), + source.to_string(), + ); + let error_buffer = ErrorBuffer::new(); + let syntax = get_syntax_for_media_type(media_type); + let input = StringInput::from(&*source_file); + let comments = SingleThreadedComments::default(); + + let handler = Handler::with_emitter_and_flags( + Box::new(error_buffer.clone()), + HandlerFlags { + can_emit_warnings: true, + dont_buffer_diagnostics: true, + ..HandlerFlags::default() + }, + ); + + let lexer = Lexer::new(syntax, TARGET, input, Some(&comments)); + let mut parser = swc_ecmascript::parser::Parser::new_from(lexer); + + let sm = &source_map; + let module = parser.parse_module().map_err(move |err| { + let mut diagnostic = err.into_diagnostic(&handler); + diagnostic.emit(); + + ErrBox::from(DiagnosticBuffer::from_error_buffer(error_buffer, |span| { + sm.lookup_char_pos(span.lo) + })) + })?; + let leading_comments = + comments.with_leading(module.span.lo, |comments| comments.to_vec()); + + Ok(ParsedModule { + leading_comments, + module, + source_map: Rc::new(source_map), + comments, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parsed_module_get_leading_comments() { + let specifier = "https://deno.land/x/mod.ts"; + let source = r#"// this is the first comment + // this is the second comment + import * as bar from "./test.ts";"#; + let parsed_module = parse(&specifier, source, MediaType::TypeScript) + .expect("could not parse module"); + let dependencies = parsed_module.get_dependencies(); + let leading_comments: Vec<&str> = dependencies[0] + .leading_comments + .iter() + .map(|c| c.text.as_str()) + .collect(); + assert_eq!( + leading_comments, + vec![" this is the first comment", " this is the second comment"] + ); + } + + #[test] + fn test_transpile() { + let specifier = "https://deno.land/x/mod.ts"; + let source = r#" + enum D { + A, + B, + C, + } + + export class A { + private b: string; + protected c: number = 1; + e: "foo"; + constructor (public d = D.A) { + const e = "foo" as const; + this.e = e; + } + } + "#; + let module = parse(&specifier, source, MediaType::TypeScript) + .expect("could not parse module"); + let (code, maybe_map) = module + .transpile(&EmitTranspileOptions::default()) + .expect("could not strip types"); + assert!(code.starts_with("var D;\n(function(D) {\n")); + assert!( + code.contains("\n//# sourceMappingURL=data:application/json;base64,") + ); + assert!(maybe_map.is_none()); + } + + #[test] + fn test_transpile_tsx() { + let specifier = "https://deno.land/x/mod.ts"; + let source = r#" + export class A { + render() { + return
+ } + } + "#; + let module = parse(&specifier, source, MediaType::TSX) + .expect("could not parse module"); + let (code, _) = module + .transpile(&EmitTranspileOptions::default()) + .expect("could not strip types"); + assert!(code.contains("React.createElement(\"div\", null")); + } + + #[test] + fn test_transpile_decorators() { + let specifier = "https://deno.land/x/mod.ts"; + let source = r#" + function enumerable(value: boolean) { + return function ( + _target: any, + _propertyKey: string, + descriptor: PropertyDescriptor, + ) { + descriptor.enumerable = value; + }; + } + + export class A { + @enumerable(false) + a() { + Test.value; + } + } + "#; + let module = parse(&specifier, source, MediaType::TypeScript) + .expect("could not parse module"); + let (code, _) = module + .transpile(&EmitTranspileOptions::default()) + .expect("could not strip types"); + assert!(code.contains("_applyDecoratedDescriptor(")); + } +} diff --git a/cli/lint.rs b/cli/lint.rs index 4e12d582ce8d5e..0e90b1d5bcb1ab 100644 --- a/cli/lint.rs +++ b/cli/lint.rs @@ -6,12 +6,12 @@ //! At the moment it is only consumed using CLI but in //! the future it can be easily extended to provide //! the same functions as ops available in JS runtime. +use crate::ast; use crate::colors; use crate::file_fetcher::map_file_extension; use crate::fmt::collect_files; use crate::fmt::run_parallelized; use crate::fmt_errors; -use crate::swc_util; use deno_core::ErrBox; use deno_lint::diagnostic::LintDiagnostic; use deno_lint::linter::Linter; @@ -179,7 +179,7 @@ fn lint_file(file_path: PathBuf) -> Result, ErrBox> { let file_name = file_path.to_string_lossy().to_string(); let source_code = fs::read_to_string(&file_path)?; let media_type = map_file_extension(&file_path); - let syntax = swc_util::get_syntax_for_media_type(media_type); + let syntax = ast::get_syntax_for_media_type(media_type); let lint_rules = get_rules(); let mut linter = create_linter(syntax, lint_rules); diff --git a/cli/main.rs b/cli/main.rs index 353b7763f95d4d..7908f84f01a94b 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -23,6 +23,7 @@ extern crate serde_derive; extern crate tokio; extern crate url; +mod ast; mod checksum; pub mod colors; pub mod deno_dir; @@ -546,7 +547,7 @@ async fn doc_command( let loader = Box::new(global_state.file_fetcher.clone()); let doc_parser = doc::DocParser::new(loader, private); let media_type = map_file_extension(&PathBuf::from(&source_file)); - let syntax = swc_util::get_syntax_for_media_type(media_type); + let syntax = ast::get_syntax_for_media_type(media_type); let parse_result = if source_file == "--builtin" { doc_parser.parse_source( diff --git a/cli/tsc.rs b/cli/tsc.rs index 5fa8b67a35bc67..8196ed3e141dcf 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -1,4 +1,5 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +use crate::ast; use crate::colors; use crate::diagnostics::Diagnostic; use crate::diagnostics::DiagnosticItem; @@ -753,12 +754,13 @@ impl TsCompiler { let mut emit_map = HashMap::new(); for source_file in source_files { - let parser = AstParser::default(); - let stripped_source = parser.strip_types( + let parsed_module = ast::parse( &source_file.file_name, - MediaType::TypeScript, &source_file.source_code, + MediaType::TypeScript, )?; + let (stripped_source, _) = + parsed_module.transpile(&ast::EmitTranspileOptions::default())?; // TODO(bartlomieju): this is superfluous, just to make caching function happy let emitted_filename = PathBuf::from(&source_file.file_name) From a1f50172aa0e6446f0a55ddfd7cd61fa03fff0ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Sun, 23 Aug 2020 14:03:45 +0200 Subject: [PATCH 2/2] bump --- cli/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 631b74b8994ea8..a85d1a619f77b1 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -55,7 +55,7 @@ serde_json = { version = "1.0.57", features = [ "preserve_order" ] } sys-info = "0.7.0" sourcemap = "6.0.1" swc_common = { version = "=0.9.1", features = ["sourcemap"] } -swc_ecmascript = { version = "=0.5.0", features = ["codegen", "parser", "transforms", "visit"] } +swc_ecmascript = { version = "=0.5.0", features = ["codegen", "parser", "transforms", "react", "visit"] } tempfile = "3.1.0" termcolor = "1.1.0" tokio = { version = "0.2.22", features = ["full"] }