From 073e14746ef062566fc9224aced66714559e201b Mon Sep 17 00:00:00 2001 From: Geoffrey Booth Date: Sat, 19 Nov 2016 11:13:30 -0800 Subject: [PATCH] Triple backticks to allow creation of JavaScript blocks (#4357) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Support JavaScript code blocks set apart by triple backticks (``` ... ```) * Add test for escaped backticks * Remove TODOs for things we’re never going to support * Convert escaped backticks to backticks; update tests * Block inline JavaScript can end with an escaped backtick character * Updated JavaScript token regexes per @lydell * In JavaScript blocks, escape backslashes when they immediately precede backticks; additional tests * Test that we don’t break backslash escaping in JavaScript literals --- lib/coffee-script/lexer.js | 15 ++++--- src/lexer.coffee | 16 +++++-- test/javascript_literals.coffee | 74 +++++++++++++++++++++++++++++---- 3 files changed, 89 insertions(+), 16 deletions(-) diff --git a/lib/coffee-script/lexer.js b/lib/coffee-script/lexer.js index e9984ee013..c6309bc0c9 100644 --- a/lib/coffee-script/lexer.js +++ b/lib/coffee-script/lexer.js @@ -1,6 +1,6 @@ // Generated by CoffeeScript 1.11.1 (function() { - var BOM, BOOL, CALLABLE, CODE, COFFEE_ALIASES, COFFEE_ALIAS_MAP, COFFEE_KEYWORDS, COMMENT, COMPARE, COMPOUND_ASSIGN, HERECOMMENT_ILLEGAL, HEREDOC_DOUBLE, HEREDOC_INDENT, HEREDOC_SINGLE, HEREGEX, HEREGEX_OMIT, IDENTIFIER, INDENTABLE_CLOSERS, INDEXABLE, INVALID_ESCAPE, INVERSES, JSTOKEN, JS_KEYWORDS, LEADING_BLANK_LINE, LINE_BREAK, LINE_CONTINUER, Lexer, MATH, MULTI_DENT, NOT_REGEX, NUMBER, OPERATOR, POSSIBLY_DIVISION, REGEX, REGEX_FLAGS, REGEX_ILLEGAL, RELATION, RESERVED, Rewriter, SHIFT, SIMPLE_STRING_OMIT, STRICT_PROSCRIBED, STRING_DOUBLE, STRING_OMIT, STRING_SINGLE, STRING_START, TRAILING_BLANK_LINE, TRAILING_SPACES, UNARY, UNARY_MATH, VALID_FLAGS, WHITESPACE, compact, count, invertLiterate, isUnassignable, key, locationDataToString, ref, ref1, repeat, starts, throwSyntaxError, + var BOM, BOOL, CALLABLE, CODE, COFFEE_ALIASES, COFFEE_ALIAS_MAP, COFFEE_KEYWORDS, COMMENT, COMPARE, COMPOUND_ASSIGN, HERECOMMENT_ILLEGAL, HEREDOC_DOUBLE, HEREDOC_INDENT, HEREDOC_SINGLE, HEREGEX, HEREGEX_OMIT, HERE_JSTOKEN, IDENTIFIER, INDENTABLE_CLOSERS, INDEXABLE, INVALID_ESCAPE, INVERSES, JSTOKEN, JS_KEYWORDS, LEADING_BLANK_LINE, LINE_BREAK, LINE_CONTINUER, Lexer, MATH, MULTI_DENT, NOT_REGEX, NUMBER, OPERATOR, POSSIBLY_DIVISION, REGEX, REGEX_FLAGS, REGEX_ILLEGAL, RELATION, RESERVED, Rewriter, SHIFT, SIMPLE_STRING_OMIT, STRICT_PROSCRIBED, STRING_DOUBLE, STRING_OMIT, STRING_SINGLE, STRING_START, TRAILING_BLANK_LINE, TRAILING_SPACES, UNARY, UNARY_MATH, VALID_FLAGS, WHITESPACE, compact, count, invertLiterate, isUnassignable, key, locationDataToString, ref, ref1, repeat, starts, throwSyntaxError, indexOf = [].indexOf || function(item) { for (var i = 0, l = this.length; i < l; i++) { if (i in this && this[i] === item) return i; } return -1; }, slice = [].slice; @@ -337,11 +337,14 @@ Lexer.prototype.jsToken = function() { var match, script; - if (!(this.chunk.charAt(0) === '`' && (match = JSTOKEN.exec(this.chunk)))) { + if (!(this.chunk.charAt(0) === '`' && (match = HERE_JSTOKEN.exec(this.chunk) || JSTOKEN.exec(this.chunk)))) { return 0; } - this.token('JS', (script = match[0]).slice(1, -1), 0, script.length); - return script.length; + script = match[1].replace(/\\+(`|$)/g, function(string) { + return string.slice(-Math.ceil(string.length / 2)); + }); + this.token('JS', script, 0, match[0].length); + return match[0].length; }; Lexer.prototype.regexToken = function() { @@ -1009,7 +1012,9 @@ MULTI_DENT = /^(?:\n[^\n\S]*)+/; - JSTOKEN = /^`[^\\`]*(?:\\.[^\\`]*)*`/; + JSTOKEN = /^`(?!``)((?:[^`\\]|\\[\s\S])*)`/; + + HERE_JSTOKEN = /^```((?:[^`\\]|\\[\s\S]|`(?!``))*)```/; STRING_START = /^(?:'''|"""|'|")/; diff --git a/src/lexer.coffee b/src/lexer.coffee index 1613a0c715..3c3962d8cf 100644 --- a/src/lexer.coffee +++ b/src/lexer.coffee @@ -292,9 +292,16 @@ exports.Lexer = class Lexer # Matches JavaScript interpolated directly into the source via backticks. jsToken: -> - return 0 unless @chunk.charAt(0) is '`' and match = JSTOKEN.exec @chunk - @token 'JS', (script = match[0])[1...-1], 0, script.length - script.length + return 0 unless @chunk.charAt(0) is '`' and + (match = HERE_JSTOKEN.exec(@chunk) or JSTOKEN.exec(@chunk)) + # Convert escaped backticks to backticks, and escaped backslashes + # just before escaped backticks to backslashes + script = match[1].replace /\\+(`|$)/g, (string) -> + # `string` is always a value like '\`', '\\\`', '\\\\\`', etc. + # By reducing it to its latter half, we turn '\`' to '`', '\\\`' to '\`', etc. + string[-Math.ceil(string.length / 2)..] + @token 'JS', script, 0, match[0].length + match[0].length # Matches regular expression literals, as well as multiline extended ones. # Lexing regular expressions is difficult to distinguish from division, so we @@ -900,7 +907,8 @@ CODE = /^[-=]>/ MULTI_DENT = /^(?:\n[^\n\S]*)+/ -JSTOKEN = /^`[^\\`]*(?:\\.[^\\`]*)*`/ +JSTOKEN = ///^ `(?!``) ((?: [^`\\] | \\[\s\S] )*) ` /// +HERE_JSTOKEN = ///^ ``` ((?: [^`\\] | \\[\s\S] | `(?!``) )*) ``` /// # String-matching-regexes. STRING_START = /^(?:'''|"""|'|")/ diff --git a/test/javascript_literals.coffee b/test/javascript_literals.coffee index 91a9efddeb..45c68535c1 100644 --- a/test/javascript_literals.coffee +++ b/test/javascript_literals.coffee @@ -1,10 +1,70 @@ -# Javascript Literals +# JavaScript Literals # ------------------- -# TODO: refactor javascript literal tests -# TODO: add indexing and method invocation tests: `[1]`[0] is 1, `function(){}`.call() +test "inline JavaScript is evaluated", -> + eq '\\`', ` + // Inline JS + "\\\\\`" + ` -eq '\\`', ` - // Inline JS - "\\\`" -` +test "escaped backticks are output correctly", -> + `var a = \`2 + 2 = ${4}\`` + eq a, '2 + 2 = 4' + +test "backslashes before a newline don’t break JavaScript blocks", -> + `var a = \`To be, or not\\ + to be.\`` + eq a, ''' + To be, or not\\ + to be.''' + +test "block inline JavaScript is evaluated", -> + ``` + var a = 1; + var b = 2; + ``` + c = 3 + ```var d = 4;``` + eq a + b + c + d, 10 + +test "block inline JavaScript containing backticks", -> + ``` + // This is a comment with `backticks` + var a = 42; + var b = `foo ${'bar'}`; + var c = 3; + var d = 'foo`bar`'; + ``` + eq a + c, 45 + eq b, 'foo bar' + eq d, 'foo`bar`' + +test "block JavaScript can end with an escaped backtick character", -> + ```var a = \`hello\```` + ``` + var b = \`world${'!'}\```` + eq a, 'hello' + eq b, 'world!' + +test "JavaScript block only escapes backslashes followed by backticks", -> + eq `'\\\n'`, '\\\n' + +test "escaped JavaScript blocks speed round", -> + # The following has escaped backslashes because they’re required in strings, but the intent is this: + # `hello` → hello; + # `\`hello\`` → `hello`; + # `\`Escaping backticks in JS: \\\`hello\\\`\`` → `Escaping backticks in JS: \`hello\``; + # `Single backslash: \ ` → Single backslash: \ ; + # `Double backslash: \\ ` → Double backslash: \\ ; + # `Single backslash at EOS: \\` → Single backslash at EOS: \; + # `Double backslash at EOS: \\\\` → Double backslash at EOS: \\; + for [input, output] in [ + ['`hello`', 'hello;'] + ['`\\`hello\\``', '`hello`;'] + ['`\\`Escaping backticks in JS: \\\\\\`hello\\\\\\`\\``', '`Escaping backticks in JS: \\`hello\\``;'] + ['`Single backslash: \\ `', 'Single backslash: \\ ;'] + ['`Double backslash: \\\\ `', 'Double backslash: \\\\ ;'] + ['`Single backslash at EOS: \\\\`', 'Single backslash at EOS: \\;'] + ['`Double backslash at EOS: \\\\\\\\`', 'Double backslash at EOS: \\\\;'] + ] + eq CoffeeScript.compile(input, bare: yes), "#{output}\n\n"