From 458fe54899c942f69f26cf789fa5339627f2fa81 Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Tue, 30 Apr 2019 13:56:34 -0400 Subject: [PATCH] Add `codemod` mode for @glimmer/syntax preprocess. Currently, setting `mode` to `'codemod'` will: * Disable `entity` parsing in simple-html-tokenizer (ensures that parsing + printing is not lossy) * Enable `ignoreStandalone` mode in handlebars parser (ensures that standalone whitespace after a block opening or closing is not stripped). In the future we can do other things to make the codemod world even better, but this is a really good first step (IMHO). --- .../@glimmer/syntax/lib/generation/print.ts | 122 +++++---- packages/@glimmer/syntax/lib/parser.ts | 7 +- .../lib/parser/tokenizer-event-handlers.ts | 42 ++- packages/@glimmer/syntax/lib/types/nodes.ts | 2 + .../syntax/test/generation/print-test.ts | 258 +++++++----------- 5 files changed, 209 insertions(+), 222 deletions(-) diff --git a/packages/@glimmer/syntax/lib/generation/print.ts b/packages/@glimmer/syntax/lib/generation/print.ts index 897242fad..9d4577de0 100644 --- a/packages/@glimmer/syntax/lib/generation/print.ts +++ b/packages/@glimmer/syntax/lib/generation/print.ts @@ -7,10 +7,63 @@ function unreachable(): never { throw new Error('unreachable'); } -export default function build(ast: AST.Node): string { +interface PrinterOptions { + entityEncoding: 'transformed' | 'raw'; +} + +export default function build( + ast: AST.Node, + options: PrinterOptions = { entityEncoding: 'transformed' } +): string { if (!ast) { return ''; } + + function buildEach(asts: AST.Node[]): string[] { + return asts.map(node => build(node, options)); + } + + function pathParams(ast: AST.Node): string { + let path: string; + + switch (ast.type) { + case 'MustacheStatement': + case 'SubExpression': + case 'ElementModifierStatement': + case 'BlockStatement': + path = build(ast.path, options); + break; + case 'PartialStatement': + path = build(ast.name, options); + break; + default: + return unreachable(); + } + + return compactJoin([path, buildEach(ast.params).join(' '), build(ast.hash, options)], ' '); + } + + function compactJoin(array: Option[], delimiter?: string): string { + return compact(array).join(delimiter || ''); + } + + function blockParams(block: AST.BlockStatement): Option { + const params = block.program.blockParams; + if (params.length) { + return ` as |${params.join(' ')}|`; + } + + return null; + } + + function openBlock(block: AST.BlockStatement): string { + return ['{{#', pathParams(block), blockParams(block), '}}'].join(''); + } + + function closeBlock(block: any): string { + return ['{{/', build(block.path, options), '}}'].join(''); + } + const output: string[] = []; switch (ast.type) { @@ -60,29 +113,33 @@ export default function build(ast: AST.Node): string { if (ast.value.type === 'TextNode') { if (ast.value.chars !== '') { output.push(ast.name, '='); - output.push('"', escapeAttrValue(ast.value.chars), '"'); + output.push( + '"', + options.entityEncoding === 'raw' ? ast.value.chars : escapeAttrValue(ast.value.chars), + '"' + ); } else { output.push(ast.name); } } else { output.push(ast.name, '='); // ast.value is mustache or concat - output.push(build(ast.value)); + output.push(build(ast.value, options)); } break; case 'ConcatStatement': output.push('"'); ast.parts.forEach((node: AST.TextNode | AST.MustacheStatement) => { if (node.type === 'TextNode') { - output.push(escapeAttrValue(node.chars)); + output.push(options.entityEncoding === 'raw' ? node.chars : escapeAttrValue(node.chars)); } else { - output.push(build(node)); + output.push(build(node, options)); } }); output.push('"'); break; case 'TextNode': - output.push(escapeText(ast.chars)); + output.push(options.entityEncoding === 'raw' ? ast.chars : escapeText(ast.chars)); break; case 'MustacheStatement': { @@ -120,13 +177,13 @@ export default function build(ast: AST.Node): string { lines.push(openBlock(ast)); } - lines.push(build(ast.program)); + lines.push(build(ast.program, options)); if (ast.inverse) { if (!ast.inverse.chained) { lines.push('{{else}}'); } - lines.push(build(ast.inverse)); + lines.push(build(ast.inverse, options)); } if (!ast.chained) { @@ -171,7 +228,7 @@ export default function build(ast: AST.Node): string { output.push( ast.pairs .map(pair => { - return build(pair); + return build(pair, options); }) .join(' ') ); @@ -179,7 +236,7 @@ export default function build(ast: AST.Node): string { break; case 'HashPair': { - output.push(`${ast.key}=${build(ast.value)}`); + output.push(`${ast.key}=${build(ast.value, options)}`); } break; } @@ -195,48 +252,3 @@ function compact(array: Option[]): string[] { }); return newArray; } - -function buildEach(asts: AST.Node[]): string[] { - return asts.map(build); -} - -function pathParams(ast: AST.Node): string { - let path: string; - - switch (ast.type) { - case 'MustacheStatement': - case 'SubExpression': - case 'ElementModifierStatement': - case 'BlockStatement': - path = build(ast.path); - break; - case 'PartialStatement': - path = build(ast.name); - break; - default: - return unreachable(); - } - - return compactJoin([path, buildEach(ast.params).join(' '), build(ast.hash)], ' '); -} - -function compactJoin(array: Option[], delimiter?: string): string { - return compact(array).join(delimiter || ''); -} - -function blockParams(block: AST.BlockStatement): Option { - const params = block.program.blockParams; - if (params.length) { - return ` as |${params.join(' ')}|`; - } - - return null; -} - -function openBlock(block: AST.BlockStatement): string { - return ['{{#', pathParams(block), blockParams(block), '}}'].join(''); -} - -function closeBlock(block: any): string { - return ['{{/', build(block.path), '}}'].join(''); -} diff --git a/packages/@glimmer/syntax/lib/parser.ts b/packages/@glimmer/syntax/lib/parser.ts index a3b233052..a64209159 100644 --- a/packages/@glimmer/syntax/lib/parser.ts +++ b/packages/@glimmer/syntax/lib/parser.ts @@ -8,8 +8,6 @@ import * as HBS from './types/handlebars-ast'; import { Option } from '@glimmer/interfaces'; import { assert, expect } from '@glimmer/util'; -const entityParser = new EntityParser(namedCharRefs); - export type Element = AST.Template | AST.Block | AST.ElementNode; export interface Tag { @@ -39,10 +37,11 @@ export abstract class Parser { public currentNode: Option< AST.CommentStatement | AST.TextNode | Tag<'StartTag' | 'EndTag'> > = null; - public tokenizer = new EventedTokenizer(this, entityParser); + public tokenizer: EventedTokenizer; - constructor(source: string) { + constructor(source: string, entityParser = new EntityParser(namedCharRefs)) { this.source = source.split(/(?:\r\n?|\n)/g); + this.tokenizer = new EventedTokenizer(this, entityParser); } abstract Program(node: HBS.Program): HBS.Output<'Program'>; diff --git a/packages/@glimmer/syntax/lib/parser/tokenizer-event-handlers.ts b/packages/@glimmer/syntax/lib/parser/tokenizer-event-handlers.ts index e9d71802e..cc236f13d 100644 --- a/packages/@glimmer/syntax/lib/parser/tokenizer-event-handlers.ts +++ b/packages/@glimmer/syntax/lib/parser/tokenizer-event-handlers.ts @@ -12,6 +12,7 @@ import Walker from '../traversal/walker'; import * as handlebars from 'handlebars'; import { assign } from '@glimmer/util'; import { NodeVisitor } from '../traversal/visitor'; +import { EntityParser } from 'simple-html-tokenizer'; export const voidMap: { [tagName: string]: boolean; @@ -338,13 +339,26 @@ export interface ASTPluginEnvironment { meta?: object; syntax: Syntax; } +interface HandlebarsParseOptions { + srcName?: string; + ignoreStandalone?: boolean; +} export interface PreprocessOptions { meta?: unknown; plugins?: { ast?: ASTPluginBuilder[]; }; - parseOptions?: object; + parseOptions?: HandlebarsParseOptions; + + /** + Useful for specifying a group of options together. + + When `'codemod'` we disable all whitespace control in handlebars + (to preserve as much as possible) and we also avoid any + escaping/unescaping of HTML entity codes. + */ + mode?: 'codemod' | 'precompile'; } export interface Syntax { @@ -363,10 +377,28 @@ const syntax: Syntax = { Walker, }; -export function preprocess(html: string, options?: PreprocessOptions): AST.Template { - const parseOptions = options ? options.parseOptions : {}; - let ast = typeof html === 'object' ? html : (handlebars.parse(html, parseOptions) as HBS.Program); - let program = new TokenizerEventHandlers(html).acceptTemplate(ast); +export function preprocess(html: string, options: PreprocessOptions = {}): AST.Template { + let mode = options.mode || 'precompile'; + + let ast: HBS.Program; + if (typeof html === 'object') { + ast = html; + } else { + let parseOptions = options.parseOptions || {}; + + if (mode === 'codemod') { + parseOptions.ignoreStandalone = true; + } + + ast = handlebars.parse(html, parseOptions) as HBS.Program; + } + + let entityParser = undefined; + if (mode === 'codemod') { + entityParser = new EntityParser({}); + } + + let program = new TokenizerEventHandlers(html, entityParser).acceptTemplate(ast); if (options && options.plugins && options.plugins.ast) { for (let i = 0, l = options.plugins.ast.length; i < l; i++) { diff --git a/packages/@glimmer/syntax/lib/types/nodes.ts b/packages/@glimmer/syntax/lib/types/nodes.ts index 337482f45..dda5c32b4 100644 --- a/packages/@glimmer/syntax/lib/types/nodes.ts +++ b/packages/@glimmer/syntax/lib/types/nodes.ts @@ -57,6 +57,8 @@ export interface Block extends CommonProgram { symbols?: BlockSymbols; } +export type EntityEncodingState = 'transformed' | 'raw'; + export interface Template extends CommonProgram { type: 'Template'; symbols?: Symbols; diff --git a/packages/@glimmer/syntax/test/generation/print-test.ts b/packages/@glimmer/syntax/test/generation/print-test.ts index 17f203091..37e0d4f5d 100644 --- a/packages/@glimmer/syntax/test/generation/print-test.ts +++ b/packages/@glimmer/syntax/test/generation/print-test.ts @@ -1,162 +1,104 @@ -import { preprocess as parse, print, builders as b } from '@glimmer/syntax'; +import { preprocess as parse, print } from '@glimmer/syntax'; const { test } = QUnit; -function printTransform(template: string) { - return print(parse(template)); -} - -function printEqual(template: string) { - QUnit.assert.equal(printTransform(template), template); -} - -QUnit.module('[glimmer-syntax] Code generation'); - -test('ElementNode: tag', function() { - printEqual('

'); -}); - -test('ElementNode: nested tags with indent', function() { - printEqual('
\n

Test

\n
'); -}); - -test('ElementNode: attributes', function() { - printEqual('

'); -}); - -test('ElementNode: attributes escaping', function() { - printEqual('

'); -}); - -test('TextNode: chars escape', assert => { - assert.equal(printTransform('< &   > ©2018'), '< &   > ©2018'); -}); - -test('TextNode: chars', function() { - printEqual('

Test

'); -}); - -test('MustacheStatement: slash in path', function() { - printEqual('{{namespace/foo "bar" baz="qux"}}'); -}); - -test('MustacheStatement: path', function() { - printEqual('

{{model.title}}

'); -}); - -test('MustacheStatement: StringLiteral param', function() { - printEqual('

{{link-to "Foo"}}

'); -}); - -test('MustacheStatement: StringLiteral path', function() { - printEqual(''); -}); - -test('MustacheStatement: BooleanLiteral path', function() { - printEqual(''); -}); - -test('MustacheStatement: NumberLiteral path', function() { - printEqual(''); -}); - -test('MustacheStatement: hash', function() { - printEqual('

{{link-to "Foo" class="bar"}}

'); -}); - -test('MustacheStatement: as element attribute', function() { - printEqual('

Test

'); -}); - -test('MustacheStatement: as element attribute with path', function() { - printEqual('

Test

'); -}); - -test('ConcatStatement: in element attribute string', function() { - printEqual('

Test

'); -}); - -test('ConcatStatement: in element attribute string escaping', function() { - printEqual('

Test

'); -}); - -test('ElementModifierStatement', function() { - printEqual('

Test

'); -}); - -test('SubExpression', function() { - printEqual( - '

{{my-component submit=(action (mut model.name) (full-name model.firstName "Smith"))}}

' - ); -}); - -test('BlockStatement: multiline', function() { - printEqual('
    {{#each foos as |foo index|}}\n
  • {{foo}}: {{index}}
    • \n{{/each}}
'); -}); - -test('BlockStatement: inline', function() { - printEqual('{{#if foo}}

{{foo}}

{{/if}}'); -}); - -test('UndefinedLiteral', assert => { - const ast = b.program([b.mustache(b.undefined())]); - assert.equal(print(ast), '{{undefined}}'); -}); - -test('NumberLiteral', assert => { - const ast = b.program([b.mustache('foo', undefined, b.hash([b.pair('bar', b.number(5))]))]); - assert.equal(print(ast), '{{foo bar=5}}'); -}); - -test('BooleanLiteral', assert => { - const ast = b.program([b.mustache('foo', undefined, b.hash([b.pair('bar', b.boolean(true))]))]); - assert.equal(print(ast), '{{foo bar=true}}'); -}); - -test('HTML comment', function() { - printEqual(''); -}); - -test('Handlebars comment', assert => { - assert.equal(printTransform('{{! foo }}'), '{{!-- foo --}}'); -}); - -test('Handlebars comment: in ElementNode', function() { - printEqual('
'); -}); - -test('Handlebars comment: in ElementNode children', function() { - printEqual('
{{!-- foo bar --}}
'); -}); - -test('Handlebars in handlebar comment', function() { - printEqual('{{!-- {{foo-bar}} --}}'); -}); - -test('Void elements', function() { - printEqual('
'); -}); - -test('Void elements self closing', function() { - printEqual('
'); -}); - -test('Angle bracket component', function() { - printEqual('{{bar}}'); -}); - -test('Angle bracket component without content', function() { - printEqual(''); -}); - -test('Self-closing angle bracket component', function() { - printEqual(''); -}); - -test('Block params', function() { - printEqual('{{bar}}'); -}); - -test('Attributes without value', function() { - printEqual(''); +let templates = [ + '

', + '

', + '

Test

', + '

{{model.title}}

', + '

{{link-to "Foo" class="bar"}}

', + '

Test

', + '

Test

', + '

Test

', + '

Test

', + '

{{my-component submit=(action (mut model.name) (full-name model.firstName "Smith"))}}

', + '
    {{#each foos as |foo index|}}\n
  • {{foo}}: {{index}}
    • \n{{/each}}
', + '{{#if foo}}

{{foo}}

{{/if}}', + '{{bar}}', + '', + '', + '{{bar}}', + + '', + + // void elements + '
', + '
', + + // comments + '', + '
', + '
{{!-- foo bar --}}
', + '{{!-- {{foo-bar}} --}}', + + // literals + '', + '', + '', + '{{panel arg="Foo"}}', + '{{panel arg=true}}', + '{{panel arg=5}}', + + // nested tags with indent + '
\n

Test

\n
', + + // attributes escaping + '

', + '

Test

', + + // slash in path + '{{namespace/foo "bar" baz="qux"}}', +]; + +QUnit.module('[glimmer-syntax] Code generation', function() { + function printTransform(template: string) { + return print(parse(template)); + } + + templates.forEach(template => { + test(`${template} is stable when printed`, function(assert) { + assert.equal(printTransform(template), template); + }); + }); + + test('TextNode: chars escape - but do not match', assert => { + assert.equal( + printTransform('< &   > ©2018'), + '< &   > ©2018' + ); + }); + + test('Handlebars comment', assert => { + assert.equal(printTransform('{{! foo }}'), '{{!-- foo --}}'); + }); +}); + +QUnit.module('[glimmer-syntax] Code generation - source -> source', function() { + function printTransform(template: string) { + let ast = parse(template, { + mode: 'codemod', + parseOptions: { ignoreStandalone: true }, + }); + + return print(ast, { entityEncoding: 'raw' }); + } + + function buildTest(template: string) { + test(`${template} is stable when printed`, function(assert) { + assert.equal(printTransform(template), template); + }); + } + + templates.forEach(buildTest); + + [ + '< &   > ©2018', + + // newlines after opening block + '{{#each}}\n
  • foo
    • \n{{/each}}', + + // TODO: fix whitespace control in codemod mode + // '\n{{~#foo-bar~}} {{~/foo-bar~}} ', + ].forEach(buildTest); });