From 90274b9739320b76f36fbe86c7d2a96e9541a786 Mon Sep 17 00:00:00 2001 From: Andreas Ehrencrona Date: Wed, 9 Dec 2020 12:46:45 +0200 Subject: [PATCH 1/6] Refactored preprocess for readability --- src/compiler/preprocess/decode_sourcemap.ts | 88 ++++ src/compiler/preprocess/index.ts | 381 +++++++----------- .../preprocess/parse_tag_attributes.ts | 14 + src/compiler/preprocess/replace_in_code.ts | 68 ++++ src/compiler/preprocess/types.ts | 18 + src/compiler/utils/string_with_sourcemap.ts | 2 +- 6 files changed, 324 insertions(+), 247 deletions(-) create mode 100644 src/compiler/preprocess/decode_sourcemap.ts create mode 100644 src/compiler/preprocess/parse_tag_attributes.ts create mode 100644 src/compiler/preprocess/replace_in_code.ts create mode 100644 src/compiler/preprocess/types.ts diff --git a/src/compiler/preprocess/decode_sourcemap.ts b/src/compiler/preprocess/decode_sourcemap.ts new file mode 100644 index 000000000000..c7e13c28537f --- /dev/null +++ b/src/compiler/preprocess/decode_sourcemap.ts @@ -0,0 +1,88 @@ +import { decode as decode_mappings } from 'sourcemap-codec'; +import { Processed } from './types'; + +/** + * Import decoded sourcemap from mozilla/source-map/SourceMapGenerator + * Forked from source-map/lib/source-map-generator.js + * from methods _serializeMappings and toJSON. + * We cannot use source-map.d.ts types, because we access hidden properties. + */ +function decoded_sourcemap_from_generator(generator: any) { + let previous_generated_line = 1; + const converted_mappings = [[]]; + let result_line; + let result_segment; + let mapping; + + const source_idx = generator._sources.toArray() + .reduce((acc, val, idx) => (acc[val] = idx, acc), {}); + + const name_idx = generator._names.toArray() + .reduce((acc, val, idx) => (acc[val] = idx, acc), {}); + + const mappings = generator._mappings.toArray(); + result_line = converted_mappings[0]; + + for (let i = 0, len = mappings.length; i < len; i++) { + mapping = mappings[i]; + + if (mapping.generatedLine > previous_generated_line) { + while (mapping.generatedLine > previous_generated_line) { + converted_mappings.push([]); + previous_generated_line++; + } + result_line = converted_mappings[mapping.generatedLine - 1]; // line is one-based + } else if (i > 0) { + const previous_mapping = mappings[i - 1]; + if ( + // sorted by selectivity + mapping.generatedColumn === previous_mapping.generatedColumn && + mapping.originalColumn === previous_mapping.originalColumn && + mapping.name === previous_mapping.name && + mapping.generatedLine === previous_mapping.generatedLine && + mapping.originalLine === previous_mapping.originalLine && + mapping.source === previous_mapping.source + ) { + continue; + } + } + result_line.push([mapping.generatedColumn]); + result_segment = result_line[result_line.length - 1]; + + if (mapping.source != null) { + result_segment.push(...[ + source_idx[mapping.source], + mapping.originalLine - 1, // line is one-based + mapping.originalColumn + ]); + if (mapping.name != null) { + result_segment.push(name_idx[mapping.name]); + } + } + } + + const map = { + version: generator._version, + sources: generator._sources.toArray(), + names: generator._names.toArray(), + mappings: converted_mappings + }; + if (generator._file != null) { + (map as any).file = generator._file; + } + // not needed: map.sourcesContent and map.sourceRoot + return map; +} + +export function decode_map(processed: Processed) { + let decoded_map = typeof processed.map === 'string' ? JSON.parse(processed.map) : processed.map; + if (typeof(decoded_map.mappings) === 'string') { + decoded_map.mappings = decode_mappings(decoded_map.mappings); + } + if ((decoded_map as any)._mappings && decoded_map.constructor.name === 'SourceMapGenerator') { + // import decoded sourcemap from mozilla/source-map/SourceMapGenerator + decoded_map = decoded_sourcemap_from_generator(decoded_map); + } + + return decoded_map; +} diff --git a/src/compiler/preprocess/index.ts b/src/compiler/preprocess/index.ts index b51b67bb2345..84537c684ac4 100644 --- a/src/compiler/preprocess/index.ts +++ b/src/compiler/preprocess/index.ts @@ -1,207 +1,153 @@ import { RawSourceMap, DecodedSourceMap } from '@ampproject/remapping/dist/types/types'; -import { decode as decode_mappings } from 'sourcemap-codec'; import { getLocator } from 'locate-character'; -import { StringWithSourcemap, sourcemap_add_offset, combine_sourcemaps } from '../utils/string_with_sourcemap'; - -export interface Processed { - code: string; - map?: string | object; // we are opaque with the type here to avoid dependency on the remapping module for our public types. +import { StringWithSourcemap, SourceLocation, sourcemap_add_offset, combine_sourcemaps } from '../utils/string_with_sourcemap'; +import { decode_map } from './decode_sourcemap'; +import parse_tag_attributes from './parse_tag_attributes'; +import { replace_in_code, Source } from './replace_in_code'; +import { Preprocessor, PreprocessorGroup, Processed } from './types'; + +interface SourceUpdate { + string: string; + map?: DecodedSourceMap; dependencies?: string[]; } -export interface PreprocessorGroup { - markup?: (options: { - content: string; - filename: string; - }) => Processed | Promise; - style?: Preprocessor; - script?: Preprocessor; -} +class PreprocessResult { + // sourcemap_list is sorted in reverse order from last map (index 0) to first map (index -1) + // so we use sourcemap_list.unshift() to add new maps + // https://github.com/ampproject/remapping#multiple-transformations-of-a-file + sourcemap_list: Array = []; + dependencies: string[] = []; + + get_location: ReturnType; + + constructor(public source: string, public filename: string) { + this.update_source({string: source}); + } -export type Preprocessor = (options: { - content: string; - attributes: Record; - filename?: string; -}) => Processed | Promise; - -function parse_attributes(str: string) { - const attrs = {}; - str.split(/\s+/).filter(Boolean).forEach(attr => { - const p = attr.indexOf('='); - if (p === -1) { - attrs[attr] = true; - } else { - attrs[attr.slice(0, p)] = '\'"'.includes(attr[p + 1]) ? - attr.slice(p + 2, -1) : - attr.slice(p + 1); + update_source({string: source, map, dependencies}: SourceUpdate) { + this.source = source; + this.get_location = getLocator(source); + + if (map) { + this.sourcemap_list.unshift(map); } - }); - return attrs; -} -interface Replacement { - offset: number; - length: number; - replacement: StringWithSourcemap; -} + if (dependencies) { + this.dependencies.push(...dependencies); + } + } -async function replace_async( - filename: string, - source: string, - get_location: ReturnType, - re: RegExp, - func: (...any) => Promise -): Promise { - const replacements: Array> = []; - source.replace(re, (...args) => { - replacements.push( - func(...args).then( - res => - ({ - offset: args[args.length - 2], - length: args[0].length, - replacement: res - }) as Replacement - ) + processed(): Processed { + // Combine all the source maps for each preprocessor function into one + const map: RawSourceMap = combine_sourcemaps( + this.filename, + this.sourcemap_list ); - return ''; - }); - const out = new StringWithSourcemap(); - let last_end = 0; - for (const { offset, length, replacement } of await Promise.all( - replacements - )) { - // content = unchanged source characters before the replaced segment - const content = StringWithSourcemap.from_source( - filename, source.slice(last_end, offset), get_location(last_end)); - out.concat(content).concat(replacement); - last_end = offset + length; + + return { + // TODO return separated output, in future version where svelte.compile supports it: + // style: { code: styleCode, map: styleMap }, + // script { code: scriptCode, map: scriptMap }, + // markup { code: markupCode, map: markupMap }, + + code: this.source, + dependencies: [...new Set(this.dependencies)], + map: (map as object), + toString: () => this.source + }; } - // final_content = unchanged source characters after last replaced segment - const final_content = StringWithSourcemap.from_source( - filename, source.slice(last_end), get_location(last_end)); - return out.concat(final_content); } /** - * Import decoded sourcemap from mozilla/source-map/SourceMapGenerator - * Forked from source-map/lib/source-map-generator.js - * from methods _serializeMappings and toJSON. - * We cannot use source-map.d.ts types, because we access hidden properties. + * Convert preprocessor output for the tag content into StringWithSourceMap */ -function decoded_sourcemap_from_generator(generator: any) { - let previous_generated_line = 1; - const converted_mappings = [[]]; - let result_line; - let result_segment; - let mapping; - - const source_idx = generator._sources.toArray() - .reduce((acc, val, idx) => (acc[val] = idx, acc), {}); - - const name_idx = generator._names.toArray() - .reduce((acc, val, idx) => (acc[val] = idx, acc), {}); - - const mappings = generator._mappings.toArray(); - result_line = converted_mappings[0]; - - for (let i = 0, len = mappings.length; i < len; i++) { - mapping = mappings[i]; - - if (mapping.generatedLine > previous_generated_line) { - while (mapping.generatedLine > previous_generated_line) { - converted_mappings.push([]); - previous_generated_line++; - } - result_line = converted_mappings[mapping.generatedLine - 1]; // line is one-based - } else if (i > 0) { - const previous_mapping = mappings[i - 1]; - if ( - // sorted by selectivity - mapping.generatedColumn === previous_mapping.generatedColumn && - mapping.originalColumn === previous_mapping.originalColumn && - mapping.name === previous_mapping.name && - mapping.generatedLine === previous_mapping.generatedLine && - mapping.originalLine === previous_mapping.originalLine && - mapping.source === previous_mapping.source - ) { - continue; - } - } - result_line.push([mapping.generatedColumn]); - result_segment = result_line[result_line.length - 1]; - - if (mapping.source != null) { - result_segment.push(...[ - source_idx[mapping.source], - mapping.originalLine - 1, // line is one-based - mapping.originalColumn - ]); - if (mapping.name != null) { - result_segment.push(name_idx[mapping.name]); - } - } +function processed_content_to_sws( + processed: Processed, + location: SourceLocation +): StringWithSourcemap { + // Convert the preprocessed code and its sourcemap to a StringWithSourcemap + let decoded_map: DecodedSourceMap; + if (processed.map) { + decoded_map = decode_map(processed); + + sourcemap_add_offset(decoded_map, location); } - const map = { - version: generator._version, - sources: generator._sources.toArray(), - names: generator._names.toArray(), - mappings: converted_mappings - }; - if (generator._file != null) { - (map as any).file = generator._file; - } - // not needed: map.sourcesContent and map.sourceRoot - return map; + return StringWithSourcemap.from_processed(processed.code, decoded_map); } /** - * Convert a preprocessor output and its leading prefix and trailing suffix into StringWithSourceMap + * Convert the whole tag including content (replacing it with `processed`) + * into a `StringWithSourcemap` representing the transformed code. */ -function get_replacement( - filename: string, - offset: number, - get_location: ReturnType, - original: string, +function processed_tag_to_sws( processed: Processed, - prefix: string, - suffix: string -): StringWithSourcemap { + tag_name: 'style' | 'script', + attributes: string, + content: string, + { filename, get_location }: Source): StringWithSourcemap { + const build_sws = (content: string, offset: number) => + StringWithSourcemap.from_source(filename, content, get_location(offset)); - // Convert the unchanged prefix and suffix to StringWithSourcemap - const prefix_with_map = StringWithSourcemap.from_source( - filename, prefix, get_location(offset)); - const suffix_with_map = StringWithSourcemap.from_source( - filename, suffix, get_location(offset + prefix.length + original.length)); + const tag_open = `<${tag_name}${attributes || ''}>`; + const tag_close = ``; - // Convert the preprocessed code and its sourcemap to a StringWithSourcemap - let decoded_map: DecodedSourceMap; - if (processed.map) { - decoded_map = typeof processed.map === 'string' ? JSON.parse(processed.map) : processed.map; - if (typeof(decoded_map.mappings) === 'string') { - decoded_map.mappings = decode_mappings(decoded_map.mappings); - } - if ((decoded_map as any)._mappings && decoded_map.constructor.name === 'SourceMapGenerator') { - // import decoded sourcemap from mozilla/source-map/SourceMapGenerator - decoded_map = decoded_sourcemap_from_generator(decoded_map); - } - sourcemap_add_offset(decoded_map, get_location(offset + prefix.length)); + const tag_open_sws = build_sws(tag_open, 0); + const tag_close_sws = build_sws(tag_close, tag_open.length + content.length); + + const content_sws = processed_content_to_sws(processed, get_location(tag_open.length)); + + return tag_open_sws.concat(content_sws).concat(tag_close_sws); +} + +async function process_tag( + tag_name: 'style' | 'script', + preprocessor: Preprocessor, + source: Source +): Promise { + const { filename, get_location } = source; + const tag_regex = + tag_name === 'style' + ? /|([^]*?)<\/style>|\/>)/gi + : /|([^]*?)<\/script>|\/>)/gi; + + const dependencies: string[] = []; + + async function process_single_tag( + tag_with_content: string, + attributes = '', + content = '', + tag_offset: number + ): Promise { + const no_change = () => + StringWithSourcemap.from_source(filename, tag_with_content, get_location(tag_offset)); + + if (!attributes && !content) return no_change(); + + const processed = await preprocessor({ + content: content || '', + attributes: parse_tag_attributes(attributes || ''), + filename + }); + + if (!processed) return no_change(); + if (processed.dependencies) dependencies.push(...processed.dependencies); + + return processed_tag_to_sws(processed, tag_name, attributes, content, + {...source, get_location: offset => source.get_location(offset + tag_offset)}); } - const processed_with_map = StringWithSourcemap.from_processed(processed.code, decoded_map); - // Surround the processed code with the prefix and suffix, retaining valid sourcemappings - return prefix_with_map.concat(processed_with_map).concat(suffix_with_map); + return {...await replace_in_code(tag_regex, process_single_tag, source), dependencies}; } export default async function preprocess( source: string, preprocessor: PreprocessorGroup | PreprocessorGroup[], options?: { filename?: string } -) { +): Promise { // @ts-ignore todo: doublecheck const filename = (options && options.filename) || preprocessor.filename; // legacy - const dependencies = []; const preprocessors = preprocessor ? Array.isArray(preprocessor) ? preprocessor : [preprocessor] @@ -211,95 +157,38 @@ export default async function preprocess( const script = preprocessors.map(p => p.script).filter(Boolean); const style = preprocessors.map(p => p.style).filter(Boolean); - // sourcemap_list is sorted in reverse order from last map (index 0) to first map (index -1) - // so we use sourcemap_list.unshift() to add new maps - // https://github.com/ampproject/remapping#multiple-transformations-of-a-file - const sourcemap_list: Array = []; - - // TODO keep track: what preprocessor generated what sourcemap? to make debugging easier = detect low-resolution sourcemaps in fn combine_mappings + const result = new PreprocessResult(source, filename); - for (const fn of markup) { + // TODO keep track: what preprocessor generated what sourcemap? + // to make debugging easier = detect low-resolution sourcemaps in fn combine_mappings - // run markup preprocessor - const processed = await fn({ - content: source, + for (const process of markup) { + const processed = await process({ + content: result.source, filename }); if (!processed) continue; - if (processed.dependencies) dependencies.push(...processed.dependencies); - source = processed.code; - if (processed.map) { - sourcemap_list.unshift( - typeof(processed.map) === 'string' + result.update_source({ + string: processed.code, + map: processed.map + ? // TODO: can we use decode_sourcemap? + typeof processed.map === 'string' ? JSON.parse(processed.map) : processed.map - ); - } - } - - async function preprocess_tag_content(tag_name: 'style' | 'script', preprocessor: Preprocessor) { - const get_location = getLocator(source); - const tag_regex = tag_name === 'style' - ? /|([^]*?)<\/style>|\/>)/gi - : /|([^]*?)<\/script>|\/>)/gi; - - const res = await replace_async( - filename, - source, - get_location, - tag_regex, - async (match, attributes = '', content = '', offset) => { - const no_change = () => StringWithSourcemap.from_source( - filename, match, get_location(offset)); - if (!attributes && !content) { - return no_change(); - } - attributes = attributes || ''; - content = content || ''; - - // run script preprocessor - const processed = await preprocessor({ - content, - attributes: parse_attributes(attributes), - filename - }); - - if (!processed) return no_change(); - if (processed.dependencies) dependencies.push(...processed.dependencies); - return get_replacement(filename, offset, get_location, content, processed, `<${tag_name}${attributes}>`, ``); - } - ); - source = res.string; - sourcemap_list.unshift(res.map); + : undefined, + dependencies: processed.dependencies + }); } - for (const fn of script) { - await preprocess_tag_content('script', fn); + for (const process of script) { + result.update_source(await process_tag('script', process, result)); } - for (const fn of style) { - await preprocess_tag_content('style', fn); + for (const preprocess of style) { + result.update_source(await process_tag('style', preprocess, result)); } - // Combine all the source maps for each preprocessor function into one - const map: RawSourceMap = combine_sourcemaps( - filename, - sourcemap_list - ); - - return { - // TODO return separated output, in future version where svelte.compile supports it: - // style: { code: styleCode, map: styleMap }, - // script { code: scriptCode, map: scriptMap }, - // markup { code: markupCode, map: markupMap }, - - code: source, - dependencies: [...new Set(dependencies)], - map: (map as object), - toString() { - return source; - } - }; + return result.processed(); } diff --git a/src/compiler/preprocess/parse_tag_attributes.ts b/src/compiler/preprocess/parse_tag_attributes.ts new file mode 100644 index 000000000000..0fe8802c7808 --- /dev/null +++ b/src/compiler/preprocess/parse_tag_attributes.ts @@ -0,0 +1,14 @@ +export default function parse_tag_attributes(str: string) { + const attrs = {}; + str.split(/\s+/).filter(Boolean).forEach(attr => { + const p = attr.indexOf('='); + if (p === -1) { + attrs[attr] = true; + } else { + attrs[attr.slice(0, p)] = '\'"'.includes(attr[p + 1]) ? + attr.slice(p + 2, -1) : + attr.slice(p + 1); + } + }); + return attrs; +} diff --git a/src/compiler/preprocess/replace_in_code.ts b/src/compiler/preprocess/replace_in_code.ts new file mode 100644 index 000000000000..96a071a1d67d --- /dev/null +++ b/src/compiler/preprocess/replace_in_code.ts @@ -0,0 +1,68 @@ +import { getLocator } from 'locate-character'; +import { StringWithSourcemap } from '../utils/string_with_sourcemap'; + +export interface Source { + source: string; + get_location: ReturnType; + filename: string; +} + +interface Replacement { + offset: number; + length: number; + replacement: StringWithSourcemap; +} + +function calculate_replacements( + re: RegExp, + get_replacement: (...match: any[]) => Promise, + source: string +) { + const replacements: Array> = []; + + source.replace(re, (...match) => { + replacements.push( + get_replacement(...match).then( + replacement => { + const matched_string = match[0]; + const offset = match[match.length-2]; + + return ({ offset, length: matched_string.length, replacement }); + } + ) + ); + return ''; + }); + + return Promise.all(replacements); +} + +function perform_replacements( + replacements: Replacement[], + { filename, source, get_location }: Source +): StringWithSourcemap { + const out = new StringWithSourcemap(); + let last_end = 0; + + for (const { offset, length, replacement } of replacements) { + const unchanged_prefix = StringWithSourcemap.from_source( + filename, source.slice(last_end, offset), get_location(last_end)); + out.concat(unchanged_prefix).concat(replacement); + last_end = offset + length; + } + + const unchanged_suffix = StringWithSourcemap.from_source( + filename, source.slice(last_end), get_location(last_end)); + + return out.concat(unchanged_suffix); +} + +export async function replace_in_code( + regex: RegExp, + get_replacement: (...match: any[]) => Promise, + location: Source +): Promise { + const replacements = await calculate_replacements(regex, get_replacement, location.source); + + return perform_replacements(replacements, location); +} diff --git a/src/compiler/preprocess/types.ts b/src/compiler/preprocess/types.ts new file mode 100644 index 000000000000..c78dc0c756e6 --- /dev/null +++ b/src/compiler/preprocess/types.ts @@ -0,0 +1,18 @@ +export interface Processed { + code: string; + map?: string | object; // we are opaque with the type here to avoid dependency on the remapping module for our public types. + dependencies?: string[]; + toString?: () => string; +} + +export interface PreprocessorGroup { + markup?: (options: { content: string; filename: string }) => Processed | Promise; + style?: Preprocessor; + script?: Preprocessor; +} + +export type Preprocessor = (options: { + content: string; + attributes: Record; + filename?: string; +}) => Processed | Promise; diff --git a/src/compiler/utils/string_with_sourcemap.ts b/src/compiler/utils/string_with_sourcemap.ts index 421a0c1fbd97..79b39a70f67d 100644 --- a/src/compiler/utils/string_with_sourcemap.ts +++ b/src/compiler/utils/string_with_sourcemap.ts @@ -2,7 +2,7 @@ import { DecodedSourceMap, RawSourceMap, SourceMapLoader } from '@ampproject/rem import remapping from '@ampproject/remapping'; import { SourceMap } from 'magic-string'; -type SourceLocation = { +export type SourceLocation = { line: number; column: number; }; From 0011123174442ca2b6fe180392dd68f6b4af3059 Mon Sep 17 00:00:00 2001 From: Andreas Ehrencrona Date: Wed, 9 Dec 2020 13:01:50 +0200 Subject: [PATCH 2/6] minor tweaks --- src/compiler/preprocess/index.ts | 40 ++++++++++++---------- src/compiler/preprocess/replace_in_code.ts | 12 +++---- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/src/compiler/preprocess/index.ts b/src/compiler/preprocess/index.ts index 84537c684ac4..3a8b8676fffa 100644 --- a/src/compiler/preprocess/index.ts +++ b/src/compiler/preprocess/index.ts @@ -12,6 +12,9 @@ interface SourceUpdate { dependencies?: string[]; } +/** + * Represents intermediate states of the preprocessing. + */ class PreprocessResult { // sourcemap_list is sorted in reverse order from last map (index 0) to first map (index -1) // so we use sourcemap_list.unshift() to add new maps @@ -22,13 +25,13 @@ class PreprocessResult { get_location: ReturnType; constructor(public source: string, public filename: string) { - this.update_source({string: source}); + this.update_source({ string: source }); } - update_source({string: source, map, dependencies}: SourceUpdate) { + update_source({ string: source, map, dependencies }: SourceUpdate) { this.source = source; this.get_location = getLocator(source); - + if (map) { this.sourcemap_list.unshift(map); } @@ -38,12 +41,9 @@ class PreprocessResult { } } - processed(): Processed { + to_processed(): Processed { // Combine all the source maps for each preprocessor function into one - const map: RawSourceMap = combine_sourcemaps( - this.filename, - this.sourcemap_list - ); + const map: RawSourceMap = combine_sourcemaps(this.filename, this.sourcemap_list); return { // TODO return separated output, in future version where svelte.compile supports it: @@ -53,7 +53,7 @@ class PreprocessResult { code: this.source, dependencies: [...new Set(this.dependencies)], - map: (map as object), + map: map as object, toString: () => this.source }; } @@ -78,29 +78,31 @@ function processed_content_to_sws( } /** - * Convert the whole tag including content (replacing it with `processed`) - * into a `StringWithSourcemap` representing the transformed code. + * Given the whole tag including content, return a `StringWithSourcemap` + * representing the tag content replaced with `processed`. */ function processed_tag_to_sws( processed: Processed, tag_name: 'style' | 'script', attributes: string, - content: string, - { filename, get_location }: Source): StringWithSourcemap { - const build_sws = (content: string, offset: number) => - StringWithSourcemap.from_source(filename, content, get_location(offset)); + { source, filename, get_location }: Source): StringWithSourcemap { + const build_sws = (source: string, offset: number) => + StringWithSourcemap.from_source(filename, source, get_location(offset)); const tag_open = `<${tag_name}${attributes || ''}>`; const tag_close = ``; const tag_open_sws = build_sws(tag_open, 0); - const tag_close_sws = build_sws(tag_close, tag_open.length + content.length); + const tag_close_sws = build_sws(tag_close, tag_open.length + source.length); const content_sws = processed_content_to_sws(processed, get_location(tag_open.length)); return tag_open_sws.concat(content_sws).concat(tag_close_sws); } +/** + * Calculate the updates required to process all instances of the specified tag. + */ async function process_tag( tag_name: 'style' | 'script', preprocessor: Preprocessor, @@ -134,8 +136,8 @@ async function process_tag( if (!processed) return no_change(); if (processed.dependencies) dependencies.push(...processed.dependencies); - return processed_tag_to_sws(processed, tag_name, attributes, content, - {...source, get_location: offset => source.get_location(offset + tag_offset)}); + return processed_tag_to_sws(processed, tag_name, attributes, + {source: content, get_location: offset => source.get_location(offset + tag_offset), filename}); } return {...await replace_in_code(tag_regex, process_single_tag, source), dependencies}; @@ -190,5 +192,5 @@ export default async function preprocess( result.update_source(await process_tag('style', preprocess, result)); } - return result.processed(); + return result.to_processed(); } diff --git a/src/compiler/preprocess/replace_in_code.ts b/src/compiler/preprocess/replace_in_code.ts index 96a071a1d67d..eb2e553547d4 100644 --- a/src/compiler/preprocess/replace_in_code.ts +++ b/src/compiler/preprocess/replace_in_code.ts @@ -2,9 +2,9 @@ import { getLocator } from 'locate-character'; import { StringWithSourcemap } from '../utils/string_with_sourcemap'; export interface Source { - source: string; + source: string; get_location: ReturnType; - filename: string; + filename: string; } interface Replacement { @@ -24,11 +24,11 @@ function calculate_replacements( replacements.push( get_replacement(...match).then( replacement => { - const matched_string = match[0]; - const offset = match[match.length-2]; + const matched_string = match[0]; + const offset = match[match.length-2]; - return ({ offset, length: matched_string.length, replacement }); - } + return ({ offset, length: matched_string.length, replacement }); + } ) ); return ''; From d17807a13a8127a38c27576df9cc8b4cbebb5355 Mon Sep 17 00:00:00 2001 From: Andreas Ehrencrona Date: Thu, 10 Dec 2020 11:04:08 +0200 Subject: [PATCH 3/6] implemented preprocess-sync --- src/compiler/preprocess/index.ts | 208 +++++++++++++++++---- src/compiler/preprocess/replace_in_code.ts | 38 +--- src/compiler/preprocess/types.ts | 14 +- 3 files changed, 178 insertions(+), 82 deletions(-) diff --git a/src/compiler/preprocess/index.ts b/src/compiler/preprocess/index.ts index 3a8b8676fffa..76aba7000d6e 100644 --- a/src/compiler/preprocess/index.ts +++ b/src/compiler/preprocess/index.ts @@ -3,8 +3,8 @@ import { getLocator } from 'locate-character'; import { StringWithSourcemap, SourceLocation, sourcemap_add_offset, combine_sourcemaps } from '../utils/string_with_sourcemap'; import { decode_map } from './decode_sourcemap'; import parse_tag_attributes from './parse_tag_attributes'; -import { replace_in_code, Source } from './replace_in_code'; -import { Preprocessor, PreprocessorGroup, Processed } from './types'; +import { perform_replacements, Source, Replacement } from './replace_in_code'; +import { Preprocessor, SyncPreprocessor, PreprocessorGroup, Processed } from './types'; interface SourceUpdate { string: string; @@ -100,65 +100,190 @@ function processed_tag_to_sws( return tag_open_sws.concat(content_sws).concat(tag_close_sws); } -/** - * Calculate the updates required to process all instances of the specified tag. - */ -async function process_tag( - tag_name: 'style' | 'script', - preprocessor: Preprocessor, - source: Source -): Promise { - const { filename, get_location } = source; + +interface TagInstance { + tag_with_content: string, + attributes: string, + content: string, + tag_offset: number; +} + +export function get_tag_instances(tag: 'style'|'script', source: string): TagInstance[] { const tag_regex = - tag_name === 'style' + tag === 'style' ? /|([^]*?)<\/style>|\/>)/gi : /|([^]*?)<\/script>|\/>)/gi; - const dependencies: string[] = []; - - async function process_single_tag( - tag_with_content: string, - attributes = '', - content = '', - tag_offset: number - ): Promise { - const no_change = () => - StringWithSourcemap.from_source(filename, tag_with_content, get_location(tag_offset)); + const instances: TagInstance[] = []; - if (!attributes && !content) return no_change(); + source.replace(tag_regex, (...match) => { + const [tag_with_content, attributes, content, tag_offset] = match; - const processed = await preprocessor({ + instances.push({ + tag_with_content, + attributes: attributes || '', content: content || '', - attributes: parse_tag_attributes(attributes || ''), - filename + tag_offset }); - if (!processed) return no_change(); - if (processed.dependencies) dependencies.push(...processed.dependencies); + return ''; + }); - return processed_tag_to_sws(processed, tag_name, attributes, - {source: content, get_location: offset => source.get_location(offset + tag_offset), filename}); - } + return instances; +} - return {...await replace_in_code(tag_regex, process_single_tag, source), dependencies}; +function to_source_update(tags: TagInstance[], processed: Processed[], tag_name: 'style' | 'script', source: Source) { + const { filename, get_location } = source; + const dependencies: string[] = []; + + const replacements: Replacement[] = processed.map((processed, idx) => { + const match = tags[idx]; + const {tag_with_content, attributes, content, tag_offset} = match; + + let sws: StringWithSourcemap; + + if (!processed) { + sws = StringWithSourcemap.from_source(filename, tag_with_content, get_location(tag_offset)); + } else { + if (processed.dependencies) dependencies.push(...processed.dependencies); + + sws = processed_tag_to_sws(processed, tag_name, attributes, { + source: content, + get_location: offset => source.get_location(offset + tag_offset), + filename + }); + } + + return { offset: tag_offset, length: tag_with_content.length, replacement: sws }; + }); + + return {...perform_replacements(replacements, source), dependencies}; } -export default async function preprocess( - source: string, +/** + * Calculate the updates required to process all instances of the specified tag. + */ +async function process_tag( + tag_name: 'style' | 'script', + preprocessor: Preprocessor | SyncPreprocessor, + source: Source +): Promise { + const { filename } = source; + + const tags = get_tag_instances(tag_name, source.source); + + const processed = await Promise.all(tags.map(({attributes, content}) => { + if (attributes || content) { + return preprocessor({ + content, + attributes: parse_tag_attributes(attributes || ''), + filename + }); + } + })); + + return to_source_update(tags, processed, tag_name, source); +} + +function process_tag_sync( + tag_name: 'style' | 'script', + preprocessor: SyncPreprocessor, + source: Source +): SourceUpdate { + const { filename } = source; + + const tags = get_tag_instances(tag_name, source.source); + + const processed: Processed[] = tags.map(({attributes, content}) => { + if (attributes || content) { + return preprocessor({ + content, + attributes: parse_tag_attributes(attributes || ''), + filename + }); + } + }); + + return to_source_update(tags, processed, tag_name, source); +} + +function decode_preprocessor_params( preprocessor: PreprocessorGroup | PreprocessorGroup[], options?: { filename?: string } -): Promise { +) { // @ts-ignore todo: doublecheck const filename = (options && options.filename) || preprocessor.filename; // legacy - const preprocessors = preprocessor - ? Array.isArray(preprocessor) ? preprocessor : [preprocessor] - : []; + const preprocessors = preprocessor ? (Array.isArray(preprocessor) ? preprocessor : [preprocessor]) : []; const markup = preprocessors.map(p => p.markup).filter(Boolean); const script = preprocessors.map(p => p.script).filter(Boolean); const style = preprocessors.map(p => p.style).filter(Boolean); + return { markup, script, style, filename }; +} + +function is_sync(processor: Preprocessor | SyncPreprocessor): processor is SyncPreprocessor { + return processor['is_sync']; +} + +export function preprocess_sync( + source: string, + preprocessor: PreprocessorGroup | PreprocessorGroup[], + options?: { filename?: string } +): Processed { + const { markup, script, style, filename } = decode_preprocessor_params(preprocessor, options); + + const result = new PreprocessResult(source, filename); + + // TODO keep track: what preprocessor generated what sourcemap? + // to make debugging easier = detect low-resolution sourcemaps in fn combine_mappings + + for (const process of markup) { + if (is_sync(process)) { + const processed = process({ + content: result.source, + filename, + attributes: null + }); + + if (!processed) continue; + + result.update_source({ + string: processed.code, + map: processed.map + ? // TODO: can we use decode_sourcemap? + typeof processed.map === 'string' + ? JSON.parse(processed.map) + : processed.map + : undefined, + dependencies: processed.dependencies + }); + } + } + + for (const process of script) { + if (is_sync(process)) { + result.update_source(process_tag_sync('script', process, result)); + } + } + + for (const process of style) { + if (is_sync(process)) { + result.update_source(process_tag_sync('style', process, result)); + } + } + + return result.to_processed(); +} + +export default async function preprocess( + source: string, + preprocessor: PreprocessorGroup | PreprocessorGroup[], + options?: { filename?: string } +): Promise { + const { markup, script, style, filename } = decode_preprocessor_params(preprocessor, options); + const result = new PreprocessResult(source, filename); // TODO keep track: what preprocessor generated what sourcemap? @@ -167,7 +292,8 @@ export default async function preprocess( for (const process of markup) { const processed = await process({ content: result.source, - filename + filename, + attributes: null }); if (!processed) continue; @@ -188,8 +314,8 @@ export default async function preprocess( result.update_source(await process_tag('script', process, result)); } - for (const preprocess of style) { - result.update_source(await process_tag('style', preprocess, result)); + for (const process of style) { + result.update_source(await process_tag('style', process, result)); } return result.to_processed(); diff --git a/src/compiler/preprocess/replace_in_code.ts b/src/compiler/preprocess/replace_in_code.ts index eb2e553547d4..89a422241f33 100644 --- a/src/compiler/preprocess/replace_in_code.ts +++ b/src/compiler/preprocess/replace_in_code.ts @@ -7,37 +7,13 @@ export interface Source { filename: string; } -interface Replacement { +export interface Replacement { offset: number; length: number; replacement: StringWithSourcemap; } -function calculate_replacements( - re: RegExp, - get_replacement: (...match: any[]) => Promise, - source: string -) { - const replacements: Array> = []; - - source.replace(re, (...match) => { - replacements.push( - get_replacement(...match).then( - replacement => { - const matched_string = match[0]; - const offset = match[match.length-2]; - - return ({ offset, length: matched_string.length, replacement }); - } - ) - ); - return ''; - }); - - return Promise.all(replacements); -} - -function perform_replacements( +export function perform_replacements( replacements: Replacement[], { filename, source, get_location }: Source ): StringWithSourcemap { @@ -56,13 +32,3 @@ function perform_replacements( return out.concat(unchanged_suffix); } - -export async function replace_in_code( - regex: RegExp, - get_replacement: (...match: any[]) => Promise, - location: Source -): Promise { - const replacements = await calculate_replacements(regex, get_replacement, location.source); - - return perform_replacements(replacements, location); -} diff --git a/src/compiler/preprocess/types.ts b/src/compiler/preprocess/types.ts index c78dc0c756e6..a4aaf215a76d 100644 --- a/src/compiler/preprocess/types.ts +++ b/src/compiler/preprocess/types.ts @@ -6,13 +6,17 @@ export interface Processed { } export interface PreprocessorGroup { - markup?: (options: { content: string; filename: string }) => Processed | Promise; - style?: Preprocessor; - script?: Preprocessor; + markup?: Preprocessor | SyncPreprocessor; + style?: Preprocessor | SyncPreprocessor; + script?: Preprocessor | SyncPreprocessor; } -export type Preprocessor = (options: { +interface PreprocessorOptions { content: string; attributes: Record; filename?: string; -}) => Processed | Promise; +} + +export declare type Preprocessor = (options: PreprocessorOptions) => Promise; + +export declare type SyncPreprocessor = ((options: PreprocessorOptions) => Processed) & { is_sync: true }; From e5c71eeb296b1ed0fe576f3f08687c920885cf5f Mon Sep 17 00:00:00 2001 From: Andreas Ehrencrona Date: Fri, 11 Dec 2020 10:13:19 +0200 Subject: [PATCH 4/6] works with typescript --- src/compiler/index.ts | 2 +- src/compiler/preprocess/index.ts | 71 ++++++++++++++++---------------- src/compiler/preprocess/types.ts | 11 +++-- 3 files changed, 43 insertions(+), 41 deletions(-) diff --git a/src/compiler/index.ts b/src/compiler/index.ts index 14d55f5470de..bb36c37c9821 100644 --- a/src/compiler/index.ts +++ b/src/compiler/index.ts @@ -1,6 +1,6 @@ export { default as compile } from './compile/index'; export { default as parse } from './parse/index'; -export { default as preprocess } from './preprocess/index'; +export { default as preprocess, preprocess_sync } from './preprocess/index'; export { walk } from 'estree-walker'; export const VERSION = '__VERSION__'; diff --git a/src/compiler/preprocess/index.ts b/src/compiler/preprocess/index.ts index 76aba7000d6e..693f5ca59d90 100644 --- a/src/compiler/preprocess/index.ts +++ b/src/compiler/preprocess/index.ts @@ -218,13 +218,10 @@ function decode_preprocessor_params( const markup = preprocessors.map(p => p.markup).filter(Boolean); const script = preprocessors.map(p => p.script).filter(Boolean); + const script_sync = preprocessors.map(p => p.script_sync).filter(Boolean); const style = preprocessors.map(p => p.style).filter(Boolean); - return { markup, script, style, filename }; -} - -function is_sync(processor: Preprocessor | SyncPreprocessor): processor is SyncPreprocessor { - return processor['is_sync']; + return { markup, script, script_sync, style, filename }; } export function preprocess_sync( @@ -232,47 +229,49 @@ export function preprocess_sync( preprocessor: PreprocessorGroup | PreprocessorGroup[], options?: { filename?: string } ): Processed { - const { markup, script, style, filename } = decode_preprocessor_params(preprocessor, options); + const { script_sync, filename } = decode_preprocessor_params(preprocessor, options); const result = new PreprocessResult(source, filename); // TODO keep track: what preprocessor generated what sourcemap? // to make debugging easier = detect low-resolution sourcemaps in fn combine_mappings - for (const process of markup) { - if (is_sync(process)) { - const processed = process({ - content: result.source, - filename, - attributes: null - }); + // for (const process of markup) { + // if (is_sync(process)) { + // const processed = process({ + // content: result.source, + // filename, + // attributes: null + // }); - if (!processed) continue; + // if (!processed) continue; - result.update_source({ - string: processed.code, - map: processed.map - ? // TODO: can we use decode_sourcemap? - typeof processed.map === 'string' - ? JSON.parse(processed.map) - : processed.map - : undefined, - dependencies: processed.dependencies - }); - } + // result.update_source({ + // string: processed.code, + // map: processed.map + // ? // TODO: can we use decode_sourcemap? + // typeof processed.map === 'string' + // ? JSON.parse(processed.map) + // : processed.map + // : undefined, + // dependencies: processed.dependencies + // }); + // } else { + // console.error(`Preprocessor is not synchronous: ${process}.`); + // } + // } + + for (const process of script_sync) { + result.update_source(process_tag_sync('script', process, result)); } - for (const process of script) { - if (is_sync(process)) { - result.update_source(process_tag_sync('script', process, result)); - } - } - - for (const process of style) { - if (is_sync(process)) { - result.update_source(process_tag_sync('style', process, result)); - } - } + // for (const process of style) { + // if (is_sync(process)) { + // result.update_source(process_tag_sync('style', process, result)); + // } else { + // console.error(`Preprocessor is not synchronous: ${process}.`); + // } + // } return result.to_processed(); } diff --git a/src/compiler/preprocess/types.ts b/src/compiler/preprocess/types.ts index a4aaf215a76d..66e0d2d94b98 100644 --- a/src/compiler/preprocess/types.ts +++ b/src/compiler/preprocess/types.ts @@ -6,9 +6,12 @@ export interface Processed { } export interface PreprocessorGroup { - markup?: Preprocessor | SyncPreprocessor; - style?: Preprocessor | SyncPreprocessor; - script?: Preprocessor | SyncPreprocessor; + markup?: Preprocessor; + style?: Preprocessor; + script?: Preprocessor; + markup_sync?: SyncPreprocessor; + script_sync?: SyncPreprocessor; + style_sync?: SyncPreprocessor; } interface PreprocessorOptions { @@ -19,4 +22,4 @@ interface PreprocessorOptions { export declare type Preprocessor = (options: PreprocessorOptions) => Promise; -export declare type SyncPreprocessor = ((options: PreprocessorOptions) => Processed) & { is_sync: true }; +export declare type SyncPreprocessor = (options: PreprocessorOptions) => Processed; From d8f02cab2495c032da63bc27baffc55f47b7f78c Mon Sep 17 00:00:00 2001 From: Andreas Ehrencrona Date: Fri, 11 Dec 2020 11:03:42 +0200 Subject: [PATCH 5/6] all transforms --- .../compile/nodes/shared/Expression.ts | 4 +- src/compiler/preprocess/index.ts | 117 ++++++++---------- 2 files changed, 54 insertions(+), 67 deletions(-) diff --git a/src/compiler/compile/nodes/shared/Expression.ts b/src/compiler/compile/nodes/shared/Expression.ts index 1fc23830bd79..89ec4dcd1a76 100644 --- a/src/compiler/compile/nodes/shared/Expression.ts +++ b/src/compiler/compile/nodes/shared/Expression.ts @@ -315,8 +315,8 @@ export default class Expression { // rename #ctx -> child_ctx; walk(func_expression, { enter(node) { - if (node.type === 'Identifier' && node.name === '#ctx') { - node.name = 'child_ctx'; + if (node.type === 'Identifier' && (node as Identifier).name === '#ctx') { + (node as Identifier).name = 'child_ctx'; } } }); diff --git a/src/compiler/preprocess/index.ts b/src/compiler/preprocess/index.ts index 693f5ca59d90..532c6793ac83 100644 --- a/src/compiler/preprocess/index.ts +++ b/src/compiler/preprocess/index.ts @@ -160,19 +160,16 @@ function to_source_update(tags: TagInstance[], processed: Processed[], tag_name: return {...perform_replacements(replacements, source), dependencies}; } -/** - * Calculate the updates required to process all instances of the specified tag. - */ -async function process_tag( +function get_processed_for_tag( tag_name: 'style' | 'script', preprocessor: Preprocessor | SyncPreprocessor, source: Source -): Promise { +) { const { filename } = source; const tags = get_tag_instances(tag_name, source.source); - const processed = await Promise.all(tags.map(({attributes, content}) => { + const processed = tags.map(({attributes, content}) => { if (attributes || content) { return preprocessor({ content, @@ -180,9 +177,22 @@ async function process_tag( filename }); } - })); + }); + + return { tags, processed }; +} + +/** + * Calculate the updates required to process all instances of the specified tag. + */ +async function process_tag( + tag_name: 'style' | 'script', + preprocessor: Preprocessor | SyncPreprocessor, + source: Source +): Promise { + const { tags, processed } = get_processed_for_tag(tag_name, preprocessor, source); - return to_source_update(tags, processed, tag_name, source); + return to_source_update(tags, await Promise.all(processed), tag_name, source); } function process_tag_sync( @@ -190,21 +200,9 @@ function process_tag_sync( preprocessor: SyncPreprocessor, source: Source ): SourceUpdate { - const { filename } = source; - - const tags = get_tag_instances(tag_name, source.source); - - const processed: Processed[] = tags.map(({attributes, content}) => { - if (attributes || content) { - return preprocessor({ - content, - attributes: parse_tag_attributes(attributes || ''), - filename - }); - } - }); + const { tags, processed } = get_processed_for_tag(tag_name, preprocessor, source); - return to_source_update(tags, processed, tag_name, source); + return to_source_update(tags, processed as Processed[], tag_name, source); } function decode_preprocessor_params( @@ -217,11 +215,26 @@ function decode_preprocessor_params( const preprocessors = preprocessor ? (Array.isArray(preprocessor) ? preprocessor : [preprocessor]) : []; const markup = preprocessors.map(p => p.markup).filter(Boolean); + const markup_sync = preprocessors.map(p => p.markup_sync).filter(Boolean); const script = preprocessors.map(p => p.script).filter(Boolean); const script_sync = preprocessors.map(p => p.script_sync).filter(Boolean); const style = preprocessors.map(p => p.style).filter(Boolean); + const style_sync = preprocessors.map(p => p.style_sync).filter(Boolean); - return { markup, script, script_sync, style, filename }; + return { markup, markup_sync, script, script_sync, style, style_sync, filename }; +} + +function processed_markup_to_source_update(processed: Processed): SourceUpdate { + return { + string: processed.code, + map: processed.map + ? // TODO: can we use decode_sourcemap? + typeof processed.map === 'string' + ? JSON.parse(processed.map) + : processed.map + : undefined, + dependencies: processed.dependencies + }; } export function preprocess_sync( @@ -229,49 +242,32 @@ export function preprocess_sync( preprocessor: PreprocessorGroup | PreprocessorGroup[], options?: { filename?: string } ): Processed { - const { script_sync, filename } = decode_preprocessor_params(preprocessor, options); + const { script_sync, style_sync, markup_sync, filename } = decode_preprocessor_params(preprocessor, options); const result = new PreprocessResult(source, filename); // TODO keep track: what preprocessor generated what sourcemap? // to make debugging easier = detect low-resolution sourcemaps in fn combine_mappings - // for (const process of markup) { - // if (is_sync(process)) { - // const processed = process({ - // content: result.source, - // filename, - // attributes: null - // }); - - // if (!processed) continue; - - // result.update_source({ - // string: processed.code, - // map: processed.map - // ? // TODO: can we use decode_sourcemap? - // typeof processed.map === 'string' - // ? JSON.parse(processed.map) - // : processed.map - // : undefined, - // dependencies: processed.dependencies - // }); - // } else { - // console.error(`Preprocessor is not synchronous: ${process}.`); - // } - // } + for (const process of markup_sync) { + const processed = process({ + content: result.source, + filename, + attributes: null + }); + + if (!processed) continue; + + result.update_source(processed_markup_to_source_update(processed)); + } for (const process of script_sync) { result.update_source(process_tag_sync('script', process, result)); } - // for (const process of style) { - // if (is_sync(process)) { - // result.update_source(process_tag_sync('style', process, result)); - // } else { - // console.error(`Preprocessor is not synchronous: ${process}.`); - // } - // } + for (const process of style_sync) { + result.update_source(process_tag_sync('style', process, result)); + } return result.to_processed(); } @@ -297,16 +293,7 @@ export default async function preprocess( if (!processed) continue; - result.update_source({ - string: processed.code, - map: processed.map - ? // TODO: can we use decode_sourcemap? - typeof processed.map === 'string' - ? JSON.parse(processed.map) - : processed.map - : undefined, - dependencies: processed.dependencies - }); + result.update_source(processed_markup_to_source_update(processed)); } for (const process of script) { From 4c48470bb80a583b07895824bc1cecb085471d19 Mon Sep 17 00:00:00 2001 From: Andreas Ehrencrona Date: Fri, 11 Dec 2020 11:05:04 +0200 Subject: [PATCH 6/6] formatting --- src/compiler/preprocess/index.ts | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/compiler/preprocess/index.ts b/src/compiler/preprocess/index.ts index 532c6793ac83..8e98aab2311b 100644 --- a/src/compiler/preprocess/index.ts +++ b/src/compiler/preprocess/index.ts @@ -70,7 +70,7 @@ function processed_content_to_sws( let decoded_map: DecodedSourceMap; if (processed.map) { decoded_map = decode_map(processed); - + sourcemap_add_offset(decoded_map, location); } @@ -102,9 +102,9 @@ function processed_tag_to_sws( interface TagInstance { - tag_with_content: string, - attributes: string, - content: string, + tag_with_content: string, + attributes: string, + content: string, tag_offset: number; } @@ -146,12 +146,12 @@ function to_source_update(tags: TagInstance[], processed: Processed[], tag_name: sws = StringWithSourcemap.from_source(filename, tag_with_content, get_location(tag_offset)); } else { if (processed.dependencies) dependencies.push(...processed.dependencies); - + sws = processed_tag_to_sws(processed, tag_name, attributes, { source: content, get_location: offset => source.get_location(offset + tag_offset), filename - }); + }); } return { offset: tag_offset, length: tag_with_content.length, replacement: sws }; @@ -166,11 +166,11 @@ function get_processed_for_tag( source: Source ) { const { filename } = source; - + const tags = get_tag_instances(tag_name, source.source); - + const processed = tags.map(({attributes, content}) => { - if (attributes || content) { + if (attributes || content) { return preprocessor({ content, attributes: parse_tag_attributes(attributes || ''), @@ -182,8 +182,8 @@ function get_processed_for_tag( return { tags, processed }; } -/** - * Calculate the updates required to process all instances of the specified tag. +/** + * Calculate the updates required to process all instances of the specified tag. */ async function process_tag( tag_name: 'style' | 'script',