Basic support for markup, style and script preprocessors

Suggestion for sveltejs#181 and sveltejs#876

package.json

@@ -46,6 +46,7 @@
     "acorn": "^5.1.1",
     "chalk": "^2.0.1",
     "codecov": "^2.2.0",
+    "coffeescript": "^2.0.2",
     "console-group": "^0.3.2",
     "css-tree": "1.0.0-alpha22",
     "eslint": "^4.3.0",
@@ -54,13 +55,16 @@
     "estree-walker": "^0.5.1",
     "glob": "^7.1.1",
     "jsdom": "^11.1.0",
+    "less": "^2.7.3",
     "locate-character": "^2.0.0",
     "magic-string": "^0.22.3",
     "mocha": "^3.2.0",
     "nightmare": "^2.10.0",
     "node-resolve": "^1.3.3",
+    "node-sass": "^4.7.1",
     "nyc": "^11.1.0",
     "prettier": "^1.7.0",
+    "pug": "^2.0.0-rc.4",
     "reify": "^0.12.3",
     "rollup": "^0.48.2",
     "rollup-plugin-buble": "^0.15.0",
@@ -73,6 +77,7 @@
     "rollup-watch": "^4.3.1",
     "source-map": "^0.5.6",
     "source-map-support": "^0.4.8",
+    "stylus": "^0.54.5",
     "ts-node": "^3.3.0",
     "tslib": "^1.8.0",
     "typescript": "^2.6.1"

src/index.ts

@@ -4,13 +4,14 @@ import generate from './generators/dom/index';
 import generateSSR from './generators/server-side-rendering/index';
 import { assign } from './shared/index.js';
 import Stylesheet from './css/Stylesheet';
-import { Parsed, CompileOptions, Warning } from './interfaces';
+import { Parsed, CompileOptions, Warning, PreprocessOptions, Preprocessor } from './interfaces';
+import { SourceMap } from 'magic-string';
 
 const version = '__VERSION__';
 
 function normalizeOptions(options: CompileOptions): CompileOptions {
-	let normalizedOptions = assign({ generate: 'dom' }, options);
-	const { onwarn, onerror } = normalizedOptions;
+	let normalizedOptions = assign({generate: 'dom', preprocessor: false}, options);
+	const {onwarn, onerror} = normalizedOptions;
 	normalizedOptions.onwarn = onwarn
 		? (warning: Warning) => onwarn(warning, defaultOnwarn)
 		: defaultOnwarn;
@@ -34,9 +35,77 @@ function defaultOnerror(error: Error) {
 	throw error;
 }
 
+function _parseAttributeValue(value: string | boolean) {
+	if (value === 'true' || value === 'false') {
+		return value === 'true';
+	}
+	return (<string>value).replace(/"/ig, '');
+}
+
+function _parseStyleAttributes(str: string) {
+	const attrs = {};
+	str.split(/\s+/).filter(Boolean).forEach(attr => {
+		const [name, value] = attr.split('=');
+		attrs[name] = _parseAttributeValue(value);
+	});
+	return attrs;
+}
+
+async function _doPreprocess(source, type: 'script' | 'style', preprocessor: Preprocessor) {
+	const exp = new RegExp(`<${type}([\\S\\s]*?)>([\\S\\s]*?)<\\/${type}>`, 'ig');
+	const match = exp.exec(source);
+	if (match) {
+		const attributes: Record<string, string | boolean> = _parseStyleAttributes(match[1]);
+		const content: string = match[2];
+		const processed: { code: string, map?: SourceMap | string } = await preprocessor({
+			content,
+			attributes
+		});
+		return source.replace(content, processed.code || content);
+	}
+}
+
+export async function preprocess(source: string, options: PreprocessOptions) {
+	const {markup, style, script} = options;
+	if (!!markup) {
+		try {
+			const processed: { code: string, map?: SourceMap | string } = await markup({content: source});
+			source = processed.code;
+		} catch (error) {
+			defaultOnerror(error);
+		}
+	}
+
+	if (!!style) {
+		try {
+			source = await _doPreprocess(source, 'style', style);
+		} catch (error) {
+			defaultOnerror(error);
+		}
+	}
+
+	if (!!script) {
+		try {
+			source = await _doPreprocess(source, 'script', script);
+		} catch (error) {
+			defaultOnerror(error);
+		}
+	}
+
+	return {
+		// TODO return separated output, in future version where svelte.compile supports it:
+		// style: { code: styleCode, map: styleMap },
+		// script { code: scriptCode, map: scriptMap },
+		// markup { code: markupCode, map: markupMap },
+
+		toString() {
+			return source;
+		}
+	};
+}
+
 export function compile(source: string, _options: CompileOptions) {
 	const options = normalizeOptions(_options);
-
 	let parsed: Parsed;
 
 	try {
@@ -53,7 +122,7 @@ export function compile(source: string, _options: CompileOptions) {
 	const compiler = options.generate === 'ssr' ? generateSSR : generate;
 
 	return compiler(parsed, source, stylesheet, options);
-}
+};
 
 export function create(source: string, _options: CompileOptions = {}) {
 	_options.format = 'eval';
@@ -65,7 +134,7 @@ export function create(source: string, _options: CompileOptions = {}) {
 	}
 
 	try {
-		return (0,eval)(compiled.code);
+		return (0, eval)(compiled.code);
 	} catch (err) {
 		if (_options.onerror) {
 			_options.onerror(err);

src/interfaces.ts

@@ -1,3 +1,5 @@
+import {SourceMap} from 'magic-string';
+
 export interface Node {
 	start: number;
 	end: number;
@@ -59,6 +61,7 @@ export interface CompileOptions {
 
 	onerror?: (error: Error) => void;
 	onwarn?: (warning: Warning) => void;
+	preprocessor?: ((raw: string) => string) | false ;
 }
 
 export interface GenerateOptions {
@@ -77,4 +80,12 @@ export interface Visitor {
 export interface CustomElementOptions {
 	tag?: string;
 	props?: string[];
-}
+}
+
+export interface PreprocessOptions {
+	markup?: (options: {content: string}) => { code: string, map?: SourceMap | string };
+	style?: Preprocessor;
+	script?: Preprocessor;
+}
+
+export type Preprocessor = (options: {content: string, attributes: Record<string, string | boolean>}) => { code: string, map?: SourceMap | string };

test/preprocess/index.js

@@ -0,0 +1,165 @@
+import assert from 'assert';
+import * as fs from 'fs';
+import {parse} from 'acorn';
+import {addLineNumbers, env, normalizeHtml, svelte} from '../helpers.js';
+
+function tryRequire(file) {
+	try {
+		const mod = require(file);
+		return mod.default || mod;
+	} catch (err) {
+		if (err.code !== 'MODULE_NOT_FOUND') throw err;
+		return null;
+	}
+}
+
+function normalizeWarning(warning) {
+	warning.frame = warning.frame.replace(/^\n/, '').
+		replace(/^\t+/gm, '').
+		replace(/\s+$/gm, '');
+	delete warning.filename;
+	delete warning.toString;
+	return warning;
+}
+
+function checkCodeIsValid(code) {
+	try {
+		parse(code);
+	} catch (err) {
+		console.error(addLineNumbers(code));
+		throw new Error(err.message);
+	}
+}
+
+describe('preprocess', () => {
+	fs.readdirSync('test/preprocess/samples').forEach(dir => {
+		if (dir[0] === '.') return;
+
+		// add .solo to a sample directory name to only run that test
+		const solo = /\.solo/.test(dir);
+		const skip = /\.skip/.test(dir);
+
+		if (solo && process.env.CI) {
+			throw new Error('Forgot to remove `solo: true` from test');
+		}
+
+		(solo ? it.only : skip ? it.skip : it)(dir, () => {
+			const config = tryRequire(`./samples/${dir}/_config.js`) || {};
+			const input = fs.existsSync(`test/preprocess/samples/${dir}/input.pug`) ?
+				fs.readFileSync(`test/preprocess/samples/${dir}/input.pug`,
+				'utf-8').replace(/\s+$/, '') :
+				fs.readFileSync(`test/preprocess/samples/${dir}/input.html`,
+				'utf-8').replace(/\s+$/, '');
+
+			svelte.preprocess(input, config).
+				then(processed => processed.toString()).
+				then(processed => {
+
+					const expectedWarnings = (config.warnings || []).map(
+						normalizeWarning);
+					const domWarnings = [];
+					const ssrWarnings = [];
+
+					const dom = svelte.compile(
+						processed,
+						Object.assign(config, {
+							format: 'iife',
+							name: 'SvelteComponent',
+							onwarn: warning => {
+								domWarnings.push(warning);
+							},
+						}),
+					);
+
+					const ssr = svelte.compile(
+						processed,
+						Object.assign(config, {
+							format: 'iife',
+							generate: 'ssr',
+							name: 'SvelteComponent',
+							onwarn: warning => {
+								ssrWarnings.push(warning);
+							},
+						}),
+					);
+
+					// check the code is valid
+					checkCodeIsValid(dom.code);
+					checkCodeIsValid(ssr.code);
+
+					assert.equal(dom.css, ssr.css);
+
+					assert.deepEqual(
+						domWarnings.map(normalizeWarning),
+						ssrWarnings.map(normalizeWarning),
+					);
+					assert.deepEqual(domWarnings.map(normalizeWarning), expectedWarnings);
+
+					const expected = {
+						html: read(`test/preprocess/samples/${dir}/expected.html`),
+						css: read(`test/preprocess/samples/${dir}/expected.css`),
+					};
+
+					if (expected.css !== null) {
+						fs.writeFileSync(`test/preprocess/samples/${dir}/_actual.css`, dom.css);
+						assert.equal(dom.css.replace(/svelte-\d+/g, 'svelte-xyz'),
+							expected.css);
+					}
+
+					// verify that the right elements have scoping selectors
+					if (expected.html !== null) {
+						const window = env();
+
+						// dom
+						try {
+							const Component = eval(
+								`(function () { ${dom.code}; return SvelteComponent; }())`
+							);
+							const target = window.document.querySelector('main');
+
+							new Component({ target, data: config.data });
+							const html = target.innerHTML;
+
+							fs.writeFileSync(`test/preprocess/samples/${dir}/_actual.html`, html);
+
+							assert.equal(
+								normalizeHtml(window, html.replace(/svelte-\d+/g, 'svelte-xyz')),
+								normalizeHtml(window, expected.html)
+							);
+						} catch (err) {
+							console.log(dom.code);
+							throw err;
+						}
+
+						// ssr
+						try {
+							const component = eval(
+								`(function () { ${ssr.code}; return SvelteComponent; }())`
+							);
+
+							assert.equal(
+								normalizeHtml(
+									window,
+									component.render(config.data).replace(/svelte-\d+/g, 'svelte-xyz')
+								),
+								normalizeHtml(window, expected.html)
+							);
+						} catch (err) {
+							console.log(ssr.code);
+							throw err;
+						}
+					}
+				}).catch(error => {
+					throw error
+				});
+		});
+	});
+});
+
+function read(file) {
+	try {
+		return fs.readFileSync(file, 'utf-8');
+	} catch (err) {
+		return null;
+	}
+}

test/preprocess/samples/use-coffeescript-preprocessor/_config.js

@@ -0,0 +1,19 @@
+import * as CoffeeScript from 'coffeescript';
+
+export default {
+	cascade: false,
+	script: ({content, attributes}) => {
+		if (attributes.type !== 'text/coffeescript') {
+			return {code: content};
+		}
+
+		return new Promise((fulfil, reject) => {
+			try {
+				const code = CoffeeScript.compile(content, {});
+				fulfil({code});
+			} catch (error) {
+				reject(error);
+			}
+		});
+	},
+};

test/preprocess/samples/use-coffeescript-preprocessor/expected.html

@@ -0,0 +1 @@
+<h1>Hello foo!</h1>

test/preprocess/samples/use-coffeescript-preprocessor/input.html

@@ -0,0 +1,8 @@
+<h1>Hello {{name}}!</h1>
+
+<script type="text/coffeescript">
+	export default {
+		data: () ->
+			name: 'foo'
+	};
+</script>

test/preprocess/samples/use-pug-preprocessor/_config.js

@@ -0,0 +1,15 @@
+import * as pug from 'pug';
+
+export default {
+	cascade: false,
+	markup: ({content}) => {
+		return new Promise((fulfil, reject) => {
+			try {
+				const code = pug.render(content);
+				fulfil({code});
+			} catch (error) {
+				reject(error);
+			}
+		});
+	},
+};

test/preprocess/samples/use-pug-preprocessor/expected.css

@@ -0,0 +1 @@
+[foo][svelte-xyz]{color:red}[baz][svelte-xyz]{color:blue}