Skip to content

Commit

Permalink
Tests: Better pretty print (#2600)
Browse files Browse the repository at this point in the history
This is a major improvement for the pretty-printer.

The main change is that `TokenStreamTransformer.prettyprint` now takes a Prism token stream instead of a simplified token stream. This means that it has a lot more information to work with and it uses that information.

It mostly behaves like the old pretty-printer with 3 main differences:

1. Tokens that contain a nested token stream that only contains a single string are printed as one line.
   Example: `["namespace", ["java"]]`
2. The pretty-printer can now glue tokens together so that they are printed in a single line.
    Example: `["number", "100"], ["operator", ".."], ["number", "100"]`
3. Line breaks in the input code may now cause empty lines in the output. Previously, only blank strings containing more at least 2 line breaks produce empty lines. This helps readability as it groups tokens together.
  • Loading branch information
RunDevelopment authored Nov 21, 2020
1 parent f0e2b70 commit 8bfcc81
Show file tree
Hide file tree
Showing 4 changed files with 435 additions and 54 deletions.
3 changes: 2 additions & 1 deletion tests/core/greedy.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,14 @@
const { assert } = require('chai');
const PrismLoader = require('../helper/prism-loader');
const TestCase = require('../helper/test-case');
const TokenStreamTransformer = require('../helper/token-stream-transformer');


function testTokens({ grammar, code, expected }) {
const Prism = PrismLoader.createEmptyPrism();
Prism.languages.test = grammar;

const simpleTokens = TestCase.simpleTokenize(Prism, code, 'test');
const simpleTokens = TokenStreamTransformer.simplify(TestCase.tokenize(Prism, code, 'test'));

assert.deepStrictEqual(simpleTokens, expected);
}
Expand Down
23 changes: 15 additions & 8 deletions tests/helper/test-case.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ const { assert } = require("chai");
const PrismLoader = require("./prism-loader");
const TokenStreamTransformer = require("./token-stream-transformer");

/**
* @typedef {import("./token-stream-transformer").TokenStream} TokenStream
*/

/**
* Handles parsing of a test case file.
*
Expand Down Expand Up @@ -58,7 +62,7 @@ module.exports = {
const Prism = PrismLoader.createInstance(usedLanguages.languages);

// the first language is the main language to highlight
const simplifiedTokenStream = this.simpleTokenize(Prism, testCase.code, usedLanguages.mainLanguage);
const tokenStream = this.tokenize(Prism, testCase.code, usedLanguages.mainLanguage);

if (testCase.expectedTokenStream === null) {
// the test case doesn't have an expected value
Expand All @@ -71,7 +75,7 @@ module.exports = {
// change the file
const lineEnd = (/\r\n/.test(testCase.code) || !/\n/.test(testCase.code)) ? '\r\n' : '\n';
const separator = "\n\n----------------------------------------------------\n\n";
const pretty = TokenStreamTransformer.prettyprint(simplifiedTokenStream)
const pretty = TokenStreamTransformer.prettyprint(tokenStream)
.replace(/^( +)/gm, m => {
return "\t".repeat(m.length / 4);
});
Expand All @@ -80,11 +84,14 @@ module.exports = {
if (testCase.comment) {
content += separator + testCase.comment;
}
//content += '\n'
content = content.replace(/\r?\n/g, lineEnd);

fs.writeFileSync(filePath, content, "utf-8");
} else {
// there is an expected value
const simplifiedTokenStream = TokenStreamTransformer.simplify(tokenStream);

const actual = JSON.stringify(simplifiedTokenStream);
const expected = JSON.stringify(testCase.expectedTokenStream);

Expand All @@ -100,7 +107,7 @@ module.exports = {
const columnNumber = expectedJsonLines.pop().length + 1;
const lineNumber = testCase.expectedLineOffset + expectedJsonLines.length;

const tokenStreamStr = TokenStreamTransformer.prettyprint(simplifiedTokenStream);
const tokenStreamStr = TokenStreamTransformer.prettyprint(tokenStream);
const message = "\n\nActual Token Stream:" +
"\n-----------------------------------------\n" +
tokenStreamStr +
Expand All @@ -112,16 +119,16 @@ module.exports = {
},

/**
* Returns the simplified token stream of the given code highlighted with `language`.
* Returns the token stream of the given code highlighted with `language`.
*
* The `before-tokenize` and `after-tokenize` hooks will also be executed.
*
* @param {import('../../components/prism-core')} Prism The Prism instance which will tokenize `code`.
* @param {string} code The code to tokenize.
* @param {string} language The language id.
* @returns {Array<string|Array<string|any[]>>}
* @returns {TokenStream}
*/
simpleTokenize(Prism, code, language) {
tokenize(Prism, code, language) {
const env = {
code,
grammar: Prism.languages[language],
Expand All @@ -132,7 +139,7 @@ module.exports = {
env.tokens = Prism.tokenize(env.code, env.grammar);
Prism.hooks.run('after-tokenize', env);

return TokenStreamTransformer.simplify(env.tokens);
return env.tokens;
},


Expand Down Expand Up @@ -202,7 +209,7 @@ module.exports = {

const code = testCaseParts[0].trim();
const expected = (testCaseParts[1] || '').trim();
const comment = (testCaseParts[2] || '').trim();
const comment = (testCaseParts[2] || '').trimStart();

const testCase = {
code,
Expand Down
Loading

0 comments on commit 8bfcc81

Please sign in to comment.