Skip to content

Commit

Permalink
feat: rework the nodes-to-string function to output correct tags
Browse files Browse the repository at this point in the history
  • Loading branch information
@cheton
cheton committed Jul 7, 2022
1 parent 1d8294d commit e13eac5
Show file tree
Hide file tree
Showing 6 changed files with 108 additions and 53 deletions.
5 changes: 3 additions & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "i18next-scanner",
"version": "3.3.0",
"version": "4.0.0",
"description": "Scan your code, extract translation keys/values, and merge them into i18n resource files.",
"homepage": "https://github.com/i18next/i18next-scanner",
"author": "Cheton Wu <[email protected]>",
Expand Down Expand Up @@ -57,13 +57,14 @@
"clone-deep": "^4.0.0",
"commander": "^9.0.0",
"deepmerge": "^4.0.0",
"ensure-array": "^1.0.0",
"ensure-type": "^1.5.0",
"eol": "^0.9.1",
"esprima-next": "^5.7.0",
"gulp-sort": "^2.0.0",
"i18next": "*",
"lodash": "^4.0.0",
"parse5": "^6.0.0",
"react-is": ">=16",
"sortobject": "^4.0.0",
"through2": "^4.0.0",
"vinyl": "^2.2.0",
Expand Down
21 changes: 17 additions & 4 deletions src/nodes-to-string.js
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { ensureArray, ensureBoolean } from 'ensure-type';
import _get from 'lodash/get';

const isJSXText = (node) => {
Expand Down Expand Up @@ -32,7 +33,10 @@ const isObjectExpression = (node) => {
return node.type === 'ObjectExpression';
};

const nodesToString = (nodes, code) => {
const nodesToString = (nodes, options) => {
const supportBasicHtmlNodes = ensureBoolean(options?.supportBasicHtmlNodes);
const keepBasicHtmlNodesFor = ensureArray(options?.keepBasicHtmlNodesFor);

let memo = '';
let nodeIndex = 0;
nodes.forEach((node, i) => {
Expand All @@ -55,14 +59,23 @@ const nodesToString = (nodes, code) => {
} if (isStringLiteral(expression)) {
memo += expression.value;
} else if (isObjectExpression(expression) && (_get(expression, 'properties[0].type') === 'Property')) {
memo += `<${nodeIndex}>{{${expression.properties[0].key.name}}}</${nodeIndex}>`;
memo += `{{${expression.properties[0].key.name}}}`;
} else {
console.error(`Unsupported JSX expression. Only static values or {{interpolation}} blocks are supported. Got ${expression.type}:`);
console.error(code.slice(node.start, node.end));
console.error(ensureString(options?.code).slice(node.start, node.end));
console.error(node.expression);
}
} else if (node.children) {
memo += `<${nodeIndex}>${nodesToString(node.children, code)}</${nodeIndex}>`;
const shouldKeepChild = supportBasicHtmlNodes && keepBasicHtmlNodesFor.indexOf(node.openingElement?.name?.name) > -1;
const selfClosing = node.openingElement?.selfClosing;
const openingElement = shouldKeepChild ? node.openingElement?.name?.name : nodeIndex;
const closingElement = shouldKeepChild ? node.closingElement?.name?.name : nodeIndex;

if (selfClosing) {
memo += `<${openingElement}/>${nodesToString(node.children, options)}`;
} else {
memo += `<${openingElement}>${nodesToString(node.children, options)}</${closingElement}>`;
}
}

++nodeIndex;
Expand Down
20 changes: 17 additions & 3 deletions src/parser.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import acornStage3 from 'acorn-stage3';
import chalk from 'chalk';
import cloneDeep from 'clone-deep';
import deepMerge from 'deepmerge';
import ensureArray from 'ensure-array';
import { ensureArray } from 'ensure-type';
import { parse } from 'esprima-next';
import _ from 'lodash';
import parse5 from 'parse5';
Expand Down Expand Up @@ -47,7 +47,9 @@ const defaults = {
ecmaVersion: 2020, // defaults to 2020
sourceType: 'module', // defaults to 'module'
// Check out https://github.com/acornjs/acorn/tree/master/acorn#interface for additional options
}
},
supportBasicHtmlNodes: true, // Enables keeping the name of simple nodes (e.g. <br/>) in translations instead of indexed keys
keepBasicHtmlNodesFor: ['br', 'strong', 'i', 'p'], // Which nodes are allowed to be kept in translations during defaultValue generation of <Trans>.
},

lngs: ['en'], // array of supported languages
Expand Down Expand Up @@ -171,6 +173,12 @@ const normalizeOptions = (options) => {
if (_.isUndefined(_.get(options, 'trans.acorn'))) {
_.set(options, 'trans.acorn', defaults.trans.acorn);
}
if (_.isUndefined(_.get(options, 'trans.supportBasicHtmlNodes'))) {
_.set(options, 'trans.supportBasicHtmlNodes', defaults.trans.supportBasicHtmlNodes);
}
if (_.isUndefined(_.get(options, 'trans.keepBasicHtmlNodesFor'))) {
_.set(options, 'trans.keepBasicHtmlNodesFor', defaults.trans.keepBasicHtmlNodesFor);
}
}

// Resource
Expand Down Expand Up @@ -538,6 +546,8 @@ class Parser {
defaultsKey = this.options.trans.defaultsKey, // string
fallbackKey, // boolean|function
acorn: acornOptions = this.options.trans.acorn, // object
supportBasicHtmlNodes = this.options.trans.supportBasicHtmlNodes, // boolean
keepBasicHtmlNodesFor = this.options.trans.keepBasicHtmlNodesFor, // array
} = { ...opts };

const parseJSXElement = (node, code) => {
Expand Down Expand Up @@ -634,7 +644,11 @@ class Parser {
const tOptions = attr.tOptions;
const options = {
...tOptions,
defaultValue: defaultsString || nodesToString(node.children, code),
defaultValue: defaultsString || nodesToString(node.children, {
code,
supportBasicHtmlNodes,
keepBasicHtmlNodesFor,
}),
fallbackKey: fallbackKey || this.options.trans.fallbackKey
};

Expand Down
32 changes: 28 additions & 4 deletions test/jsx-parser.test.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { Parser } from 'acorn';
import jsx from 'acorn-jsx';
import ensureArray from 'ensure-array';
import { ensureArray } from 'ensure-type';
import _get from 'lodash/get';
import nodesToString from '../src/nodes-to-string';

Expand All @@ -13,7 +13,13 @@ const jsxToString = (code) => {
return '';
}

return nodesToString(nodes, code);
const options = {
code,
supportBasicHtmlNodes: true,
keepBasicHtmlNodesFor: ['br', 'strong', 'i', 'p'],
};

return nodesToString(nodes, options);
} catch (e) {
console.error(e);
return '';
Expand All @@ -22,11 +28,29 @@ const jsxToString = (code) => {

test('JSX to i18next', () => {
expect(jsxToString('Basic text')).toBe('Basic text');
expect(jsxToString('Hello, {{name}}')).toBe('Hello, <1>{{name}}</1>');
expect(jsxToString('Hello, {{name}}')).toBe('Hello, {{name}}');
expect(jsxToString('I agree to the <Link>terms</Link>.')).toBe('I agree to the <1>terms</1>.');
expect(jsxToString('One &amp; two')).toBe('One & two');
});

test('HTML entities', () => {
expect(jsxToString('Don&apos;t do this <strong>Dave</strong>')).toBe('Don\'t do this <1>Dave</1>');
expect(jsxToString('Don&apos;t do this <strong>Dave</strong>')).toBe('Don\'t do this <strong>Dave</strong>');
});

test('HTML nodes', () => {
expect(jsxToString('Hello, <span>{{name}}</span>')).toBe('Hello, <1>{{name}}</1>');
expect(jsxToString(`
<>
<div><strong>title</strong></div>
<div>content</div>
</>
`)).toBe('<0><0><strong>title</strong></0><1>content</1></0>');
expect(jsxToString(`
<>
line 1
<br/>
line 2
<br/>
</>
`)).toBe('<0>line 1<br/>line 2<br/></0>');
});
57 changes: 29 additions & 28 deletions test/parser.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,40 +160,41 @@ test('Parse Trans components', () => {
expect(parser.get()).toEqual({
en: {
dev: {
'Hello <1>World</1>, you have <3>{{count}}</3> unread message.': 'Hello <1>World</1>, you have <3>{{count}}</3> unread message.',
'Hello <1>World</1>, you have <3>{{count}}</3> unread message._plural': 'Hello <1>World</1>, you have <3>{{count}}</3> unread message.'
'Hello <strong>World</strong>, you have {{count}} unread message.': 'Hello <strong>World</strong>, you have {{count}} unread message.',
'Hello <strong>World</strong>, you have {{count}} unread message._plural': 'Hello <strong>World</strong>, you have {{count}} unread message.'
},
translation: {
// quote style
'jsx-quotes-double': 'Use double quotes for the i18nKey attribute',
'jsx-quotes-single': 'Use single quote for the i18nKey attribute',

// plural
'plural': 'You have <1>{{count}}</1> apples',
'plural_plural': 'You have <1>{{count}}</1> apples',
'plural': 'You have {{count}} apples',
'plural_plural': 'You have {{count}} apples',

// context
'context': 'A boyfriend',
'context_male': 'A boyfriend',

// i18nKey
'multiline-text-string': 'multiline text string',
'string-literal': 'This is a <1>test</1>',
'object-expression': 'This is a <1><0>{{test}}</0></1>',
'arithmetic-expression': '2 + 2 = <1>{{result}}</1>',
'string-literal': 'This is a <strong>test</strong>',
'object-expression': 'This is a <strong>{{test}}</strong>',
'arithmetic-expression': '2 + 2 = {{result}}',
'components': 'Go to <1>Administration > Tools</1> to download administrative tools.',
'lorem-ipsum': '<0>Lorem Ipsum is simply dummy text of the printing and typesetting industry.</0>Lorem Ipsum is simply dummy text of the printing and typesetting industry.<2>Lorem Ipsum has been the industry\'s standard dummy text ever since the 1500s</2>',
'lorem-ipsum-nested': 'Lorem Ipsum is simply dummy text of the printing and typesetting industry.<1>Lorem Ipsum is simply dummy text of the printing and typesetting industry.<1>Lorem Ipsum is simply dummy text of the printing and typesetting industry.</1></1><2>Lorem Ipsum has been the industry\'s standard dummy text ever since the 1500s</2>',

"lorem-ipsum": "<p>Lorem Ipsum is simply dummy text of the printing and typesetting industry.</p>Lorem Ipsum is simply dummy text of the printing and typesetting industry.<p>Lorem Ipsum has been the industry's standard dummy text ever since the 1500s</p>",
"lorem-ipsum-nested": "Lorem Ipsum is simply dummy text of the printing and typesetting industry.<p>Lorem Ipsum is simply dummy text of the printing and typesetting industry.<p>Lorem Ipsum is simply dummy text of the printing and typesetting industry.</p></p><p>Lorem Ipsum has been the industry's standard dummy text ever since the 1500s</p>",

// fallback key
'Hello, World!': 'Hello, World!',
'multiline text string': 'multiline text string',
'This is a <1>test</1>': 'This is a <1>test</1>',
'This is a <1><0>{{test}}</0></1>': 'This is a <1><0>{{test}}</0></1>',
'2 + 2 = <1>{{result}}</1>': '2 + 2 = <1>{{result}}</1>',
'This is a <strong>test</strong>': 'This is a <strong>test</strong>',
'This is a <strong>{{test}}</strong>': 'This is a <strong>{{test}}</strong>',
'2 + 2 = {{result}}': '2 + 2 = {{result}}',
'Go to <1>Administration > Tools</1> to download administrative tools.': 'Go to <1>Administration > Tools</1> to download administrative tools.',
'<0>Lorem Ipsum is simply dummy text of the printing and typesetting industry.</0>Lorem Ipsum is simply dummy text of the printing and typesetting industry.<2>Lorem Ipsum has been the industry\'s standard dummy text ever since the 1500s</2>': '<0>Lorem Ipsum is simply dummy text of the printing and typesetting industry.</0>Lorem Ipsum is simply dummy text of the printing and typesetting industry.<2>Lorem Ipsum has been the industry\'s standard dummy text ever since the 1500s</2>',
'Lorem Ipsum is simply dummy text of the printing and typesetting industry.<1>Lorem Ipsum is simply dummy text of the printing and typesetting industry.<1>Lorem Ipsum is simply dummy text of the printing and typesetting industry.</1></1><2>Lorem Ipsum has been the industry\'s standard dummy text ever since the 1500s</2>': 'Lorem Ipsum is simply dummy text of the printing and typesetting industry.<1>Lorem Ipsum is simply dummy text of the printing and typesetting industry.<1>Lorem Ipsum is simply dummy text of the printing and typesetting industry.</1></1><2>Lorem Ipsum has been the industry\'s standard dummy text ever since the 1500s</2>',
"<p>Lorem Ipsum is simply dummy text of the printing and typesetting industry.</p>Lorem Ipsum is simply dummy text of the printing and typesetting industry.<p>Lorem Ipsum has been the industry's standard dummy text ever since the 1500s</p>": "<p>Lorem Ipsum is simply dummy text of the printing and typesetting industry.</p>Lorem Ipsum is simply dummy text of the printing and typesetting industry.<p>Lorem Ipsum has been the industry's standard dummy text ever since the 1500s</p>",
"Lorem Ipsum is simply dummy text of the printing and typesetting industry.<p>Lorem Ipsum is simply dummy text of the printing and typesetting industry.<p>Lorem Ipsum is simply dummy text of the printing and typesetting industry.</p></p><p>Lorem Ipsum has been the industry's standard dummy text ever since the 1500s</p>": "Lorem Ipsum is simply dummy text of the printing and typesetting industry.<p>Lorem Ipsum is simply dummy text of the printing and typesetting industry.<p>Lorem Ipsum is simply dummy text of the printing and typesetting industry.</p></p><p>Lorem Ipsum has been the industry's standard dummy text ever since the 1500s</p>",

// defaults
'The component might be self-closing': 'The component might be self-closing',
Expand Down Expand Up @@ -233,40 +234,40 @@ test('Parse Trans components with fallback key', () => {
expect(parser.get()).toEqual({
en: {
dev: {
'2290678f8f33c49494499fe5e32b4ebd124d9292': 'Hello <1>World</1>, you have <3>{{count}}</3> unread message.',
'2290678f8f33c49494499fe5e32b4ebd124d9292_plural': 'Hello <1>World</1>, you have <3>{{count}}</3> unread message.'
"2f9bdac18af14a38ed4c147f8076c70b740c2ca6": "Hello <strong>World</strong>, you have {{count}} unread message.",
"2f9bdac18af14a38ed4c147f8076c70b740c2ca6_plural": "Hello <strong>World</strong>, you have {{count}} unread message.",
},
translation: {
// quote style
'jsx-quotes-double': 'Use double quotes for the i18nKey attribute',
'jsx-quotes-single': 'Use single quote for the i18nKey attribute',

// plural
'plural': 'You have <1>{{count}}</1> apples',
'plural_plural': 'You have <1>{{count}}</1> apples',
'plural': 'You have {{count}} apples',
'plural_plural': 'You have {{count}} apples',

// context
'context': 'A boyfriend',
'context_male': 'A boyfriend',

// i18nKey
'multiline-text-string': 'multiline text string',
'string-literal': 'This is a <1>test</1>',
'object-expression': 'This is a <1><0>{{test}}</0></1>',
'arithmetic-expression': '2 + 2 = <1>{{result}}</1>',
"string-literal": "This is a <strong>test</strong>",
"object-expression": "This is a <strong>{{test}}</strong>",
'arithmetic-expression': '2 + 2 = {{result}}',
'components': 'Go to <1>Administration > Tools</1> to download administrative tools.',
'lorem-ipsum': '<0>Lorem Ipsum is simply dummy text of the printing and typesetting industry.</0>Lorem Ipsum is simply dummy text of the printing and typesetting industry.<2>Lorem Ipsum has been the industry\'s standard dummy text ever since the 1500s</2>',
'lorem-ipsum-nested': 'Lorem Ipsum is simply dummy text of the printing and typesetting industry.<1>Lorem Ipsum is simply dummy text of the printing and typesetting industry.<1>Lorem Ipsum is simply dummy text of the printing and typesetting industry.</1></1><2>Lorem Ipsum has been the industry\'s standard dummy text ever since the 1500s</2>',
"lorem-ipsum": "<p>Lorem Ipsum is simply dummy text of the printing and typesetting industry.</p>Lorem Ipsum is simply dummy text of the printing and typesetting industry.<p>Lorem Ipsum has been the industry's standard dummy text ever since the 1500s</p>",
"lorem-ipsum-nested": "Lorem Ipsum is simply dummy text of the printing and typesetting industry.<p>Lorem Ipsum is simply dummy text of the printing and typesetting industry.<p>Lorem Ipsum is simply dummy text of the printing and typesetting industry.</p></p><p>Lorem Ipsum has been the industry's standard dummy text ever since the 1500s</p>",

// fallback key
'0a0a9f2a6772942557ab5355d76af442f8f65e01': 'Hello, World!',
'32876cbad378f3153c900c297ed2efa06243e0e2': 'multiline text string',
'e4ca61dff6bc759d214e32c4e37c8ae594ca163d': 'This is a <1>test</1>',
'0ce90193dd25c93cdc12f25a36d31004a74c63de': 'This is a <1><0>{{test}}</0></1>',
'493781e20cd3cfd5b3137963519571c3d97ab383': '2 + 2 = <1>{{result}}</1>',
"e2eff612e7754fb3954034c9be7e600a0456b84b": "This is a <strong>test</strong>",
"453d2d7753427d2693a0c49758dbcfa30cd71834": "This is a <strong>{{test}}</strong>",
'd9ad3431d982619e3b7bd34ed248205312e95bff': '2 + 2 = {{result}}',
'083eac6b4f73ec317824caaaeea57fba3b83c1d9': 'Go to <1>Administration > Tools</1> to download administrative tools.',
'938c04be9e14562b7532a19458fe92b65c6ef941': '<0>Lorem Ipsum is simply dummy text of the printing and typesetting industry.</0>Lorem Ipsum is simply dummy text of the printing and typesetting industry.<2>Lorem Ipsum has been the industry\'s standard dummy text ever since the 1500s</2>',
'9c3ca5d5d8089e96135c8c7c9f42ba34a635fb47': 'Lorem Ipsum is simply dummy text of the printing and typesetting industry.<1>Lorem Ipsum is simply dummy text of the printing and typesetting industry.<1>Lorem Ipsum is simply dummy text of the printing and typesetting industry.</1></1><2>Lorem Ipsum has been the industry\'s standard dummy text ever since the 1500s</2>',
"11a5b6b7b39fc588da3a1a82d77051bc0e80ad71": "<p>Lorem Ipsum is simply dummy text of the printing and typesetting industry.</p>Lorem Ipsum is simply dummy text of the printing and typesetting industry.<p>Lorem Ipsum has been the industry's standard dummy text ever since the 1500s</p>",
"99dbaae55f815e76bdd1b375068c5cc734a7c5b4": "Lorem Ipsum is simply dummy text of the printing and typesetting industry.<p>Lorem Ipsum is simply dummy text of the printing and typesetting industry.<p>Lorem Ipsum is simply dummy text of the printing and typesetting industry.</p></p><p>Lorem Ipsum has been the industry's standard dummy text ever since the 1500s</p>",

// defaults
'7551746c2d33a1d0a24658c22821c8700fa58a0d': 'Hello <1>{{planet}}</1>!',
Expand Down
Loading

0 comments on commit e13eac5

Please sign in to comment.