diff --git a/lib/JSON2CSVAsyncParser.js b/lib/JSON2CSVAsyncParser.js index d260f60a..2a07d040 100644 --- a/lib/JSON2CSVAsyncParser.js +++ b/lib/JSON2CSVAsyncParser.js @@ -2,6 +2,7 @@ const { Transform } = require('stream'); const JSON2CSVTransform = require('./JSON2CSVTransform'); +const { fastJoin } = require('./utils'); class JSON2CSVAsyncParser { constructor(opts, transformOpts) { @@ -40,10 +41,10 @@ class JSON2CSVAsyncParser { promise() { return new Promise((resolve, reject) => { - let csv = ''; + let csvBuffer = []; this.processor - .on('data', chunk => (csv += chunk.toString())) - .on('finish', () => resolve(csv)) + .on('data', chunk => csvBuffer.push(chunk.toString())) + .on('finish', () => resolve(fastJoin(csvBuffer, ''))) .on('error', err => reject(err)); }); } diff --git a/lib/JSON2CSVBase.js b/lib/JSON2CSVBase.js index 167a509a..d79429cb 100644 --- a/lib/JSON2CSVBase.js +++ b/lib/JSON2CSVBase.js @@ -2,7 +2,7 @@ const os = require('os'); const lodashGet = require('lodash.get'); -const { setProp, flattenReducer } = require('./utils'); +const { getProp, setProp, fastJoin, flattenReducer } = require('./utils'); class JSON2CSVBase { constructor(opts) { @@ -29,7 +29,7 @@ class JSON2CSVBase { : '"'; processedOpts.doubleQuote = typeof processedOpts.doubleQuote === 'string' ? processedOpts.doubleQuote - : Array(3).join(processedOpts.quote); + : processedOpts.quote + processedOpts.quote; processedOpts.header = processedOpts.header !== false; processedOpts.includeEmptyRows = processedOpts.includeEmptyRows || false; processedOpts.withBOM = processedOpts.withBOM || false; @@ -49,7 +49,9 @@ class JSON2CSVBase { if (typeof fieldInfo === 'string') { return { label: fieldInfo, - value: row => lodashGet(row, fieldInfo, this.opts.defaultValue), + value: (fieldInfo.includes('.') || fieldInfo.includes('[')) + ? row => lodashGet(row, fieldInfo, this.opts.defaultValue) + : row => getProp(row, fieldInfo, this.opts.defaultValue), stringify: true, }; } @@ -62,7 +64,9 @@ class JSON2CSVBase { if (typeof fieldInfo.value === 'string') { return { label: fieldInfo.label || fieldInfo.value, - value: row => lodashGet(row, fieldInfo.value, defaultValue), + value: (fieldInfo.value.includes('.') || fieldInfo.value.includes('[')) + ? row => lodashGet(row, fieldInfo.value, defaultValue) + : row => getProp(row, fieldInfo.value, defaultValue), stringify: fieldInfo.stringify !== undefined ? fieldInfo.stringify : true, }; } @@ -93,9 +97,10 @@ class JSON2CSVBase { * @returns {String} titles as a string */ getHeader() { - return this.opts.fields - .map(fieldInfo => this.processValue(fieldInfo.label, true)) - .join(this.opts.delimiter); + return fastJoin( + this.opts.fields.map(fieldInfo => this.processValue(fieldInfo.label, true)), + this.opts.delimiter + ); } memoizePreprocessRow() { @@ -139,15 +144,20 @@ class JSON2CSVBase { * @returns {String} CSV string (row) */ processRow(row) { - if (!row - || (Object.getOwnPropertyNames(row).length === 0 - && !this.opts.includeEmptyRows)) { + if (!row) { return undefined; } - return this.opts.fields - .map(fieldInfo => this.processCell(row, fieldInfo)) - .join(this.opts.delimiter); + const processedRow = this.opts.fields.map(fieldInfo => this.processCell(row, fieldInfo)); + + if (!this.opts.includeEmptyRows && processedRow.every(field => field === undefined)) { + return undefined; + } + + return fastJoin( + processedRow, + this.opts.delimiter + ); } /** @@ -173,61 +183,39 @@ class JSON2CSVBase { return undefined; } - const isValueString = typeof value === 'string'; - if (isValueString) { - value = value - .replace(/\n/g, '\u2028') - .replace(/\r/g, '\u2029') - .replace(/\t/g, '\u21E5'); - } - - //JSON.stringify('\\') results in a string with two backslash - //characters in it. I.e. '\\\\'. - let stringifiedValue = (stringify - ? JSON.stringify(value) - : value); - - if (typeof value === 'object' && !/^".*"$/.test(stringifiedValue)) { - // Stringify object that are not stringified to a - // JSON string (like Date) to escape commas, quotes, etc. - stringifiedValue = JSON.stringify(stringifiedValue); - } + const valueType = typeof value; + if (valueType !== 'boolean' && valueType !== 'number' && valueType !== 'string') { + value = JSON.stringify(value); - if (stringifiedValue === undefined) { - return undefined; - } + if (value === undefined) { + return undefined; + } - if (isValueString) { - stringifiedValue = stringifiedValue - .replace(/\u2028/g, '\n') - .replace(/\u2029/g, '\r') - .replace(/\u21E5/g, '\t'); + if (value[0] === '"') { + value = value.replace(/^"(.+)"$/,'$1'); + } } - if (this.opts.quote === '"') { - // Replace automatically escaped single quotes by doubleQuotes - stringifiedValue = stringifiedValue - .replace(/\\"(?!$)/g, this.opts.doubleQuote); - } else { - // Unescape double quotes ('"') - // Replace single quote with double quote - // Replace wrapping quotes - stringifiedValue = stringifiedValue - .replace(/\\"(?!$)/g, '"') - .replace(new RegExp(this.opts.quote, 'g'), this.opts.doubleQuote) - .replace(/^"/, this.opts.quote) - .replace(/"$/, this.opts.quote); - } + if (typeof value === 'string') { + if(value.includes(this.opts.quote)) { + value = value.replace(new RegExp(this.opts.quote, 'g'), this.opts.doubleQuote); + } - // Remove double backslashes - stringifiedValue = stringifiedValue - .replace(/\\\\/g, '\\'); + // This should probably be remove together with the whole strignify option + if (stringify) { + value = `${this.opts.quote}${value}${this.opts.quote}`; + } else { + value = value + .replace(new RegExp(`^${this.opts.doubleQuote}`), this.opts.quote) + .replace(new RegExp(`${this.opts.doubleQuote}$`), this.opts.quote); + } - if (this.opts.excelStrings && typeof value === 'string') { - stringifiedValue = '"="' + stringifiedValue + '""'; + if (this.opts.excelStrings) { + value = `"="${value}""`; + } } - return stringifiedValue; + return value; } /** diff --git a/lib/JSON2CSVParser.js b/lib/JSON2CSVParser.js index de128359..6d284be9 100644 --- a/lib/JSON2CSVParser.js +++ b/lib/JSON2CSVParser.js @@ -1,7 +1,7 @@ 'use strict'; const JSON2CSVBase = require('./JSON2CSVBase'); -const { flattenReducer } = require('./utils'); +const { fastJoin, flattenReducer } = require('./utils'); class JSON2CSVParser extends JSON2CSVBase { constructor(opts) { @@ -73,10 +73,10 @@ class JSON2CSVParser extends JSON2CSVBase { * @returns {String} CSV string (body) */ processData(data) { - return data - .map(row => this.processRow(row)) - .filter(row => row) // Filter empty rows - .join(this.opts.eol); + return fastJoin( + data.map(row => this.processRow(row)).filter(row => row), // Filter empty rows + this.opts.eol + ); } } diff --git a/lib/utils.js b/lib/utils.js index e2a7420e..de22b554 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -1,5 +1,9 @@ 'use strict'; +function getProp(obj, path, defaultValue) { + return obj[path] === undefined ? defaultValue : obj[path]; +} + function setProp(obj, path, value) { const pathArray = Array.isArray(path) ? path : path.split('.'); const key = pathArray[0]; @@ -18,7 +22,25 @@ function flattenReducer(acc, arr) { } } +function fastJoin(arr, separator) { + let isFirst = true; + return arr.reduce((acc, elem) => { + if (elem === null || elem === undefined) { + elem = ''; + } + + if (isFirst) { + isFirst = false; + return `${elem}`; + } + + return `${acc}${separator}${elem}`; + }, ''); +} + module.exports = { + getProp, setProp, + fastJoin, flattenReducer }; \ No newline at end of file diff --git a/test/CLI.js b/test/CLI.js index 2b3604c1..bc49ac41 100644 --- a/test/CLI.js +++ b/test/CLI.js @@ -560,8 +560,8 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { const csv = stdout; t.equal(csv, [ '"a string"', - '"with a \ndescription\\n and\na new line"', - '"with a \r\ndescription and\r\nanother new line"' + '"with a \u2028description\\n and\na new line"', + '"with a \u2029\u2028description and\r\nanother new line"' ].join('\r\n')); t.end(); }); diff --git a/test/JSON2CSVAsyncParser.js b/test/JSON2CSVAsyncParser.js index d7e02a1a..db0cff73 100644 --- a/test/JSON2CSVAsyncParser.js +++ b/test/JSON2CSVAsyncParser.js @@ -745,8 +745,8 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) = parser.fromInput(jsonFixtures.escapeEOL()).promise() .then(csv => t.equal(csv, [ '"a string"', - '"with a \ndescription\\n and\na new line"', - '"with a \r\ndescription and\r\nanother new line"' + '"with a \u2028description\\n and\na new line"', + '"with a \u2029\u2028description and\r\nanother new line"' ].join('\r\n'))) .catch(err => t.notOk(true, err.message)) .then(() => t.end()); diff --git a/test/JSON2CSVParser.js b/test/JSON2CSVParser.js index 07e6103f..9e62f0be 100644 --- a/test/JSON2CSVParser.js +++ b/test/JSON2CSVParser.js @@ -657,8 +657,8 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => { t.equal(csv, [ '"a string"', - '"with a \ndescription\\n and\na new line"', - '"with a \r\ndescription and\r\nanother new line"' + '"with a \u2028description\\n and\na new line"', + '"with a \u2029\u2028description and\r\nanother new line"' ].join('\r\n')); t.end(); }); diff --git a/test/JSON2CSVTransform.js b/test/JSON2CSVTransform.js index e384bac5..3a5c23d8 100644 --- a/test/JSON2CSVTransform.js +++ b/test/JSON2CSVTransform.js @@ -1081,8 +1081,8 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) = .on('end', () => { t.equal(csv, [ '"a string"', - '"with a \ndescription\\n and\na new line"', - '"with a \r\ndescription and\r\nanother new line"' + '"with a \u2028description\\n and\na new line"', + '"with a \u2029\u2028description and\r\nanother new line"' ].join('\r\n')); t.end(); })