Performance improvements.

zemirco · Apr 1, 2019 · 260fe53 · 260fe53
1 parent d56c752
commit 260fe53
Show file tree

Hide file tree

Showing 9 changed files with 91 additions and 78 deletions.
diff --git a/lib/JSON2CSVAsyncParser.js b/lib/JSON2CSVAsyncParser.js
@@ -2,6 +2,7 @@
 
 const { Transform } = require('stream');
 const JSON2CSVTransform = require('./JSON2CSVTransform');
+const { fastJoin } = require('./utils');
 
 class JSON2CSVAsyncParser {
   constructor(opts, transformOpts) {
@@ -40,10 +41,10 @@ class JSON2CSVAsyncParser {
 
   promise() {
     return new Promise((resolve, reject) => {
-      let csv = '';
+      let csvBuffer = [];
       this.processor
-        .on('data', chunk => (csv += chunk.toString()))
-        .on('finish', () => resolve(csv))
+        .on('data', chunk => csvBuffer.push(chunk.toString()))
+        .on('finish', () => resolve(fastJoin(csvBuffer, '')))
         .on('error', err => reject(err));
     });
   }

diff --git a/lib/JSON2CSVBase.js b/lib/JSON2CSVBase.js
@@ -2,7 +2,7 @@
 
 const os = require('os');
 const lodashGet = require('lodash.get');
-const { setProp, flattenReducer } = require('./utils');
+const { getProp, setProp, fastJoin, flattenReducer } = require('./utils');
 
 class JSON2CSVBase {
   constructor(opts) {
@@ -29,7 +29,7 @@ class JSON2CSVBase {
       : '"';
     processedOpts.doubleQuote = typeof processedOpts.doubleQuote === 'string'
       ? processedOpts.doubleQuote
-      : Array(3).join(processedOpts.quote);
+      : processedOpts.quote + processedOpts.quote;
     processedOpts.header = processedOpts.header !== false;
     processedOpts.includeEmptyRows = processedOpts.includeEmptyRows || false;
     processedOpts.withBOM = processedOpts.withBOM || false;
@@ -49,7 +49,9 @@ class JSON2CSVBase {
       if (typeof fieldInfo === 'string') {
         return {
           label: fieldInfo,
-          value: row => lodashGet(row, fieldInfo, this.opts.defaultValue),
+          value: (fieldInfo.includes('.') || fieldInfo.includes('['))
+            ? row => lodashGet(row, fieldInfo, this.opts.defaultValue)
+            : row => getProp(row, fieldInfo, this.opts.defaultValue),
           stringify: true,
         };
       }
@@ -62,7 +64,9 @@ class JSON2CSVBase {
         if (typeof fieldInfo.value === 'string') {
           return {
             label: fieldInfo.label || fieldInfo.value,
-            value: row => lodashGet(row, fieldInfo.value, defaultValue),
+            value: (fieldInfo.value.includes('.') || fieldInfo.value.includes('['))
+              ? row => lodashGet(row, fieldInfo.value, defaultValue)
+              : row => getProp(row, fieldInfo.value, defaultValue),
             stringify: fieldInfo.stringify !== undefined ? fieldInfo.stringify : true,
           };
         }
@@ -93,9 +97,10 @@ class JSON2CSVBase {
    * @returns {String} titles as a string
    */
   getHeader() {
-    return this.opts.fields
-      .map(fieldInfo => this.processValue(fieldInfo.label, true))
-      .join(this.opts.delimiter);
+    return fastJoin(
+      this.opts.fields.map(fieldInfo => this.processValue(fieldInfo.label, true)),
+      this.opts.delimiter
+    );
   }
 
   memoizePreprocessRow() {
@@ -139,15 +144,20 @@ class JSON2CSVBase {
    * @returns {String} CSV string (row)
    */
   processRow(row) {
-    if (!row
-        || (Object.getOwnPropertyNames(row).length === 0
-          && !this.opts.includeEmptyRows)) {
+    if (!row) {
       return undefined;
     }
 
-    return this.opts.fields
-      .map(fieldInfo => this.processCell(row, fieldInfo))
-      .join(this.opts.delimiter);
+    const processedRow = this.opts.fields.map(fieldInfo => this.processCell(row, fieldInfo));
+
+    if (!this.opts.includeEmptyRows && processedRow.every(field => field === undefined)) {
+      return undefined;
+    }
+
+    return fastJoin(
+      processedRow,
+      this.opts.delimiter
+    );
   }
 
   /**
@@ -173,61 +183,39 @@ class JSON2CSVBase {
       return undefined;
     }
 
-    const isValueString = typeof value === 'string';
-    if (isValueString) {
-      value = value
-        .replace(/\n/g, '\u2028')
-        .replace(/\r/g, '\u2029')
-        .replace(/\t/g, '\u21E5');
-    }
-
-    //JSON.stringify('\\') results in a string with two backslash
-    //characters in it. I.e. '\\\\'.
-    let stringifiedValue = (stringify
-      ? JSON.stringify(value)
-      : value);
-
-    if (typeof value === 'object' && !/^".*"$/.test(stringifiedValue)) {
-      // Stringify object that are not stringified to a
-      // JSON string (like Date) to escape commas, quotes, etc.
-      stringifiedValue = JSON.stringify(stringifiedValue);
-    }
+    const valueType = typeof value;
+    if (valueType !== 'boolean' && valueType !== 'number' && valueType !== 'string') {
+      value = JSON.stringify(value);
 
-    if (stringifiedValue === undefined) {
-      return undefined;
-    }
+      if (value === undefined) {
+        return undefined;
+      }
 
-    if (isValueString) {
-      stringifiedValue = stringifiedValue
-        .replace(/\u2028/g, '\n')
-        .replace(/\u2029/g, '\r')
-        .replace(/\u21E5/g, '\t');
+      if (value[0] === '"') {
+        value = value.replace(/^"(.+)"$/,'$1');
+      }
     }
 
-    if (this.opts.quote === '"') {
-      // Replace automatically escaped single quotes by doubleQuotes
-      stringifiedValue = stringifiedValue
-        .replace(/\\"(?!$)/g, this.opts.doubleQuote);
-    } else {
-      // Unescape double quotes ('"')
-      // Replace single quote with double quote
-      // Replace wrapping quotes
-      stringifiedValue = stringifiedValue
-        .replace(/\\"(?!$)/g, '"')
-        .replace(new RegExp(this.opts.quote, 'g'), this.opts.doubleQuote)
-        .replace(/^"/, this.opts.quote)
-        .replace(/"$/, this.opts.quote);
-    }
+    if (typeof value === 'string') {
+      if(value.includes(this.opts.quote)) {
+        value = value.replace(new RegExp(this.opts.quote, 'g'), this.opts.doubleQuote);
+      }
 
-    // Remove double backslashes
-    stringifiedValue = stringifiedValue
-      .replace(/\\\\/g, '\\');
+      // This should probably be remove together with the whole strignify option
+      if (stringify) {
+        value = `${this.opts.quote}${value}${this.opts.quote}`;
+      } else {
+        value = value
+          .replace(new RegExp(`^${this.opts.doubleQuote}`), this.opts.quote)
+          .replace(new RegExp(`${this.opts.doubleQuote}$`), this.opts.quote);
+      }
 
-    if (this.opts.excelStrings && typeof value === 'string') {
-      stringifiedValue = '"="' + stringifiedValue + '""';
+      if (this.opts.excelStrings) {
+        value = `"="${value}""`;
+      }
     }
 
-    return stringifiedValue;
+    return value;
   }
 
   /**

diff --git a/lib/JSON2CSVParser.js b/lib/JSON2CSVParser.js
@@ -1,7 +1,7 @@
 'use strict';
 
 const JSON2CSVBase = require('./JSON2CSVBase');
-const { flattenReducer } = require('./utils');
+const { fastJoin, flattenReducer } = require('./utils');
 
 class JSON2CSVParser extends JSON2CSVBase {
   constructor(opts) {
@@ -73,10 +73,10 @@ class JSON2CSVParser extends JSON2CSVBase {
    * @returns {String} CSV string (body)
    */
   processData(data) {
-    return data
-      .map(row => this.processRow(row))
-      .filter(row => row) // Filter empty rows
-      .join(this.opts.eol);
+    return fastJoin(
+      data.map(row => this.processRow(row)).filter(row => row), // Filter empty rows
+      this.opts.eol
+    );
   }
 }
 

diff --git a/lib/JSON2CSVTransform.js b/lib/JSON2CSVTransform.js
@@ -184,7 +184,9 @@ class JSON2CSVTransform extends Transform {
    * @param {Object} data JSON object to be converted in a CSV row
    */
   pushLine(data) {
-    const processedData = this.preprocessRow(data);
+    const processedData = ((!this.opts.unwind || !this.opts.unwind.length) && !this.opts.flatten)
+      ? [data]
+      : this.preprocessRow(data);
 
     if (!this._hasWritten) {
       this.opts.fields = this.opts.fields || this.preprocessFieldsInfo(Object.keys(processedData[0]));
@@ -195,7 +197,7 @@ class JSON2CSVTransform extends Transform {
       const line = this.processRow(row, this.opts);
       if (line === undefined) return;
       this.emit('line', line);
-      this.push(this._hasWritten ? this.opts.eol + line : line);
+      this.push(this._hasWritten ? `${this.opts.eol}${line}` : line);
       this._hasWritten = true;
     });
   }

diff --git a/lib/utils.js b/lib/utils.js
@@ -1,5 +1,9 @@
 'use strict';
 
+function getProp(obj, path, defaultValue) {
+  return obj[path] === undefined ? defaultValue : obj[path];
+}
+
 function setProp(obj, path, value) {
   const pathArray = Array.isArray(path) ? path : path.split('.');
   const key = pathArray[0];
@@ -18,7 +22,25 @@ function flattenReducer(acc, arr) {
   }
 }
 
+function fastJoin(arr, separator) {
+  let isFirst = true;
+  return arr.reduce((acc, elem) => {
+    if (elem === null || elem === undefined) {
+      elem = '';
+    }
+
+    if (isFirst) {
+      isFirst = false;
+      return `${elem}`;
+    }
+
+    return `${acc}${separator}${elem}`;
+  }, '');
+}
+
 module.exports = {
+  getProp,
   setProp,
+  fastJoin,
   flattenReducer
 };
diff --git a/test/CLI.js b/test/CLI.js
@@ -560,8 +560,8 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
       const csv = stdout;
       t.equal(csv, [
       '"a string"',
-      '"with a \ndescription\\n and\na new line"',
-      '"with a \r\ndescription and\r\nanother new line"'
+      '"with a \u2028description\\n and\na new line"',
+      '"with a \u2029\u2028description and\r\nanother new line"'
     ].join('\r\n'));
       t.end();
     });

diff --git a/test/JSON2CSVAsyncParser.js b/test/JSON2CSVAsyncParser.js
@@ -716,8 +716,8 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) =
     parser.fromInput(jsonFixtures.escapeEOL()).promise()
       .then(csv => t.equal(csv, [
       '"a string"',
-      '"with a \ndescription\\n and\na new line"',
-      '"with a \r\ndescription and\r\nanother new line"'
+      '"with a \u2028description\\n and\na new line"',
+      '"with a \u2029\u2028description and\r\nanother new line"'
     ].join('\r\n')))
       .catch(err => t.notOk(true, err.message))
       .then(() => t.end());

diff --git a/test/JSON2CSVParser.js b/test/JSON2CSVParser.js
@@ -657,8 +657,8 @@ module.exports = (testRunner, jsonFixtures, csvFixtures) => {
 
     t.equal(csv, [
       '"a string"',
-      '"with a \ndescription\\n and\na new line"',
-      '"with a \r\ndescription and\r\nanother new line"'
+      '"with a \u2028description\\n and\na new line"',
+      '"with a \u2029\u2028description and\r\nanother new line"'
     ].join('\r\n'));
     t.end();
   });

diff --git a/test/JSON2CSVTransform.js b/test/JSON2CSVTransform.js
@@ -1081,8 +1081,8 @@ module.exports = (testRunner, jsonFixtures, csvFixtures, inMemoryJsonFixtures) =
       .on('end', () => {
         t.equal(csv, [
       '"a string"',
-      '"with a \ndescription\\n and\na new line"',
-      '"with a \r\ndescription and\r\nanother new line"'
+      '"with a \u2028description\\n and\na new line"',
+      '"with a \u2029\u2028description and\r\nanother new line"'
     ].join('\r\n'));
         t.end();
       })