Feature/small improvements (#105)

* small improvements

* updated dependencies

* small improvements, following upgrade to TS5

* added comments regarding connections configuration

README.md

@@ -109,7 +109,7 @@ Or, if you have moved <code>config</code> folder out from Nmig's directory:<br /
 <br /><b>Note:</b> "logs_directory" will be created during script execution.</p>
 
 <h3>VERSION</h3>
-<p>Current version is 5.6.0<br />
+<p>Current version is 5.7.0<br />
 
 <h3>LICENSE</h3>
 <p>NMIG is available under "GNU GENERAL PUBLIC LICENSE" (v. 3) <br />

config/config.json

@@ -8,10 +8,11 @@
         "Connection parameters to your MySQL database",
         "Please ensure, that you have defined all parameters properly.",
         "Ensure, that details like 'charset' are included (if necessary).",
-        "Notice, any parameter, acceptable by the node mysql module can be placed here."
+        "Note, any parameter, acceptable by the node mysql module can be placed here.",
+        "Check the 'source.host' parameter, that can be either '127.0.0.1' or 'localhost', depends on your setup."
     ],
     "source" : {
-        "host"             : "localhost",
+        "host"             : "127.0.0.1",
         "port"             : 3306,
         "database"         : "test_db",
         "charset"          : "utf8mb4",
@@ -24,10 +25,11 @@
         "Connection parameters to your PostgreSQL database",
         "Please ensure, that you have defined all parameters properly.",
         "Ensure, that details like 'charset' are included (if necessary).",
-        "Notice, any parameter, acceptable by the node pg module can be placed here."
+        "Note, any parameter, acceptable by the node pg module can be placed here.",
+        "Check the 'target.host' parameter, that can be either '127.0.0.1' or 'localhost', depends on your setup."
     ],
     "target" : {
-        "host"     : "localhost",
+        "host"     : "127.0.0.1",
         "port"     : 5432,
         "database" : "test_db",
         "charset"  : "UTF8",
@@ -48,7 +50,7 @@
         "Acceptable values:",
         "1. 'DEFAULT' - when set to 'DEFAULT', Nmig will run 2 data-loader processes.",
         "2. Any positive integer.",
-        "Notice:",
+        "Note:",
         "1.",
         "Usually, migration gets accomplished faster with only 2 data-loader processes,",
         "even if more CPU cores are available.",

package.json

@@ -1,6 +1,6 @@
 {
     "name": "nmig",
-    "version": "5.6.0",
+    "version": "5.7.0",
     "description": "The database migration app",
     "author": "Anatoly Khaytovich<[email protected]>",
     "license": "GPL-3.0",
@@ -12,19 +12,19 @@
         "node": ">=10.0.0"
     },
     "dependencies": {
-        "@types/node": "^17.0.16",
-        "@types/pg": "^8.6.4",
-        "@types/uuid": "^8.3.4",
-        "json2csv": "^5.0.6",
-        "mysql2": "^2.3.3",
-        "pg": "^8.7.3",
-        "pg-copy-streams": "^6.0.2",
-        "uuid": "^8.3.2"
+        "@types/node": "18.15.13",
+        "@types/pg": "8.6.6",
+        "@types/uuid": "9.0.1",
+        "json2csv": "5.0.7",
+        "mysql2": "3.2.3",
+        "pg": "8.10.0",
+        "pg-copy-streams": "6.0.5",
+        "uuid": "9.0.0"
     },
     "devDependencies": {
-        "@types/tape": "^4.13.2",
-        "tape": "^5.5.0",
-        "typescript": "4.5.5"
+        "@types/tape": "4.13.4",
+        "tape": "5.6.3",
+        "typescript": "5.0.4"
     },
     "scripts": {
         "build": "tsc --incremental -p tsconfig.json",

src/ColumnsDataArranger.ts

@@ -33,23 +33,17 @@ const isSpacial = (type: string): boolean => {
 /**
  * Defines if given type is one of MySQL binary types.
  */
-const isBinary = (type: string): boolean => {
-    return type.indexOf('blob') !== -1 || type.indexOf('binary') !== -1;
-};
+const isBinary = (type: string): boolean => type.indexOf('blob') !== -1 || type.indexOf('binary') !== -1;
 
 /**
  * Defines if given type is one of MySQL bit types.
  */
-const isBit = (type: string): boolean => {
-    return type.indexOf('bit') !== -1;
-};
+const isBit = (type: string): boolean => type.indexOf('bit') !== -1;
 
 /**
  * Defines if given type is one of MySQL date-time types.
  */
-const isDateTime = (type: string): boolean => {
-    return type.indexOf('timestamp') !== -1 || type.indexOf('date') !== -1;
-};
+const isDateTime = (type: string): boolean => type.indexOf('timestamp') !== -1 || type.indexOf('date') !== -1;
 
 /**
  * Defines if given type is one of MySQL numeric types.
@@ -63,16 +57,24 @@ const isNumeric = (type: string): boolean => {
         || type.indexOf('point') !== -1;
 };
 
+/**
+ * Returns type name, while dropping unnecessary parts.
+ * Sample:
+ * 1. "enum" instead of "enum('val1','val2','val3')".
+ * 2. "int" instead of "int(11)".
+ */
+const getSanitizedType = (type: string): string => type.split('(')[0];
+
 /**
  * Arranges columns data before loading.
  */
-export default (arrTableColumns: any[], mysqlVersion: string | number, encoding: Encoding): string => {
+export default (arrTableColumns: any[], mysqlVersion: number, encoding: Encoding): string => {
     let strRetVal: string = '';
     const wkbFunc: string = mysqlVersion >= 5.76 ? 'ST_AsWKB' : 'AsWKB';
 
     arrTableColumns.forEach((column: any) => {
         const field: string = column.Field;
-        const type: string  = column.Type;
+        const type: string = getSanitizedType(column.Type);
 
         if (isSpacial(type)) {
             // Apply HEX(ST_AsWKB(...)) due to the issue, described at https://bugs.mysql.com/bug.php?id=69798

src/Conversion.ts

@@ -123,7 +123,7 @@ export default class Conversion {
     /**
      * Current version of source (MySQL) db.
      */
-    public _mysqlVersion: string | number;
+    public _mysqlVersion: string;
 
     /**
      * Node-MySQL connections pool.
@@ -309,7 +309,7 @@ export default class Conversion {
      */
     private _setLogger = (): ChildProcess => {
         // A path to the FsOps.js file.
-        // !!!Notice, in runtime it points to ../dist/src/LogsProcessor.js and not FsOps.ts
+        // !!!Notice, in runtime it points to ../dist/src/LogsProcessor.js and not LogsProcessor.ts
         const loggerPath: string = path.join(__dirname, 'LogsProcessor.js');
 
         const logger: ChildProcess = fork(loggerPath)

src/DataChunksProcessor.ts

@@ -38,7 +38,12 @@ export default async (conversion: Conversion, tableName: string, haveDataChunksP
 
     const originalTableName: string = extraConfigProcessor.getTableName(conversion, tableName, true);
     const logTitle: string = 'DataChunksProcessor::default';
-    const selectFieldList: string = arrangeColumnsData(conversion._dicTables[tableName].arrTableColumns, conversion._mysqlVersion, conversion._encoding);
+    const selectFieldList: string = arrangeColumnsData(
+        conversion._dicTables[tableName].arrTableColumns,
+        +(conversion._mysqlVersion.split(".").slice(0, 2).join(".")),
+        conversion._encoding,
+    );
+
     const sqlRowsCnt: string = `SELECT COUNT(1) AS rows_count FROM \`${ originalTableName }\`;`;
     const params: IDBAccessQueryParams = {
         conversion: conversion,

src/DataPipeManager.ts

@@ -80,6 +80,7 @@ const dataPoolProcessed = (conversion: Conversion): boolean => {
 const getNumberOfSimultaneouslyRunningLoaderProcesses = (conversion: Conversion): number => {
     if (conversion._numberOfSimultaneouslyRunningLoaderProcesses !== 'DEFAULT') {
         return Math.min(
+            (os.cpus().length || 1),
             conversion._dataPool.length,
             conversion._maxEachDbConnectionPoolSize,
             <number>conversion._numberOfSimultaneouslyRunningLoaderProcesses,

src/FsOps.ts

@@ -20,7 +20,7 @@
  */
 import * as fs from 'fs';
 import * as path from 'path';
-import {ChildProcess} from 'child_process';
+import { ChildProcess } from 'child_process';
 
 import Conversion from './Conversion';
 import { LogMessage } from './LogMessage';

src/LogsProcessor.ts

@@ -19,11 +19,11 @@
  * @author Anatoly Khaytovich <[email protected]>
  */
 import * as fs from 'fs';
+import ErrnoException = NodeJS.ErrnoException;
 
 import Conversion from './Conversion';
 import { LogMessage } from './LogMessage';
 import { LogMessageType } from './LogMessageType';
-import ErrnoException = NodeJS.ErrnoException;
 
 /**
  * Writes a detailed error message to the "/errors-only.log" file.
@@ -63,7 +63,7 @@ const _generateErrorInBackground = (
  */
 const _logInBackground = (
     conversion: Conversion,
-    log: string | NodeJS.ErrnoException,
+    log: string | ErrnoException,
     tableLogPath?: string,
 ): Promise<void> => {
     return new Promise<void>(resolve => {
@@ -119,7 +119,7 @@ let conv: Conversion;
 process.on('message', async (_log: LogMessage) => {
     try {
         if (_log.type === LogMessageType.CONFIG) {
-            // Create Conversion instance, but avoid circular recursion,
+            // Create Conversion instance, but avoid recursion,
             // which might lead to redundant logger processes creation.
             const avoidLogger: boolean = true;
             conv = conv || new Conversion(_log.config, avoidLogger);

src/StructureLoader.ts

@@ -41,10 +41,10 @@ const processTableBeforeDataLoading = async (conversion: Conversion, tableName:
 /**
  * Retrieves the source db (MySQL) version.
  */
-const getMySqlVersion = async (conversion: Conversion): Promise<void> => {
+const setMySqlVersion = async (conversion: Conversion): Promise<void> => {
     const params: IDBAccessQueryParams = {
         conversion: conversion,
-        caller: 'StructureLoader::getMySqlVersion',
+        caller: 'StructureLoader::setMySqlVersion',
         sql: 'SELECT VERSION() AS mysql_version;',
         vendor: DBVendors.MYSQL,
         processExitOnError: false,
@@ -60,15 +60,15 @@ const getMySqlVersion = async (conversion: Conversion): Promise<void> => {
     const arrVersion: string[] = result.data[0].mysql_version.split('.');
     const majorVersion: string = arrVersion[0];
     const minorVersion: string = arrVersion.slice(1).join('');
-    conversion._mysqlVersion = +(`${ majorVersion }.${ minorVersion }`);
+    conversion._mysqlVersion = `${ majorVersion }.${ minorVersion }`;
 };
 
 /**
  * Loads source tables and views, that need to be migrated.
  */
 export default async (conversion: Conversion): Promise<Conversion> => {
     const logTitle: string = 'StructureLoader::default';
-    await getMySqlVersion(conversion);
+    await setMySqlVersion(conversion);
     const haveTablesLoaded: boolean = await migrationStateManager.get(conversion, 'tables_loaded');
     let sql: string = `SHOW FULL TABLES IN \`${ conversion._mySqlDbName }\` WHERE 1 = 1`;