diff --git a/README.md b/README.md
index dd6f9449..a97e09cc 100644
--- a/README.md
+++ b/README.md
@@ -89,7 +89,7 @@ from MySQL to PostgreSQL as easy and smooth as possible.
Note: "logs_directory" will be created during script execution.
VERSION
-Current version is 4.0.0
+
Current version is 4.0.1
(major version . improvements . bug fixes)
KNOWN ISSUES
diff --git a/config/config.json b/config/config.json
index 000a33a0..16b52c46 100644
--- a/config/config.json
+++ b/config/config.json
@@ -13,7 +13,7 @@
"host" : "localhost",
"port" : 3306,
"database" : "test_db",
- "charset" : "UTF8",
+ "charset" : "utf8mb4",
"user" : "root",
"password" : "0123456789"
},
@@ -82,7 +82,7 @@
"exclude_tables": [],
"include_tables_description": [
- "List (Array) of tables, that will not be migrated.",
+ "List (Array) of tables, that will be migrated.",
"By default, nmig will migrate all tables."
],
"include_tables": [],
diff --git a/config/test_config.json b/config/test_config.json
index fc8cb29a..c01864c0 100644
--- a/config/test_config.json
+++ b/config/test_config.json
@@ -11,7 +11,7 @@
"source" : {
"host" : "localhost",
"port" : 3306,
- "charset" : "UTF8",
+ "charset" : "utf8mb4",
"database" : "nmig_test_db",
"user" : "root",
"password" : "0123456789"
diff --git a/package-lock.json b/package-lock.json
index 252eb9d4..fcfb1ba5 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,22 +1,22 @@
{
"name": "nmig",
- "version": "4.0.0",
+ "version": "4.0.1",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"@types/mysql": {
- "version": "2.15.5",
- "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.5.tgz",
- "integrity": "sha512-4QAISTUGZbcFh7bqdndo08xRdES5OTU+JODy8VCZbe1yiXyGjqw1H83G43XjQ3IbC10wn9xlGd44A5RXJwNh0Q==",
+ "version": "2.15.6",
+ "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.6.tgz",
+ "integrity": "sha512-PJBY2R3jGJwGrmFgGAJ+1nj4S/PLkF6nT+HvUygniq9ZcVht0mTH1TLAjjyfIXf9FfrELs8mbqOrWa/Tn89NCA==",
"dev": true,
"requires": {
"@types/node": "*"
}
},
"@types/node": {
- "version": "11.13.4",
- "resolved": "https://registry.npmjs.org/@types/node/-/node-11.13.4.tgz",
- "integrity": "sha512-+rabAZZ3Yn7tF/XPGHupKIL5EcAbrLxnTr/hgQICxbeuAfWtT0UZSfULE+ndusckBItcv4o6ZeOJplQikVcLvQ==",
+ "version": "12.6.8",
+ "resolved": "https://registry.npmjs.org/@types/node/-/node-12.6.8.tgz",
+ "integrity": "sha512-aX+gFgA5GHcDi89KG5keey2zf0WfZk/HAQotEamsK2kbey+8yGKcson0hbK8E+v0NArlCJQCqMP161YhV6ZXLg==",
"dev": true
},
"@types/pg": {
@@ -54,9 +54,9 @@
"dev": true
},
"bignumber.js": {
- "version": "4.1.0",
- "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-4.1.0.tgz",
- "integrity": "sha512-eJzYkFYy9L4JzXsbymsFn3p54D+llV27oTQ+ziJG7WFRheJcNZilgVXMG0LoZtlQSKBsJdWtLFqOD0u+U0jZKA=="
+ "version": "7.2.1",
+ "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz",
+ "integrity": "sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ=="
},
"brace-expansion": {
"version": "1.1.11",
@@ -152,9 +152,9 @@
"dev": true
},
"glob": {
- "version": "7.1.3",
- "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz",
- "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==",
+ "version": "7.1.4",
+ "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz",
+ "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==",
"dev": true,
"requires": {
"fs.realpath": "^1.0.0",
@@ -252,11 +252,11 @@
"dev": true
},
"mysql": {
- "version": "2.16.0",
- "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.16.0.tgz",
- "integrity": "sha512-dPbN2LHonQp7D5ja5DJXNbCLe/HRdu+f3v61aguzNRQIrmZLOeRoymBYyeThrR6ug+FqzDL95Gc9maqZUJS+Gw==",
+ "version": "2.17.1",
+ "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.17.1.tgz",
+ "integrity": "sha512-7vMqHQ673SAk5C8fOzTG2LpPcf3bNt0oL3sFpxPEEFp1mdlDcrLK0On7z8ZYKaaHrHwNcQ/MTUz7/oobZ2OyyA==",
"requires": {
- "bignumber.js": "4.1.0",
+ "bignumber.js": "7.2.1",
"readable-stream": "2.3.6",
"safe-buffer": "5.1.2",
"sqlstring": "2.3.1"
@@ -301,9 +301,9 @@
"dev": true
},
"pg": {
- "version": "7.9.0",
- "resolved": "https://registry.npmjs.org/pg/-/pg-7.9.0.tgz",
- "integrity": "sha512-GkzteBFpsIoIBCSuomqik3IGvhqAtTr32jclR24RmUg170Jrn6ypwR97YalFHrsE1iaW8T0aAH13dmij8QUQ0g==",
+ "version": "7.12.0",
+ "resolved": "https://registry.npmjs.org/pg/-/pg-7.12.0.tgz",
+ "integrity": "sha512-q54Ic0oBXfDZMwheP8ALeUX32TUXvF7SNgAlZjyhkDuFCJkQCgcLBz0Be5uOrAj3ljSok/CI9lRbYzEko0z1Zw==",
"requires": {
"buffer-writer": "2.0.0",
"packet-reader": "1.0.0",
@@ -320,9 +320,9 @@
"integrity": "sha1-2hhHsglA5C7hSSvq9l1J2RskXfc="
},
"pg-copy-streams": {
- "version": "2.2.0",
- "resolved": "https://registry.npmjs.org/pg-copy-streams/-/pg-copy-streams-2.2.0.tgz",
- "integrity": "sha512-w1gy/KAD49vaCCUfqcjh4fy1V4TF6fvtZ+EaRAU0QKtADMxihb+qYRgbWTDbi6Lh6v31CsWC0ru8vCDHofrRQQ=="
+ "version": "2.2.2",
+ "resolved": "https://registry.npmjs.org/pg-copy-streams/-/pg-copy-streams-2.2.2.tgz",
+ "integrity": "sha512-mjSqs6hrsRhBojCuY2hxyg48B+3th5ARBjMxBCEisIqBvdRD0g5ETdbts20TzrOfha8ueJQOmQCJCprtczJtGQ=="
},
"pg-int8": {
"version": "1.0.1",
@@ -330,9 +330,9 @@
"integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw=="
},
"pg-pool": {
- "version": "2.0.6",
- "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-2.0.6.tgz",
- "integrity": "sha512-hod2zYQxM8Gt482q+qONGTYcg/qVcV32VHVPtktbBJs0us3Dj7xibISw0BAAXVMCzt8A/jhfJvpZaxUlqtqs0g=="
+ "version": "2.0.7",
+ "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-2.0.7.tgz",
+ "integrity": "sha512-UiJyO5B9zZpu32GSlP0tXy8J2NsJ9EFGFfz5v6PSbdz/1hBLX1rNiiy5+mAm5iJJYwfCv4A0EBcQLGWwjbpzZw=="
},
"pg-types": {
"version": "2.0.1",
@@ -378,9 +378,9 @@
}
},
"process-nextick-args": {
- "version": "2.0.0",
- "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
- "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw=="
+ "version": "2.0.1",
+ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
+ "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"readable-stream": {
"version": "2.3.6",
@@ -397,9 +397,9 @@
}
},
"resolve": {
- "version": "1.10.0",
- "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.10.0.tgz",
- "integrity": "sha512-3sUr9aq5OfSg2S9pNtPA9hL1FVEAjvfOC4leW0SNf/mpnaakz2a9femSd6LqAww2RaFctwyf1lCqnTHuF1rxDg==",
+ "version": "1.11.1",
+ "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.1.tgz",
+ "integrity": "sha512-vIpgF6wfuJOZI7KKKSP+HmiKggadPQAdsp5HiC1mvqnfp0gF1vdwgBWZIdrVft9pgqoMFQN+R7BSWZiBxx+BBw==",
"dev": true,
"requires": {
"path-parse": "^1.0.6"
@@ -457,24 +457,32 @@
}
},
"tape": {
- "version": "4.10.1",
- "resolved": "https://registry.npmjs.org/tape/-/tape-4.10.1.tgz",
- "integrity": "sha512-G0DywYV1jQeY3axeYnXUOt6ktnxS9OPJh97FGR3nrua8lhWi1zPflLxcAHavZ7Jf3qUfY7cxcVIVFa4mY2IY1w==",
+ "version": "4.11.0",
+ "resolved": "https://registry.npmjs.org/tape/-/tape-4.11.0.tgz",
+ "integrity": "sha512-yixvDMX7q7JIs/omJSzSZrqulOV51EC9dK8dM0TzImTIkHWfe2/kFyL5v+d9C+SrCMaICk59ujsqFAVidDqDaA==",
"dev": true,
"requires": {
"deep-equal": "~1.0.1",
"defined": "~1.0.0",
"for-each": "~0.3.3",
"function-bind": "~1.1.1",
- "glob": "~7.1.3",
+ "glob": "~7.1.4",
"has": "~1.0.3",
- "inherits": "~2.0.3",
+ "inherits": "~2.0.4",
"minimist": "~1.2.0",
"object-inspect": "~1.6.0",
- "resolve": "~1.10.0",
+ "resolve": "~1.11.1",
"resumer": "~0.0.0",
"string.prototype.trim": "~1.1.2",
"through": "~2.3.8"
+ },
+ "dependencies": {
+ "inherits": {
+ "version": "2.0.4",
+ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
+ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
+ "dev": true
+ }
}
},
"through": {
@@ -483,9 +491,9 @@
"integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
},
"typescript": {
- "version": "3.4.3",
- "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.4.3.tgz",
- "integrity": "sha512-FFgHdPt4T/duxx6Ndf7hwgMZZjZpB+U0nMNGVCYPq0rEzWKjEDobm4J6yb3CS7naZ0yURFqdw9Gwc7UOh/P9oQ==",
+ "version": "3.5.3",
+ "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.5.3.tgz",
+ "integrity": "sha512-ACzBtm/PhXBDId6a6sDJfroT2pOWt/oOnk4/dElG5G33ZL776N3Y6/6bKZJBFpd+b05F3Ct9qDjMeJmRWtE2/g==",
"dev": true
},
"util-deprecate": {
@@ -500,9 +508,9 @@
"dev": true
},
"xtend": {
- "version": "4.0.1",
- "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
- "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68="
+ "version": "4.0.2",
+ "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
+ "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="
}
}
}
diff --git a/package.json b/package.json
index 6d182e49..a62fe880 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "nmig",
- "version": "4.0.0",
+ "version": "4.0.1",
"description": "The database migration app",
"author": "Anatoly Khaytovich",
"license": "GPL-3.0",
@@ -12,17 +12,17 @@
"node": ">=8.0.0"
},
"dependencies": {
- "mysql": "^2.16.0",
- "pg": "^7.9.0",
- "pg-copy-streams": "^2.2.0"
+ "mysql": "^2.17.1",
+ "pg": "^7.12.0",
+ "pg-copy-streams": "^2.2.2"
},
"devDependencies": {
- "@types/mysql": "^2.15.5",
- "@types/node": "^11.13.4",
+ "@types/mysql": "^2.15.6",
+ "@types/node": "^12.6.8",
"@types/pg": "^7.4.14",
"@types/tape": "^4.2.33",
- "tape": "^4.10.1",
- "typescript": "^3.4.3"
+ "tape": "^4.11.0",
+ "typescript": "^3.5.3"
},
"scripts": {
"build": "tsc",
diff --git a/src/DataPipeManager.ts b/src/DataPipeManager.ts
index 2a45d345..c0f1c153 100644
--- a/src/DataPipeManager.ts
+++ b/src/DataPipeManager.ts
@@ -61,7 +61,7 @@ function getSmallestDataChunkSizeInMb(conversion: Conversion): number {
/**
* Creates an array of indexes, that point to data chunks, that will be processed during current COPY operation.
*/
-function fillBandwidth(conversion: Conversion): number[] {
+async function fillBandwidth(conversion: Conversion): Promise {
const dataChunkIndexes: number[] = [];
// Loop through the data pool from the beginning to the end.
@@ -113,7 +113,7 @@ function fillBandwidth(conversion: Conversion): number[] {
if (firstUnprocessedChunkIndex === -1) {
const msg: string = 'Something went wrong with DataPipeManager.';
- log(conversion, msg, undefined, true);
+ await generateError(conversion, msg);
process.exit();
}
@@ -135,7 +135,8 @@ async function pipeData(conversion: Conversion, dataLoaderPath: string, options:
return processConstraints(conversion);
}
- const chunksToLoad: any[] = fillBandwidth(conversion).map((index: number) => conversion._dataPool[index]);
+ const chunksIndexes: number[] = await fillBandwidth(conversion);
+ const chunksToLoad: any[] = chunksIndexes.map((index: number) => conversion._dataPool[index]);
const loaderProcess: ChildProcess = fork(dataLoaderPath, options);
loaderProcess.on('message', async (signal: any) => {
diff --git a/src/FsOps.ts b/src/FsOps.ts
index 2f43eebd..85fada29 100644
--- a/src/FsOps.ts
+++ b/src/FsOps.ts
@@ -29,7 +29,7 @@ export function generateError(conversion: Conversion, message: string, sql: stri
return new Promise(resolve => {
message += `\n\n\tSQL: ${sql}\n\n`;
const buffer: Buffer = Buffer.from(message, conversion._encoding);
- log(conversion, message, undefined, true);
+ log(conversion, message, undefined);
fs.open(conversion._errorLogsPath, 'a', conversion._0777, (error: Error, fd: number) => {
if (error) {
@@ -48,13 +48,10 @@ export function generateError(conversion: Conversion, message: string, sql: stri
* Writes given log to the "/all.log" file.
* If necessary, writes given log to the "/{tableName}.log" file.
*/
-export function log(conversion: Conversion, log: string | NodeJS.ErrnoException, tableLogPath?: string, isErrorLog?: boolean): void {
+export function log(conversion: Conversion, log: string | NodeJS.ErrnoException, tableLogPath?: string): void {
+ console.log(log);
const buffer: Buffer = Buffer.from(`${ log }\n\n`, conversion._encoding);
- if (!isErrorLog) {
- console.log(log);
- }
-
fs.open(conversion._allLogsPath, 'a', conversion._0777, (error: Error, fd: number) => {
if (!error) {
fs.write(fd, buffer, 0, buffer.length, null, () => {
@@ -124,28 +121,37 @@ export function readExtraConfig(config: any, baseDir: string): Promise {
/**
* Creates logs directory.
*/
-export function createLogsDirectory(conversion: Conversion): Promise {
- return new Promise(resolve => {
- const logTitle: string = 'FsOps::createLogsDirectory';
- console.log(`\t--[${ logTitle }] Creating logs directory...`);
+export async function createLogsDirectory(conversion: Conversion): Promise {
+ const logTitle: string = 'FsOps::createLogsDirectory';
+ await createDirectory(conversion, conversion._logsDirPath, logTitle);
+ await createDirectory(conversion, conversion._notCreatedViewsPath, logTitle);
+ return conversion;
+}
+
+/**
+ * Creates a directory at the specified path.
+ */
+function createDirectory(conversion: Conversion, directoryPath: string, logTitle: string): Promise {
+ return new Promise(resolve => {
+ console.log(`\t--[${ logTitle }] Creating directory ${ directoryPath }...`);
- fs.stat(conversion._logsDirPath, (directoryDoesNotExist: Error, stat: fs.Stats) => {
+ fs.stat(directoryPath, (directoryDoesNotExist: Error, stat: fs.Stats) => {
if (directoryDoesNotExist) {
- fs.mkdir(conversion._logsDirPath, conversion._0777, e => {
+ fs.mkdir(directoryPath, conversion._0777, e => {
if (e) {
- console.log(`\t--[${ logTitle }] Cannot perform a migration due to impossibility to create "logs_directory": ${ conversion._logsDirPath }`);
+ console.log(`\t--[${ logTitle }] Cannot perform a migration due to impossibility to create directory: ${ directoryPath }`);
process.exit();
} else {
- log(conversion, '\t--[logTitle] Logs directory is created...');
- resolve(conversion);
+ log(conversion, `\t--[${ logTitle }] Directory ${ directoryPath } is created...`);
+ resolve();
}
});
} else if (!stat.isDirectory()) {
console.log(`\t--[${ logTitle }] Cannot perform a migration due to unexpected error`);
process.exit();
} else {
- log(conversion, `\t--[${ logTitle }] Logs directory already exists...`);
- resolve(conversion);
+ log(conversion, `\t--[${ logTitle }] Directory ${ directoryPath } already exists...`);
+ resolve();
}
});
});
diff --git a/src/ViewGenerator.ts b/src/ViewGenerator.ts
index 0142124e..85551aa3 100644
--- a/src/ViewGenerator.ts
+++ b/src/ViewGenerator.ts
@@ -51,55 +51,19 @@ function generateView(schema: string, viewName: string, mysqlViewCode: string):
*/
function logNotCreatedView(conversion: Conversion, viewName: string, sql: string): Promise {
return new Promise((resolve) => {
- fs.stat(conversion._notCreatedViewsPath, (directoryDoesNotExist: NodeJS.ErrnoException, stat: Stats) => {
- if (directoryDoesNotExist) {
- fs.mkdir(conversion._notCreatedViewsPath, conversion._0777, (e: NodeJS.ErrnoException) => {
- if (e) {
- log(conversion, `\t--[logNotCreatedView] ${ e }`);
- return resolve();
- }
-
- log(conversion, '\t--[logNotCreatedView] "not_created_views" directory is created...');
- // "not_created_views" directory is created. Can write the log...
- fs.open(
- path.join(conversion._notCreatedViewsPath, `${ viewName }.sql`),
- 'w',
- conversion._0777,
- (error: NodeJS.ErrnoException, fd: number
- ) => {
- if (error) {
- log(conversion, error);
- return resolve();
- }
-
- const buffer = Buffer.from(sql, conversion._encoding);
- fs.write(fd, buffer, 0, buffer.length, null, () => {
- fs.close(fd, () => {
- return resolve();
- });
- });
- });
- });
- } else if (!stat.isDirectory()) {
- log(conversion, '\t--[logNotCreatedView] Cannot write the log due to unexpected error');
+ const viewFilePath: string = path.join(conversion._notCreatedViewsPath, `${ viewName }.sql`);
+ fs.open(viewFilePath,'w', conversion._0777, (error: NodeJS.ErrnoException, fd: number) => {
+ if (error) {
+ log(conversion, error);
return resolve();
- } else {
- // "not_created_views" directory already exists. Can write the log...
- const viewFilePath: string = path.join(conversion._notCreatedViewsPath, `${ viewName }.sql`);
- fs.open(viewFilePath,'w', conversion._0777, (error: NodeJS.ErrnoException, fd: number) => {
- if (error) {
- log(conversion, error);
- return resolve();
- }
+ }
- const buffer = Buffer.from(sql, conversion._encoding);
- fs.write(fd, buffer, 0, buffer.length, null, () => {
- fs.close(fd, () => {
- return resolve();
- });
- });
+ const buffer = Buffer.from(sql, conversion._encoding);
+ fs.write(fd, buffer, 0, buffer.length, null, () => {
+ fs.close(fd, () => {
+ return resolve();
});
- }
+ });
});
});
}
diff --git a/test/test_schema.sql b/test/test_schema.sql
index 91a016e2..fad8583c 100644
--- a/test/test_schema.sql
+++ b/test/test_schema.sql
@@ -65,3 +65,15 @@ CREATE TABLE IF NOT EXISTS `table_c`(
CONSTRAINT `table_c_table_a_id_foreign` FOREIGN KEY(`table_a_id`) REFERENCES `table_a`(`id_test_sequence`) ON UPDATE RESTRICT ON DELETE CASCADE,
CONSTRAINT `table_c_table_b_id_1_2_foreign` FOREIGN KEY(`table_b_id1`, `table_b_id2`) REFERENCES `table_b`(`id1`, `id2`)
) ENGINE = innodb;
+
+CREATE TABLE IF NOT EXISTS `category_company`(
+ `id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT,
+ `company_id` INT(10) UNSIGNED NOT NULL,
+ `category_id` INT(10) UNSIGNED NOT NULL,
+ PRIMARY KEY (`id`)
+)
+CHARACTER SET = utf8mb4
+COLLATE = utf8mb4_unicode_ci
+ENGINE = innodb;
+
+INSERT INTO `category_company`(`company_id`, `category_id`) VALUES(111, 2), (333, 2);